1//===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file implements the LegalizerHelper class to legalize
10/// individual instructions and the LegalizeMachineIR wrapper pass for the
11/// primary legalization.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
16#include "llvm/CodeGen/GlobalISel/CallLowering.h"
17#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
18#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
19#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
20#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
21#include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h"
22#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
23#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24#include "llvm/CodeGen/GlobalISel/Utils.h"
25#include "llvm/CodeGen/LowLevelTypeUtils.h"
26#include "llvm/CodeGen/MachineConstantPool.h"
27#include "llvm/CodeGen/MachineFrameInfo.h"
28#include "llvm/CodeGen/MachineRegisterInfo.h"
29#include "llvm/CodeGen/RuntimeLibcallUtil.h"
30#include "llvm/CodeGen/TargetFrameLowering.h"
31#include "llvm/CodeGen/TargetInstrInfo.h"
32#include "llvm/CodeGen/TargetLowering.h"
33#include "llvm/CodeGen/TargetOpcodes.h"
34#include "llvm/CodeGen/TargetSubtargetInfo.h"
35#include "llvm/IR/Instructions.h"
36#include "llvm/Support/Debug.h"
37#include "llvm/Support/MathExtras.h"
38#include "llvm/Support/raw_ostream.h"
39#include "llvm/Target/TargetMachine.h"
40#include <cassert>
41#include <numeric>
42#include <optional>
43
44#define DEBUG_TYPE "legalizer"
45
46using namespace llvm;
47using namespace LegalizeActions;
48using namespace MIPatternMatch;
49
50/// Choose an insertion point for a conversion of an incoming PHI value.
51/// Values available before an invoke must be converted before it, so the
52/// conversion also executes on the unwind edge. An invoke result is defined
53/// after G_INVOKE_REGION_START and is only available on the normal edge.
54static MachineBasicBlock::iterator
55getPHIIncomingInsertPoint(MachineBasicBlock &MBB, Register Reg,
56 const MachineRegisterInfo &MRI) {
57 auto InsertPt = MBB.getFirstTerminatorForward();
58 // Only invoke regions need the extra definition-order check.
59 if (InsertPt == MBB.end() ||
60 InsertPt->getOpcode() != TargetOpcode::G_INVOKE_REGION_START)
61 return InsertPt;
62
63 MachineInstr *Def = MRI.getVRegDef(Reg);
64 if (Def->getParent() == &MBB)
65 for (auto I = InsertPt, E = MBB.end(); I != E; ++I)
66 if (&*I == Def)
67 return std::next(x: I);
68 return InsertPt;
69}
70
71/// Try to break down \p OrigTy into \p NarrowTy sized pieces.
72///
73/// Returns the number of \p NarrowTy elements needed to reconstruct \p OrigTy,
74/// with any leftover piece as type \p LeftoverTy
75///
76/// Returns -1 in the first element of the pair if the breakdown is not
77/// satisfiable.
78static std::pair<int, int>
79getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) {
80 assert(!LeftoverTy.isValid() && "this is an out argument");
81
82 unsigned Size = OrigTy.getSizeInBits();
83 unsigned NarrowSize = NarrowTy.getSizeInBits();
84 unsigned NumParts = Size / NarrowSize;
85 unsigned LeftoverSize = Size - NumParts * NarrowSize;
86 assert(Size > NarrowSize);
87
88 if (LeftoverSize == 0)
89 return {NumParts, 0};
90
91 if (NarrowTy.isVector()) {
92 unsigned EltSize = OrigTy.getScalarSizeInBits();
93 if (LeftoverSize % EltSize != 0)
94 return {-1, -1};
95 LeftoverTy = OrigTy.changeElementCount(
96 EC: ElementCount::getFixed(MinVal: LeftoverSize / EltSize));
97 } else {
98 LeftoverTy = LLT::integer(SizeInBits: LeftoverSize);
99 }
100
101 int NumLeftover = LeftoverSize / LeftoverTy.getSizeInBits();
102 return std::make_pair(x&: NumParts, y&: NumLeftover);
103}
104
105static Type *getFloatTypeForLLT(LLVMContext &Ctx, LLT Ty) {
106
107 if (!Ty.isScalar())
108 return nullptr;
109
110 switch (Ty.getSizeInBits()) {
111 case 16:
112 return Type::getHalfTy(C&: Ctx);
113 case 32:
114 return Type::getFloatTy(C&: Ctx);
115 case 64:
116 return Type::getDoubleTy(C&: Ctx);
117 case 80:
118 return Type::getX86_FP80Ty(C&: Ctx);
119 case 128:
120 return Type::getFP128Ty(C&: Ctx);
121 default:
122 return nullptr;
123 }
124}
125
126LegalizerHelper::LegalizerHelper(MachineFunction &MF,
127 GISelChangeObserver &Observer,
128 MachineIRBuilder &Builder,
129 const LibcallLoweringInfo *Libcalls)
130 : MIRBuilder(Builder), Observer(Observer), MRI(MF.getRegInfo()),
131 LI(*MF.getSubtarget().getLegalizerInfo()),
132 TLI(*MF.getSubtarget().getTargetLowering()), Libcalls(Libcalls) {}
133
134LegalizerHelper::LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI,
135 GISelChangeObserver &Observer,
136 MachineIRBuilder &B,
137 const LibcallLoweringInfo *Libcalls,
138 GISelValueTracking *VT)
139 : MIRBuilder(B), Observer(Observer), MRI(MF.getRegInfo()), LI(LI),
140 TLI(*MF.getSubtarget().getTargetLowering()), Libcalls(Libcalls), VT(VT) {}
141
142LegalizerHelper::LegalizeResult
143LegalizerHelper::legalizeInstrStep(MachineInstr &MI,
144 LostDebugLocObserver &LocObserver) {
145 LLVM_DEBUG(dbgs() << "\nLegalizing: " << MI);
146
147 MIRBuilder.setInstrAndDebugLoc(MI);
148
149 if (isa<GIntrinsic>(Val: MI))
150 return LI.legalizeIntrinsic(Helper&: *this, MI) ? Legalized : UnableToLegalize;
151 auto Step = LI.getAction(MI, MRI);
152 switch (Step.Action) {
153 case Legal:
154 LLVM_DEBUG(dbgs() << ".. Already legal\n");
155 return AlreadyLegal;
156 case Libcall:
157 LLVM_DEBUG(dbgs() << ".. Convert to libcall\n");
158 return libcall(MI, LocObserver);
159 case NarrowScalar:
160 LLVM_DEBUG(dbgs() << ".. Narrow scalar\n");
161 return narrowScalar(MI, TypeIdx: Step.TypeIdx, NarrowTy: Step.NewType);
162 case WidenScalar:
163 LLVM_DEBUG(dbgs() << ".. Widen scalar\n");
164 return widenScalar(MI, TypeIdx: Step.TypeIdx, WideTy: Step.NewType);
165 case Bitcast:
166 LLVM_DEBUG(dbgs() << ".. Bitcast type\n");
167 return bitcast(MI, TypeIdx: Step.TypeIdx, Ty: Step.NewType);
168 case Lower:
169 LLVM_DEBUG(dbgs() << ".. Lower\n");
170 return lower(MI, TypeIdx: Step.TypeIdx, Ty: Step.NewType);
171 case FewerElements:
172 LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n");
173 return fewerElementsVector(MI, TypeIdx: Step.TypeIdx, NarrowTy: Step.NewType);
174 case MoreElements:
175 LLVM_DEBUG(dbgs() << ".. Increase number of elements\n");
176 return moreElementsVector(MI, TypeIdx: Step.TypeIdx, MoreTy: Step.NewType);
177 case Custom:
178 LLVM_DEBUG(dbgs() << ".. Custom legalization\n");
179 return LI.legalizeCustom(Helper&: *this, MI, LocObserver) ? Legalized
180 : UnableToLegalize;
181 default:
182 LLVM_DEBUG(dbgs() << ".. Unable to legalize\n");
183 return UnableToLegalize;
184 }
185}
186
187void LegalizerHelper::insertParts(Register DstReg,
188 LLT ResultTy, LLT PartTy,
189 ArrayRef<Register> PartRegs,
190 LLT LeftoverTy,
191 ArrayRef<Register> LeftoverRegs) {
192 if (!LeftoverTy.isValid()) {
193 assert(LeftoverRegs.empty());
194
195 if (!ResultTy.isVector()) {
196 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: PartRegs);
197 return;
198 }
199
200 if (PartTy.isVector())
201 MIRBuilder.buildConcatVectors(Res: DstReg, Ops: PartRegs);
202 else
203 MIRBuilder.buildBuildVector(Res: DstReg, Ops: PartRegs);
204 return;
205 }
206
207 // Merge sub-vectors with different number of elements and insert into DstReg.
208 if (ResultTy.isVector()) {
209 assert(LeftoverRegs.size() == 1 && "Expected one leftover register");
210 SmallVector<Register, 8> AllRegs(PartRegs);
211 AllRegs.append(in_start: LeftoverRegs.begin(), in_end: LeftoverRegs.end());
212 return mergeMixedSubvectors(DstReg, PartRegs: AllRegs);
213 }
214
215 SmallVector<Register> GCDRegs;
216 LLT GCDTy = getGCDType(OrigTy: getGCDType(OrigTy: ResultTy, TargetTy: LeftoverTy), TargetTy: PartTy);
217 for (auto PartReg : concat<const Register>(Ranges&: PartRegs, Ranges&: LeftoverRegs))
218 extractGCDType(Parts&: GCDRegs, GCDTy, SrcReg: PartReg);
219 LLT ResultLCMTy = buildLCMMergePieces(DstTy: ResultTy, NarrowTy: LeftoverTy, GCDTy, VRegs&: GCDRegs);
220 buildWidenedRemergeToDst(DstReg, LCMTy: ResultLCMTy, RemergeRegs: GCDRegs);
221}
222
223void LegalizerHelper::appendVectorElts(SmallVectorImpl<Register> &Elts,
224 Register Reg) {
225 LLT Ty = MRI.getType(Reg);
226 SmallVector<Register, 8> RegElts;
227 extractParts(Reg, Ty: Ty.getScalarType(), NumParts: Ty.getNumElements(), VRegs&: RegElts,
228 MIRBuilder, MRI);
229 Elts.append(RHS: RegElts);
230}
231
232/// Merge \p PartRegs with different types into \p DstReg.
233void LegalizerHelper::mergeMixedSubvectors(Register DstReg,
234 ArrayRef<Register> PartRegs) {
235 SmallVector<Register, 8> AllElts;
236 for (unsigned i = 0; i < PartRegs.size() - 1; ++i)
237 appendVectorElts(Elts&: AllElts, Reg: PartRegs[i]);
238
239 Register Leftover = PartRegs[PartRegs.size() - 1];
240 if (!MRI.getType(Reg: Leftover).isVector())
241 AllElts.push_back(Elt: Leftover);
242 else
243 appendVectorElts(Elts&: AllElts, Reg: Leftover);
244
245 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: AllElts);
246}
247
248/// Append the result registers of G_UNMERGE_VALUES \p MI to \p Regs.
249static void getUnmergeResults(SmallVectorImpl<Register> &Regs,
250 const MachineInstr &MI) {
251 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
252
253 const int StartIdx = Regs.size();
254 const int NumResults = MI.getNumOperands() - 1;
255 Regs.resize(N: Regs.size() + NumResults);
256 for (int I = 0; I != NumResults; ++I)
257 Regs[StartIdx + I] = MI.getOperand(i: I).getReg();
258}
259
260void LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts,
261 LLT GCDTy, Register SrcReg) {
262 LLT SrcTy = MRI.getType(Reg: SrcReg);
263 if (SrcTy == GCDTy) {
264 // If the source already evenly divides the result type, we don't need to do
265 // anything.
266 Parts.push_back(Elt: SrcReg);
267 } else {
268 // Need to split into common type sized pieces.
269 auto Unmerge = MIRBuilder.buildUnmerge(Res: GCDTy, Op: SrcReg);
270 getUnmergeResults(Regs&: Parts, MI: *Unmerge);
271 }
272}
273
274LLT LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts, LLT DstTy,
275 LLT NarrowTy, Register SrcReg) {
276 LLT SrcTy = MRI.getType(Reg: SrcReg);
277 LLT GCDTy = getGCDType(OrigTy: getGCDType(OrigTy: SrcTy, TargetTy: NarrowTy), TargetTy: DstTy);
278 extractGCDType(Parts, GCDTy, SrcReg);
279 return GCDTy;
280}
281
282LLT LegalizerHelper::buildLCMMergePieces(LLT DstTy, LLT NarrowTy, LLT GCDTy,
283 SmallVectorImpl<Register> &VRegs,
284 unsigned PadStrategy) {
285 LLT LCMTy = getLCMType(OrigTy: DstTy, TargetTy: NarrowTy);
286
287 int NumParts = LCMTy.getSizeInBits() / NarrowTy.getSizeInBits();
288 int NumSubParts = NarrowTy.getSizeInBits() / GCDTy.getSizeInBits();
289 int NumOrigSrc = VRegs.size();
290
291 Register PadReg;
292
293 // Get a value we can use to pad the source value if the sources won't evenly
294 // cover the result type.
295 if (NumOrigSrc < NumParts * NumSubParts) {
296 if (PadStrategy == TargetOpcode::G_ZEXT)
297 PadReg = MIRBuilder.buildConstant(Res: GCDTy, Val: 0).getReg(Idx: 0);
298 else if (PadStrategy == TargetOpcode::G_ANYEXT)
299 PadReg = MIRBuilder.buildUndef(Res: GCDTy).getReg(Idx: 0);
300 else {
301 assert(PadStrategy == TargetOpcode::G_SEXT);
302
303 // Shift the sign bit of the low register through the high register.
304 auto ShiftAmt =
305 MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: GCDTy.getSizeInBits() - 1);
306 PadReg = MIRBuilder.buildAShr(Dst: GCDTy, Src0: VRegs.back(), Src1: ShiftAmt).getReg(Idx: 0);
307 }
308 }
309
310 // Registers for the final merge to be produced.
311 SmallVector<Register, 4> Remerge(NumParts);
312
313 // Registers needed for intermediate merges, which will be merged into a
314 // source for Remerge.
315 SmallVector<Register, 4> SubMerge(NumSubParts);
316
317 // Once we've fully read off the end of the original source bits, we can reuse
318 // the same high bits for remaining padding elements.
319 Register AllPadReg;
320
321 // Build merges to the LCM type to cover the original result type.
322 for (int I = 0; I != NumParts; ++I) {
323 bool AllMergePartsArePadding = true;
324
325 // Build the requested merges to the requested type.
326 for (int J = 0; J != NumSubParts; ++J) {
327 int Idx = I * NumSubParts + J;
328 if (Idx >= NumOrigSrc) {
329 SubMerge[J] = PadReg;
330 continue;
331 }
332
333 SubMerge[J] = VRegs[Idx];
334
335 // There are meaningful bits here we can't reuse later.
336 AllMergePartsArePadding = false;
337 }
338
339 // If we've filled up a complete piece with padding bits, we can directly
340 // emit the natural sized constant if applicable, rather than a merge of
341 // smaller constants.
342 if (AllMergePartsArePadding && !AllPadReg) {
343 if (PadStrategy == TargetOpcode::G_ANYEXT)
344 AllPadReg = MIRBuilder.buildUndef(Res: NarrowTy).getReg(Idx: 0);
345 else if (PadStrategy == TargetOpcode::G_ZEXT)
346 AllPadReg = MIRBuilder.buildConstant(Res: NarrowTy, Val: 0).getReg(Idx: 0);
347
348 // If this is a sign extension, we can't materialize a trivial constant
349 // with the right type and have to produce a merge.
350 }
351
352 if (AllPadReg) {
353 // Avoid creating additional instructions if we're just adding additional
354 // copies of padding bits.
355 Remerge[I] = AllPadReg;
356 continue;
357 }
358
359 if (NumSubParts == 1)
360 Remerge[I] = SubMerge[0];
361 else
362 Remerge[I] = MIRBuilder.buildMergeLikeInstr(Res: NarrowTy, Ops: SubMerge).getReg(Idx: 0);
363
364 // In the sign extend padding case, re-use the first all-signbit merge.
365 if (AllMergePartsArePadding && !AllPadReg)
366 AllPadReg = Remerge[I];
367 }
368
369 VRegs = std::move(Remerge);
370 return LCMTy;
371}
372
373void LegalizerHelper::buildWidenedRemergeToDst(Register DstReg, LLT LCMTy,
374 ArrayRef<Register> RemergeRegs) {
375 LLT DstTy = MRI.getType(Reg: DstReg);
376
377 // Create the merge to the widened source, and extract the relevant bits into
378 // the result.
379
380 if (DstTy == LCMTy) {
381 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: RemergeRegs);
382 return;
383 }
384
385 auto Remerge = MIRBuilder.buildMergeLikeInstr(Res: LCMTy, Ops: RemergeRegs);
386 if (DstTy.isScalar() && LCMTy.isScalar()) {
387 MIRBuilder.buildTrunc(Res: DstReg, Op: Remerge);
388 return;
389 }
390
391 if (LCMTy.isVector()) {
392 unsigned NumDefs = LCMTy.getSizeInBits() / DstTy.getSizeInBits();
393 SmallVector<Register, 8> UnmergeDefs(NumDefs);
394 UnmergeDefs[0] = DstReg;
395 for (unsigned I = 1; I != NumDefs; ++I)
396 UnmergeDefs[I] = MRI.createGenericVirtualRegister(Ty: DstTy);
397
398 MIRBuilder.buildUnmerge(Res: UnmergeDefs,
399 Op: MIRBuilder.buildMergeLikeInstr(Res: LCMTy, Ops: RemergeRegs));
400 return;
401 }
402
403 llvm_unreachable("unhandled case");
404}
405
406static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
407#define RTLIBCASE_INT(LibcallPrefix) \
408 do { \
409 switch (Size) { \
410 case 32: \
411 return RTLIB::LibcallPrefix##32; \
412 case 64: \
413 return RTLIB::LibcallPrefix##64; \
414 case 128: \
415 return RTLIB::LibcallPrefix##128; \
416 default: \
417 llvm_unreachable("unexpected size"); \
418 } \
419 } while (0)
420
421#define RTLIBCASE(LibcallPrefix) \
422 do { \
423 switch (Size) { \
424 case 32: \
425 return RTLIB::LibcallPrefix##32; \
426 case 64: \
427 return RTLIB::LibcallPrefix##64; \
428 case 80: \
429 return RTLIB::LibcallPrefix##80; \
430 case 128: \
431 return RTLIB::LibcallPrefix##128; \
432 default: \
433 llvm_unreachable("unexpected size"); \
434 } \
435 } while (0)
436
437 switch (Opcode) {
438 case TargetOpcode::G_LROUND:
439 RTLIBCASE(LROUND_F);
440 case TargetOpcode::G_LLROUND:
441 RTLIBCASE(LLROUND_F);
442 case TargetOpcode::G_MUL:
443 RTLIBCASE_INT(MUL_I);
444 case TargetOpcode::G_SDIV:
445 RTLIBCASE_INT(SDIV_I);
446 case TargetOpcode::G_UDIV:
447 RTLIBCASE_INT(UDIV_I);
448 case TargetOpcode::G_SREM:
449 RTLIBCASE_INT(SREM_I);
450 case TargetOpcode::G_UREM:
451 RTLIBCASE_INT(UREM_I);
452 case TargetOpcode::G_CTLZ_ZERO_POISON:
453 RTLIBCASE_INT(CTLZ_I);
454 case TargetOpcode::G_FADD:
455 RTLIBCASE(ADD_F);
456 case TargetOpcode::G_FSUB:
457 RTLIBCASE(SUB_F);
458 case TargetOpcode::G_FMUL:
459 RTLIBCASE(MUL_F);
460 case TargetOpcode::G_FDIV:
461 RTLIBCASE(DIV_F);
462 case TargetOpcode::G_FEXP:
463 RTLIBCASE(EXP_F);
464 case TargetOpcode::G_FEXP2:
465 RTLIBCASE(EXP2_F);
466 case TargetOpcode::G_FEXP10:
467 RTLIBCASE(EXP10_F);
468 case TargetOpcode::G_FREM:
469 RTLIBCASE(REM_F);
470 case TargetOpcode::G_FPOW:
471 RTLIBCASE(POW_F);
472 case TargetOpcode::G_FPOWI:
473 RTLIBCASE(POWI_F);
474 case TargetOpcode::G_FMA:
475 RTLIBCASE(FMA_F);
476 case TargetOpcode::G_FSIN:
477 RTLIBCASE(SIN_F);
478 case TargetOpcode::G_FCOS:
479 RTLIBCASE(COS_F);
480 case TargetOpcode::G_FTAN:
481 RTLIBCASE(TAN_F);
482 case TargetOpcode::G_FASIN:
483 RTLIBCASE(ASIN_F);
484 case TargetOpcode::G_FACOS:
485 RTLIBCASE(ACOS_F);
486 case TargetOpcode::G_FATAN:
487 RTLIBCASE(ATAN_F);
488 case TargetOpcode::G_FATAN2:
489 RTLIBCASE(ATAN2_F);
490 case TargetOpcode::G_FSINH:
491 RTLIBCASE(SINH_F);
492 case TargetOpcode::G_FCOSH:
493 RTLIBCASE(COSH_F);
494 case TargetOpcode::G_FTANH:
495 RTLIBCASE(TANH_F);
496 case TargetOpcode::G_FSINCOS:
497 RTLIBCASE(SINCOS_F);
498 case TargetOpcode::G_FMODF:
499 RTLIBCASE(MODF_F);
500 case TargetOpcode::G_FLOG10:
501 RTLIBCASE(LOG10_F);
502 case TargetOpcode::G_FLOG:
503 RTLIBCASE(LOG_F);
504 case TargetOpcode::G_FLOG2:
505 RTLIBCASE(LOG2_F);
506 case TargetOpcode::G_FLDEXP:
507 RTLIBCASE(LDEXP_F);
508 case TargetOpcode::G_FCEIL:
509 RTLIBCASE(CEIL_F);
510 case TargetOpcode::G_FFLOOR:
511 RTLIBCASE(FLOOR_F);
512 case TargetOpcode::G_FMINNUM:
513 RTLIBCASE(FMIN_F);
514 case TargetOpcode::G_FMAXNUM:
515 RTLIBCASE(FMAX_F);
516 case TargetOpcode::G_FMINIMUMNUM:
517 RTLIBCASE(FMINIMUM_NUM_F);
518 case TargetOpcode::G_FMAXIMUMNUM:
519 RTLIBCASE(FMAXIMUM_NUM_F);
520 case TargetOpcode::G_FSQRT:
521 RTLIBCASE(SQRT_F);
522 case TargetOpcode::G_FRINT:
523 RTLIBCASE(RINT_F);
524 case TargetOpcode::G_FNEARBYINT:
525 RTLIBCASE(NEARBYINT_F);
526 case TargetOpcode::G_INTRINSIC_TRUNC:
527 RTLIBCASE(TRUNC_F);
528 case TargetOpcode::G_INTRINSIC_ROUND:
529 RTLIBCASE(ROUND_F);
530 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
531 RTLIBCASE(ROUNDEVEN_F);
532 case TargetOpcode::G_INTRINSIC_LRINT:
533 RTLIBCASE(LRINT_F);
534 case TargetOpcode::G_INTRINSIC_LLRINT:
535 RTLIBCASE(LLRINT_F);
536 }
537 llvm_unreachable("Unknown libcall function");
538#undef RTLIBCASE_INT
539#undef RTLIBCASE
540}
541
542static bool hasSwiftErrorArg(MachineFunction &MF) {
543 const TargetLowering &TLI = *MF.getSubtarget().getTargetLowering();
544 return TLI.supportSwiftError() &&
545 MF.getFunction().getAttributes().hasAttrSomewhere(
546 Kind: Attribute::SwiftError);
547}
548
549/// True if an instruction is in tail position in its caller. Intended for
550/// legalizing libcalls as tail calls when possible.
551static bool isLibCallInTailPosition(const CallLowering::ArgInfo &Result,
552 MachineInstr &MI,
553 const TargetInstrInfo &TII,
554 MachineRegisterInfo &MRI) {
555 MachineBasicBlock &MBB = *MI.getParent();
556 const Function &F = MBB.getParent()->getFunction();
557
558 // Conservatively require the attributes of the call to match those of
559 // the return. Ignore NoAlias and NonNull because they don't affect the
560 // call sequence.
561 AttributeList CallerAttrs = F.getAttributes();
562 if (AttrBuilder(F.getContext(), CallerAttrs.getRetAttrs())
563 .removeAttribute(Val: Attribute::NoAlias)
564 .removeAttribute(Val: Attribute::NonNull)
565 .hasAttributes())
566 return false;
567
568 // It's not safe to eliminate the sign / zero extension of the return value.
569 if (CallerAttrs.hasRetAttr(Kind: Attribute::ZExt) ||
570 CallerAttrs.hasRetAttr(Kind: Attribute::SExt))
571 return false;
572
573 // Check if tail calls have been disabled in this function.
574 if (F.getFnAttribute(Kind: "disable-tail-calls").getValueAsBool())
575 return false;
576
577 // Only tail call if the following instruction is a standard return or if we
578 // have a `thisreturn` callee, and a sequence like:
579 //
580 // G_MEMCPY %0, %1, %2
581 // $x0 = COPY %0
582 // RET_ReallyLR implicit $x0
583 auto Next = next_nodbg(It: MI.getIterator(), End: MBB.instr_end());
584 if (Next != MBB.instr_end() && Next->isCopy()) {
585 if (MI.getOpcode() == TargetOpcode::G_BZERO)
586 return false;
587
588 // For MEMCPY/MOMMOVE/MEMSET these will be the first use (the dst), as the
589 // mempy/etc routines return the same parameter. For other it will be the
590 // returned value.
591 Register VReg = MI.getOperand(i: 0).getReg();
592 if (!VReg.isVirtual() || VReg != Next->getOperand(i: 1).getReg())
593 return false;
594
595 Register PReg = Next->getOperand(i: 0).getReg();
596 if (!PReg.isPhysical())
597 return false;
598
599 auto Ret = next_nodbg(It: Next, End: MBB.instr_end());
600 if (Ret == MBB.instr_end() || !Ret->isReturn())
601 return false;
602
603 if (Ret->getNumImplicitOperands() != 1)
604 return false;
605
606 if (!Ret->getOperand(i: 0).isReg() || PReg != Ret->getOperand(i: 0).getReg())
607 return false;
608
609 // Skip over the COPY that we just validated.
610 Next = Ret;
611 }
612
613 if (Next == MBB.instr_end() || TII.isTailCall(Inst: *Next) || !Next->isReturn())
614 return false;
615
616 return true;
617}
618
619LegalizerHelper::LegalizeResult LegalizerHelper::createLibcall(
620 const char *Name, const CallLowering::ArgInfo &Result,
621 ArrayRef<CallLowering::ArgInfo> Args, const CallingConv::ID CC,
622 LostDebugLocObserver &LocObserver, MachineInstr *MI) const {
623 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
624
625 CallLowering::CallLoweringInfo Info;
626 Info.CallConv = CC;
627 Info.Callee = MachineOperand::CreateES(SymName: Name);
628 Info.OrigRet = Result;
629 if (MI)
630 Info.IsTailCall =
631 (Result.Ty->isVoidTy() ||
632 Result.Ty == MIRBuilder.getMF().getFunction().getReturnType()) &&
633 isLibCallInTailPosition(Result, MI&: *MI, TII: MIRBuilder.getTII(),
634 MRI&: *MIRBuilder.getMRI()) &&
635 // Lowering doesn't support tail calling inside a function with
636 // a swifterror argument yet.
637 !hasSwiftErrorArg(MF&: MIRBuilder.getMF());
638
639 llvm::append_range(C&: Info.OrigArgs, R&: Args);
640 if (!CLI.lowerCall(MIRBuilder, Info))
641 return LegalizerHelper::UnableToLegalize;
642
643 if (MI && Info.LoweredTailCall) {
644 assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?");
645
646 // Check debug locations before removing the return.
647 LocObserver.checkpoint(CheckDebugLocs: true);
648
649 // We must have a return following the call (or debug insts) to get past
650 // isLibCallInTailPosition.
651 do {
652 MachineInstr *Next = MI->getNextNode();
653 assert(Next &&
654 (Next->isCopy() || Next->isReturn() || Next->isDebugInstr()) &&
655 "Expected instr following MI to be return or debug inst?");
656 // We lowered a tail call, so the call is now the return from the block.
657 // Delete the old return.
658 Next->eraseFromParent();
659 } while (MI->getNextNode());
660
661 // We expect to lose the debug location from the return.
662 LocObserver.checkpoint(CheckDebugLocs: false);
663 }
664 return LegalizerHelper::Legalized;
665}
666
667LegalizerHelper::LegalizeResult LegalizerHelper::createLibcall(
668 RTLIB::Libcall Libcall, const CallLowering::ArgInfo &Result,
669 ArrayRef<CallLowering::ArgInfo> Args, LostDebugLocObserver &LocObserver,
670 MachineInstr *MI) const {
671 if (!Libcalls)
672 return LegalizerHelper::UnableToLegalize;
673
674 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: Libcall);
675 if (LibcallImpl == RTLIB::Unsupported)
676 return LegalizerHelper::UnableToLegalize;
677
678 StringRef Name = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: LibcallImpl);
679 const CallingConv::ID CC = Libcalls->getLibcallImplCallingConv(Call: LibcallImpl);
680 return createLibcall(Name: Name.data(), Result, Args, CC, LocObserver, MI);
681}
682
683// Useful for libcalls where all operands have the same type.
684LegalizerHelper::LegalizeResult
685LegalizerHelper::simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder,
686 unsigned Size, Type *OpType,
687 LostDebugLocObserver &LocObserver) const {
688 auto Libcall = getRTLibDesc(Opcode: MI.getOpcode(), Size);
689
690 // FIXME: What does the original arg index mean here?
691 SmallVector<CallLowering::ArgInfo, 3> Args;
692 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands()))
693 Args.push_back(Elt: {MO.getReg(), OpType, 0});
694 return createLibcall(Libcall, Result: {MI.getOperand(i: 0).getReg(), OpType, 0}, Args,
695 LocObserver, MI: &MI);
696}
697
698LegalizerHelper::LegalizeResult LegalizerHelper::emitSincosLibcall(
699 MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size, Type *OpType,
700 LostDebugLocObserver &LocObserver) {
701 MachineFunction &MF = *MI.getMF();
702 MachineRegisterInfo &MRI = MF.getRegInfo();
703
704 Register DstSin = MI.getOperand(i: 0).getReg();
705 Register DstCos = MI.getOperand(i: 1).getReg();
706 Register Src = MI.getOperand(i: 2).getReg();
707 LLT DstTy = MRI.getType(Reg: DstSin);
708
709 int MemSize = DstTy.getSizeInBytes();
710 Align Alignment = getStackTemporaryAlignment(Type: DstTy);
711 const DataLayout &DL = MIRBuilder.getDataLayout();
712 unsigned AddrSpace = DL.getAllocaAddrSpace();
713 MachinePointerInfo PtrInfo;
714
715 Register StackPtrSin =
716 createStackTemporary(Bytes: TypeSize::getFixed(ExactSize: MemSize), Alignment, PtrInfo)
717 .getReg(Idx: 0);
718 Register StackPtrCos =
719 createStackTemporary(Bytes: TypeSize::getFixed(ExactSize: MemSize), Alignment, PtrInfo)
720 .getReg(Idx: 0);
721
722 auto &Ctx = MF.getFunction().getContext();
723 auto LibcallResult = createLibcall(
724 Libcall: getRTLibDesc(Opcode: MI.getOpcode(), Size), Result: {{0}, Type::getVoidTy(C&: Ctx), 0},
725 Args: {{Src, OpType, 0},
726 {StackPtrSin, PointerType::get(C&: Ctx, AddressSpace: AddrSpace), 1},
727 {StackPtrCos, PointerType::get(C&: Ctx, AddressSpace: AddrSpace), 2}},
728 LocObserver, MI: &MI);
729
730 if (LibcallResult != LegalizeResult::Legalized)
731 return LegalizerHelper::UnableToLegalize;
732
733 MachineMemOperand *LoadMMOSin = MF.getMachineMemOperand(
734 PtrInfo, F: MachineMemOperand::MOLoad, Size: MemSize, BaseAlignment: Alignment);
735 MachineMemOperand *LoadMMOCos = MF.getMachineMemOperand(
736 PtrInfo, F: MachineMemOperand::MOLoad, Size: MemSize, BaseAlignment: Alignment);
737
738 MIRBuilder.buildLoad(Res: DstSin, Addr: StackPtrSin, MMO&: *LoadMMOSin);
739 MIRBuilder.buildLoad(Res: DstCos, Addr: StackPtrCos, MMO&: *LoadMMOCos);
740 MI.eraseFromParent();
741
742 return LegalizerHelper::Legalized;
743}
744
745LegalizerHelper::LegalizeResult
746LegalizerHelper::emitModfLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder,
747 unsigned Size, Type *OpType,
748 LostDebugLocObserver &LocObserver) {
749 MachineFunction &MF = MIRBuilder.getMF();
750 MachineRegisterInfo &MRI = MF.getRegInfo();
751
752 Register DstFrac = MI.getOperand(i: 0).getReg();
753 Register DstInt = MI.getOperand(i: 1).getReg();
754 Register Src = MI.getOperand(i: 2).getReg();
755 LLT DstTy = MRI.getType(Reg: DstFrac);
756
757 int MemSize = DstTy.getSizeInBytes();
758 Align Alignment = getStackTemporaryAlignment(Type: DstTy);
759 const DataLayout &DL = MIRBuilder.getDataLayout();
760 unsigned AddrSpace = DL.getAllocaAddrSpace();
761 MachinePointerInfo PtrInfo;
762
763 Register StackPtrInt =
764 createStackTemporary(Bytes: TypeSize::getFixed(ExactSize: MemSize), Alignment, PtrInfo)
765 .getReg(Idx: 0);
766
767 auto &Ctx = MF.getFunction().getContext();
768 auto LibcallResult = createLibcall(
769 Libcall: getRTLibDesc(Opcode: MI.getOpcode(), Size), Result: {DstFrac, OpType, 0},
770 Args: {{Src, OpType, 0}, {StackPtrInt, PointerType::get(C&: Ctx, AddressSpace: AddrSpace), 1}},
771 LocObserver, MI: &MI);
772
773 if (LibcallResult != LegalizeResult::Legalized)
774 return LegalizerHelper::UnableToLegalize;
775
776 MachineMemOperand *LoadMMOInt = MF.getMachineMemOperand(
777 PtrInfo, F: MachineMemOperand::MOLoad, Size: MemSize, BaseAlignment: Alignment);
778
779 MIRBuilder.buildLoad(Res: DstInt, Addr: StackPtrInt, MMO&: *LoadMMOInt);
780 MI.eraseFromParent();
781
782 return LegalizerHelper::Legalized;
783}
784
785static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
786 Type *FromType) {
787 auto ToMVT = MVT::getVT(Ty: ToType);
788 auto FromMVT = MVT::getVT(Ty: FromType);
789
790 switch (Opcode) {
791 case TargetOpcode::G_FPEXT:
792 return RTLIB::getFPEXT(OpVT: FromMVT, RetVT: ToMVT);
793 case TargetOpcode::G_FPTRUNC:
794 return RTLIB::getFPROUND(OpVT: FromMVT, RetVT: ToMVT);
795 case TargetOpcode::G_FPTOSI:
796 return RTLIB::getFPTOSINT(OpVT: FromMVT, RetVT: ToMVT);
797 case TargetOpcode::G_FPTOUI:
798 return RTLIB::getFPTOUINT(OpVT: FromMVT, RetVT: ToMVT);
799 case TargetOpcode::G_SITOFP:
800 return RTLIB::getSINTTOFP(OpVT: FromMVT, RetVT: ToMVT);
801 case TargetOpcode::G_UITOFP:
802 return RTLIB::getUINTTOFP(OpVT: FromMVT, RetVT: ToMVT);
803 }
804 llvm_unreachable("Unsupported libcall function");
805}
806
807LegalizerHelper::LegalizeResult LegalizerHelper::conversionLibcall(
808 MachineInstr &MI, Type *ToType, Type *FromType,
809 LostDebugLocObserver &LocObserver, bool IsSigned) const {
810 CallLowering::ArgInfo Arg = {MI.getOperand(i: 1).getReg(), FromType, 0};
811 if (FromType->isIntegerTy()) {
812 if (TLI.shouldSignExtendTypeInLibCall(Ty: FromType, IsSigned))
813 Arg.Flags[0].setSExt();
814 else
815 Arg.Flags[0].setZExt();
816 }
817
818 RTLIB::Libcall Libcall = getConvRTLibDesc(Opcode: MI.getOpcode(), ToType, FromType);
819 return createLibcall(Libcall, Result: {MI.getOperand(i: 0).getReg(), ToType, 0}, Args: Arg,
820 LocObserver, MI: &MI);
821}
822
823LegalizerHelper::LegalizeResult
824LegalizerHelper::createMemLibcall(MachineRegisterInfo &MRI, MachineInstr &MI,
825 LostDebugLocObserver &LocObserver) const {
826 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
827
828 SmallVector<CallLowering::ArgInfo, 3> Args;
829 // Add all the args, except for the last which is an imm denoting 'tail'.
830 for (unsigned i = 0; i < MI.getNumOperands() - 1; ++i) {
831 Register Reg = MI.getOperand(i).getReg();
832
833 // Need derive an IR type for call lowering.
834 LLT OpLLT = MRI.getType(Reg);
835 Type *OpTy = nullptr;
836 if (OpLLT.isPointer())
837 OpTy = PointerType::get(C&: Ctx, AddressSpace: OpLLT.getAddressSpace());
838 else
839 OpTy = IntegerType::get(C&: Ctx, NumBits: OpLLT.getSizeInBits());
840 Args.push_back(Elt: {Reg, OpTy, 0});
841 }
842
843 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
844 RTLIB::Libcall RTLibcall;
845 unsigned Opc = MI.getOpcode();
846 switch (Opc) {
847 case TargetOpcode::G_BZERO:
848 RTLibcall = RTLIB::BZERO;
849 break;
850 case TargetOpcode::G_MEMCPY:
851 RTLibcall = RTLIB::MEMCPY;
852 Args[0].Flags[0].setReturned();
853 break;
854 case TargetOpcode::G_MEMMOVE:
855 RTLibcall = RTLIB::MEMMOVE;
856 Args[0].Flags[0].setReturned();
857 break;
858 case TargetOpcode::G_MEMSET:
859 RTLibcall = RTLIB::MEMSET;
860 Args[0].Flags[0].setReturned();
861 break;
862 default:
863 llvm_unreachable("unsupported opcode");
864 }
865
866 if (!Libcalls) // FIXME: Should be mandatory
867 return LegalizerHelper::UnableToLegalize;
868
869 RTLIB::LibcallImpl RTLibcallImpl = Libcalls->getLibcallImpl(Call: RTLibcall);
870
871 // Unsupported libcall on the target.
872 if (RTLibcallImpl == RTLIB::Unsupported) {
873 LLVM_DEBUG(dbgs() << ".. .. Could not find libcall name for "
874 << MIRBuilder.getTII().getName(Opc) << "\n");
875 return LegalizerHelper::UnableToLegalize;
876 }
877
878 CallLowering::CallLoweringInfo Info;
879 Info.CallConv = Libcalls->getLibcallImplCallingConv(Call: RTLibcallImpl);
880
881 StringRef LibcallName =
882 RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: RTLibcallImpl);
883 Info.Callee = MachineOperand::CreateES(SymName: LibcallName.data());
884 Info.OrigRet = CallLowering::ArgInfo({0}, Type::getVoidTy(C&: Ctx), 0);
885 Info.IsTailCall =
886 MI.getOperand(i: MI.getNumOperands() - 1).getImm() &&
887 isLibCallInTailPosition(Result: Info.OrigRet, MI, TII: MIRBuilder.getTII(), MRI) &&
888 // Lowering doesn't support tail calling inside a function with
889 // a swifterror argument yet.
890 !hasSwiftErrorArg(MF&: MIRBuilder.getMF());
891
892 llvm::append_range(C&: Info.OrigArgs, R&: Args);
893 if (!CLI.lowerCall(MIRBuilder, Info))
894 return LegalizerHelper::UnableToLegalize;
895
896 if (Info.LoweredTailCall) {
897 assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?");
898
899 // Check debug locations before removing the return.
900 LocObserver.checkpoint(CheckDebugLocs: true);
901
902 // We must have a return following the call (or debug insts) to get past
903 // isLibCallInTailPosition.
904 do {
905 MachineInstr *Next = MI.getNextNode();
906 assert(Next &&
907 (Next->isCopy() || Next->isReturn() || Next->isDebugInstr()) &&
908 "Expected instr following MI to be return or debug inst?");
909 // We lowered a tail call, so the call is now the return from the block.
910 // Delete the old return.
911 Next->eraseFromParent();
912 } while (MI.getNextNode());
913
914 // We expect to lose the debug location from the return.
915 LocObserver.checkpoint(CheckDebugLocs: false);
916 }
917
918 return LegalizerHelper::Legalized;
919}
920
921static unsigned getAtomicLibcallISDOpcode(unsigned Opc) {
922 switch (Opc) {
923 case TargetOpcode::G_ATOMIC_CMPXCHG:
924 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS:
925 return ISD::ATOMIC_CMP_SWAP;
926 case TargetOpcode::G_ATOMICRMW_XCHG:
927 return ISD::ATOMIC_SWAP;
928 case TargetOpcode::G_ATOMICRMW_ADD:
929 case TargetOpcode::G_ATOMICRMW_SUB:
930 return ISD::ATOMIC_LOAD_ADD;
931 case TargetOpcode::G_ATOMICRMW_AND:
932 return ISD::ATOMIC_LOAD_CLR;
933 case TargetOpcode::G_ATOMICRMW_OR:
934 return ISD::ATOMIC_LOAD_OR;
935 case TargetOpcode::G_ATOMICRMW_XOR:
936 return ISD::ATOMIC_LOAD_XOR;
937 default:
938 llvm_unreachable("unsupported opcode");
939 }
940}
941
942LegalizerHelper::LegalizeResult
943LegalizerHelper::createAtomicLibcall(MachineInstr &MI) const {
944 auto &Ctx = MIRBuilder.getContext();
945 unsigned Opc = MI.getOpcode();
946
947 if (!Libcalls) // FIXME: Should be mandatory
948 return LegalizerHelper::UnableToLegalize;
949
950 const MachineMemOperand &MMO = cast<GMemOperation>(Val&: MI).getMMO();
951 LLT MemType = MMO.getMemoryType();
952 MVT VT =
953 MemType.isVector() ? MVT() : MVT::getIntegerVT(BitWidth: MemType.getSizeInBits());
954 RTLIB::AtomicLibcall LC = RTLIB::selectAtomicLibcall(
955 Libcalls: *Libcalls, Opc: getAtomicLibcallISDOpcode(Opc), Order: MMO.getMergedOrdering(), VT);
956
957 // Unsupported libcall on the target.
958 if (LC.Impl == RTLIB::Unsupported) {
959 LLVM_DEBUG(dbgs() << ".. .. Could not find libcall name for "
960 << MIRBuilder.getTII().getName(Opc) << "\n");
961 return LegalizerHelper::UnableToLegalize;
962 }
963
964 // TODO: Support __sync_* calls, which take the pointer first.
965 if (!LC.IsOutline)
966 return LegalizerHelper::UnableToLegalize;
967
968 Type *RetTy;
969 SmallVector<Register> RetRegs;
970 SmallVector<CallLowering::ArgInfo, 3> Args;
971 switch (Opc) {
972 case TargetOpcode::G_ATOMIC_CMPXCHG:
973 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
974 Register Success;
975 LLT SuccessLLT;
976 auto [Ret, RetLLT, Mem, MemLLT, Cmp, CmpLLT, New, NewLLT] =
977 MI.getFirst4RegLLTs();
978 RetRegs.push_back(Elt: Ret);
979 RetTy = IntegerType::get(C&: Ctx, NumBits: RetLLT.getSizeInBits());
980 if (Opc == TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS) {
981 std::tie(args&: Ret, args&: RetLLT, args&: Success, args&: SuccessLLT, args&: Mem, args&: MemLLT, args&: Cmp, args&: CmpLLT, args&: New,
982 args&: NewLLT) = MI.getFirst5RegLLTs();
983 RetRegs.push_back(Elt: Success);
984 RetTy = StructType::get(
985 Context&: Ctx, Elements: {RetTy, IntegerType::get(C&: Ctx, NumBits: SuccessLLT.getSizeInBits())});
986 }
987 Args.push_back(Elt: {Cmp, IntegerType::get(C&: Ctx, NumBits: CmpLLT.getSizeInBits()), 0});
988 Args.push_back(Elt: {New, IntegerType::get(C&: Ctx, NumBits: NewLLT.getSizeInBits()), 0});
989 Args.push_back(Elt: {Mem, PointerType::get(C&: Ctx, AddressSpace: MemLLT.getAddressSpace()), 0});
990 break;
991 }
992 case TargetOpcode::G_ATOMICRMW_XCHG:
993 case TargetOpcode::G_ATOMICRMW_ADD:
994 case TargetOpcode::G_ATOMICRMW_SUB:
995 case TargetOpcode::G_ATOMICRMW_AND:
996 case TargetOpcode::G_ATOMICRMW_OR:
997 case TargetOpcode::G_ATOMICRMW_XOR: {
998 auto [Ret, RetLLT, Mem, MemLLT, Val, ValLLT] = MI.getFirst3RegLLTs();
999 RetRegs.push_back(Elt: Ret);
1000 RetTy = IntegerType::get(C&: Ctx, NumBits: RetLLT.getSizeInBits());
1001 if (Opc == TargetOpcode::G_ATOMICRMW_AND)
1002 Val =
1003 MIRBuilder.buildXor(Dst: ValLLT, Src0: MIRBuilder.buildConstant(Res: ValLLT, Val: -1), Src1: Val)
1004 .getReg(Idx: 0);
1005 else if (Opc == TargetOpcode::G_ATOMICRMW_SUB)
1006 Val =
1007 MIRBuilder.buildSub(Dst: ValLLT, Src0: MIRBuilder.buildConstant(Res: ValLLT, Val: 0), Src1: Val)
1008 .getReg(Idx: 0);
1009 Args.push_back(Elt: {Val, IntegerType::get(C&: Ctx, NumBits: ValLLT.getSizeInBits()), 0});
1010 Args.push_back(Elt: {Mem, PointerType::get(C&: Ctx, AddressSpace: MemLLT.getAddressSpace()), 0});
1011 break;
1012 }
1013 default:
1014 llvm_unreachable("unsupported opcode");
1015 }
1016
1017 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
1018 CallLowering::CallLoweringInfo Info;
1019 Info.CallConv = Libcalls->getLibcallImplCallingConv(Call: LC.Impl);
1020
1021 StringRef LibcallName =
1022 RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: LC.Impl);
1023 Info.Callee = MachineOperand::CreateES(SymName: LibcallName.data());
1024 Info.OrigRet = CallLowering::ArgInfo(RetRegs, RetTy, 0);
1025
1026 llvm::append_range(C&: Info.OrigArgs, R&: Args);
1027 if (!CLI.lowerCall(MIRBuilder, Info))
1028 return LegalizerHelper::UnableToLegalize;
1029
1030 return LegalizerHelper::Legalized;
1031}
1032
1033static RTLIB::Libcall
1034getStateLibraryFunctionFor(MachineInstr &MI, const TargetLowering &TLI) {
1035 RTLIB::Libcall RTLibcall;
1036 switch (MI.getOpcode()) {
1037 case TargetOpcode::G_GET_FPENV:
1038 RTLibcall = RTLIB::FEGETENV;
1039 break;
1040 case TargetOpcode::G_SET_FPENV:
1041 case TargetOpcode::G_RESET_FPENV:
1042 RTLibcall = RTLIB::FESETENV;
1043 break;
1044 case TargetOpcode::G_GET_FPMODE:
1045 RTLibcall = RTLIB::FEGETMODE;
1046 break;
1047 case TargetOpcode::G_SET_FPMODE:
1048 case TargetOpcode::G_RESET_FPMODE:
1049 RTLibcall = RTLIB::FESETMODE;
1050 break;
1051 default:
1052 llvm_unreachable("Unexpected opcode");
1053 }
1054 return RTLibcall;
1055}
1056
1057// Some library functions that read FP state (fegetmode, fegetenv) write the
1058// state into a region in memory. IR intrinsics that do the same operations
1059// (get_fpmode, get_fpenv) return the state as integer value. To implement these
1060// intrinsics via the library functions, we need to use temporary variable,
1061// for example:
1062//
1063// %0:_(s32) = G_GET_FPMODE
1064//
1065// is transformed to:
1066//
1067// %1:_(p0) = G_FRAME_INDEX %stack.0
1068// BL &fegetmode
1069// %0:_(s32) = G_LOAD % 1
1070//
1071LegalizerHelper::LegalizeResult
1072LegalizerHelper::createGetStateLibcall(MachineInstr &MI,
1073 LostDebugLocObserver &LocObserver) {
1074 const DataLayout &DL = MIRBuilder.getDataLayout();
1075 auto &MF = MIRBuilder.getMF();
1076 auto &MRI = *MIRBuilder.getMRI();
1077 auto &Ctx = MF.getFunction().getContext();
1078
1079 // Create temporary, where library function will put the read state.
1080 Register Dst = MI.getOperand(i: 0).getReg();
1081 LLT StateTy = MRI.getType(Reg: Dst);
1082 TypeSize StateSize = StateTy.getSizeInBytes();
1083 Align TempAlign = getStackTemporaryAlignment(Type: StateTy);
1084 MachinePointerInfo TempPtrInfo;
1085 auto Temp = createStackTemporary(Bytes: StateSize, Alignment: TempAlign, PtrInfo&: TempPtrInfo);
1086
1087 // Create a call to library function, with the temporary as an argument.
1088 unsigned TempAddrSpace = DL.getAllocaAddrSpace();
1089 Type *StatePtrTy = PointerType::get(C&: Ctx, AddressSpace: TempAddrSpace);
1090 RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI);
1091 auto Res = createLibcall(
1092 Libcall: RTLibcall, Result: CallLowering::ArgInfo({0}, Type::getVoidTy(C&: Ctx), 0),
1093 Args: CallLowering::ArgInfo({Temp.getReg(Idx: 0), StatePtrTy, 0}), LocObserver,
1094 MI: nullptr);
1095 if (Res != LegalizerHelper::Legalized)
1096 return Res;
1097
1098 // Create a load from the temporary.
1099 MachineMemOperand *MMO = MF.getMachineMemOperand(
1100 PtrInfo: TempPtrInfo, F: MachineMemOperand::MOLoad, MemTy: StateTy, BaseAlignment: TempAlign);
1101 MIRBuilder.buildLoadInstr(Opcode: TargetOpcode::G_LOAD, Res: Dst, Addr: Temp, MMO&: *MMO);
1102
1103 return LegalizerHelper::Legalized;
1104}
1105
1106// Similar to `createGetStateLibcall` the function calls a library function
1107// using transient space in stack. In this case the library function reads
1108// content of memory region.
1109LegalizerHelper::LegalizeResult
1110LegalizerHelper::createSetStateLibcall(MachineInstr &MI,
1111 LostDebugLocObserver &LocObserver) {
1112 const DataLayout &DL = MIRBuilder.getDataLayout();
1113 auto &MF = MIRBuilder.getMF();
1114 auto &MRI = *MIRBuilder.getMRI();
1115 auto &Ctx = MF.getFunction().getContext();
1116
1117 // Create temporary, where library function will get the new state.
1118 Register Src = MI.getOperand(i: 0).getReg();
1119 LLT StateTy = MRI.getType(Reg: Src);
1120 TypeSize StateSize = StateTy.getSizeInBytes();
1121 Align TempAlign = getStackTemporaryAlignment(Type: StateTy);
1122 MachinePointerInfo TempPtrInfo;
1123 auto Temp = createStackTemporary(Bytes: StateSize, Alignment: TempAlign, PtrInfo&: TempPtrInfo);
1124
1125 // Put the new state into the temporary.
1126 MachineMemOperand *MMO = MF.getMachineMemOperand(
1127 PtrInfo: TempPtrInfo, F: MachineMemOperand::MOStore, MemTy: StateTy, BaseAlignment: TempAlign);
1128 MIRBuilder.buildStore(Val: Src, Addr: Temp, MMO&: *MMO);
1129
1130 // Create a call to library function, with the temporary as an argument.
1131 unsigned TempAddrSpace = DL.getAllocaAddrSpace();
1132 Type *StatePtrTy = PointerType::get(C&: Ctx, AddressSpace: TempAddrSpace);
1133 RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI);
1134 return createLibcall(Libcall: RTLibcall,
1135 Result: CallLowering::ArgInfo({0}, Type::getVoidTy(C&: Ctx), 0),
1136 Args: CallLowering::ArgInfo({Temp.getReg(Idx: 0), StatePtrTy, 0}),
1137 LocObserver, MI: nullptr);
1138}
1139
1140/// Returns the corresponding libcall for the given Pred and
1141/// the ICMP predicate that should be generated to compare with #0
1142/// after the libcall.
1143static std::pair<RTLIB::Libcall, CmpInst::Predicate>
1144getFCMPLibcallDesc(const CmpInst::Predicate Pred, unsigned Size) {
1145#define RTLIBCASE_CMP(LibcallPrefix, ICmpPred) \
1146 do { \
1147 switch (Size) { \
1148 case 32: \
1149 return {RTLIB::LibcallPrefix##32, ICmpPred}; \
1150 case 64: \
1151 return {RTLIB::LibcallPrefix##64, ICmpPred}; \
1152 case 128: \
1153 return {RTLIB::LibcallPrefix##128, ICmpPred}; \
1154 default: \
1155 llvm_unreachable("unexpected size"); \
1156 } \
1157 } while (0)
1158
1159 // These use the three-way (-1/0/1) compare libcalls, whose result is tested
1160 // against 0 with a signed integer predicate. Unordered (UO) is a boolean.
1161 switch (Pred) {
1162 case CmpInst::FCMP_OEQ:
1163 RTLIBCASE_CMP(FCMP3_PRED_OEQ_F, CmpInst::ICMP_EQ);
1164 case CmpInst::FCMP_UNE:
1165 RTLIBCASE_CMP(FCMP3_PRED_UNE_F, CmpInst::ICMP_NE);
1166 case CmpInst::FCMP_OGE:
1167 RTLIBCASE_CMP(FCMP3_PRED_OGE_F, CmpInst::ICMP_SGE);
1168 case CmpInst::FCMP_OLT:
1169 RTLIBCASE_CMP(FCMP3_PRED_OLT_F, CmpInst::ICMP_SLT);
1170 case CmpInst::FCMP_OLE:
1171 RTLIBCASE_CMP(FCMP3_PRED_OLE_F, CmpInst::ICMP_SLE);
1172 case CmpInst::FCMP_OGT:
1173 RTLIBCASE_CMP(FCMP3_PRED_OGT_F, CmpInst::ICMP_SGT);
1174 case CmpInst::FCMP_UNO:
1175 RTLIBCASE_CMP(UO_F, CmpInst::ICMP_NE);
1176 default:
1177 return {RTLIB::UNKNOWN_LIBCALL, CmpInst::BAD_ICMP_PREDICATE};
1178 }
1179}
1180
1181LegalizerHelper::LegalizeResult
1182LegalizerHelper::createFCMPLibcall(MachineInstr &MI,
1183 LostDebugLocObserver &LocObserver) {
1184 auto &MF = MIRBuilder.getMF();
1185 auto &Ctx = MF.getFunction().getContext();
1186 const GFCmp *Cmp = cast<GFCmp>(Val: &MI);
1187
1188 LLT OpLLT = MRI.getType(Reg: Cmp->getLHSReg());
1189 unsigned Size = OpLLT.getSizeInBits();
1190 if ((Size != 32 && Size != 64 && Size != 128) ||
1191 OpLLT != MRI.getType(Reg: Cmp->getRHSReg()))
1192 return UnableToLegalize;
1193
1194 Type *OpType = getFloatTypeForLLT(Ctx, Ty: OpLLT);
1195
1196 // DstReg type is s32
1197 const Register DstReg = Cmp->getReg(Idx: 0);
1198 LLT DstTy = MRI.getType(Reg: DstReg);
1199 const auto Cond = Cmp->getCond();
1200 Type *RetTy = TLI.getCmpLibcallReturnType(Ctx);
1201
1202 // Reference:
1203 // https://gcc.gnu.org/onlinedocs/gccint/Soft-float-library-routines.html#Comparison-functions-1
1204 // Generates a libcall followed by ICMP.
1205 const auto BuildLibcall = [&](const RTLIB::Libcall Libcall,
1206 const CmpInst::Predicate ICmpPred,
1207 const DstOp &Res) -> Register {
1208 // FCMP libcall returns an integer, and needs an ICMP with #0.
1209 LLT TempLLT = LLT::integer(SizeInBits: RetTy->getIntegerBitWidth());
1210 Register Temp = MRI.createGenericVirtualRegister(Ty: TempLLT);
1211 // Generate libcall, holding result in Temp
1212 const auto Status = createLibcall(
1213 Libcall, Result: {Temp, RetTy, 0},
1214 Args: {{Cmp->getLHSReg(), OpType, 0}, {Cmp->getRHSReg(), OpType, 1}},
1215 LocObserver, MI: &MI);
1216 if (Status != Legalized)
1217 return {};
1218
1219 // Compare temp with #0 to get the final result.
1220 return MIRBuilder
1221 .buildICmp(Pred: ICmpPred, Res, Op0: Temp, Op1: MIRBuilder.buildConstant(Res: TempLLT, Val: 0))
1222 .getReg(Idx: 0);
1223 };
1224
1225 // Simple case if we have a direct mapping from predicate to libcall
1226 if (const auto [Libcall, ICmpPred] = getFCMPLibcallDesc(Pred: Cond, Size);
1227 Libcall != RTLIB::UNKNOWN_LIBCALL &&
1228 ICmpPred != CmpInst::BAD_ICMP_PREDICATE) {
1229 if (BuildLibcall(Libcall, ICmpPred, DstReg)) {
1230 return Legalized;
1231 }
1232 return UnableToLegalize;
1233 }
1234
1235 // No direct mapping found, should be generated as combination of libcalls.
1236
1237 switch (Cond) {
1238 case CmpInst::FCMP_UEQ: {
1239 // FCMP_UEQ: unordered or equal
1240 // Convert into (FCMP_OEQ || FCMP_UNO).
1241
1242 const auto [OeqLibcall, OeqPred] =
1243 getFCMPLibcallDesc(Pred: CmpInst::FCMP_OEQ, Size);
1244 const auto Oeq = BuildLibcall(OeqLibcall, OeqPred, DstTy);
1245
1246 const auto [UnoLibcall, UnoPred] =
1247 getFCMPLibcallDesc(Pred: CmpInst::FCMP_UNO, Size);
1248 const auto Uno = BuildLibcall(UnoLibcall, UnoPred, DstTy);
1249 if (Oeq && Uno)
1250 MIRBuilder.buildOr(Dst: DstReg, Src0: Oeq, Src1: Uno);
1251 else
1252 return UnableToLegalize;
1253
1254 break;
1255 }
1256 case CmpInst::FCMP_ONE: {
1257 // FCMP_ONE: ordered and operands are unequal
1258 // Convert into (!FCMP_OEQ && !FCMP_UNO).
1259
1260 // We inverse the predicate instead of generating a NOT
1261 // to save one instruction.
1262 // On AArch64 isel can even select two cmp into a single ccmp.
1263 const auto [OeqLibcall, OeqPred] =
1264 getFCMPLibcallDesc(Pred: CmpInst::FCMP_OEQ, Size);
1265 const auto NotOeq =
1266 BuildLibcall(OeqLibcall, CmpInst::getInversePredicate(pred: OeqPred), DstTy);
1267
1268 const auto [UnoLibcall, UnoPred] =
1269 getFCMPLibcallDesc(Pred: CmpInst::FCMP_UNO, Size);
1270 const auto NotUno =
1271 BuildLibcall(UnoLibcall, CmpInst::getInversePredicate(pred: UnoPred), DstTy);
1272
1273 if (NotOeq && NotUno)
1274 MIRBuilder.buildAnd(Dst: DstReg, Src0: NotOeq, Src1: NotUno);
1275 else
1276 return UnableToLegalize;
1277
1278 break;
1279 }
1280 case CmpInst::FCMP_ULT:
1281 case CmpInst::FCMP_UGE:
1282 case CmpInst::FCMP_UGT:
1283 case CmpInst::FCMP_ULE:
1284 case CmpInst::FCMP_ORD: {
1285 // Convert into: !(inverse(Pred))
1286 // E.g. FCMP_ULT becomes !FCMP_OGE
1287 // This is equivalent to the following, but saves some instructions.
1288 // MIRBuilder.buildNot(
1289 // PredTy,
1290 // MIRBuilder.buildFCmp(CmpInst::getInversePredicate(Pred), PredTy,
1291 // Op1, Op2));
1292 const auto [InversedLibcall, InversedPred] =
1293 getFCMPLibcallDesc(Pred: CmpInst::getInversePredicate(pred: Cond), Size);
1294 if (!BuildLibcall(InversedLibcall,
1295 CmpInst::getInversePredicate(pred: InversedPred), DstReg))
1296 return UnableToLegalize;
1297 break;
1298 }
1299 default:
1300 return UnableToLegalize;
1301 }
1302
1303 return Legalized;
1304}
1305
1306// The function is used to legalize operations that set default environment
1307// state. In C library a call like `fesetmode(FE_DFL_MODE)` is used for that.
1308// On most targets supported in glibc FE_DFL_MODE is defined as
1309// `((const femode_t *) -1)`. Such assumption is used here. If for some target
1310// it is not true, the target must provide custom lowering.
1311LegalizerHelper::LegalizeResult
1312LegalizerHelper::createResetStateLibcall(MachineInstr &MI,
1313 LostDebugLocObserver &LocObserver) {
1314 const DataLayout &DL = MIRBuilder.getDataLayout();
1315 auto &MF = MIRBuilder.getMF();
1316 auto &Ctx = MF.getFunction().getContext();
1317
1318 // Create an argument for the library function.
1319 unsigned AddrSpace = DL.getDefaultGlobalsAddressSpace();
1320 Type *StatePtrTy = PointerType::get(C&: Ctx, AddressSpace: AddrSpace);
1321 unsigned PtrSize = DL.getPointerSizeInBits(AS: AddrSpace);
1322 LLT MemTy = LLT::pointer(AddressSpace: AddrSpace, SizeInBits: PtrSize);
1323 auto DefValue = MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: PtrSize), Val: -1LL);
1324 DstOp Dest(MRI.createGenericVirtualRegister(Ty: MemTy));
1325 MIRBuilder.buildIntToPtr(Dst: Dest, Src: DefValue);
1326
1327 RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI);
1328 return createLibcall(
1329 Libcall: RTLibcall, Result: CallLowering::ArgInfo({0}, Type::getVoidTy(C&: Ctx), 0),
1330 Args: CallLowering::ArgInfo({Dest.getReg(), StatePtrTy, 0}), LocObserver, MI: &MI);
1331}
1332
1333LegalizerHelper::LegalizeResult
1334LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
1335 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
1336
1337 switch (MI.getOpcode()) {
1338 default:
1339 return UnableToLegalize;
1340 case TargetOpcode::G_MUL:
1341 case TargetOpcode::G_SDIV:
1342 case TargetOpcode::G_UDIV:
1343 case TargetOpcode::G_SREM:
1344 case TargetOpcode::G_UREM:
1345 case TargetOpcode::G_CTLZ_ZERO_POISON: {
1346 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1347 unsigned Size = LLTy.getSizeInBits();
1348 Type *HLTy = IntegerType::get(C&: Ctx, NumBits: Size);
1349 auto Status = simpleLibcall(MI, MIRBuilder, Size, OpType: HLTy, LocObserver);
1350 if (Status != Legalized)
1351 return Status;
1352 break;
1353 }
1354 case TargetOpcode::G_FADD:
1355 case TargetOpcode::G_FSUB:
1356 case TargetOpcode::G_FMUL:
1357 case TargetOpcode::G_FDIV:
1358 case TargetOpcode::G_FMA:
1359 case TargetOpcode::G_FPOW:
1360 case TargetOpcode::G_FREM:
1361 case TargetOpcode::G_FCOS:
1362 case TargetOpcode::G_FSIN:
1363 case TargetOpcode::G_FTAN:
1364 case TargetOpcode::G_FACOS:
1365 case TargetOpcode::G_FASIN:
1366 case TargetOpcode::G_FATAN:
1367 case TargetOpcode::G_FATAN2:
1368 case TargetOpcode::G_FCOSH:
1369 case TargetOpcode::G_FSINH:
1370 case TargetOpcode::G_FTANH:
1371 case TargetOpcode::G_FLOG10:
1372 case TargetOpcode::G_FLOG:
1373 case TargetOpcode::G_FLOG2:
1374 case TargetOpcode::G_FEXP:
1375 case TargetOpcode::G_FEXP2:
1376 case TargetOpcode::G_FEXP10:
1377 case TargetOpcode::G_FCEIL:
1378 case TargetOpcode::G_FFLOOR:
1379 case TargetOpcode::G_FMINNUM:
1380 case TargetOpcode::G_FMAXNUM:
1381 case TargetOpcode::G_FMINIMUMNUM:
1382 case TargetOpcode::G_FMAXIMUMNUM:
1383 case TargetOpcode::G_FSQRT:
1384 case TargetOpcode::G_FRINT:
1385 case TargetOpcode::G_FNEARBYINT:
1386 case TargetOpcode::G_INTRINSIC_TRUNC:
1387 case TargetOpcode::G_INTRINSIC_ROUND:
1388 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: {
1389 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1390 unsigned Size = LLTy.getSizeInBits();
1391 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1392 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1393 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1394 return UnableToLegalize;
1395 }
1396 auto Status = simpleLibcall(MI, MIRBuilder, Size, OpType: HLTy, LocObserver);
1397 if (Status != Legalized)
1398 return Status;
1399 break;
1400 }
1401 case TargetOpcode::G_FSINCOS: {
1402 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1403 unsigned Size = LLTy.getSizeInBits();
1404 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1405 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1406 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1407 return UnableToLegalize;
1408 }
1409 return emitSincosLibcall(MI, MIRBuilder, Size, OpType: HLTy, LocObserver);
1410 }
1411 case TargetOpcode::G_FMODF: {
1412 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1413 unsigned Size = LLTy.getSizeInBits();
1414 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1415 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1416 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1417 return UnableToLegalize;
1418 }
1419 return emitModfLibcall(MI, MIRBuilder, Size, OpType: HLTy, LocObserver);
1420 }
1421 case TargetOpcode::G_LROUND:
1422 case TargetOpcode::G_LLROUND:
1423 case TargetOpcode::G_INTRINSIC_LRINT:
1424 case TargetOpcode::G_INTRINSIC_LLRINT: {
1425 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
1426 unsigned Size = LLTy.getSizeInBits();
1427 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1428 Type *ITy = IntegerType::get(
1429 C&: Ctx, NumBits: MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits());
1430 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1431 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1432 return UnableToLegalize;
1433 }
1434 auto Libcall = getRTLibDesc(Opcode: MI.getOpcode(), Size);
1435 LegalizeResult Status =
1436 createLibcall(Libcall, Result: {MI.getOperand(i: 0).getReg(), ITy, 0},
1437 Args: {{MI.getOperand(i: 1).getReg(), HLTy, 0}}, LocObserver, MI: &MI);
1438 if (Status != Legalized)
1439 return Status;
1440 MI.eraseFromParent();
1441 return Legalized;
1442 }
1443 case TargetOpcode::G_FPOWI:
1444 case TargetOpcode::G_FLDEXP: {
1445 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1446 unsigned Size = LLTy.getSizeInBits();
1447 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1448 Type *ITy = IntegerType::get(
1449 C&: Ctx, NumBits: MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits());
1450 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1451 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1452 return UnableToLegalize;
1453 }
1454 auto Libcall = getRTLibDesc(Opcode: MI.getOpcode(), Size);
1455 SmallVector<CallLowering::ArgInfo, 2> Args = {
1456 {MI.getOperand(i: 1).getReg(), HLTy, 0},
1457 {MI.getOperand(i: 2).getReg(), ITy, 1}};
1458 Args[1].Flags[0].setSExt();
1459 LegalizeResult Status = createLibcall(
1460 Libcall, Result: {MI.getOperand(i: 0).getReg(), HLTy, 0}, Args, LocObserver, MI: &MI);
1461 if (Status != Legalized)
1462 return Status;
1463 break;
1464 }
1465 case TargetOpcode::G_FPEXT:
1466 case TargetOpcode::G_FPTRUNC: {
1467 Type *FromTy = getFloatTypeForLLT(Ctx, Ty: MRI.getType(Reg: MI.getOperand(i: 1).getReg()));
1468 Type *ToTy = getFloatTypeForLLT(Ctx, Ty: MRI.getType(Reg: MI.getOperand(i: 0).getReg()));
1469 if (!FromTy || !ToTy)
1470 return UnableToLegalize;
1471 LegalizeResult Status = conversionLibcall(MI, ToType: ToTy, FromType: FromTy, LocObserver);
1472 if (Status != Legalized)
1473 return Status;
1474 break;
1475 }
1476 case TargetOpcode::G_FCMP: {
1477 LegalizeResult Status = createFCMPLibcall(MI, LocObserver);
1478 if (Status != Legalized)
1479 return Status;
1480 MI.eraseFromParent();
1481 return Status;
1482 }
1483 case TargetOpcode::G_FPTOSI:
1484 case TargetOpcode::G_FPTOUI: {
1485 // FIXME: Support other types
1486 Type *FromTy =
1487 getFloatTypeForLLT(Ctx, Ty: MRI.getType(Reg: MI.getOperand(i: 1).getReg()));
1488 unsigned ToSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
1489 if ((ToSize != 32 && ToSize != 64 && ToSize != 128) || !FromTy)
1490 return UnableToLegalize;
1491 LegalizeResult Status = conversionLibcall(MI, ToType: Type::getIntNTy(C&: Ctx, N: ToSize),
1492 FromType: FromTy, LocObserver);
1493 if (Status != Legalized)
1494 return Status;
1495 break;
1496 }
1497 case TargetOpcode::G_SITOFP:
1498 case TargetOpcode::G_UITOFP: {
1499 unsigned FromSize = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
1500 Type *ToTy =
1501 getFloatTypeForLLT(Ctx, Ty: MRI.getType(Reg: MI.getOperand(i: 0).getReg()));
1502 if ((FromSize != 32 && FromSize != 64 && FromSize != 128) || !ToTy)
1503 return UnableToLegalize;
1504 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SITOFP;
1505 LegalizeResult Status = conversionLibcall(
1506 MI, ToType: ToTy, FromType: Type::getIntNTy(C&: Ctx, N: FromSize), LocObserver, IsSigned);
1507 if (Status != Legalized)
1508 return Status;
1509 break;
1510 }
1511 case TargetOpcode::G_ATOMICRMW_XCHG:
1512 case TargetOpcode::G_ATOMICRMW_ADD:
1513 case TargetOpcode::G_ATOMICRMW_SUB:
1514 case TargetOpcode::G_ATOMICRMW_AND:
1515 case TargetOpcode::G_ATOMICRMW_OR:
1516 case TargetOpcode::G_ATOMICRMW_XOR:
1517 case TargetOpcode::G_ATOMIC_CMPXCHG:
1518 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
1519 auto Status = createAtomicLibcall(MI);
1520 if (Status != Legalized)
1521 return Status;
1522 break;
1523 }
1524 case TargetOpcode::G_BZERO:
1525 case TargetOpcode::G_MEMCPY:
1526 case TargetOpcode::G_MEMMOVE:
1527 case TargetOpcode::G_MEMSET: {
1528 LegalizeResult Result =
1529 createMemLibcall(MRI&: *MIRBuilder.getMRI(), MI, LocObserver);
1530 if (Result != Legalized)
1531 return Result;
1532 MI.eraseFromParent();
1533 return Result;
1534 }
1535 case TargetOpcode::G_GET_FPENV:
1536 case TargetOpcode::G_GET_FPMODE: {
1537 LegalizeResult Result = createGetStateLibcall(MI, LocObserver);
1538 if (Result != Legalized)
1539 return Result;
1540 break;
1541 }
1542 case TargetOpcode::G_SET_FPENV:
1543 case TargetOpcode::G_SET_FPMODE: {
1544 LegalizeResult Result = createSetStateLibcall(MI, LocObserver);
1545 if (Result != Legalized)
1546 return Result;
1547 break;
1548 }
1549 case TargetOpcode::G_RESET_FPENV:
1550 case TargetOpcode::G_RESET_FPMODE: {
1551 LegalizeResult Result = createResetStateLibcall(MI, LocObserver);
1552 if (Result != Legalized)
1553 return Result;
1554 break;
1555 }
1556 }
1557
1558 MI.eraseFromParent();
1559 return Legalized;
1560}
1561
1562LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
1563 unsigned TypeIdx,
1564 LLT NarrowTy) {
1565 uint64_t SizeOp0 = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
1566 uint64_t NarrowSize = NarrowTy.getSizeInBits();
1567
1568 switch (MI.getOpcode()) {
1569 default:
1570 return UnableToLegalize;
1571 case TargetOpcode::G_IMPLICIT_DEF: {
1572 Register DstReg = MI.getOperand(i: 0).getReg();
1573 LLT DstTy = MRI.getType(Reg: DstReg);
1574
1575 // If SizeOp0 is not an exact multiple of NarrowSize, emit
1576 // G_ANYEXT(G_IMPLICIT_DEF). Cast result to vector if needed.
1577 // FIXME: Although this would also be legal for the general case, it causes
1578 // a lot of regressions in the emitted code (superfluous COPYs, artifact
1579 // combines not being hit). This seems to be a problem related to the
1580 // artifact combiner.
1581 if (SizeOp0 % NarrowSize != 0) {
1582 LLT ImplicitTy = DstTy.changeElementType(NewEltTy: NarrowTy);
1583 Register ImplicitReg = MIRBuilder.buildUndef(Res: ImplicitTy).getReg(Idx: 0);
1584 MIRBuilder.buildAnyExt(Res: DstReg, Op: ImplicitReg);
1585
1586 MI.eraseFromParent();
1587 return Legalized;
1588 }
1589
1590 int NumParts = SizeOp0 / NarrowSize;
1591
1592 SmallVector<Register, 2> DstRegs;
1593 for (int i = 0; i < NumParts; ++i)
1594 DstRegs.push_back(Elt: MIRBuilder.buildUndef(Res: NarrowTy).getReg(Idx: 0));
1595
1596 if (DstTy.isVector())
1597 MIRBuilder.buildBuildVector(Res: DstReg, Ops: DstRegs);
1598 else
1599 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
1600 MI.eraseFromParent();
1601 return Legalized;
1602 }
1603 case TargetOpcode::G_CONSTANT: {
1604 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1605 const APInt &Val = MI.getOperand(i: 1).getCImm()->getValue();
1606 unsigned TotalSize = Ty.getSizeInBits();
1607 unsigned NarrowSize = NarrowTy.getSizeInBits();
1608 int NumParts = TotalSize / NarrowSize;
1609
1610 SmallVector<Register, 4> PartRegs;
1611 for (int I = 0; I != NumParts; ++I) {
1612 unsigned Offset = I * NarrowSize;
1613 auto K = MIRBuilder.buildConstant(Res: NarrowTy,
1614 Val: Val.lshr(shiftAmt: Offset).trunc(width: NarrowSize));
1615 PartRegs.push_back(Elt: K.getReg(Idx: 0));
1616 }
1617
1618 LLT LeftoverTy;
1619 unsigned LeftoverBits = TotalSize - NumParts * NarrowSize;
1620 SmallVector<Register, 1> LeftoverRegs;
1621 if (LeftoverBits != 0) {
1622 LeftoverTy = LLT::scalar(SizeInBits: LeftoverBits);
1623 auto K = MIRBuilder.buildConstant(
1624 Res: LeftoverTy,
1625 Val: Val.lshr(shiftAmt: NumParts * NarrowSize).trunc(width: LeftoverBits));
1626 LeftoverRegs.push_back(Elt: K.getReg(Idx: 0));
1627 }
1628
1629 insertParts(DstReg: MI.getOperand(i: 0).getReg(),
1630 ResultTy: Ty, PartTy: NarrowTy, PartRegs, LeftoverTy, LeftoverRegs);
1631
1632 MI.eraseFromParent();
1633 return Legalized;
1634 }
1635 case TargetOpcode::G_BITCAST: {
1636 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
1637 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1638 if (SrcTy.isVector() || DstTy.isVector())
1639 return UnableToLegalize;
1640
1641 auto Unmerge = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: MI.getOperand(i: 1));
1642 unsigned NumParts = Unmerge->getNumOperands() - 1;
1643 SmallVector<Register> Parts(NumParts);
1644 for (unsigned I = 0; I < NumParts; ++I)
1645 Parts[I] = Unmerge.getReg(Idx: I);
1646 MIRBuilder.buildMergeValues(Res: MI.getOperand(i: 0), Ops: Parts);
1647 MI.eraseFromParent();
1648 return Legalized;
1649 }
1650 case TargetOpcode::G_SEXT:
1651 case TargetOpcode::G_ZEXT:
1652 case TargetOpcode::G_ANYEXT:
1653 return narrowScalarExt(MI, TypeIdx, Ty: NarrowTy);
1654 case TargetOpcode::G_TRUNC: {
1655 if (TypeIdx != 1)
1656 return UnableToLegalize;
1657
1658 uint64_t SizeOp1 = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
1659 if (NarrowTy.getSizeInBits() * 2 != SizeOp1) {
1660 LLVM_DEBUG(dbgs() << "Can't narrow trunc to type " << NarrowTy << "\n");
1661 return UnableToLegalize;
1662 }
1663
1664 auto Unmerge = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: MI.getOperand(i: 1));
1665 MIRBuilder.buildCopy(Res: MI.getOperand(i: 0), Op: Unmerge.getReg(Idx: 0));
1666 MI.eraseFromParent();
1667 return Legalized;
1668 }
1669 case TargetOpcode::G_CONSTANT_FOLD_BARRIER:
1670 case TargetOpcode::G_FREEZE: {
1671 if (TypeIdx != 0)
1672 return UnableToLegalize;
1673
1674 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1675 // Should widen scalar first
1676 if (Ty.getSizeInBits() % NarrowTy.getSizeInBits() != 0)
1677 return UnableToLegalize;
1678
1679 auto Unmerge = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: MI.getOperand(i: 1).getReg());
1680 SmallVector<Register, 8> Parts;
1681 for (unsigned i = 0; i < Unmerge->getNumDefs(); ++i) {
1682 Parts.push_back(
1683 Elt: MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {NarrowTy}, SrcOps: {Unmerge.getReg(Idx: i)})
1684 .getReg(Idx: 0));
1685 }
1686
1687 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0).getReg(), Ops: Parts);
1688 MI.eraseFromParent();
1689 return Legalized;
1690 }
1691 case TargetOpcode::G_ADD:
1692 case TargetOpcode::G_SUB:
1693 case TargetOpcode::G_SADDO:
1694 case TargetOpcode::G_SSUBO:
1695 case TargetOpcode::G_SADDE:
1696 case TargetOpcode::G_SSUBE:
1697 case TargetOpcode::G_UADDO:
1698 case TargetOpcode::G_USUBO:
1699 case TargetOpcode::G_UADDE:
1700 case TargetOpcode::G_USUBE:
1701 return narrowScalarAddSub(MI, TypeIdx, NarrowTy);
1702 case TargetOpcode::G_MUL:
1703 case TargetOpcode::G_UMULH:
1704 case TargetOpcode::G_SMULH:
1705 return narrowScalarMul(MI, Ty: NarrowTy);
1706 case TargetOpcode::G_EXTRACT:
1707 return narrowScalarExtract(MI, TypeIdx, Ty: NarrowTy);
1708 case TargetOpcode::G_INSERT:
1709 return narrowScalarInsert(MI, TypeIdx, Ty: NarrowTy);
1710 case TargetOpcode::G_LOAD: {
1711 auto &LoadMI = cast<GLoad>(Val&: MI);
1712 Register DstReg = LoadMI.getDstReg();
1713 LLT DstTy = MRI.getType(Reg: DstReg);
1714 if (DstTy.isVector())
1715 return UnableToLegalize;
1716
1717 if (8 * LoadMI.getMemSize().getValue() != DstTy.getSizeInBits()) {
1718 Register TmpReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
1719 MIRBuilder.buildLoad(Res: TmpReg, Addr: LoadMI.getPointerReg(), MMO&: LoadMI.getMMO());
1720 MIRBuilder.buildAnyExt(Res: DstReg, Op: TmpReg);
1721 LoadMI.eraseFromParent();
1722 return Legalized;
1723 }
1724
1725 return reduceLoadStoreWidth(MI&: LoadMI, TypeIdx, NarrowTy);
1726 }
1727 case TargetOpcode::G_ZEXTLOAD:
1728 case TargetOpcode::G_SEXTLOAD:
1729 case TargetOpcode::G_FPEXTLOAD: {
1730 auto &LoadMI = cast<GExtLoad>(Val&: MI);
1731 Register DstReg = LoadMI.getDstReg();
1732 Register PtrReg = LoadMI.getPointerReg();
1733
1734 Register TmpReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
1735 auto &MMO = LoadMI.getMMO();
1736 unsigned MemSize = MMO.getSizeInBits().getValue();
1737
1738 if (MemSize == NarrowSize) {
1739 MIRBuilder.buildLoad(Res: TmpReg, Addr: PtrReg, MMO);
1740 } else if (MemSize < NarrowSize) {
1741 MIRBuilder.buildLoadInstr(Opcode: LoadMI.getOpcode(), Res: TmpReg, Addr: PtrReg, MMO);
1742 } else if (MemSize > NarrowSize) {
1743 // FIXME: Need to split the load.
1744 return UnableToLegalize;
1745 }
1746
1747 if (isa<GZExtLoad>(Val: LoadMI))
1748 MIRBuilder.buildZExt(Res: DstReg, Op: TmpReg);
1749 else if (isa<GSExtLoad>(Val: LoadMI))
1750 MIRBuilder.buildSExt(Res: DstReg, Op: TmpReg);
1751 else
1752 MIRBuilder.buildFPExt(Res: DstReg, Op: TmpReg);
1753
1754 LoadMI.eraseFromParent();
1755 return Legalized;
1756 }
1757 case TargetOpcode::G_STORE: {
1758 auto &StoreMI = cast<GStore>(Val&: MI);
1759
1760 Register SrcReg = StoreMI.getValueReg();
1761 LLT SrcTy = MRI.getType(Reg: SrcReg);
1762 if (SrcTy.isVector())
1763 return UnableToLegalize;
1764
1765 int NumParts = SizeOp0 / NarrowSize;
1766 unsigned HandledSize = NumParts * NarrowTy.getSizeInBits();
1767 unsigned LeftoverBits = SrcTy.getSizeInBits() - HandledSize;
1768 if (SrcTy.isVector() && LeftoverBits != 0)
1769 return UnableToLegalize;
1770
1771 if (8 * StoreMI.getMemSize().getValue() != SrcTy.getSizeInBits()) {
1772 Register TmpReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
1773 MIRBuilder.buildTrunc(Res: TmpReg, Op: SrcReg);
1774 MIRBuilder.buildStore(Val: TmpReg, Addr: StoreMI.getPointerReg(), MMO&: StoreMI.getMMO());
1775 StoreMI.eraseFromParent();
1776 return Legalized;
1777 }
1778
1779 return reduceLoadStoreWidth(MI&: StoreMI, TypeIdx: 0, NarrowTy);
1780 }
1781 case TargetOpcode::G_FPTRUNCSTORE: {
1782 auto &StoreMI = cast<GFPTruncStore>(Val&: MI);
1783 Register SrcReg = StoreMI.getValueReg();
1784 Register PtrReg = StoreMI.getPointerReg();
1785
1786 auto &MMO = StoreMI.getMMO();
1787 unsigned MemSize = MMO.getSizeInBits().getValue();
1788 if (MemSize > NarrowSize) {
1789 return UnableToLegalize;
1790 }
1791
1792 auto TmpReg = MIRBuilder.buildFPTrunc(Res: NarrowTy, Op: SrcReg);
1793 if (MemSize == NarrowSize) {
1794 MIRBuilder.buildStore(Val: TmpReg, Addr: PtrReg, MMO);
1795 } else if (MemSize < NarrowSize) {
1796 MIRBuilder.buildStoreInstr(Opcode: TargetOpcode::G_FPTRUNCSTORE, Val: TmpReg, Addr: PtrReg,
1797 MMO);
1798 }
1799
1800 StoreMI.eraseFromParent();
1801 return Legalized;
1802 }
1803 case TargetOpcode::G_SELECT:
1804 return narrowScalarSelect(MI, TypeIdx, Ty: NarrowTy);
1805 case TargetOpcode::G_AND:
1806 case TargetOpcode::G_OR:
1807 case TargetOpcode::G_XOR: {
1808 // Legalize bitwise operation:
1809 // A = BinOp<Ty> B, C
1810 // into:
1811 // B1, ..., BN = G_UNMERGE_VALUES B
1812 // C1, ..., CN = G_UNMERGE_VALUES C
1813 // A1 = BinOp<Ty/N> B1, C2
1814 // ...
1815 // AN = BinOp<Ty/N> BN, CN
1816 // A = G_MERGE_VALUES A1, ..., AN
1817 return narrowScalarBasic(MI, TypeIdx, Ty: NarrowTy);
1818 }
1819 case TargetOpcode::G_SHL:
1820 case TargetOpcode::G_LSHR:
1821 case TargetOpcode::G_ASHR:
1822 return narrowScalarShift(MI, TypeIdx, Ty: NarrowTy);
1823 case TargetOpcode::G_CTLZ:
1824 case TargetOpcode::G_CTLZ_ZERO_POISON:
1825 case TargetOpcode::G_CTTZ:
1826 case TargetOpcode::G_CTTZ_ZERO_POISON:
1827 case TargetOpcode::G_CTLS:
1828 case TargetOpcode::G_CTPOP:
1829 if (TypeIdx == 1)
1830 switch (MI.getOpcode()) {
1831 case TargetOpcode::G_CTLZ:
1832 case TargetOpcode::G_CTLZ_ZERO_POISON:
1833 return narrowScalarCTLZ(MI, TypeIdx, Ty: NarrowTy);
1834 case TargetOpcode::G_CTTZ:
1835 case TargetOpcode::G_CTTZ_ZERO_POISON:
1836 return narrowScalarCTTZ(MI, TypeIdx, Ty: NarrowTy);
1837 case TargetOpcode::G_CTPOP:
1838 return narrowScalarCTPOP(MI, TypeIdx, Ty: NarrowTy);
1839 case TargetOpcode::G_CTLS:
1840 return narrowScalarCTLS(MI, TypeIdx, Ty: NarrowTy);
1841 default:
1842 return UnableToLegalize;
1843 }
1844
1845 Observer.changingInstr(MI);
1846 narrowScalarDst(MI, NarrowTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_ZEXT);
1847 Observer.changedInstr(MI);
1848 return Legalized;
1849 case TargetOpcode::G_INTTOPTR:
1850 if (TypeIdx != 1)
1851 return UnableToLegalize;
1852
1853 Observer.changingInstr(MI);
1854 narrowScalarSrc(MI, NarrowTy, OpIdx: 1);
1855 Observer.changedInstr(MI);
1856 return Legalized;
1857 case TargetOpcode::G_PTRTOINT:
1858 if (TypeIdx != 0)
1859 return UnableToLegalize;
1860
1861 Observer.changingInstr(MI);
1862 narrowScalarDst(MI, NarrowTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_ZEXT);
1863 Observer.changedInstr(MI);
1864 return Legalized;
1865 case TargetOpcode::G_PHI: {
1866 // FIXME: add support for when SizeOp0 isn't an exact multiple of
1867 // NarrowSize.
1868 if (SizeOp0 % NarrowSize != 0)
1869 return UnableToLegalize;
1870
1871 unsigned NumParts = SizeOp0 / NarrowSize;
1872 SmallVector<Register, 2> DstRegs(NumParts);
1873 SmallVector<SmallVector<Register, 2>, 2> SrcRegs(MI.getNumOperands() / 2);
1874 Observer.changingInstr(MI);
1875 for (unsigned i = 1; i < MI.getNumOperands(); i += 2) {
1876 MachineBasicBlock &OpMBB = *MI.getOperand(i: i + 1).getMBB();
1877 MIRBuilder.setInsertPt(MBB&: OpMBB, II: getPHIIncomingInsertPoint(
1878 MBB&: OpMBB, Reg: MI.getOperand(i).getReg(), MRI));
1879 extractParts(Reg: MI.getOperand(i).getReg(), Ty: NarrowTy, NumParts,
1880 VRegs&: SrcRegs[i / 2], MIRBuilder, MRI);
1881 }
1882 MachineBasicBlock &MBB = *MI.getParent();
1883 MIRBuilder.setInsertPt(MBB, II: MI);
1884 for (unsigned i = 0; i < NumParts; ++i) {
1885 DstRegs[i] = MRI.createGenericVirtualRegister(Ty: NarrowTy);
1886 MachineInstrBuilder MIB =
1887 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_PHI).addDef(RegNo: DstRegs[i]);
1888 for (unsigned j = 1; j < MI.getNumOperands(); j += 2)
1889 MIB.addUse(RegNo: SrcRegs[j / 2][i]).add(MO: MI.getOperand(i: j + 1));
1890 }
1891 MIRBuilder.setInsertPt(MBB, II: MBB.getFirstNonPHI());
1892 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0), Ops: DstRegs);
1893 Observer.changedInstr(MI);
1894 MI.eraseFromParent();
1895 return Legalized;
1896 }
1897 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1898 case TargetOpcode::G_INSERT_VECTOR_ELT: {
1899 if (TypeIdx != 2)
1900 return UnableToLegalize;
1901
1902 int OpIdx = MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
1903 Observer.changingInstr(MI);
1904 narrowScalarSrc(MI, NarrowTy, OpIdx);
1905 Observer.changedInstr(MI);
1906 return Legalized;
1907 }
1908 case TargetOpcode::G_ICMP: {
1909 Register LHS = MI.getOperand(i: 2).getReg();
1910 LLT SrcTy = MRI.getType(Reg: LHS);
1911 CmpInst::Predicate Pred =
1912 static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
1913
1914 LLT LeftoverTy; // Example: s88 -> s64 (NarrowTy) + s24 (leftover)
1915 SmallVector<Register, 4> LHSPartRegs, LHSLeftoverRegs;
1916 if (!extractParts(Reg: LHS, RegTy: SrcTy, MainTy: NarrowTy, LeftoverTy, VRegs&: LHSPartRegs,
1917 LeftoverVRegs&: LHSLeftoverRegs, MIRBuilder, MRI))
1918 return UnableToLegalize;
1919
1920 LLT Unused; // Matches LeftoverTy; G_ICMP LHS and RHS are the same type.
1921 SmallVector<Register, 4> RHSPartRegs, RHSLeftoverRegs;
1922 if (!extractParts(Reg: MI.getOperand(i: 3).getReg(), RegTy: SrcTy, MainTy: NarrowTy, LeftoverTy&: Unused,
1923 VRegs&: RHSPartRegs, LeftoverVRegs&: RHSLeftoverRegs, MIRBuilder, MRI))
1924 return UnableToLegalize;
1925
1926 // We now have the LHS and RHS of the compare split into narrow-type
1927 // registers, plus potentially some leftover type.
1928 Register Dst = MI.getOperand(i: 0).getReg();
1929 LLT ResTy = MRI.getType(Reg: Dst);
1930 if (ICmpInst::isEquality(P: Pred)) {
1931 // For each part on the LHS and RHS, keep track of the result of XOR-ing
1932 // them together. For each equal part, the result should be all 0s. For
1933 // each non-equal part, we'll get at least one 1.
1934 auto Zero = MIRBuilder.buildConstant(Res: NarrowTy, Val: 0);
1935 SmallVector<Register, 4> Xors;
1936 for (auto LHSAndRHS : zip(t&: LHSPartRegs, u&: RHSPartRegs)) {
1937 auto LHS = std::get<0>(t&: LHSAndRHS);
1938 auto RHS = std::get<1>(t&: LHSAndRHS);
1939 auto Xor = MIRBuilder.buildXor(Dst: NarrowTy, Src0: LHS, Src1: RHS).getReg(Idx: 0);
1940 Xors.push_back(Elt: Xor);
1941 }
1942
1943 // Build a G_XOR for each leftover register. Each G_XOR must be widened
1944 // to the desired narrow type so that we can OR them together later.
1945 SmallVector<Register, 4> WidenedXors;
1946 for (auto LHSAndRHS : zip(t&: LHSLeftoverRegs, u&: RHSLeftoverRegs)) {
1947 auto LHS = std::get<0>(t&: LHSAndRHS);
1948 auto RHS = std::get<1>(t&: LHSAndRHS);
1949 auto Xor = MIRBuilder.buildXor(Dst: LeftoverTy, Src0: LHS, Src1: RHS).getReg(Idx: 0);
1950 LLT GCDTy = extractGCDType(Parts&: WidenedXors, DstTy: NarrowTy, NarrowTy: LeftoverTy, SrcReg: Xor);
1951 buildLCMMergePieces(DstTy: LeftoverTy, NarrowTy, GCDTy, VRegs&: WidenedXors,
1952 /* PadStrategy = */ TargetOpcode::G_ZEXT);
1953 llvm::append_range(C&: Xors, R&: WidenedXors);
1954 }
1955
1956 // Now, for each part we broke up, we know if they are equal/not equal
1957 // based off the G_XOR. We can OR these all together and compare against
1958 // 0 to get the result.
1959 assert(Xors.size() >= 2 && "Should have gotten at least two Xors?");
1960 auto Or = MIRBuilder.buildOr(Dst: NarrowTy, Src0: Xors[0], Src1: Xors[1]);
1961 for (unsigned I = 2, E = Xors.size(); I < E; ++I)
1962 Or = MIRBuilder.buildOr(Dst: NarrowTy, Src0: Or, Src1: Xors[I]);
1963 MIRBuilder.buildICmp(Pred, Res: Dst, Op0: Or, Op1: Zero);
1964 } else {
1965 Register CmpIn;
1966 for (unsigned I = 0, E = LHSPartRegs.size(); I != E; ++I) {
1967 Register CmpOut;
1968 CmpInst::Predicate PartPred;
1969
1970 if (I == E - 1 && LHSLeftoverRegs.empty()) {
1971 PartPred = Pred;
1972 CmpOut = Dst;
1973 } else {
1974 PartPred = ICmpInst::getUnsignedPredicate(Pred);
1975 CmpOut = MRI.createGenericVirtualRegister(Ty: ResTy);
1976 }
1977
1978 if (!CmpIn) {
1979 MIRBuilder.buildICmp(Pred: PartPred, Res: CmpOut, Op0: LHSPartRegs[I],
1980 Op1: RHSPartRegs[I]);
1981 } else {
1982 auto Cmp = MIRBuilder.buildICmp(Pred: PartPred, Res: ResTy, Op0: LHSPartRegs[I],
1983 Op1: RHSPartRegs[I]);
1984 auto CmpEq = MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: ResTy,
1985 Op0: LHSPartRegs[I], Op1: RHSPartRegs[I]);
1986 MIRBuilder.buildSelect(Res: CmpOut, Tst: CmpEq, Op0: CmpIn, Op1: Cmp);
1987 }
1988
1989 CmpIn = CmpOut;
1990 }
1991
1992 for (unsigned I = 0, E = LHSLeftoverRegs.size(); I != E; ++I) {
1993 Register CmpOut;
1994 CmpInst::Predicate PartPred;
1995
1996 if (I == E - 1) {
1997 PartPred = Pred;
1998 CmpOut = Dst;
1999 } else {
2000 PartPred = ICmpInst::getUnsignedPredicate(Pred);
2001 CmpOut = MRI.createGenericVirtualRegister(Ty: ResTy);
2002 }
2003
2004 if (!CmpIn) {
2005 MIRBuilder.buildICmp(Pred: PartPred, Res: CmpOut, Op0: LHSLeftoverRegs[I],
2006 Op1: RHSLeftoverRegs[I]);
2007 } else {
2008 auto Cmp = MIRBuilder.buildICmp(Pred: PartPred, Res: ResTy, Op0: LHSLeftoverRegs[I],
2009 Op1: RHSLeftoverRegs[I]);
2010 auto CmpEq =
2011 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: ResTy,
2012 Op0: LHSLeftoverRegs[I], Op1: RHSLeftoverRegs[I]);
2013 MIRBuilder.buildSelect(Res: CmpOut, Tst: CmpEq, Op0: CmpIn, Op1: Cmp);
2014 }
2015
2016 CmpIn = CmpOut;
2017 }
2018 }
2019 MI.eraseFromParent();
2020 return Legalized;
2021 }
2022 case TargetOpcode::G_FCMP:
2023 if (TypeIdx != 0)
2024 return UnableToLegalize;
2025
2026 Observer.changingInstr(MI);
2027 narrowScalarDst(MI, NarrowTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_ZEXT);
2028 Observer.changedInstr(MI);
2029 return Legalized;
2030
2031 case TargetOpcode::G_SEXT_INREG: {
2032 if (TypeIdx != 0)
2033 return UnableToLegalize;
2034
2035 int64_t SizeInBits = MI.getOperand(i: 2).getImm();
2036
2037 // So long as the new type has more bits than the bits we're extending we
2038 // don't need to break it apart.
2039 if (NarrowTy.getScalarSizeInBits() > SizeInBits) {
2040 Observer.changingInstr(MI);
2041 // We don't lose any non-extension bits by truncating the src and
2042 // sign-extending the dst.
2043 MachineOperand &MO1 = MI.getOperand(i: 1);
2044 auto TruncMIB = MIRBuilder.buildTrunc(Res: NarrowTy, Op: MO1);
2045 MO1.setReg(TruncMIB.getReg(Idx: 0));
2046
2047 MachineOperand &MO2 = MI.getOperand(i: 0);
2048 Register DstExt = MRI.createGenericVirtualRegister(Ty: NarrowTy);
2049 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2050 MIRBuilder.buildSExt(Res: MO2, Op: DstExt);
2051 MO2.setReg(DstExt);
2052 Observer.changedInstr(MI);
2053 return Legalized;
2054 }
2055
2056 // Break it apart. Components below the extension point are unmodified. The
2057 // component containing the extension point becomes a narrower SEXT_INREG.
2058 // Components above it are ashr'd from the component containing the
2059 // extension point.
2060 if (SizeOp0 % NarrowSize != 0)
2061 return UnableToLegalize;
2062 int NumParts = SizeOp0 / NarrowSize;
2063
2064 // List the registers where the destination will be scattered.
2065 SmallVector<Register, 2> DstRegs;
2066 // List the registers where the source will be split.
2067 SmallVector<Register, 2> SrcRegs;
2068
2069 // Create all the temporary registers.
2070 for (int i = 0; i < NumParts; ++i) {
2071 Register SrcReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
2072
2073 SrcRegs.push_back(Elt: SrcReg);
2074 }
2075
2076 // Explode the big arguments into smaller chunks.
2077 MIRBuilder.buildUnmerge(Res: SrcRegs, Op: MI.getOperand(i: 1));
2078
2079 Register AshrCstReg =
2080 MIRBuilder.buildConstant(Res: NarrowTy, Val: NarrowTy.getScalarSizeInBits() - 1)
2081 .getReg(Idx: 0);
2082 Register FullExtensionReg;
2083 Register PartialExtensionReg;
2084
2085 // Do the operation on each small part.
2086 for (int i = 0; i < NumParts; ++i) {
2087 if ((i + 1) * NarrowTy.getScalarSizeInBits() <= SizeInBits) {
2088 DstRegs.push_back(Elt: SrcRegs[i]);
2089 PartialExtensionReg = DstRegs.back();
2090 } else if (i * NarrowTy.getScalarSizeInBits() >= SizeInBits) {
2091 assert(PartialExtensionReg &&
2092 "Expected to visit partial extension before full");
2093 if (FullExtensionReg) {
2094 DstRegs.push_back(Elt: FullExtensionReg);
2095 continue;
2096 }
2097 DstRegs.push_back(
2098 Elt: MIRBuilder.buildAShr(Dst: NarrowTy, Src0: PartialExtensionReg, Src1: AshrCstReg)
2099 .getReg(Idx: 0));
2100 FullExtensionReg = DstRegs.back();
2101 } else {
2102 DstRegs.push_back(
2103 Elt: MIRBuilder
2104 .buildInstr(
2105 Opc: TargetOpcode::G_SEXT_INREG, DstOps: {NarrowTy},
2106 SrcOps: {SrcRegs[i], SizeInBits % NarrowTy.getScalarSizeInBits()})
2107 .getReg(Idx: 0));
2108 PartialExtensionReg = DstRegs.back();
2109 }
2110 }
2111
2112 // Gather the destination registers into the final destination.
2113 Register DstReg = MI.getOperand(i: 0).getReg();
2114 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
2115 MI.eraseFromParent();
2116 return Legalized;
2117 }
2118 case TargetOpcode::G_BSWAP:
2119 case TargetOpcode::G_BITREVERSE: {
2120 if (SizeOp0 % NarrowSize != 0)
2121 return UnableToLegalize;
2122
2123 Observer.changingInstr(MI);
2124 SmallVector<Register, 2> SrcRegs, DstRegs;
2125 unsigned NumParts = SizeOp0 / NarrowSize;
2126 extractParts(Reg: MI.getOperand(i: 1).getReg(), Ty: NarrowTy, NumParts, VRegs&: SrcRegs,
2127 MIRBuilder, MRI);
2128
2129 for (unsigned i = 0; i < NumParts; ++i) {
2130 auto DstPart = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {NarrowTy},
2131 SrcOps: {SrcRegs[NumParts - 1 - i]});
2132 DstRegs.push_back(Elt: DstPart.getReg(Idx: 0));
2133 }
2134
2135 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0), Ops: DstRegs);
2136
2137 Observer.changedInstr(MI);
2138 MI.eraseFromParent();
2139 return Legalized;
2140 }
2141 case TargetOpcode::G_PTR_ADD:
2142 case TargetOpcode::G_PTRMASK: {
2143 if (TypeIdx != 1)
2144 return UnableToLegalize;
2145 Observer.changingInstr(MI);
2146 narrowScalarSrc(MI, NarrowTy, OpIdx: 2);
2147 Observer.changedInstr(MI);
2148 return Legalized;
2149 }
2150 case TargetOpcode::G_FPTOUI:
2151 case TargetOpcode::G_FPTOSI:
2152 case TargetOpcode::G_FPTOUI_SAT:
2153 case TargetOpcode::G_FPTOSI_SAT:
2154 return narrowScalarFPTOI(MI, TypeIdx, Ty: NarrowTy);
2155 case TargetOpcode::G_FPEXT:
2156 if (TypeIdx != 0)
2157 return UnableToLegalize;
2158 Observer.changingInstr(MI);
2159 narrowScalarDst(MI, NarrowTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_FPEXT);
2160 Observer.changedInstr(MI);
2161 return Legalized;
2162 case TargetOpcode::G_FLDEXP:
2163 case TargetOpcode::G_STRICT_FLDEXP:
2164 return narrowScalarFLDEXP(MI, TypeIdx, Ty: NarrowTy);
2165 case TargetOpcode::G_VSCALE: {
2166 Register Dst = MI.getOperand(i: 0).getReg();
2167 LLT Ty = MRI.getType(Reg: Dst);
2168
2169 // Assume VSCALE(1) fits into a legal integer
2170 const APInt One(NarrowTy.getSizeInBits(), 1);
2171 auto VScaleBase = MIRBuilder.buildVScale(Res: NarrowTy, MinElts: One);
2172 auto ZExt = MIRBuilder.buildZExt(Res: Ty, Op: VScaleBase);
2173 auto C = MIRBuilder.buildConstant(Res: Ty, Val: *MI.getOperand(i: 1).getCImm());
2174 MIRBuilder.buildMul(Dst, Src0: ZExt, Src1: C);
2175
2176 MI.eraseFromParent();
2177 return Legalized;
2178 }
2179 }
2180}
2181
2182Register LegalizerHelper::coerceToInteger(Register Val) {
2183 LLT Ty = MRI.getType(Reg: Val);
2184 if (Ty.isScalar() && !Ty.isFloat())
2185 return Val;
2186
2187 const DataLayout &DL = MIRBuilder.getDataLayout();
2188 LLT NewTy = LLT::integer(SizeInBits: Ty.getSizeInBits());
2189
2190 if (Ty.isFloat())
2191 return MIRBuilder.buildBitcast(Dst: NewTy, Src: Val).getReg(Idx: 0);
2192
2193 if (Ty.isPointer()) {
2194 if (DL.isNonIntegralAddressSpace(AddrSpace: Ty.getAddressSpace()))
2195 return Register();
2196 return MIRBuilder.buildPtrToInt(Dst: NewTy, Src: Val).getReg(Idx: 0);
2197 }
2198
2199 Register NewVal = Val;
2200
2201 assert(Ty.isVector());
2202 if (Ty.isPointerVector())
2203 NewVal = MIRBuilder.buildPtrToInt(Dst: NewTy, Src: NewVal).getReg(Idx: 0);
2204 return MIRBuilder.buildBitcast(Dst: NewTy, Src: NewVal).getReg(Idx: 0);
2205}
2206
2207void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy,
2208 unsigned OpIdx, unsigned ExtOpcode) {
2209 MachineOperand &MO = MI.getOperand(i: OpIdx);
2210 auto ExtB = MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {WideTy}, SrcOps: {MO});
2211 MO.setReg(ExtB.getReg(Idx: 0));
2212}
2213
2214void LegalizerHelper::widenScalarSrcUsingFPExt(MachineInstr &MI, LLT WideTy,
2215 unsigned OpIdx) {
2216 MachineOperand &MO = MI.getOperand(i: OpIdx);
2217 auto ExtB = MIRBuilder.buildInstr(Opc: TargetOpcode::G_FPEXT, DstOps: {WideTy}, SrcOps: {MO},
2218 Flags: MI.getFlags());
2219 MO.setReg(ExtB.getReg(Idx: 0));
2220}
2221
2222void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy,
2223 unsigned OpIdx) {
2224 MachineOperand &MO = MI.getOperand(i: OpIdx);
2225 auto ExtB = MIRBuilder.buildTrunc(Res: NarrowTy, Op: MO);
2226 MO.setReg(ExtB.getReg(Idx: 0));
2227}
2228
2229void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy,
2230 unsigned OpIdx, unsigned TruncOpcode) {
2231 MachineOperand &MO = MI.getOperand(i: OpIdx);
2232 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2233 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2234 MIRBuilder.buildInstr(Opc: TruncOpcode, DstOps: {MO}, SrcOps: {DstExt});
2235 MO.setReg(DstExt);
2236}
2237
2238void LegalizerHelper::widenScalarDstUsingFPTrunc(MachineInstr &MI, LLT WideTy,
2239 unsigned OpIdx) {
2240 MachineOperand &MO = MI.getOperand(i: OpIdx);
2241 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2242 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2243 MIRBuilder.buildInstr(Opc: TargetOpcode::G_FPTRUNC, DstOps: {MO}, SrcOps: {DstExt}, Flags: MI.getFlags());
2244 MO.setReg(DstExt);
2245}
2246
2247void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy,
2248 unsigned OpIdx, unsigned ExtOpcode) {
2249 MachineOperand &MO = MI.getOperand(i: OpIdx);
2250 Register DstTrunc = MRI.createGenericVirtualRegister(Ty: NarrowTy);
2251 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2252 MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {MO}, SrcOps: {DstTrunc});
2253 MO.setReg(DstTrunc);
2254}
2255
2256void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy,
2257 unsigned OpIdx) {
2258 MachineOperand &MO = MI.getOperand(i: OpIdx);
2259 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2260 Register Dst = MO.getReg();
2261 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2262 MO.setReg(DstExt);
2263 MIRBuilder.buildDeleteTrailingVectorElements(Res: Dst, Op0: DstExt);
2264}
2265
2266void LegalizerHelper::moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy,
2267 unsigned OpIdx) {
2268 MachineOperand &MO = MI.getOperand(i: OpIdx);
2269 MO.setReg(MIRBuilder.buildPadVectorWithUndefElements(Res: MoreTy, Op0: MO).getReg(Idx: 0));
2270}
2271
2272void LegalizerHelper::bitcastSrc(MachineInstr &MI, LLT CastTy, unsigned OpIdx) {
2273 MachineOperand &Op = MI.getOperand(i: OpIdx);
2274 Op.setReg(MIRBuilder.buildBitcast(Dst: CastTy, Src: Op).getReg(Idx: 0));
2275}
2276
2277void LegalizerHelper::bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx) {
2278 MachineOperand &MO = MI.getOperand(i: OpIdx);
2279 Register CastDst = MRI.createGenericVirtualRegister(Ty: CastTy);
2280 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2281 MIRBuilder.buildBitcast(Dst: MO, Src: CastDst);
2282 MO.setReg(CastDst);
2283}
2284
2285LegalizerHelper::LegalizeResult
2286LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx,
2287 LLT WideTy) {
2288 if (TypeIdx != 1)
2289 return UnableToLegalize;
2290
2291 auto [DstReg, DstTy, Src1Reg, Src1Ty] = MI.getFirst2RegLLTs();
2292 if (DstTy.isVector())
2293 return UnableToLegalize;
2294
2295 LLT SrcTy = MRI.getType(Reg: Src1Reg);
2296 const int DstSize = DstTy.getSizeInBits();
2297 const int SrcSize = SrcTy.getSizeInBits();
2298 const int WideSize = WideTy.getSizeInBits();
2299 const int NumMerge = (DstSize + WideSize - 1) / WideSize;
2300
2301 unsigned NumOps = MI.getNumOperands();
2302 unsigned NumSrc = MI.getNumOperands() - 1;
2303 unsigned PartSize = DstTy.getSizeInBits() / NumSrc;
2304
2305 if (WideSize >= DstSize) {
2306 // Directly pack the bits in the target type.
2307 Register ResultReg = MIRBuilder.buildZExt(Res: WideTy, Op: Src1Reg).getReg(Idx: 0);
2308
2309 for (unsigned I = 2; I != NumOps; ++I) {
2310 const unsigned Offset = (I - 1) * PartSize;
2311
2312 Register SrcReg = MI.getOperand(i: I).getReg();
2313 assert(MRI.getType(SrcReg) == LLT::scalar(PartSize));
2314
2315 auto ZextInput = MIRBuilder.buildZExt(Res: WideTy, Op: SrcReg);
2316
2317 Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg :
2318 MRI.createGenericVirtualRegister(Ty: WideTy);
2319
2320 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: Offset);
2321 auto Shl = MIRBuilder.buildShl(Dst: WideTy, Src0: ZextInput, Src1: ShiftAmt);
2322 MIRBuilder.buildOr(Dst: NextResult, Src0: ResultReg, Src1: Shl);
2323 ResultReg = NextResult;
2324 }
2325
2326 if (WideSize > DstSize)
2327 MIRBuilder.buildTrunc(Res: DstReg, Op: ResultReg);
2328 else if (DstTy.isPointer())
2329 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: ResultReg);
2330 else if (DstTy != WideTy)
2331 MIRBuilder.buildBitcast(Dst: DstReg, Src: ResultReg);
2332
2333 MI.eraseFromParent();
2334 return Legalized;
2335 }
2336
2337 // Unmerge the original values to the GCD type, and recombine to the next
2338 // multiple greater than the original type.
2339 //
2340 // %3:_(s12) = G_MERGE_VALUES %0:_(s4), %1:_(s4), %2:_(s4) -> s6
2341 // %4:_(s2), %5:_(s2) = G_UNMERGE_VALUES %0
2342 // %6:_(s2), %7:_(s2) = G_UNMERGE_VALUES %1
2343 // %8:_(s2), %9:_(s2) = G_UNMERGE_VALUES %2
2344 // %10:_(s6) = G_MERGE_VALUES %4, %5, %6
2345 // %11:_(s6) = G_MERGE_VALUES %7, %8, %9
2346 // %12:_(s12) = G_MERGE_VALUES %10, %11
2347 //
2348 // Padding with undef if necessary:
2349 //
2350 // %2:_(s8) = G_MERGE_VALUES %0:_(s4), %1:_(s4) -> s6
2351 // %3:_(s2), %4:_(s2) = G_UNMERGE_VALUES %0
2352 // %5:_(s2), %6:_(s2) = G_UNMERGE_VALUES %1
2353 // %7:_(s2) = G_IMPLICIT_DEF
2354 // %8:_(s6) = G_MERGE_VALUES %3, %4, %5
2355 // %9:_(s6) = G_MERGE_VALUES %6, %7, %7
2356 // %10:_(s12) = G_MERGE_VALUES %8, %9
2357
2358 const int GCD = std::gcd(m: SrcSize, n: WideSize);
2359 LLT GCDTy = WideTy.changeElementSize(NewEltSize: GCD);
2360
2361 SmallVector<Register, 8> NewMergeRegs;
2362 SmallVector<Register, 8> Unmerges;
2363 LLT WideDstTy = WideTy.changeElementSize(NewEltSize: NumMerge * WideSize);
2364
2365 // Decompose the original operands if they don't evenly divide.
2366 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands())) {
2367 Register SrcReg = MO.getReg();
2368 if (GCD == SrcSize) {
2369 Unmerges.push_back(Elt: SrcReg);
2370 } else {
2371 auto Unmerge = MIRBuilder.buildUnmerge(Res: GCDTy, Op: SrcReg);
2372 for (int J = 0, JE = Unmerge->getNumOperands() - 1; J != JE; ++J)
2373 Unmerges.push_back(Elt: Unmerge.getReg(Idx: J));
2374 }
2375 }
2376
2377 // Pad with undef to the next size that is a multiple of the requested size.
2378 if (static_cast<int>(Unmerges.size()) != NumMerge * WideSize) {
2379 Register UndefReg = MIRBuilder.buildUndef(Res: GCDTy).getReg(Idx: 0);
2380 for (int I = Unmerges.size(); I != NumMerge * WideSize; ++I)
2381 Unmerges.push_back(Elt: UndefReg);
2382 }
2383
2384 const int PartsPerGCD = WideSize / GCD;
2385
2386 // Build merges of each piece.
2387 ArrayRef<Register> Slicer(Unmerges);
2388 for (int I = 0; I != NumMerge; ++I, Slicer = Slicer.drop_front(N: PartsPerGCD)) {
2389 auto Merge =
2390 MIRBuilder.buildMergeLikeInstr(Res: WideTy, Ops: Slicer.take_front(N: PartsPerGCD));
2391 NewMergeRegs.push_back(Elt: Merge.getReg(Idx: 0));
2392 }
2393
2394 // A truncate may be necessary if the requested type doesn't evenly divide the
2395 // original result type.
2396 if (DstTy.getSizeInBits() == WideDstTy.getSizeInBits()) {
2397 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: NewMergeRegs);
2398 } else {
2399 auto FinalMerge = MIRBuilder.buildMergeLikeInstr(Res: WideDstTy, Ops: NewMergeRegs);
2400 MIRBuilder.buildTrunc(Res: DstReg, Op: FinalMerge.getReg(Idx: 0));
2401 }
2402
2403 MI.eraseFromParent();
2404 return Legalized;
2405}
2406
2407LegalizerHelper::LegalizeResult
2408LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx,
2409 LLT WideTy) {
2410 if (TypeIdx != 0)
2411 return UnableToLegalize;
2412
2413 int NumDst = MI.getNumOperands() - 1;
2414 Register SrcReg = MI.getOperand(i: NumDst).getReg();
2415 LLT SrcTy = MRI.getType(Reg: SrcReg);
2416 if (SrcTy.isVector())
2417 return UnableToLegalize;
2418
2419 Register Dst0Reg = MI.getOperand(i: 0).getReg();
2420 LLT DstTy = MRI.getType(Reg: Dst0Reg);
2421 if (!DstTy.isScalar())
2422 return UnableToLegalize;
2423
2424 if (WideTy.getSizeInBits() >= SrcTy.getSizeInBits()) {
2425 if (SrcTy.isPointer()) {
2426 const DataLayout &DL = MIRBuilder.getDataLayout();
2427 if (DL.isNonIntegralAddressSpace(AddrSpace: SrcTy.getAddressSpace())) {
2428 LLVM_DEBUG(
2429 dbgs() << "Not casting non-integral address space integer\n");
2430 return UnableToLegalize;
2431 }
2432
2433 SrcTy = LLT::scalar(SizeInBits: SrcTy.getSizeInBits());
2434 SrcReg = MIRBuilder.buildPtrToInt(Dst: SrcTy, Src: SrcReg).getReg(Idx: 0);
2435 }
2436
2437 // Widen SrcTy to WideTy. This does not affect the result, but since the
2438 // user requested this size, it is probably better handled than SrcTy and
2439 // should reduce the total number of legalization artifacts.
2440 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
2441 SrcTy = WideTy;
2442 SrcReg = MIRBuilder.buildAnyExt(Res: WideTy, Op: SrcReg).getReg(Idx: 0);
2443 }
2444
2445 // Theres no unmerge type to target. Directly extract the bits from the
2446 // source type
2447 unsigned DstSize = DstTy.getSizeInBits();
2448
2449 if (SrcTy.isFloat()) {
2450 SrcReg = coerceToInteger(Val: SrcReg);
2451 SrcTy = MRI.getType(Reg: SrcReg);
2452 }
2453
2454 MIRBuilder.buildTrunc(Res: Dst0Reg, Op: SrcReg);
2455 for (int I = 1; I != NumDst; ++I) {
2456 auto ShiftAmt = MIRBuilder.buildConstant(Res: SrcTy, Val: DstSize * I);
2457 auto Shr = MIRBuilder.buildLShr(Dst: SrcTy, Src0: SrcReg, Src1: ShiftAmt);
2458 MIRBuilder.buildTrunc(Res: MI.getOperand(i: I), Op: Shr);
2459 }
2460
2461 MI.eraseFromParent();
2462 return Legalized;
2463 }
2464
2465 // Extend the source to a wider type.
2466 LLT LCMTy = getLCMType(OrigTy: SrcTy, TargetTy: WideTy);
2467
2468 Register WideSrc = SrcReg;
2469 if (LCMTy.getSizeInBits() != SrcTy.getSizeInBits()) {
2470 // TODO: If this is an integral address space, cast to integer and anyext.
2471 if (SrcTy.isPointer()) {
2472 LLVM_DEBUG(dbgs() << "Widening pointer source types not implemented\n");
2473 return UnableToLegalize;
2474 }
2475
2476 WideSrc = MIRBuilder.buildAnyExt(Res: LCMTy, Op: WideSrc).getReg(Idx: 0);
2477 }
2478
2479 auto Unmerge = MIRBuilder.buildUnmerge(Res: WideTy, Op: WideSrc);
2480
2481 // Create a sequence of unmerges and merges to the original results. Since we
2482 // may have widened the source, we will need to pad the results with dead defs
2483 // to cover the source register.
2484 // e.g. widen s48 to s64:
2485 // %1:_(s48), %2:_(s48) = G_UNMERGE_VALUES %0:_(s96)
2486 //
2487 // =>
2488 // %4:_(s192) = G_ANYEXT %0:_(s96)
2489 // %5:_(s64), %6, %7 = G_UNMERGE_VALUES %4 ; Requested unmerge
2490 // ; unpack to GCD type, with extra dead defs
2491 // %8:_(s16), %9, %10, %11 = G_UNMERGE_VALUES %5:_(s64)
2492 // %12:_(s16), %13, dead %14, dead %15 = G_UNMERGE_VALUES %6:_(s64)
2493 // dead %16:_(s16), dead %17, dead %18, dead %18 = G_UNMERGE_VALUES %7:_(s64)
2494 // %1:_(s48) = G_MERGE_VALUES %8:_(s16), %9, %10 ; Remerge to destination
2495 // %2:_(s48) = G_MERGE_VALUES %11:_(s16), %12, %13 ; Remerge to destination
2496 const LLT GCDTy = getGCDType(OrigTy: WideTy, TargetTy: DstTy);
2497 const int NumUnmerge = Unmerge->getNumOperands() - 1;
2498 const int PartsPerRemerge = DstTy.getSizeInBits() / GCDTy.getSizeInBits();
2499
2500 // Directly unmerge to the destination without going through a GCD type
2501 // if possible
2502 if (PartsPerRemerge == 1) {
2503 const int PartsPerUnmerge = WideTy.getSizeInBits() / DstTy.getSizeInBits();
2504
2505 for (int I = 0; I != NumUnmerge; ++I) {
2506 auto MIB = MIRBuilder.buildInstr(Opcode: TargetOpcode::G_UNMERGE_VALUES);
2507
2508 for (int J = 0; J != PartsPerUnmerge; ++J) {
2509 int Idx = I * PartsPerUnmerge + J;
2510 if (Idx < NumDst)
2511 MIB.addDef(RegNo: MI.getOperand(i: Idx).getReg());
2512 else {
2513 // Create dead def for excess components.
2514 MIB.addDef(RegNo: MRI.createGenericVirtualRegister(Ty: DstTy));
2515 }
2516 }
2517
2518 MIB.addUse(RegNo: Unmerge.getReg(Idx: I));
2519 }
2520 } else {
2521 SmallVector<Register, 16> Parts;
2522 for (int J = 0; J != NumUnmerge; ++J)
2523 extractGCDType(Parts, GCDTy, SrcReg: Unmerge.getReg(Idx: J));
2524
2525 SmallVector<Register, 8> RemergeParts;
2526 for (int I = 0; I != NumDst; ++I) {
2527 for (int J = 0; J < PartsPerRemerge; ++J) {
2528 const int Idx = I * PartsPerRemerge + J;
2529 RemergeParts.emplace_back(Args&: Parts[Idx]);
2530 }
2531
2532 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: I).getReg(), Ops: RemergeParts);
2533 RemergeParts.clear();
2534 }
2535 }
2536
2537 MI.eraseFromParent();
2538 return Legalized;
2539}
2540
2541LegalizerHelper::LegalizeResult
2542LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx,
2543 LLT WideTy) {
2544 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
2545 unsigned Offset = MI.getOperand(i: 2).getImm();
2546
2547 if (TypeIdx == 0) {
2548 if (SrcTy.isVector() || DstTy.isVector())
2549 return UnableToLegalize;
2550
2551 SrcOp Src(SrcReg);
2552 if (SrcTy.isPointer()) {
2553 // Extracts from pointers can be handled only if they are really just
2554 // simple integers.
2555 const DataLayout &DL = MIRBuilder.getDataLayout();
2556 if (DL.isNonIntegralAddressSpace(AddrSpace: SrcTy.getAddressSpace()))
2557 return UnableToLegalize;
2558
2559 LLT SrcAsIntTy = LLT::scalar(SizeInBits: SrcTy.getSizeInBits());
2560 Src = MIRBuilder.buildPtrToInt(Dst: SrcAsIntTy, Src);
2561 SrcTy = SrcAsIntTy;
2562 }
2563
2564 if (DstTy.isPointer())
2565 return UnableToLegalize;
2566
2567 if (Offset == 0) {
2568 // Avoid a shift in the degenerate case.
2569 MIRBuilder.buildTrunc(Res: DstReg,
2570 Op: MIRBuilder.buildAnyExtOrTrunc(Res: WideTy, Op: Src));
2571 MI.eraseFromParent();
2572 return Legalized;
2573 }
2574
2575 // Do a shift in the source type.
2576 LLT ShiftTy = SrcTy;
2577 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
2578 Src = MIRBuilder.buildAnyExt(Res: WideTy, Op: Src);
2579 ShiftTy = WideTy;
2580 }
2581
2582 auto LShr = MIRBuilder.buildLShr(
2583 Dst: ShiftTy, Src0: Src, Src1: MIRBuilder.buildConstant(Res: ShiftTy, Val: Offset));
2584 MIRBuilder.buildTrunc(Res: DstReg, Op: LShr);
2585 MI.eraseFromParent();
2586 return Legalized;
2587 }
2588
2589 if (SrcTy.isScalar()) {
2590 Observer.changingInstr(MI);
2591 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2592 Observer.changedInstr(MI);
2593 return Legalized;
2594 }
2595
2596 if (!SrcTy.isVector())
2597 return UnableToLegalize;
2598
2599 if (DstTy != SrcTy.getElementType())
2600 return UnableToLegalize;
2601
2602 if (Offset % SrcTy.getScalarSizeInBits() != 0)
2603 return UnableToLegalize;
2604
2605 Observer.changingInstr(MI);
2606 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2607
2608 MI.getOperand(i: 2).setImm((WideTy.getSizeInBits() / SrcTy.getSizeInBits()) *
2609 Offset);
2610 widenScalarDst(MI, WideTy: WideTy.getScalarType(), OpIdx: 0);
2611 Observer.changedInstr(MI);
2612 return Legalized;
2613}
2614
2615LegalizerHelper::LegalizeResult
2616LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx,
2617 LLT WideTy) {
2618 if (TypeIdx != 0 || WideTy.isVector())
2619 return UnableToLegalize;
2620 Observer.changingInstr(MI);
2621 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2622 widenScalarDst(MI, WideTy);
2623 Observer.changedInstr(MI);
2624 return Legalized;
2625}
2626
2627LegalizerHelper::LegalizeResult
2628LegalizerHelper::widenScalarAddSubOverflow(MachineInstr &MI, unsigned TypeIdx,
2629 LLT WideTy) {
2630 unsigned Opcode;
2631 unsigned ExtOpcode;
2632 std::optional<Register> CarryIn;
2633 switch (MI.getOpcode()) {
2634 default:
2635 llvm_unreachable("Unexpected opcode!");
2636 case TargetOpcode::G_SADDO:
2637 Opcode = TargetOpcode::G_ADD;
2638 ExtOpcode = TargetOpcode::G_SEXT;
2639 break;
2640 case TargetOpcode::G_SSUBO:
2641 Opcode = TargetOpcode::G_SUB;
2642 ExtOpcode = TargetOpcode::G_SEXT;
2643 break;
2644 case TargetOpcode::G_UADDO:
2645 Opcode = TargetOpcode::G_ADD;
2646 ExtOpcode = TargetOpcode::G_ZEXT;
2647 break;
2648 case TargetOpcode::G_USUBO:
2649 Opcode = TargetOpcode::G_SUB;
2650 ExtOpcode = TargetOpcode::G_ZEXT;
2651 break;
2652 case TargetOpcode::G_SADDE:
2653 Opcode = TargetOpcode::G_UADDE;
2654 ExtOpcode = TargetOpcode::G_SEXT;
2655 CarryIn = MI.getOperand(i: 4).getReg();
2656 break;
2657 case TargetOpcode::G_SSUBE:
2658 Opcode = TargetOpcode::G_USUBE;
2659 ExtOpcode = TargetOpcode::G_SEXT;
2660 CarryIn = MI.getOperand(i: 4).getReg();
2661 break;
2662 case TargetOpcode::G_UADDE:
2663 Opcode = TargetOpcode::G_UADDE;
2664 ExtOpcode = TargetOpcode::G_ZEXT;
2665 CarryIn = MI.getOperand(i: 4).getReg();
2666 break;
2667 case TargetOpcode::G_USUBE:
2668 Opcode = TargetOpcode::G_USUBE;
2669 ExtOpcode = TargetOpcode::G_ZEXT;
2670 CarryIn = MI.getOperand(i: 4).getReg();
2671 break;
2672 }
2673
2674 if (TypeIdx == 1) {
2675 unsigned BoolExtOp = MIRBuilder.getBoolExtOp(IsVec: WideTy.isVector(), IsFP: false);
2676
2677 Observer.changingInstr(MI);
2678 if (CarryIn)
2679 widenScalarSrc(MI, WideTy, OpIdx: 4, ExtOpcode: BoolExtOp);
2680 widenScalarDst(MI, WideTy, OpIdx: 1);
2681
2682 Observer.changedInstr(MI);
2683 return Legalized;
2684 }
2685
2686 auto LHSExt = MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {WideTy}, SrcOps: {MI.getOperand(i: 2)});
2687 auto RHSExt = MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {WideTy}, SrcOps: {MI.getOperand(i: 3)});
2688 // Do the arithmetic in the larger type.
2689 Register NewOp;
2690 if (CarryIn) {
2691 LLT CarryOutTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
2692 NewOp = MIRBuilder
2693 .buildInstr(Opc: Opcode, DstOps: {WideTy, CarryOutTy},
2694 SrcOps: {LHSExt, RHSExt, *CarryIn})
2695 .getReg(Idx: 0);
2696 } else {
2697 NewOp = MIRBuilder.buildInstr(Opc: Opcode, DstOps: {WideTy}, SrcOps: {LHSExt, RHSExt}).getReg(Idx: 0);
2698 }
2699 LLT OrigTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2700 auto TruncOp = MIRBuilder.buildTrunc(Res: OrigTy, Op: NewOp);
2701 auto ExtOp = MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {WideTy}, SrcOps: {TruncOp});
2702 // There is no overflow if the ExtOp is the same as NewOp.
2703 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: MI.getOperand(i: 1), Op0: NewOp, Op1: ExtOp);
2704 // Now trunc the NewOp to the original result.
2705 MIRBuilder.buildTrunc(Res: MI.getOperand(i: 0), Op: NewOp);
2706 MI.eraseFromParent();
2707 return Legalized;
2708}
2709
2710LegalizerHelper::LegalizeResult
2711LegalizerHelper::widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx,
2712 LLT WideTy) {
2713 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SADDSAT ||
2714 MI.getOpcode() == TargetOpcode::G_SSUBSAT ||
2715 MI.getOpcode() == TargetOpcode::G_SSHLSAT;
2716 bool IsShift = MI.getOpcode() == TargetOpcode::G_SSHLSAT ||
2717 MI.getOpcode() == TargetOpcode::G_USHLSAT;
2718 // We can convert this to:
2719 // 1. Any extend iN to iM
2720 // 2. SHL by M-N
2721 // 3. [US][ADD|SUB|SHL]SAT
2722 // 4. L/ASHR by M-N
2723 //
2724 // It may be more efficient to lower this to a min and a max operation in
2725 // the higher precision arithmetic if the promoted operation isn't legal,
2726 // but this decision is up to the target's lowering request.
2727 Register DstReg = MI.getOperand(i: 0).getReg();
2728
2729 unsigned NewBits = WideTy.getScalarSizeInBits();
2730 unsigned SHLAmount = NewBits - MRI.getType(Reg: DstReg).getScalarSizeInBits();
2731
2732 // Shifts must zero-extend the RHS to preserve the unsigned quantity, and
2733 // must not left shift the RHS to preserve the shift amount.
2734 auto LHS = MIRBuilder.buildAnyExt(Res: WideTy, Op: MI.getOperand(i: 1));
2735 auto RHS = IsShift ? MIRBuilder.buildZExt(Res: WideTy, Op: MI.getOperand(i: 2))
2736 : MIRBuilder.buildAnyExt(Res: WideTy, Op: MI.getOperand(i: 2));
2737 auto ShiftK = MIRBuilder.buildConstant(Res: WideTy, Val: SHLAmount);
2738 auto ShiftL = MIRBuilder.buildShl(Dst: WideTy, Src0: LHS, Src1: ShiftK);
2739 auto ShiftR = IsShift ? RHS : MIRBuilder.buildShl(Dst: WideTy, Src0: RHS, Src1: ShiftK);
2740
2741 auto WideInst = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {WideTy},
2742 SrcOps: {ShiftL, ShiftR}, Flags: MI.getFlags());
2743
2744 // Use a shift that will preserve the number of sign bits when the trunc is
2745 // folded away.
2746 auto Result = IsSigned ? MIRBuilder.buildAShr(Dst: WideTy, Src0: WideInst, Src1: ShiftK)
2747 : MIRBuilder.buildLShr(Dst: WideTy, Src0: WideInst, Src1: ShiftK);
2748
2749 MIRBuilder.buildTrunc(Res: DstReg, Op: Result);
2750 MI.eraseFromParent();
2751 return Legalized;
2752}
2753
2754LegalizerHelper::LegalizeResult
2755LegalizerHelper::widenScalarMulo(MachineInstr &MI, unsigned TypeIdx,
2756 LLT WideTy) {
2757 if (TypeIdx == 1) {
2758 Observer.changingInstr(MI);
2759 widenScalarDst(MI, WideTy, OpIdx: 1);
2760 Observer.changedInstr(MI);
2761 return Legalized;
2762 }
2763
2764 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SMULO;
2765 auto [Result, OriginalOverflow, LHS, RHS] = MI.getFirst4Regs();
2766 LLT SrcTy = MRI.getType(Reg: LHS);
2767 LLT OverflowTy = MRI.getType(Reg: OriginalOverflow);
2768 unsigned SrcBitWidth = SrcTy.getScalarSizeInBits();
2769
2770 // To determine if the result overflowed in the larger type, we extend the
2771 // input to the larger type, do the multiply (checking if it overflows),
2772 // then also check the high bits of the result to see if overflow happened
2773 // there.
2774 unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
2775 auto LeftOperand = MIRBuilder.buildInstr(Opc: ExtOp, DstOps: {WideTy}, SrcOps: {LHS});
2776 auto RightOperand = MIRBuilder.buildInstr(Opc: ExtOp, DstOps: {WideTy}, SrcOps: {RHS});
2777
2778 // Multiplication cannot overflow if the WideTy is >= 2 * original width,
2779 // so we don't need to check the overflow result of larger type Mulo.
2780 bool WideMulCanOverflow = WideTy.getScalarSizeInBits() < 2 * SrcBitWidth;
2781
2782 unsigned MulOpc =
2783 WideMulCanOverflow ? MI.getOpcode() : (unsigned)TargetOpcode::G_MUL;
2784
2785 MachineInstrBuilder Mulo;
2786 if (WideMulCanOverflow)
2787 Mulo = MIRBuilder.buildInstr(Opc: MulOpc, DstOps: {WideTy, OverflowTy},
2788 SrcOps: {LeftOperand, RightOperand});
2789 else
2790 Mulo = MIRBuilder.buildInstr(Opc: MulOpc, DstOps: {WideTy}, SrcOps: {LeftOperand, RightOperand});
2791
2792 auto Mul = Mulo->getOperand(i: 0);
2793 MIRBuilder.buildTrunc(Res: Result, Op: Mul);
2794
2795 MachineInstrBuilder ExtResult;
2796 // Overflow occurred if it occurred in the larger type, or if the high part
2797 // of the result does not zero/sign-extend the low part. Check this second
2798 // possibility first.
2799 if (IsSigned) {
2800 // For signed, overflow occurred when the high part does not sign-extend
2801 // the low part.
2802 ExtResult = MIRBuilder.buildSExtInReg(Res: WideTy, Op: Mul, ImmOp: SrcBitWidth);
2803 } else {
2804 // Unsigned overflow occurred when the high part does not zero-extend the
2805 // low part.
2806 ExtResult = MIRBuilder.buildZExtInReg(Res: WideTy, Op: Mul, ImmOp: SrcBitWidth);
2807 }
2808
2809 if (WideMulCanOverflow) {
2810 auto Overflow =
2811 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: OverflowTy, Op0: Mul, Op1: ExtResult);
2812 // Finally check if the multiplication in the larger type itself overflowed.
2813 MIRBuilder.buildOr(Dst: OriginalOverflow, Src0: Mulo->getOperand(i: 1), Src1: Overflow);
2814 } else {
2815 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: OriginalOverflow, Op0: Mul, Op1: ExtResult);
2816 }
2817 MI.eraseFromParent();
2818 return Legalized;
2819}
2820
2821LegalizerHelper::LegalizeResult
2822LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
2823 unsigned Opcode = MI.getOpcode();
2824 switch (Opcode) {
2825 default:
2826 return UnableToLegalize;
2827 case TargetOpcode::G_ATOMICRMW_XCHG:
2828 case TargetOpcode::G_ATOMICRMW_ADD:
2829 case TargetOpcode::G_ATOMICRMW_SUB:
2830 case TargetOpcode::G_ATOMICRMW_AND:
2831 case TargetOpcode::G_ATOMICRMW_OR:
2832 case TargetOpcode::G_ATOMICRMW_XOR:
2833 case TargetOpcode::G_ATOMICRMW_MIN:
2834 case TargetOpcode::G_ATOMICRMW_MAX:
2835 case TargetOpcode::G_ATOMICRMW_UMIN:
2836 case TargetOpcode::G_ATOMICRMW_UMAX:
2837 assert(TypeIdx == 0 && "atomicrmw with second scalar type");
2838 Observer.changingInstr(MI);
2839 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
2840 widenScalarDst(MI, WideTy, OpIdx: 0);
2841 Observer.changedInstr(MI);
2842 return Legalized;
2843 case TargetOpcode::G_ATOMIC_CMPXCHG:
2844 assert(TypeIdx == 0 && "G_ATOMIC_CMPXCHG with second scalar type");
2845 Observer.changingInstr(MI);
2846 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
2847 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ANYEXT);
2848 widenScalarDst(MI, WideTy, OpIdx: 0);
2849 Observer.changedInstr(MI);
2850 return Legalized;
2851 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS:
2852 if (TypeIdx == 0) {
2853 Observer.changingInstr(MI);
2854 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ANYEXT);
2855 widenScalarSrc(MI, WideTy, OpIdx: 4, ExtOpcode: TargetOpcode::G_ANYEXT);
2856 widenScalarDst(MI, WideTy, OpIdx: 0);
2857 Observer.changedInstr(MI);
2858 return Legalized;
2859 }
2860 assert(TypeIdx == 1 &&
2861 "G_ATOMIC_CMPXCHG_WITH_SUCCESS with third scalar type");
2862 Observer.changingInstr(MI);
2863 widenScalarDst(MI, WideTy, OpIdx: 1);
2864 Observer.changedInstr(MI);
2865 return Legalized;
2866 case TargetOpcode::G_EXTRACT:
2867 return widenScalarExtract(MI, TypeIdx, WideTy);
2868 case TargetOpcode::G_INSERT:
2869 return widenScalarInsert(MI, TypeIdx, WideTy);
2870 case TargetOpcode::G_MERGE_VALUES:
2871 return widenScalarMergeValues(MI, TypeIdx, WideTy);
2872 case TargetOpcode::G_UNMERGE_VALUES:
2873 return widenScalarUnmergeValues(MI, TypeIdx, WideTy);
2874 case TargetOpcode::G_SADDO:
2875 case TargetOpcode::G_SSUBO:
2876 case TargetOpcode::G_UADDO:
2877 case TargetOpcode::G_USUBO:
2878 case TargetOpcode::G_SADDE:
2879 case TargetOpcode::G_SSUBE:
2880 case TargetOpcode::G_UADDE:
2881 case TargetOpcode::G_USUBE:
2882 return widenScalarAddSubOverflow(MI, TypeIdx, WideTy);
2883 case TargetOpcode::G_UMULO:
2884 case TargetOpcode::G_SMULO:
2885 return widenScalarMulo(MI, TypeIdx, WideTy);
2886 case TargetOpcode::G_SADDSAT:
2887 case TargetOpcode::G_SSUBSAT:
2888 case TargetOpcode::G_SSHLSAT:
2889 case TargetOpcode::G_UADDSAT:
2890 case TargetOpcode::G_USUBSAT:
2891 case TargetOpcode::G_USHLSAT:
2892 return widenScalarAddSubShlSat(MI, TypeIdx, WideTy);
2893 case TargetOpcode::G_CTTZ:
2894 case TargetOpcode::G_CTTZ_ZERO_POISON:
2895 case TargetOpcode::G_CTLZ:
2896 case TargetOpcode::G_CTLZ_ZERO_POISON:
2897 case TargetOpcode::G_CTLS:
2898 case TargetOpcode::G_CTPOP: {
2899 if (TypeIdx == 0) {
2900 Observer.changingInstr(MI);
2901 widenScalarDst(MI, WideTy, OpIdx: 0);
2902 Observer.changedInstr(MI);
2903 return Legalized;
2904 }
2905
2906 Register SrcReg = MI.getOperand(i: 1).getReg();
2907
2908 // First extend the input.
2909 unsigned ExtOpc;
2910 switch (Opcode) {
2911 case TargetOpcode::G_CTTZ:
2912 case TargetOpcode::G_CTTZ_ZERO_POISON:
2913 case TargetOpcode::G_CTLZ_ZERO_POISON: // poison shifted out below
2914 ExtOpc = TargetOpcode::G_ANYEXT;
2915 break;
2916 case TargetOpcode::G_CTLS:
2917 ExtOpc = TargetOpcode::G_SEXT;
2918 break;
2919 default:
2920 ExtOpc = TargetOpcode::G_ZEXT;
2921 }
2922
2923 auto MIBSrc = MIRBuilder.buildInstr(Opc: ExtOpc, DstOps: {WideTy}, SrcOps: {SrcReg});
2924 LLT CurTy = MRI.getType(Reg: SrcReg);
2925 unsigned NewOpc = Opcode;
2926 if (NewOpc == TargetOpcode::G_CTTZ) {
2927 // The count is the same in the larger type except if the original
2928 // value was zero. This can be handled by setting the bit just off
2929 // the top of the original type.
2930 auto TopBit = APInt::getOneBitSet(numBits: WideTy.getScalarSizeInBits(),
2931 BitNo: CurTy.getScalarSizeInBits());
2932 MIBSrc = MIRBuilder.buildOr(
2933 Dst: WideTy, Src0: MIBSrc, Src1: MIRBuilder.buildConstant(Res: WideTy, Val: TopBit));
2934 // Now we know the operand is non-zero, use the more relaxed opcode.
2935 NewOpc = TargetOpcode::G_CTTZ_ZERO_POISON;
2936 }
2937
2938 unsigned SizeDiff =
2939 WideTy.getScalarSizeInBits() - CurTy.getScalarSizeInBits();
2940
2941 if (Opcode == TargetOpcode::G_CTLZ_ZERO_POISON) {
2942 // An optimization where the result is the CTLZ after the left shift by
2943 // (Difference in widety and current ty), that is,
2944 // MIBSrc = MIBSrc << (sizeinbits(WideTy) - sizeinbits(CurTy))
2945 // Result = ctlz MIBSrc
2946 MIBSrc = MIRBuilder.buildShl(Dst: WideTy, Src0: MIBSrc,
2947 Src1: MIRBuilder.buildConstant(Res: WideTy, Val: SizeDiff));
2948 }
2949
2950 // Perform the operation at the larger size.
2951 auto MIBNewOp = MIRBuilder.buildInstr(Opc: NewOpc, DstOps: {WideTy}, SrcOps: {MIBSrc});
2952 // This is already the correct result for CTPOP and CTTZs
2953 if (Opcode == TargetOpcode::G_CTLZ || Opcode == TargetOpcode::G_CTLS) {
2954 // The correct result is NewOp - (Difference in widety and current ty).
2955 // At this stage SUB is guaranteed to be positive no-wrap,
2956 // that to be used in further KnownBits optimizations for CTLZ.
2957 MIBNewOp = MIRBuilder.buildSub(
2958 Dst: WideTy, Src0: MIBNewOp, Src1: MIRBuilder.buildConstant(Res: WideTy, Val: SizeDiff),
2959 Flags: Opcode == TargetOpcode::G_CTLZ
2960 ? std::optional<unsigned>(MachineInstr::NoUWrap)
2961 : std::nullopt);
2962 }
2963
2964 MIRBuilder.buildZExtOrTrunc(Res: MI.getOperand(i: 0), Op: MIBNewOp);
2965 MI.eraseFromParent();
2966 return Legalized;
2967 }
2968 case TargetOpcode::G_BSWAP: {
2969 Observer.changingInstr(MI);
2970 Register DstReg = MI.getOperand(i: 0).getReg();
2971
2972 Register ShrReg = MRI.createGenericVirtualRegister(Ty: WideTy);
2973 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2974 Register ShiftAmtReg = MRI.createGenericVirtualRegister(Ty: WideTy);
2975 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2976
2977 MI.getOperand(i: 0).setReg(DstExt);
2978
2979 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2980
2981 LLT Ty = MRI.getType(Reg: DstReg);
2982 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
2983 MIRBuilder.buildConstant(Res: ShiftAmtReg, Val: DiffBits);
2984 MIRBuilder.buildLShr(Dst: ShrReg, Src0: DstExt, Src1: ShiftAmtReg);
2985
2986 MIRBuilder.buildTrunc(Res: DstReg, Op: ShrReg);
2987 Observer.changedInstr(MI);
2988 return Legalized;
2989 }
2990 case TargetOpcode::G_BITREVERSE: {
2991 Observer.changingInstr(MI);
2992
2993 Register DstReg = MI.getOperand(i: 0).getReg();
2994 LLT Ty = MRI.getType(Reg: DstReg);
2995 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
2996
2997 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2998 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2999 MI.getOperand(i: 0).setReg(DstExt);
3000 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
3001
3002 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: DiffBits);
3003 auto Shift = MIRBuilder.buildLShr(Dst: WideTy, Src0: DstExt, Src1: ShiftAmt);
3004 MIRBuilder.buildTrunc(Res: DstReg, Op: Shift);
3005 Observer.changedInstr(MI);
3006 return Legalized;
3007 }
3008 case TargetOpcode::G_FREEZE:
3009 case TargetOpcode::G_CONSTANT_FOLD_BARRIER:
3010 Observer.changingInstr(MI);
3011 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3012 widenScalarDst(MI, WideTy);
3013 Observer.changedInstr(MI);
3014 return Legalized;
3015
3016 case TargetOpcode::G_ABS:
3017 Observer.changingInstr(MI);
3018 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_SEXT);
3019 widenScalarDst(MI, WideTy);
3020 Observer.changedInstr(MI);
3021 return Legalized;
3022
3023 case TargetOpcode::G_ADD:
3024 case TargetOpcode::G_AND:
3025 case TargetOpcode::G_MUL:
3026 case TargetOpcode::G_OR:
3027 case TargetOpcode::G_XOR:
3028 case TargetOpcode::G_SUB:
3029 case TargetOpcode::G_SHUFFLE_VECTOR:
3030 // Perform operation at larger width (any extension is fines here, high bits
3031 // don't affect the result) and then truncate the result back to the
3032 // original type.
3033 Observer.changingInstr(MI);
3034 // The G_ANYEXTs below leave the new high bits unconstrained, so no-wrap and
3035 // disjoint claims proved at the narrow width no longer hold. Paths that
3036 // widen with value-preserving G_ZEXT/G_SEXT keep their flags.
3037 MI.clearFlags(flags: MachineInstr::NoUWrap | MachineInstr::NoSWrap |
3038 MachineInstr::Disjoint);
3039 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3040 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
3041 widenScalarDst(MI, WideTy);
3042 Observer.changedInstr(MI);
3043 return Legalized;
3044
3045 case TargetOpcode::G_SBFX:
3046 case TargetOpcode::G_UBFX:
3047 Observer.changingInstr(MI);
3048
3049 if (TypeIdx == 0) {
3050 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3051 widenScalarDst(MI, WideTy);
3052 } else {
3053 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3054 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ZEXT);
3055 }
3056
3057 Observer.changedInstr(MI);
3058 return Legalized;
3059
3060 case TargetOpcode::G_SHL:
3061 Observer.changingInstr(MI);
3062
3063 if (TypeIdx == 0) {
3064 // Widening the result with G_ANYEXT invalidates the no-wrap flags, as in
3065 // the G_ADD/G_SUB/G_MUL case above. TypeIdx 1 widens only the shift
3066 // amount, which is value-preserving, so it keeps them.
3067 MI.clearFlags(flags: MachineInstr::NoUWrap | MachineInstr::NoSWrap);
3068 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3069 widenScalarDst(MI, WideTy);
3070 } else {
3071 assert(TypeIdx == 1);
3072 // The "number of bits to shift" operand must preserve its value as an
3073 // unsigned integer:
3074 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3075 }
3076
3077 Observer.changedInstr(MI);
3078 return Legalized;
3079
3080 case TargetOpcode::G_ROTR:
3081 case TargetOpcode::G_ROTL:
3082 if (TypeIdx != 1)
3083 return UnableToLegalize;
3084
3085 Observer.changingInstr(MI);
3086 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3087 Observer.changedInstr(MI);
3088 return Legalized;
3089
3090 case TargetOpcode::G_SDIV:
3091 case TargetOpcode::G_SREM:
3092 case TargetOpcode::G_SMIN:
3093 case TargetOpcode::G_SMAX:
3094 case TargetOpcode::G_ABDS:
3095 Observer.changingInstr(MI);
3096 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_SEXT);
3097 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_SEXT);
3098 widenScalarDst(MI, WideTy);
3099 Observer.changedInstr(MI);
3100 return Legalized;
3101
3102 case TargetOpcode::G_SDIVREM:
3103 Observer.changingInstr(MI);
3104 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_SEXT);
3105 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_SEXT);
3106 widenScalarDst(MI, WideTy);
3107 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: --MIRBuilder.getInsertPt());
3108 widenScalarDst(MI, WideTy, OpIdx: 1);
3109 Observer.changedInstr(MI);
3110 return Legalized;
3111
3112 case TargetOpcode::G_ASHR:
3113 case TargetOpcode::G_LSHR:
3114 Observer.changingInstr(MI);
3115
3116 if (TypeIdx == 0) {
3117 unsigned CvtOp = Opcode == TargetOpcode::G_ASHR ? TargetOpcode::G_SEXT
3118 : TargetOpcode::G_ZEXT;
3119
3120 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: CvtOp);
3121 widenScalarDst(MI, WideTy);
3122 } else {
3123 assert(TypeIdx == 1);
3124 // The "number of bits to shift" operand must preserve its value as an
3125 // unsigned integer:
3126 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3127 }
3128
3129 Observer.changedInstr(MI);
3130 return Legalized;
3131 case TargetOpcode::G_UDIV:
3132 case TargetOpcode::G_UREM:
3133 case TargetOpcode::G_ABDU:
3134 Observer.changingInstr(MI);
3135 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ZEXT);
3136 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3137 widenScalarDst(MI, WideTy);
3138 Observer.changedInstr(MI);
3139 return Legalized;
3140 case TargetOpcode::G_UDIVREM:
3141 Observer.changingInstr(MI);
3142 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3143 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ZEXT);
3144 widenScalarDst(MI, WideTy);
3145 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: --MIRBuilder.getInsertPt());
3146 widenScalarDst(MI, WideTy, OpIdx: 1);
3147 Observer.changedInstr(MI);
3148 return Legalized;
3149 case TargetOpcode::G_UMIN:
3150 case TargetOpcode::G_UMAX: {
3151 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3152
3153 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
3154 unsigned ExtOpc =
3155 TLI.isSExtCheaperThanZExt(FromTy: getApproximateEVTForLLT(Ty, Ctx),
3156 ToTy: getApproximateEVTForLLT(Ty: WideTy, Ctx))
3157 ? TargetOpcode::G_SEXT
3158 : TargetOpcode::G_ZEXT;
3159
3160 Observer.changingInstr(MI);
3161 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: ExtOpc);
3162 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: ExtOpc);
3163 widenScalarDst(MI, WideTy);
3164 Observer.changedInstr(MI);
3165 return Legalized;
3166 }
3167
3168 case TargetOpcode::G_SELECT:
3169 Observer.changingInstr(MI);
3170 if (TypeIdx == 0) {
3171 // Perform operation at larger width (any extension is fine here, high
3172 // bits don't affect the result) and then truncate the result back to the
3173 // original type.
3174 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
3175 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ANYEXT);
3176 widenScalarDst(MI, WideTy);
3177 } else {
3178 bool IsVec = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).isVector();
3179 // Explicit extension is required here since high bits affect the result.
3180 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: MIRBuilder.getBoolExtOp(IsVec, IsFP: false));
3181 }
3182 Observer.changedInstr(MI);
3183 return Legalized;
3184
3185 case TargetOpcode::G_FPEXT:
3186 if (TypeIdx != 1)
3187 return UnableToLegalize;
3188
3189 Observer.changingInstr(MI);
3190 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3191 Observer.changedInstr(MI);
3192 return Legalized;
3193 case TargetOpcode::G_FPTOSI:
3194 case TargetOpcode::G_FPTOUI:
3195 case TargetOpcode::G_INTRINSIC_LRINT:
3196 case TargetOpcode::G_INTRINSIC_LLRINT:
3197 case TargetOpcode::G_IS_FPCLASS:
3198 Observer.changingInstr(MI);
3199
3200 if (TypeIdx == 0)
3201 widenScalarDst(MI, WideTy);
3202 else
3203 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3204
3205 Observer.changedInstr(MI);
3206 return Legalized;
3207 case TargetOpcode::G_SITOFP:
3208 Observer.changingInstr(MI);
3209
3210 if (TypeIdx == 0)
3211 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3212 else
3213 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_SEXT);
3214
3215 Observer.changedInstr(MI);
3216 return Legalized;
3217 case TargetOpcode::G_UITOFP:
3218 Observer.changingInstr(MI);
3219
3220 if (TypeIdx == 0)
3221 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3222 else
3223 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ZEXT);
3224
3225 Observer.changedInstr(MI);
3226 return Legalized;
3227 case TargetOpcode::G_FPTOSI_SAT:
3228 case TargetOpcode::G_FPTOUI_SAT:
3229 Observer.changingInstr(MI);
3230
3231 if (TypeIdx == 0) {
3232 Register OldDst = MI.getOperand(i: 0).getReg();
3233 LLT Ty = MRI.getType(Reg: OldDst);
3234 Register ExtReg = MRI.createGenericVirtualRegister(Ty: WideTy);
3235 Register NewDst;
3236 MI.getOperand(i: 0).setReg(ExtReg);
3237 uint64_t ShortBits = Ty.getScalarSizeInBits();
3238 uint64_t WideBits = WideTy.getScalarSizeInBits();
3239 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
3240 if (Opcode == TargetOpcode::G_FPTOSI_SAT) {
3241 // z = i16 fptosi_sat(a)
3242 // ->
3243 // x = i32 fptosi_sat(a)
3244 // y = smin(x, 32767)
3245 // z = smax(y, -32768)
3246 auto MaxVal = MIRBuilder.buildConstant(
3247 Res: WideTy, Val: APInt::getSignedMaxValue(numBits: ShortBits).sext(width: WideBits));
3248 auto MinVal = MIRBuilder.buildConstant(
3249 Res: WideTy, Val: APInt::getSignedMinValue(numBits: ShortBits).sext(width: WideBits));
3250 Register MidReg =
3251 MIRBuilder.buildSMin(Dst: WideTy, Src0: ExtReg, Src1: MaxVal).getReg(Idx: 0);
3252 NewDst = MIRBuilder.buildSMax(Dst: WideTy, Src0: MidReg, Src1: MinVal).getReg(Idx: 0);
3253 } else {
3254 // z = i16 fptoui_sat(a)
3255 // ->
3256 // x = i32 fptoui_sat(a)
3257 // y = smin(x, 65535)
3258 auto MaxVal = MIRBuilder.buildConstant(
3259 Res: WideTy, Val: APInt::getAllOnes(numBits: ShortBits).zext(width: WideBits));
3260 NewDst = MIRBuilder.buildUMin(Dst: WideTy, Src0: ExtReg, Src1: MaxVal).getReg(Idx: 0);
3261 }
3262 MIRBuilder.buildTrunc(Res: OldDst, Op: NewDst);
3263 } else
3264 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3265
3266 Observer.changedInstr(MI);
3267 return Legalized;
3268 case TargetOpcode::G_LOAD:
3269 case TargetOpcode::G_SEXTLOAD:
3270 case TargetOpcode::G_ZEXTLOAD:
3271 case TargetOpcode::G_FPEXTLOAD:
3272 Observer.changingInstr(MI);
3273 widenScalarDst(MI, WideTy);
3274 Observer.changedInstr(MI);
3275 return Legalized;
3276
3277 case TargetOpcode::G_STORE: {
3278 if (TypeIdx != 0)
3279 return UnableToLegalize;
3280
3281 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3282 assert(!Ty.isPointerOrPointerVector() && "Can't widen type");
3283 if (!Ty.isScalar()) {
3284 // We need to widen the vector element type.
3285 Observer.changingInstr(MI);
3286 widenScalarSrc(MI, WideTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_ANYEXT);
3287 // We also need to adjust the MMO to turn this into a truncating store.
3288 MachineMemOperand &MMO = **MI.memoperands_begin();
3289 MachineFunction &MF = MIRBuilder.getMF();
3290 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo: MMO.getPointerInfo(), Ty);
3291 MI.setMemRefs(MF, MemRefs: {NewMMO});
3292 Observer.changedInstr(MI);
3293 return Legalized;
3294 }
3295
3296 Observer.changingInstr(MI);
3297
3298 unsigned ExtType = Ty.getScalarSizeInBits() == 1 ?
3299 TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT;
3300 widenScalarSrc(MI, WideTy, OpIdx: 0, ExtOpcode: ExtType);
3301
3302 Observer.changedInstr(MI);
3303 return Legalized;
3304 }
3305 case TargetOpcode::G_FPTRUNCSTORE:
3306 if (TypeIdx != 0)
3307 return UnableToLegalize;
3308 Observer.changingInstr(MI);
3309 widenScalarSrc(MI, WideTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_FPEXT);
3310 Observer.changedInstr(MI);
3311 return Legalized;
3312 case TargetOpcode::G_CONSTANT: {
3313 MachineOperand &SrcMO = MI.getOperand(i: 1);
3314 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
3315 unsigned ExtOpc = LI.getExtOpcodeForWideningConstant(
3316 SmallTy: MRI.getType(Reg: MI.getOperand(i: 0).getReg()));
3317 assert((ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::G_SEXT ||
3318 ExtOpc == TargetOpcode::G_ANYEXT) &&
3319 "Illegal Extend");
3320 const APInt &SrcVal = SrcMO.getCImm()->getValue();
3321 const APInt &Val = (ExtOpc == TargetOpcode::G_SEXT)
3322 ? SrcVal.sext(width: WideTy.getSizeInBits())
3323 : SrcVal.zext(width: WideTy.getSizeInBits());
3324 Observer.changingInstr(MI);
3325 SrcMO.setCImm(ConstantInt::get(Context&: Ctx, V: Val));
3326
3327 widenScalarDst(MI, WideTy);
3328 Observer.changedInstr(MI);
3329 return Legalized;
3330 }
3331 case TargetOpcode::G_FCONSTANT: {
3332 // To avoid changing the bits of the constant due to extension to a larger
3333 // type and then using G_FPTRUNC, we simply convert to a G_CONSTANT.
3334 MachineOperand &SrcMO = MI.getOperand(i: 1);
3335 APInt Val = SrcMO.getFPImm()->getValueAPF().bitcastToAPInt();
3336 MIRBuilder.setInstrAndDebugLoc(MI);
3337 auto IntCst = MIRBuilder.buildConstant(Res: MI.getOperand(i: 0).getReg(), Val);
3338 widenScalarDst(MI&: *IntCst, WideTy, OpIdx: 0, TruncOpcode: TargetOpcode::G_TRUNC);
3339 MI.eraseFromParent();
3340 return Legalized;
3341 }
3342 case TargetOpcode::G_IMPLICIT_DEF: {
3343 Observer.changingInstr(MI);
3344 widenScalarDst(MI, WideTy);
3345 Observer.changedInstr(MI);
3346 return Legalized;
3347 }
3348 case TargetOpcode::G_BRCOND:
3349 Observer.changingInstr(MI);
3350 widenScalarSrc(MI, WideTy, OpIdx: 0, ExtOpcode: MIRBuilder.getBoolExtOp(IsVec: false, IsFP: false));
3351 Observer.changedInstr(MI);
3352 return Legalized;
3353
3354 case TargetOpcode::G_FCMP:
3355 Observer.changingInstr(MI);
3356 if (TypeIdx == 0)
3357 widenScalarDst(MI, WideTy);
3358 else {
3359 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 2);
3360 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 3);
3361 }
3362 Observer.changedInstr(MI);
3363 return Legalized;
3364
3365 case TargetOpcode::G_ICMP:
3366 Observer.changingInstr(MI);
3367 if (TypeIdx == 0)
3368 widenScalarDst(MI, WideTy);
3369 else {
3370 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
3371 CmpInst::Predicate Pred =
3372 static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
3373
3374 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
3375 unsigned ExtOpcode =
3376 (CmpInst::isSigned(Pred) ||
3377 TLI.isSExtCheaperThanZExt(FromTy: getApproximateEVTForLLT(Ty: SrcTy, Ctx),
3378 ToTy: getApproximateEVTForLLT(Ty: WideTy, Ctx)))
3379 ? TargetOpcode::G_SEXT
3380 : TargetOpcode::G_ZEXT;
3381 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode);
3382 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode);
3383 }
3384 Observer.changedInstr(MI);
3385 return Legalized;
3386
3387 case TargetOpcode::G_PTR_ADD:
3388 assert(TypeIdx == 1 && "unable to legalize pointer of G_PTR_ADD");
3389 Observer.changingInstr(MI);
3390 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_SEXT);
3391 Observer.changedInstr(MI);
3392 return Legalized;
3393
3394 case TargetOpcode::G_PHI: {
3395 assert(TypeIdx == 0 && "Expecting only Idx 0");
3396
3397 Observer.changingInstr(MI);
3398 for (unsigned I = 1; I < MI.getNumOperands(); I += 2) {
3399 MachineBasicBlock &OpMBB = *MI.getOperand(i: I + 1).getMBB();
3400 MIRBuilder.setInsertPt(MBB&: OpMBB, II: getPHIIncomingInsertPoint(
3401 MBB&: OpMBB, Reg: MI.getOperand(i: I).getReg(), MRI));
3402 widenScalarSrc(MI, WideTy, OpIdx: I, ExtOpcode: TargetOpcode::G_ANYEXT);
3403 }
3404
3405 MachineBasicBlock &MBB = *MI.getParent();
3406 MIRBuilder.setInsertPt(MBB, II: --MBB.getFirstNonPHI());
3407 widenScalarDst(MI, WideTy);
3408 Observer.changedInstr(MI);
3409 return Legalized;
3410 }
3411 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
3412 if (TypeIdx == 0) {
3413 Register VecReg = MI.getOperand(i: 1).getReg();
3414 LLT VecTy = MRI.getType(Reg: VecReg);
3415 Observer.changingInstr(MI);
3416
3417 widenScalarSrc(MI, WideTy: LLT::vector(EC: VecTy.getElementCount(), ScalarTy: WideTy), OpIdx: 1,
3418 ExtOpcode: TargetOpcode::G_ANYEXT);
3419
3420 widenScalarDst(MI, WideTy, OpIdx: 0);
3421 Observer.changedInstr(MI);
3422 return Legalized;
3423 }
3424
3425 if (TypeIdx != 2)
3426 return UnableToLegalize;
3427 Observer.changingInstr(MI);
3428 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3429 Observer.changedInstr(MI);
3430 return Legalized;
3431 }
3432 case TargetOpcode::G_INSERT_VECTOR_ELT: {
3433 if (TypeIdx == 0) {
3434 Observer.changingInstr(MI);
3435 const LLT WideEltTy = WideTy.getElementType();
3436
3437 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3438 widenScalarSrc(MI, WideTy: WideEltTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
3439 widenScalarDst(MI, WideTy, OpIdx: 0);
3440 Observer.changedInstr(MI);
3441 return Legalized;
3442 }
3443
3444 if (TypeIdx == 1) {
3445 Observer.changingInstr(MI);
3446
3447 Register VecReg = MI.getOperand(i: 1).getReg();
3448 LLT VecTy = MRI.getType(Reg: VecReg);
3449 LLT WideVecTy = VecTy.changeVectorElementType(NewEltTy: WideTy);
3450
3451 widenScalarSrc(MI, WideTy: WideVecTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3452 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
3453 widenScalarDst(MI, WideTy: WideVecTy, OpIdx: 0);
3454 Observer.changedInstr(MI);
3455 return Legalized;
3456 }
3457
3458 if (TypeIdx == 2) {
3459 Observer.changingInstr(MI);
3460 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ZEXT);
3461 Observer.changedInstr(MI);
3462 return Legalized;
3463 }
3464
3465 return UnableToLegalize;
3466 }
3467 case TargetOpcode::G_FADD:
3468 case TargetOpcode::G_FMUL:
3469 case TargetOpcode::G_FSUB:
3470 case TargetOpcode::G_FMA:
3471 case TargetOpcode::G_FMAD:
3472 case TargetOpcode::G_FNEG:
3473 case TargetOpcode::G_FABS:
3474 case TargetOpcode::G_FCANONICALIZE:
3475 case TargetOpcode::G_FMINNUM:
3476 case TargetOpcode::G_FMAXNUM:
3477 case TargetOpcode::G_FMINNUM_IEEE:
3478 case TargetOpcode::G_FMAXNUM_IEEE:
3479 case TargetOpcode::G_FMINIMUM:
3480 case TargetOpcode::G_FMAXIMUM:
3481 case TargetOpcode::G_FMINIMUMNUM:
3482 case TargetOpcode::G_FMAXIMUMNUM:
3483 case TargetOpcode::G_FDIV:
3484 case TargetOpcode::G_FREM:
3485 case TargetOpcode::G_FCEIL:
3486 case TargetOpcode::G_FFLOOR:
3487 case TargetOpcode::G_FCOS:
3488 case TargetOpcode::G_FSIN:
3489 case TargetOpcode::G_FTAN:
3490 case TargetOpcode::G_FACOS:
3491 case TargetOpcode::G_FASIN:
3492 case TargetOpcode::G_FATAN:
3493 case TargetOpcode::G_FATAN2:
3494 case TargetOpcode::G_FCOSH:
3495 case TargetOpcode::G_FSINH:
3496 case TargetOpcode::G_FTANH:
3497 case TargetOpcode::G_FLOG10:
3498 case TargetOpcode::G_FLOG:
3499 case TargetOpcode::G_FLOG2:
3500 case TargetOpcode::G_FRINT:
3501 case TargetOpcode::G_FNEARBYINT:
3502 case TargetOpcode::G_FSQRT:
3503 case TargetOpcode::G_FEXP:
3504 case TargetOpcode::G_FEXP2:
3505 case TargetOpcode::G_FEXP10:
3506 case TargetOpcode::G_FPOW:
3507 case TargetOpcode::G_INTRINSIC_TRUNC:
3508 case TargetOpcode::G_INTRINSIC_ROUND:
3509 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
3510 assert(TypeIdx == 0);
3511 Observer.changingInstr(MI);
3512
3513 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I)
3514 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: I);
3515
3516 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3517 Observer.changedInstr(MI);
3518 return Legalized;
3519 case TargetOpcode::G_FMODF: {
3520 Observer.changingInstr(MI);
3521 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 2);
3522
3523 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 1);
3524 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: --MIRBuilder.getInsertPt());
3525 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3526 Observer.changedInstr(MI);
3527 return Legalized;
3528 }
3529 case TargetOpcode::G_FPOWI:
3530 case TargetOpcode::G_FLDEXP:
3531 case TargetOpcode::G_STRICT_FLDEXP: {
3532 if (TypeIdx == 0) {
3533 if (Opcode == TargetOpcode::G_STRICT_FLDEXP)
3534 return UnableToLegalize;
3535
3536 Observer.changingInstr(MI);
3537 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3538 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3539 Observer.changedInstr(MI);
3540 return Legalized;
3541 }
3542
3543 if (TypeIdx == 1) {
3544 // For some reason SelectionDAG tries to promote to a libcall without
3545 // actually changing the integer type for promotion.
3546 Observer.changingInstr(MI);
3547 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_SEXT);
3548 Observer.changedInstr(MI);
3549 return Legalized;
3550 }
3551
3552 return UnableToLegalize;
3553 }
3554 case TargetOpcode::G_FFREXP: {
3555 Observer.changingInstr(MI);
3556
3557 if (TypeIdx == 0) {
3558 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 2);
3559 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3560 } else {
3561 widenScalarDst(MI, WideTy, OpIdx: 1);
3562 }
3563
3564 Observer.changedInstr(MI);
3565 return Legalized;
3566 }
3567 case TargetOpcode::G_LROUND:
3568 case TargetOpcode::G_LLROUND:
3569 Observer.changingInstr(MI);
3570
3571 if (TypeIdx == 0)
3572 widenScalarDst(MI, WideTy);
3573 else
3574 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3575
3576 Observer.changedInstr(MI);
3577 return Legalized;
3578
3579 case TargetOpcode::G_INTTOPTR:
3580 if (TypeIdx != 1)
3581 return UnableToLegalize;
3582
3583 Observer.changingInstr(MI);
3584 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ZEXT);
3585 Observer.changedInstr(MI);
3586 return Legalized;
3587 case TargetOpcode::G_PTRTOINT:
3588 if (TypeIdx != 0)
3589 return UnableToLegalize;
3590
3591 Observer.changingInstr(MI);
3592 widenScalarDst(MI, WideTy, OpIdx: 0);
3593 Observer.changedInstr(MI);
3594 return Legalized;
3595 case TargetOpcode::G_BUILD_VECTOR: {
3596 Observer.changingInstr(MI);
3597
3598 const LLT WideEltTy = TypeIdx == 1 ? WideTy : WideTy.getElementType();
3599 for (int I = 1, E = MI.getNumOperands(); I != E; ++I)
3600 widenScalarSrc(MI, WideTy: WideEltTy, OpIdx: I, ExtOpcode: TargetOpcode::G_ANYEXT);
3601
3602 // Avoid changing the result vector type if the source element type was
3603 // requested.
3604 if (TypeIdx == 1) {
3605 MI.setDesc(MIRBuilder.getTII().get(Opcode: TargetOpcode::G_BUILD_VECTOR_TRUNC));
3606 } else {
3607 widenScalarDst(MI, WideTy, OpIdx: 0);
3608 }
3609
3610 Observer.changedInstr(MI);
3611 return Legalized;
3612 }
3613 case TargetOpcode::G_SEXT_INREG:
3614 if (TypeIdx != 0)
3615 return UnableToLegalize;
3616
3617 Observer.changingInstr(MI);
3618 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3619 widenScalarDst(MI, WideTy, OpIdx: 0, TruncOpcode: TargetOpcode::G_TRUNC);
3620 Observer.changedInstr(MI);
3621 return Legalized;
3622 case TargetOpcode::G_PTRMASK: {
3623 if (TypeIdx != 1)
3624 return UnableToLegalize;
3625 Observer.changingInstr(MI);
3626 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3627 Observer.changedInstr(MI);
3628 return Legalized;
3629 }
3630 case TargetOpcode::G_VECREDUCE_ADD: {
3631 if (TypeIdx != 1)
3632 return UnableToLegalize;
3633 Observer.changingInstr(MI);
3634 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3635 widenScalarDst(MI, WideTy: WideTy.getScalarType(), OpIdx: 0, TruncOpcode: TargetOpcode::G_TRUNC);
3636 Observer.changedInstr(MI);
3637 return Legalized;
3638 }
3639 case TargetOpcode::G_VECREDUCE_FADD:
3640 case TargetOpcode::G_VECREDUCE_FMUL:
3641 case TargetOpcode::G_VECREDUCE_FMIN:
3642 case TargetOpcode::G_VECREDUCE_FMAX:
3643 case TargetOpcode::G_VECREDUCE_FMINIMUM:
3644 case TargetOpcode::G_VECREDUCE_FMAXIMUM: {
3645 if (TypeIdx != 0)
3646 return UnableToLegalize;
3647 Observer.changingInstr(MI);
3648 Register VecReg = MI.getOperand(i: 1).getReg();
3649 LLT VecTy = MRI.getType(Reg: VecReg);
3650 LLT WideVecTy = VecTy.changeElementType(NewEltTy: WideTy);
3651 widenScalarSrcUsingFPExt(MI, WideTy: WideVecTy, OpIdx: 1);
3652 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3653 Observer.changedInstr(MI);
3654 return Legalized;
3655 }
3656 case TargetOpcode::G_VSCALE: {
3657 MachineOperand &SrcMO = MI.getOperand(i: 1);
3658 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
3659 const APInt &SrcVal = SrcMO.getCImm()->getValue();
3660 // The CImm is always a signed value
3661 const APInt Val = SrcVal.sext(width: WideTy.getSizeInBits());
3662 Observer.changingInstr(MI);
3663 SrcMO.setCImm(ConstantInt::get(Context&: Ctx, V: Val));
3664 widenScalarDst(MI, WideTy);
3665 Observer.changedInstr(MI);
3666 return Legalized;
3667 }
3668 case TargetOpcode::G_SPLAT_VECTOR: {
3669 if (TypeIdx != 1)
3670 return UnableToLegalize;
3671
3672 Observer.changingInstr(MI);
3673 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3674 Observer.changedInstr(MI);
3675 return Legalized;
3676 }
3677 case TargetOpcode::G_INSERT_SUBVECTOR: {
3678 if (TypeIdx != 0)
3679 return UnableToLegalize;
3680
3681 GInsertSubvector &IS = cast<GInsertSubvector>(Val&: MI);
3682 Register BigVec = IS.getBigVec();
3683 Register SubVec = IS.getSubVec();
3684
3685 LLT SubVecTy = MRI.getType(Reg: SubVec);
3686 LLT SubVecWideTy = SubVecTy.changeElementType(NewEltTy: WideTy.getElementType());
3687
3688 // Widen the G_INSERT_SUBVECTOR
3689 auto BigZExt = MIRBuilder.buildZExt(Res: WideTy, Op: BigVec);
3690 auto SubZExt = MIRBuilder.buildZExt(Res: SubVecWideTy, Op: SubVec);
3691 auto WideInsert = MIRBuilder.buildInsertSubvector(Res: WideTy, Src0: BigZExt, Src1: SubZExt,
3692 Index: IS.getIndexImm());
3693
3694 // Truncate back down
3695 auto SplatZero = MIRBuilder.buildSplatVector(
3696 Res: WideTy, Val: MIRBuilder.buildConstant(Res: WideTy.getElementType(), Val: 0));
3697 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_NE, Res: IS.getReg(Idx: 0), Op0: WideInsert,
3698 Op1: SplatZero);
3699
3700 MI.eraseFromParent();
3701
3702 return Legalized;
3703 }
3704 case TargetOpcode::G_BITCAST:
3705 if (WideTy.isVector())
3706 return UnableToLegalize;
3707 Observer.changingInstr(MI);
3708 if (TypeIdx == 0)
3709 widenScalarDst(MI, WideTy, OpIdx: 0, TruncOpcode: TargetOpcode::G_TRUNC);
3710 else
3711 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3712 Observer.changedInstr(MI);
3713
3714 Register Dst = MI.getOperand(i: 0).getReg();
3715 Register Src = MI.getOperand(i: 1).getReg();
3716 if (MRI.getType(Reg: Dst) == MRI.getType(Reg: Src)) {
3717 Observer.changingAllUsesOfReg(MRI, Reg: Dst);
3718 MRI.replaceRegWith(FromReg: Dst, ToReg: Src);
3719 Observer.finishedChangingAllUsesOfReg();
3720 MI.eraseFromParent();
3721 }
3722
3723 return Legalized;
3724 }
3725}
3726
3727static void getUnmergePieces(SmallVectorImpl<Register> &Pieces,
3728 MachineIRBuilder &B, Register Src, LLT Ty) {
3729 auto Unmerge = B.buildUnmerge(Res: Ty, Op: Src);
3730 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
3731 Pieces.push_back(Elt: Unmerge.getReg(Idx: I));
3732}
3733
3734static void emitLoadFromConstantPool(Register DstReg, const Constant *ConstVal,
3735 MachineIRBuilder &MIRBuilder) {
3736 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
3737 MachineFunction &MF = MIRBuilder.getMF();
3738 const DataLayout &DL = MIRBuilder.getDataLayout();
3739 unsigned AddrSpace = DL.getDefaultGlobalsAddressSpace();
3740 LLT AddrPtrTy = LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL.getPointerSizeInBits(AS: AddrSpace));
3741 LLT DstLLT = MRI.getType(Reg: DstReg);
3742
3743 Align Alignment(DL.getABITypeAlign(Ty: ConstVal->getType()));
3744
3745 auto Addr = MIRBuilder.buildConstantPool(
3746 Res: AddrPtrTy,
3747 Idx: MF.getConstantPool()->getConstantPoolIndex(C: ConstVal, Alignment));
3748
3749 MachineMemOperand *MMO =
3750 MF.getMachineMemOperand(PtrInfo: MachinePointerInfo::getConstantPool(MF),
3751 F: MachineMemOperand::MOLoad, MemTy: DstLLT, BaseAlignment: Alignment);
3752
3753 MIRBuilder.buildLoadInstr(Opcode: TargetOpcode::G_LOAD, Res: DstReg, Addr, MMO&: *MMO);
3754}
3755
3756LegalizerHelper::LegalizeResult
3757LegalizerHelper::lowerConstant(MachineInstr &MI) {
3758 const MachineOperand &ConstOperand = MI.getOperand(i: 1);
3759 const Constant *ConstantVal = ConstOperand.getCImm();
3760
3761 emitLoadFromConstantPool(DstReg: MI.getOperand(i: 0).getReg(), ConstVal: ConstantVal, MIRBuilder);
3762 MI.eraseFromParent();
3763
3764 return Legalized;
3765}
3766
3767LegalizerHelper::LegalizeResult
3768LegalizerHelper::lowerFConstant(MachineInstr &MI) {
3769 Register Dst = MI.getOperand(i: 0).getReg();
3770 const ConstantFP *ConstantVal = MI.getOperand(i: 1).getFPImm();
3771 const APFloat &Imm = ConstantVal->getValueAPF();
3772
3773 bool ForCodeSize = MIRBuilder.getMF().getFunction().hasOptSize();
3774 if (TLI.getFPImmAction(Imm, MRI.getType(Reg: Dst), ForCodeSize) ==
3775 TargetLowering::FPImmAction::Bitcast)
3776 MIRBuilder.buildConstant(Res: Dst, Val: Imm.bitcastToAPInt());
3777 else
3778 emitLoadFromConstantPool(DstReg: Dst, ConstVal: ConstantVal, MIRBuilder);
3779
3780 MI.eraseFromParent();
3781 return Legalized;
3782}
3783
3784LegalizerHelper::LegalizeResult
3785LegalizerHelper::lowerBitcast(MachineInstr &MI) {
3786 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
3787 if (SrcTy.isVector()) {
3788 LLT SrcEltTy = SrcTy.getElementType();
3789 SmallVector<Register, 8> SrcRegs;
3790
3791 if (DstTy.isVector()) {
3792 LLT DstEltTy = DstTy.getElementType();
3793 ElementCount DstEC = DstTy.getElementCount();
3794 ElementCount SrcEC = SrcTy.getElementCount();
3795
3796 if (!SrcEC.isKnownMultipleOf(RHS: DstEC) && !DstEC.isKnownMultipleOf(RHS: SrcEC)) {
3797 // Split non-integer element ratio bitcast
3798 //
3799 // %1:_(<3 x s16>) = G_BITCAST %0:_(<2 x s24>)
3800 //
3801 // =>
3802 //
3803 // %2:_(<6 x s8>) = G_BITCAST %0:_(<2 x s24>)
3804 // %1:_(<3 x s16>) = G_BITCAST %2:_(<6 x s8>)
3805 unsigned SrcEltSize = SrcEltTy.getScalarSizeInBits();
3806 unsigned PieceSize =
3807 std::gcd(m: SrcEltSize, n: DstEltTy.getScalarSizeInBits());
3808 LLT PieceTy = LLT::integer(SizeInBits: PieceSize);
3809
3810 if (!PieceTy.isByteSized()) {
3811 // Split bitcast whose pieces are not whole bytes through a scalar
3812 //
3813 // %1:_(<3 x s8>) = G_BITCAST %0:_(<2 x s12>)
3814 //
3815 // =>
3816 //
3817 // %2:_(s24) = G_BITCAST %0:_(<2 x s12>)
3818 // %1:_(<3 x s8>) = G_BITCAST %2:_(s24)
3819 LLT ScalarTy = LLT::integer(SizeInBits: SrcTy.getSizeInBits());
3820 Register ScalarReg = MIRBuilder.buildBitcast(Dst: ScalarTy, Src).getReg(Idx: 0);
3821 MIRBuilder.buildBitcast(Dst, Src: ScalarReg);
3822 MI.eraseFromParent();
3823 return Legalized;
3824 }
3825
3826 LLT PiecesVecTy =
3827 LLT::vector(EC: SrcEC * (SrcEltSize / PieceSize), ScalarTy: PieceTy);
3828 Register PiecesReg =
3829 MIRBuilder.buildBitcast(Dst: PiecesVecTy, Src).getReg(Idx: 0);
3830 MIRBuilder.buildBitcast(Dst, Src: PiecesReg);
3831 MI.eraseFromParent();
3832 return Legalized;
3833 }
3834
3835 unsigned NumDstElt = DstEC.getKnownMinValue();
3836 unsigned NumSrcElt = SrcEC.getKnownMinValue();
3837
3838 LLT DstCastTy = DstEltTy; // Intermediate bitcast result type
3839 LLT SrcPartTy = SrcEltTy; // Original unmerge result type.
3840
3841 // If there's an element size mismatch, insert intermediate casts to match
3842 // the result element type.
3843 if (NumSrcElt < NumDstElt) { // Source element type is larger.
3844 // %1:_(<4 x s8>) = G_BITCAST %0:_(<2 x s16>)
3845 //
3846 // =>
3847 //
3848 // %2:_(s16), %3:_(s16) = G_UNMERGE_VALUES %0
3849 // %3:_(<2 x s8>) = G_BITCAST %2
3850 // %4:_(<2 x s8>) = G_BITCAST %3
3851 // %1:_(<4 x s8>) = G_CONCAT_VECTORS %3, %4
3852 DstCastTy = DstTy.changeVectorElementCount(
3853 EC: ElementCount::getFixed(MinVal: NumDstElt / NumSrcElt));
3854 SrcPartTy = SrcEltTy;
3855 } else if (NumSrcElt > NumDstElt) { // Source element type is smaller.
3856 //
3857 // %1:_(<2 x s16>) = G_BITCAST %0:_(<4 x s8>)
3858 //
3859 // =>
3860 //
3861 // %2:_(<2 x s8>), %3:_(<2 x s8>) = G_UNMERGE_VALUES %0
3862 // %3:_(s16) = G_BITCAST %2
3863 // %4:_(s16) = G_BITCAST %3
3864 // %1:_(<2 x s16>) = G_BUILD_VECTOR %3, %4
3865 SrcPartTy = SrcTy.changeVectorElementCount(
3866 EC: ElementCount::getFixed(MinVal: NumSrcElt / NumDstElt));
3867 DstCastTy = DstEltTy;
3868 }
3869
3870 getUnmergePieces(Pieces&: SrcRegs, B&: MIRBuilder, Src, Ty: SrcPartTy);
3871 for (Register &SrcReg : SrcRegs)
3872 SrcReg = MIRBuilder.buildBitcast(Dst: DstCastTy, Src: SrcReg).getReg(Idx: 0);
3873 } else
3874 getUnmergePieces(Pieces&: SrcRegs, B&: MIRBuilder, Src, Ty: SrcEltTy);
3875
3876 MIRBuilder.buildMergeLikeInstr(Res: Dst, Ops: SrcRegs);
3877 MI.eraseFromParent();
3878 return Legalized;
3879 }
3880
3881 if (DstTy.isVector()) {
3882 SmallVector<Register, 8> SrcRegs;
3883 getUnmergePieces(Pieces&: SrcRegs, B&: MIRBuilder, Src, Ty: DstTy.getElementType());
3884 MIRBuilder.buildMergeLikeInstr(Res: Dst, Ops: SrcRegs);
3885 MI.eraseFromParent();
3886 return Legalized;
3887 }
3888
3889 return UnableToLegalize;
3890}
3891
3892/// Figure out the bit offset into a register when coercing a vector index for
3893/// the wide element type. This is only for the case when promoting vector to
3894/// one with larger elements.
3895//
3896///
3897/// %offset_idx = G_AND %idx, ~(-1 << Log2(DstEltSize / SrcEltSize))
3898/// %offset_bits = G_SHL %offset_idx, Log2(SrcEltSize)
3899static Register getBitcastWiderVectorElementOffset(MachineIRBuilder &B,
3900 Register Idx,
3901 unsigned NewEltSize,
3902 unsigned OldEltSize) {
3903 const unsigned Log2EltRatio = Log2_32(Value: NewEltSize / OldEltSize);
3904 LLT IdxTy = B.getMRI()->getType(Reg: Idx);
3905
3906 // Now figure out the amount we need to shift to get the target bits.
3907 auto OffsetMask = B.buildConstant(
3908 Res: IdxTy, Val: ~(APInt::getAllOnes(numBits: IdxTy.getSizeInBits()) << Log2EltRatio));
3909 auto OffsetIdx = B.buildAnd(Dst: IdxTy, Src0: Idx, Src1: OffsetMask);
3910 return B.buildShl(Dst: IdxTy, Src0: OffsetIdx,
3911 Src1: B.buildConstant(Res: IdxTy, Val: Log2_32(Value: OldEltSize))).getReg(Idx: 0);
3912}
3913
3914/// Perform a G_EXTRACT_VECTOR_ELT in a different sized vector element. If this
3915/// is casting to a vector with a smaller element size, perform multiple element
3916/// extracts and merge the results. If this is coercing to a vector with larger
3917/// elements, index the bitcasted vector and extract the target element with bit
3918/// operations. This is intended to force the indexing in the native register
3919/// size for architectures that can dynamically index the register file.
3920LegalizerHelper::LegalizeResult
3921LegalizerHelper::bitcastExtractVectorElt(MachineInstr &MI, unsigned TypeIdx,
3922 LLT CastTy) {
3923 if (TypeIdx != 1)
3924 return UnableToLegalize;
3925
3926 auto [Dst, DstTy, SrcVec, SrcVecTy, Idx, IdxTy] = MI.getFirst3RegLLTs();
3927
3928 LLT SrcEltTy = SrcVecTy.getElementType();
3929 unsigned NewNumElts = CastTy.isVector() ? CastTy.getNumElements() : 1;
3930 unsigned OldNumElts = SrcVecTy.getNumElements();
3931
3932 LLT NewEltTy = CastTy.getScalarType();
3933 Register CastVec = MIRBuilder.buildBitcast(Dst: CastTy, Src: SrcVec).getReg(Idx: 0);
3934
3935 const unsigned NewEltSize = NewEltTy.getSizeInBits();
3936 const unsigned OldEltSize = SrcEltTy.getSizeInBits();
3937 if (NewNumElts > OldNumElts) {
3938 // Decreasing the vector element size
3939 //
3940 // e.g. i64 = extract_vector_elt x:v2i64, y:i32
3941 // =>
3942 // v4i32:castx = bitcast x:v2i64
3943 //
3944 // i64 = bitcast
3945 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
3946 // (i32 (extract_vector_elt castx, (2 * y + 1)))
3947 //
3948 if (NewNumElts % OldNumElts != 0)
3949 return UnableToLegalize;
3950
3951 // Type of the intermediate result vector.
3952 const unsigned NewEltsPerOldElt = NewNumElts / OldNumElts;
3953 LLT MidTy =
3954 CastTy.changeElementCount(EC: ElementCount::getFixed(MinVal: NewEltsPerOldElt));
3955
3956 auto NewEltsPerOldEltK = MIRBuilder.buildConstant(Res: IdxTy, Val: NewEltsPerOldElt);
3957
3958 SmallVector<Register, 8> NewOps(NewEltsPerOldElt);
3959 auto NewBaseIdx = MIRBuilder.buildMul(Dst: IdxTy, Src0: Idx, Src1: NewEltsPerOldEltK);
3960
3961 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
3962 auto IdxOffset = MIRBuilder.buildConstant(Res: IdxTy, Val: I);
3963 auto TmpIdx = MIRBuilder.buildAdd(Dst: IdxTy, Src0: NewBaseIdx, Src1: IdxOffset);
3964 auto Elt = MIRBuilder.buildExtractVectorElement(Res: NewEltTy, Val: CastVec, Idx: TmpIdx);
3965 NewOps[I] = Elt.getReg(Idx: 0);
3966 }
3967
3968 auto NewVec = MIRBuilder.buildBuildVector(Res: MidTy, Ops: NewOps);
3969 MIRBuilder.buildBitcast(Dst, Src: NewVec);
3970 MI.eraseFromParent();
3971 return Legalized;
3972 }
3973
3974 if (NewNumElts < OldNumElts) {
3975 if (NewEltSize % OldEltSize != 0)
3976 return UnableToLegalize;
3977
3978 // This only depends on powers of 2 because we use bit tricks to figure out
3979 // the bit offset we need to shift to get the target element. A general
3980 // expansion could emit division/multiply.
3981 if (!isPowerOf2_32(Value: NewEltSize / OldEltSize))
3982 return UnableToLegalize;
3983
3984 // Increasing the vector element size.
3985 // %elt:_(small_elt) = G_EXTRACT_VECTOR_ELT %vec:_(<N x small_elt>), %idx
3986 //
3987 // =>
3988 //
3989 // %cast = G_BITCAST %vec
3990 // %scaled_idx = G_LSHR %idx, Log2(DstEltSize / SrcEltSize)
3991 // %wide_elt = G_EXTRACT_VECTOR_ELT %cast, %scaled_idx
3992 // %offset_idx = G_AND %idx, ~(-1 << Log2(DstEltSize / SrcEltSize))
3993 // %offset_bits = G_SHL %offset_idx, Log2(SrcEltSize)
3994 // %elt_bits = G_LSHR %wide_elt, %offset_bits
3995 // %elt = G_TRUNC %elt_bits
3996
3997 const unsigned Log2EltRatio = Log2_32(Value: NewEltSize / OldEltSize);
3998 auto Log2Ratio = MIRBuilder.buildConstant(Res: IdxTy, Val: Log2EltRatio);
3999
4000 // Divide to get the index in the wider element type.
4001 auto ScaledIdx = MIRBuilder.buildLShr(Dst: IdxTy, Src0: Idx, Src1: Log2Ratio);
4002
4003 Register WideElt = CastVec;
4004 if (CastTy.isVector()) {
4005 WideElt = MIRBuilder.buildExtractVectorElement(Res: NewEltTy, Val: CastVec,
4006 Idx: ScaledIdx).getReg(Idx: 0);
4007 }
4008
4009 // Compute the bit offset into the register of the target element.
4010 Register OffsetBits = getBitcastWiderVectorElementOffset(
4011 B&: MIRBuilder, Idx, NewEltSize, OldEltSize);
4012
4013 // Shift the wide element to get the target element.
4014 auto ExtractedBits = MIRBuilder.buildLShr(Dst: NewEltTy, Src0: WideElt, Src1: OffsetBits);
4015 MIRBuilder.buildTrunc(Res: Dst, Op: ExtractedBits);
4016 MI.eraseFromParent();
4017 return Legalized;
4018 }
4019
4020 return UnableToLegalize;
4021}
4022
4023/// Emit code to insert \p InsertReg into \p TargetRet at \p OffsetBits in \p
4024/// TargetReg, while preserving other bits in \p TargetReg.
4025///
4026/// (InsertReg << Offset) | (TargetReg & ~(-1 >> InsertReg.size()) << Offset)
4027static Register buildBitFieldInsert(MachineIRBuilder &B,
4028 Register TargetReg, Register InsertReg,
4029 Register OffsetBits) {
4030 LLT TargetTy = B.getMRI()->getType(Reg: TargetReg);
4031 LLT InsertTy = B.getMRI()->getType(Reg: InsertReg);
4032 auto ZextVal = B.buildZExt(Res: TargetTy, Op: InsertReg);
4033 auto ShiftedInsertVal = B.buildShl(Dst: TargetTy, Src0: ZextVal, Src1: OffsetBits);
4034
4035 // Produce a bitmask of the value to insert
4036 auto EltMask = B.buildConstant(
4037 Res: TargetTy, Val: APInt::getLowBitsSet(numBits: TargetTy.getSizeInBits(),
4038 loBitsSet: InsertTy.getSizeInBits()));
4039 // Shift it into position
4040 auto ShiftedMask = B.buildShl(Dst: TargetTy, Src0: EltMask, Src1: OffsetBits);
4041 auto InvShiftedMask = B.buildNot(Dst: TargetTy, Src0: ShiftedMask);
4042
4043 // Clear out the bits in the wide element
4044 auto MaskedOldElt = B.buildAnd(Dst: TargetTy, Src0: TargetReg, Src1: InvShiftedMask);
4045
4046 // The value to insert has all zeros already, so stick it into the masked
4047 // wide element.
4048 return B.buildOr(Dst: TargetTy, Src0: MaskedOldElt, Src1: ShiftedInsertVal).getReg(Idx: 0);
4049}
4050
4051/// Perform a G_INSERT_VECTOR_ELT in a different sized vector element. If this
4052/// is increasing the element size, perform the indexing in the target element
4053/// type, and use bit operations to insert at the element position. This is
4054/// intended for architectures that can dynamically index the register file and
4055/// want to force indexing in the native register size.
4056LegalizerHelper::LegalizeResult
4057LegalizerHelper::bitcastInsertVectorElt(MachineInstr &MI, unsigned TypeIdx,
4058 LLT CastTy) {
4059 if (TypeIdx != 0)
4060 return UnableToLegalize;
4061
4062 auto [Dst, DstTy, SrcVec, SrcVecTy, Val, ValTy, Idx, IdxTy] =
4063 MI.getFirst4RegLLTs();
4064 LLT VecTy = DstTy;
4065
4066 LLT VecEltTy = VecTy.getElementType();
4067 LLT NewEltTy = CastTy.isVector() ? CastTy.getElementType() : CastTy;
4068 const unsigned NewEltSize = NewEltTy.getSizeInBits();
4069 const unsigned OldEltSize = VecEltTy.getSizeInBits();
4070
4071 unsigned NewNumElts = CastTy.isVector() ? CastTy.getNumElements() : 1;
4072 unsigned OldNumElts = VecTy.getNumElements();
4073
4074 Register CastVec = MIRBuilder.buildBitcast(Dst: CastTy, Src: SrcVec).getReg(Idx: 0);
4075 if (NewNumElts < OldNumElts) {
4076 if (NewEltSize % OldEltSize != 0)
4077 return UnableToLegalize;
4078
4079 // This only depends on powers of 2 because we use bit tricks to figure out
4080 // the bit offset we need to shift to get the target element. A general
4081 // expansion could emit division/multiply.
4082 if (!isPowerOf2_32(Value: NewEltSize / OldEltSize))
4083 return UnableToLegalize;
4084
4085 const unsigned Log2EltRatio = Log2_32(Value: NewEltSize / OldEltSize);
4086 auto Log2Ratio = MIRBuilder.buildConstant(Res: IdxTy, Val: Log2EltRatio);
4087
4088 // Divide to get the index in the wider element type.
4089 auto ScaledIdx = MIRBuilder.buildLShr(Dst: IdxTy, Src0: Idx, Src1: Log2Ratio);
4090
4091 Register ExtractedElt = CastVec;
4092 if (CastTy.isVector()) {
4093 ExtractedElt = MIRBuilder.buildExtractVectorElement(Res: NewEltTy, Val: CastVec,
4094 Idx: ScaledIdx).getReg(Idx: 0);
4095 }
4096
4097 // Compute the bit offset into the register of the target element.
4098 Register OffsetBits = getBitcastWiderVectorElementOffset(
4099 B&: MIRBuilder, Idx, NewEltSize, OldEltSize);
4100
4101 Register InsertedElt = buildBitFieldInsert(B&: MIRBuilder, TargetReg: ExtractedElt,
4102 InsertReg: Val, OffsetBits);
4103 if (CastTy.isVector()) {
4104 InsertedElt = MIRBuilder.buildInsertVectorElement(
4105 Res: CastTy, Val: CastVec, Elt: InsertedElt, Idx: ScaledIdx).getReg(Idx: 0);
4106 }
4107
4108 MIRBuilder.buildBitcast(Dst, Src: InsertedElt);
4109 MI.eraseFromParent();
4110 return Legalized;
4111 }
4112
4113 return UnableToLegalize;
4114}
4115
4116// This attempts to handle G_CONCAT_VECTORS with illegal operands, particularly
4117// those that have smaller than legal operands.
4118//
4119// <16 x s8> = G_CONCAT_VECTORS <4 x s8>, <4 x s8>, <4 x s8>, <4 x s8>
4120//
4121// ===>
4122//
4123// s32 = G_BITCAST <4 x s8>
4124// s32 = G_BITCAST <4 x s8>
4125// s32 = G_BITCAST <4 x s8>
4126// s32 = G_BITCAST <4 x s8>
4127// <4 x s32> = G_BUILD_VECTOR s32, s32, s32, s32
4128// <16 x s8> = G_BITCAST <4 x s32>
4129LegalizerHelper::LegalizeResult
4130LegalizerHelper::bitcastConcatVector(MachineInstr &MI, unsigned TypeIdx,
4131 LLT CastTy) {
4132 // Convert it to CONCAT instruction
4133 auto ConcatMI = dyn_cast<GConcatVectors>(Val: &MI);
4134 if (!ConcatMI) {
4135 return UnableToLegalize;
4136 }
4137
4138 // Check if bitcast is Legal
4139 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
4140 LLT SrcScalTy = CastTy.getScalarType();
4141
4142 // Check if the build vector is Legal
4143 if (!LI.isLegal(Query: {TargetOpcode::G_BUILD_VECTOR, {CastTy, SrcScalTy}})) {
4144 return UnableToLegalize;
4145 }
4146
4147 // Bitcast the sources
4148 SmallVector<Register> BitcastRegs;
4149 for (unsigned i = 0; i < ConcatMI->getNumSources(); i++) {
4150 BitcastRegs.push_back(
4151 Elt: MIRBuilder.buildBitcast(Dst: SrcScalTy, Src: ConcatMI->getSourceReg(I: i))
4152 .getReg(Idx: 0));
4153 }
4154
4155 // Build the scalar values into a vector
4156 Register BuildReg =
4157 MIRBuilder.buildBuildVector(Res: CastTy, Ops: BitcastRegs).getReg(Idx: 0);
4158 MIRBuilder.buildBitcast(Dst: DstReg, Src: BuildReg);
4159
4160 MI.eraseFromParent();
4161 return Legalized;
4162}
4163
4164// This bitcasts a shuffle vector to a different type currently of the same
4165// element size. Mostly used to legalize ptr vectors, where ptrtoint/inttoptr
4166// will be used instead.
4167//
4168// <16 x p0> = G_CONCAT_VECTORS <4 x p0>, <4 x p0>, mask
4169// ===>
4170// <4 x s64> = G_PTRTOINT <4 x p0>
4171// <4 x s64> = G_PTRTOINT <4 x p0>
4172// <16 x s64> = G_CONCAT_VECTORS <4 x s64>, <4 x s64>, mask
4173// <16 x p0> = G_INTTOPTR <16 x s64>
4174LegalizerHelper::LegalizeResult
4175LegalizerHelper::bitcastShuffleVector(MachineInstr &MI, unsigned TypeIdx,
4176 LLT CastTy) {
4177 auto ShuffleMI = cast<GShuffleVector>(Val: &MI);
4178 LLT DstTy = MRI.getType(Reg: ShuffleMI->getReg(Idx: 0));
4179 LLT SrcTy = MRI.getType(Reg: ShuffleMI->getReg(Idx: 1));
4180
4181 // We currently only handle vectors of the same size.
4182 if (TypeIdx != 0 ||
4183 CastTy.getScalarSizeInBits() != DstTy.getScalarSizeInBits() ||
4184 CastTy.getElementCount() != DstTy.getElementCount())
4185 return UnableToLegalize;
4186
4187 LLT NewSrcTy = SrcTy.changeElementType(NewEltTy: CastTy.getScalarType());
4188
4189 auto Inp1 = MIRBuilder.buildCast(Dst: NewSrcTy, Src: ShuffleMI->getReg(Idx: 1));
4190 auto Inp2 = MIRBuilder.buildCast(Dst: NewSrcTy, Src: ShuffleMI->getReg(Idx: 2));
4191 auto Shuf =
4192 MIRBuilder.buildShuffleVector(Res: CastTy, Src1: Inp1, Src2: Inp2, Mask: ShuffleMI->getMask());
4193 MIRBuilder.buildCast(Dst: ShuffleMI->getReg(Idx: 0), Src: Shuf);
4194
4195 MI.eraseFromParent();
4196 return Legalized;
4197}
4198
4199/// This attempts to bitcast G_EXTRACT_SUBVECTOR to CastTy.
4200///
4201/// <vscale x 8 x i1> = G_EXTRACT_SUBVECTOR <vscale x 16 x i1>, N
4202///
4203/// ===>
4204///
4205/// <vscale x 2 x i1> = G_BITCAST <vscale x 16 x i1>
4206/// <vscale x 1 x i8> = G_EXTRACT_SUBVECTOR <vscale x 2 x i1>, N / 8
4207/// <vscale x 8 x i1> = G_BITCAST <vscale x 1 x i8>
4208LegalizerHelper::LegalizeResult
4209LegalizerHelper::bitcastExtractSubvector(MachineInstr &MI, unsigned TypeIdx,
4210 LLT CastTy) {
4211 auto ES = cast<GExtractSubvector>(Val: &MI);
4212
4213 if (!CastTy.isVector())
4214 return UnableToLegalize;
4215
4216 if (TypeIdx != 0)
4217 return UnableToLegalize;
4218
4219 Register Dst = ES->getReg(Idx: 0);
4220 Register Src = ES->getSrcVec();
4221 uint64_t Idx = ES->getIndexImm();
4222
4223 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4224
4225 LLT DstTy = MRI.getType(Reg: Dst);
4226 LLT SrcTy = MRI.getType(Reg: Src);
4227 ElementCount DstTyEC = DstTy.getElementCount();
4228 ElementCount SrcTyEC = SrcTy.getElementCount();
4229 auto DstTyMinElts = DstTyEC.getKnownMinValue();
4230 auto SrcTyMinElts = SrcTyEC.getKnownMinValue();
4231
4232 if (DstTy == CastTy)
4233 return Legalized;
4234
4235 if (DstTy.getSizeInBits() != CastTy.getSizeInBits())
4236 return UnableToLegalize;
4237
4238 unsigned CastEltSize = CastTy.getElementType().getSizeInBits();
4239 unsigned DstEltSize = DstTy.getElementType().getSizeInBits();
4240 if (CastEltSize < DstEltSize)
4241 return UnableToLegalize;
4242
4243 auto AdjustAmt = CastEltSize / DstEltSize;
4244 if (Idx % AdjustAmt != 0 || DstTyMinElts % AdjustAmt != 0 ||
4245 SrcTyMinElts % AdjustAmt != 0)
4246 return UnableToLegalize;
4247
4248 Idx /= AdjustAmt;
4249 SrcTy = LLT::vector(EC: SrcTyEC.divideCoefficientBy(RHS: AdjustAmt), ScalarSizeInBits: AdjustAmt);
4250 auto CastVec = MIRBuilder.buildBitcast(Dst: SrcTy, Src);
4251 auto PromotedES = MIRBuilder.buildExtractSubvector(Res: CastTy, Src: CastVec, Index: Idx);
4252 MIRBuilder.buildBitcast(Dst, Src: PromotedES);
4253
4254 ES->eraseFromParent();
4255 return Legalized;
4256}
4257
4258/// This attempts to bitcast G_INSERT_SUBVECTOR to CastTy.
4259///
4260/// <vscale x 16 x i1> = G_INSERT_SUBVECTOR <vscale x 16 x i1>,
4261/// <vscale x 8 x i1>,
4262/// N
4263///
4264/// ===>
4265///
4266/// <vscale x 2 x i8> = G_BITCAST <vscale x 16 x i1>
4267/// <vscale x 1 x i8> = G_BITCAST <vscale x 8 x i1>
4268/// <vscale x 2 x i8> = G_INSERT_SUBVECTOR <vscale x 2 x i8>,
4269/// <vscale x 1 x i8>, N / 8
4270/// <vscale x 16 x i1> = G_BITCAST <vscale x 2 x i8>
4271LegalizerHelper::LegalizeResult
4272LegalizerHelper::bitcastInsertSubvector(MachineInstr &MI, unsigned TypeIdx,
4273 LLT CastTy) {
4274 auto ES = cast<GInsertSubvector>(Val: &MI);
4275
4276 if (!CastTy.isVector())
4277 return UnableToLegalize;
4278
4279 if (TypeIdx != 0)
4280 return UnableToLegalize;
4281
4282 Register Dst = ES->getReg(Idx: 0);
4283 Register BigVec = ES->getBigVec();
4284 Register SubVec = ES->getSubVec();
4285 uint64_t Idx = ES->getIndexImm();
4286
4287 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4288
4289 LLT DstTy = MRI.getType(Reg: Dst);
4290 LLT BigVecTy = MRI.getType(Reg: BigVec);
4291 LLT SubVecTy = MRI.getType(Reg: SubVec);
4292
4293 if (DstTy == CastTy)
4294 return Legalized;
4295
4296 if (DstTy.getSizeInBits() != CastTy.getSizeInBits())
4297 return UnableToLegalize;
4298
4299 ElementCount DstTyEC = DstTy.getElementCount();
4300 ElementCount BigVecTyEC = BigVecTy.getElementCount();
4301 ElementCount SubVecTyEC = SubVecTy.getElementCount();
4302 auto DstTyMinElts = DstTyEC.getKnownMinValue();
4303 auto BigVecTyMinElts = BigVecTyEC.getKnownMinValue();
4304 auto SubVecTyMinElts = SubVecTyEC.getKnownMinValue();
4305
4306 unsigned CastEltSize = CastTy.getElementType().getSizeInBits();
4307 unsigned DstEltSize = DstTy.getElementType().getSizeInBits();
4308 if (CastEltSize < DstEltSize)
4309 return UnableToLegalize;
4310
4311 auto AdjustAmt = CastEltSize / DstEltSize;
4312 if (Idx % AdjustAmt != 0 || DstTyMinElts % AdjustAmt != 0 ||
4313 BigVecTyMinElts % AdjustAmt != 0 || SubVecTyMinElts % AdjustAmt != 0)
4314 return UnableToLegalize;
4315
4316 Idx /= AdjustAmt;
4317 BigVecTy = LLT::vector(EC: BigVecTyEC.divideCoefficientBy(RHS: AdjustAmt), ScalarSizeInBits: AdjustAmt);
4318 SubVecTy = LLT::vector(EC: SubVecTyEC.divideCoefficientBy(RHS: AdjustAmt), ScalarSizeInBits: AdjustAmt);
4319 auto CastBigVec = MIRBuilder.buildBitcast(Dst: BigVecTy, Src: BigVec);
4320 auto CastSubVec = MIRBuilder.buildBitcast(Dst: SubVecTy, Src: SubVec);
4321 auto PromotedIS =
4322 MIRBuilder.buildInsertSubvector(Res: CastTy, Src0: CastBigVec, Src1: CastSubVec, Index: Idx);
4323 MIRBuilder.buildBitcast(Dst, Src: PromotedIS);
4324
4325 ES->eraseFromParent();
4326 return Legalized;
4327}
4328
4329LegalizerHelper::LegalizeResult LegalizerHelper::lowerLoad(GAnyLoad &LoadMI) {
4330 // Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT
4331 Register DstReg = LoadMI.getDstReg();
4332 Register PtrReg = LoadMI.getPointerReg();
4333 LLT DstTy = MRI.getType(Reg: DstReg);
4334 MachineMemOperand &MMO = LoadMI.getMMO();
4335 LLT MemTy = MMO.getMemoryType();
4336 MachineFunction &MF = MIRBuilder.getMF();
4337
4338 LLT EltTy = MemTy.getScalarType();
4339
4340 unsigned MemSizeInBits = MemTy.getSizeInBits();
4341 unsigned MemStoreSizeInBits = 8 * MemTy.getSizeInBytes();
4342
4343 if (MemSizeInBits != MemStoreSizeInBits) {
4344 if (MemTy.isVector())
4345 return UnableToLegalize;
4346
4347 // Promote to a byte-sized load if not loading an integral number of
4348 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
4349 LLT WideMemTy = EltTy.changeElementSize(NewEltSize: MemStoreSizeInBits);
4350 MachineMemOperand *NewMMO =
4351 MF.getMachineMemOperand(MMO: &MMO, PtrInfo: MMO.getPointerInfo(), Ty: WideMemTy);
4352
4353 Register LoadReg = DstReg;
4354 LLT LoadTy = DstTy;
4355
4356 // If this wasn't already an extending load, we need to widen the result
4357 // register to avoid creating a load with a narrower result than the source.
4358 if (MemStoreSizeInBits > DstTy.getSizeInBits()) {
4359 LoadTy = WideMemTy;
4360 LoadReg = MRI.createGenericVirtualRegister(Ty: WideMemTy);
4361 }
4362
4363 if (isa<GSExtLoad>(Val: LoadMI)) {
4364 auto NewLoad = MIRBuilder.buildLoad(Res: LoadTy, Addr: PtrReg, MMO&: *NewMMO);
4365 MIRBuilder.buildSExtInReg(Res: LoadReg, Op: NewLoad, ImmOp: MemSizeInBits);
4366 } else if (isa<GZExtLoad>(Val: LoadMI) || WideMemTy == LoadTy) {
4367 auto NewLoad = MIRBuilder.buildLoad(Res: LoadTy, Addr: PtrReg, MMO&: *NewMMO);
4368 // The extra bits are guaranteed to be zero, since we stored them that
4369 // way. A zext load from Wide thus automatically gives zext from MemVT.
4370 MIRBuilder.buildAssertZExt(Res: LoadReg, Op: NewLoad, Size: MemSizeInBits);
4371 } else {
4372 MIRBuilder.buildLoad(Res: LoadReg, Addr: PtrReg, MMO&: *NewMMO);
4373 }
4374
4375 if (DstTy != LoadTy)
4376 MIRBuilder.buildTrunc(Res: DstReg, Op: LoadReg);
4377
4378 LoadMI.eraseFromParent();
4379 return Legalized;
4380 }
4381
4382 // Big endian lowering not implemented.
4383 if (MIRBuilder.getDataLayout().isBigEndian())
4384 return UnableToLegalize;
4385
4386 // This load needs splitting into power of 2 sized loads.
4387 //
4388 // Our strategy here is to generate anyextending loads for the smaller
4389 // types up to next power-2 result type, and then combine the two larger
4390 // result values together, before truncating back down to the non-pow-2
4391 // type.
4392 // E.g. v1 = i24 load =>
4393 // v2 = i32 zextload (2 byte)
4394 // v3 = i32 load (1 byte)
4395 // v4 = i32 shl v3, 16
4396 // v5 = i32 or v4, v2
4397 // v1 = i24 trunc v5
4398 // By doing this we generate the correct truncate which should get
4399 // combined away as an artifact with a matching extend.
4400
4401 uint64_t LargeSplitSize, SmallSplitSize;
4402
4403 if (!isPowerOf2_32(Value: MemSizeInBits)) {
4404 // This load needs splitting into power of 2 sized loads.
4405 LargeSplitSize = llvm::bit_floor(Value: MemSizeInBits);
4406 SmallSplitSize = MemSizeInBits - LargeSplitSize;
4407 } else {
4408 // This is already a power of 2, but we still need to split this in half.
4409 //
4410 // Assume we're being asked to decompose an unaligned load.
4411 // TODO: If this requires multiple splits, handle them all at once.
4412 auto &Ctx = MF.getFunction().getContext();
4413 if (TLI.allowsMemoryAccess(Context&: Ctx, DL: MIRBuilder.getDataLayout(), Ty: MemTy, MMO))
4414 return UnableToLegalize;
4415
4416 SmallSplitSize = LargeSplitSize = MemSizeInBits / 2;
4417 }
4418
4419 if (MemTy.isVector()) {
4420 // TODO: Handle vector extloads
4421 if (MemTy != DstTy)
4422 return UnableToLegalize;
4423
4424 Align Alignment = LoadMI.getAlign();
4425 // Given an alignment larger than the size of the memory, we can increase
4426 // the size of the load without needing to scalarize it.
4427 if (Alignment.value() * 8 > MemSizeInBits &&
4428 isPowerOf2_64(Value: DstTy.getScalarSizeInBits())) {
4429 LLT MoreTy = DstTy.changeVectorElementCount(
4430 EC: ElementCount::getFixed(MinVal: NextPowerOf2(A: DstTy.getNumElements())));
4431 MachineMemOperand *NewMMO = MF.getMachineMemOperand(MMO: &MMO, Offset: 0, Ty: MoreTy);
4432 auto NewLoad = MIRBuilder.buildLoad(Res: MoreTy, Addr: PtrReg, MMO&: *NewMMO);
4433 MIRBuilder.buildDeleteTrailingVectorElements(Res: LoadMI.getReg(Idx: 0),
4434 Op0: NewLoad.getReg(Idx: 0));
4435 LoadMI.eraseFromParent();
4436 return Legalized;
4437 }
4438
4439 // TODO: We can do better than scalarizing the vector and at least split it
4440 // in half.
4441 return reduceLoadStoreWidth(MI&: LoadMI, TypeIdx: 0, NarrowTy: DstTy.getElementType());
4442 }
4443
4444 MachineMemOperand *LargeMMO =
4445 MF.getMachineMemOperand(MMO: &MMO, Offset: 0, Size: LargeSplitSize / 8);
4446 MachineMemOperand *SmallMMO =
4447 MF.getMachineMemOperand(MMO: &MMO, Offset: LargeSplitSize / 8, Size: SmallSplitSize / 8);
4448
4449 LLT PtrTy = MRI.getType(Reg: PtrReg);
4450 unsigned AnyExtSize = PowerOf2Ceil(A: DstTy.getSizeInBits());
4451
4452 LLT AnyExtTy = LLT::integer(SizeInBits: AnyExtSize);
4453 LLT OffsetCstRes = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
4454
4455 auto LargeLoad = MIRBuilder.buildLoadInstr(Opcode: TargetOpcode::G_ZEXTLOAD, Res: AnyExtTy,
4456 Addr: PtrReg, MMO&: *LargeMMO);
4457
4458 auto OffsetCst = MIRBuilder.buildConstant(Res: OffsetCstRes, Val: LargeSplitSize / 8);
4459 Register PtrAddReg = MRI.createGenericVirtualRegister(Ty: PtrTy);
4460 auto SmallPtr = MIRBuilder.buildObjectPtrOffset(Res: PtrAddReg, Op0: PtrReg, Op1: OffsetCst);
4461 auto SmallLoad = MIRBuilder.buildLoadInstr(Opcode: LoadMI.getOpcode(), Res: AnyExtTy,
4462 Addr: SmallPtr, MMO&: *SmallMMO);
4463
4464 auto ShiftAmt = MIRBuilder.buildConstant(Res: AnyExtTy, Val: LargeSplitSize);
4465 auto Shift = MIRBuilder.buildShl(Dst: AnyExtTy, Src0: SmallLoad, Src1: ShiftAmt);
4466
4467 if (AnyExtTy == DstTy)
4468 MIRBuilder.buildOr(Dst: DstReg, Src0: Shift, Src1: LargeLoad);
4469 else if (AnyExtTy.getSizeInBits() != DstTy.getSizeInBits()) {
4470 auto Or = MIRBuilder.buildOr(Dst: AnyExtTy, Src0: Shift, Src1: LargeLoad);
4471 LLT IntDstTy = DstTy.changeToInteger();
4472 if (IntDstTy == DstTy) {
4473 MIRBuilder.buildTrunc(Res: DstReg, Op: {Or});
4474 } else {
4475 auto Trunc = MIRBuilder.buildTrunc(Res: IntDstTy, Op: Or);
4476 MIRBuilder.buildBitcast(Dst: DstReg, Src: Trunc);
4477 }
4478 } else if (DstTy.isPointer()) {
4479 auto Or = MIRBuilder.buildOr(Dst: AnyExtTy, Src0: Shift, Src1: LargeLoad);
4480
4481 // FIXME: We currently consider this to be illegal for non-integral address
4482 // spaces, but we need still need a way to reinterpret the bits.
4483 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: Or);
4484 } else {
4485 auto Or = MIRBuilder.buildOr(Dst: AnyExtTy, Src0: Shift, Src1: LargeLoad);
4486 MIRBuilder.buildBitcast(Dst: DstReg, Src: Or);
4487 }
4488
4489 LoadMI.eraseFromParent();
4490 return Legalized;
4491}
4492
4493LegalizerHelper::LegalizeResult LegalizerHelper::lowerStore(GStore &StoreMI) {
4494 // Lower a non-power of 2 store into multiple pow-2 stores.
4495 // E.g. split an i24 store into an i16 store + i8 store.
4496 // We do this by first extending the stored value to the next largest power
4497 // of 2 type, and then using truncating stores to store the components.
4498 // By doing this, likewise with G_LOAD, generate an extend that can be
4499 // artifact-combined away instead of leaving behind extracts.
4500 Register SrcReg = StoreMI.getValueReg();
4501 Register PtrReg = StoreMI.getPointerReg();
4502 LLT SrcTy = MRI.getType(Reg: SrcReg);
4503 MachineFunction &MF = MIRBuilder.getMF();
4504 MachineMemOperand &MMO = **StoreMI.memoperands_begin();
4505 LLT MemTy = MMO.getMemoryType();
4506
4507 unsigned StoreWidth = MemTy.getSizeInBits();
4508 unsigned StoreSizeInBits = 8 * MemTy.getSizeInBytes();
4509
4510 if (StoreWidth != StoreSizeInBits && !SrcTy.isVector()) {
4511 // Promote to a byte-sized store with upper bits zero if not
4512 // storing an integral number of bytes. For example, promote
4513 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
4514 LLT WideTy = LLT::integer(SizeInBits: StoreSizeInBits);
4515
4516 if (StoreSizeInBits > SrcTy.getSizeInBits()) {
4517 // Avoid creating a store with a narrower source than result.
4518 SrcReg = MIRBuilder.buildAnyExt(Res: WideTy, Op: SrcReg).getReg(Idx: 0);
4519 SrcTy = WideTy;
4520 }
4521
4522 auto ZextInReg = MIRBuilder.buildZExtInReg(Res: SrcTy, Op: SrcReg, ImmOp: StoreWidth);
4523
4524 MachineMemOperand *NewMMO =
4525 MF.getMachineMemOperand(MMO: &MMO, PtrInfo: MMO.getPointerInfo(), Ty: WideTy);
4526 MIRBuilder.buildStore(Val: ZextInReg, Addr: PtrReg, MMO&: *NewMMO);
4527 StoreMI.eraseFromParent();
4528 return Legalized;
4529 }
4530
4531 if (MemTy.isVector()) {
4532 if (MemTy != SrcTy)
4533 return scalarizeVectorBooleanStore(MI&: StoreMI);
4534
4535 // TODO: We can do better than scalarizing the vector and at least split it
4536 // in half.
4537 return reduceLoadStoreWidth(MI&: StoreMI, TypeIdx: 0, NarrowTy: SrcTy.getElementType());
4538 }
4539
4540 unsigned MemSizeInBits = MemTy.getSizeInBits();
4541 uint64_t LargeSplitSize, SmallSplitSize;
4542
4543 if (!isPowerOf2_32(Value: MemSizeInBits)) {
4544 LargeSplitSize = llvm::bit_floor<uint64_t>(Value: MemTy.getSizeInBits());
4545 SmallSplitSize = MemTy.getSizeInBits() - LargeSplitSize;
4546 } else {
4547 auto &Ctx = MF.getFunction().getContext();
4548 if (TLI.allowsMemoryAccess(Context&: Ctx, DL: MIRBuilder.getDataLayout(), Ty: MemTy, MMO))
4549 return UnableToLegalize; // Don't know what we're being asked to do.
4550
4551 SmallSplitSize = LargeSplitSize = MemSizeInBits / 2;
4552 }
4553
4554 // Extend to the next pow-2. If this store was itself the result of lowering,
4555 // e.g. an s56 store being broken into s32 + s24, we might have a stored type
4556 // that's wider than the stored size.
4557 unsigned AnyExtSize = PowerOf2Ceil(A: MemTy.getSizeInBits());
4558 const LLT NewSrcTy = LLT::integer(SizeInBits: AnyExtSize);
4559
4560 if (SrcTy.isPointer()) {
4561 const LLT IntPtrTy = LLT::integer(SizeInBits: SrcTy.getSizeInBits());
4562 SrcReg = MIRBuilder.buildPtrToInt(Dst: IntPtrTy, Src: SrcReg).getReg(Idx: 0);
4563 }
4564
4565 auto ExtVal = MIRBuilder.buildAnyExtOrTrunc(Res: NewSrcTy, Op: SrcReg);
4566
4567 // Obtain the smaller value by shifting away the larger value.
4568 auto ShiftAmt = MIRBuilder.buildConstant(Res: NewSrcTy, Val: LargeSplitSize);
4569 auto SmallVal = MIRBuilder.buildLShr(Dst: NewSrcTy, Src0: ExtVal, Src1: ShiftAmt);
4570
4571 // Generate the PtrAdd and truncating stores.
4572 LLT PtrTy = MRI.getType(Reg: PtrReg);
4573 auto OffsetCst = MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: PtrTy.getSizeInBits()),
4574 Val: LargeSplitSize / 8);
4575 auto SmallPtr = MIRBuilder.buildObjectPtrOffset(Res: PtrTy, Op0: PtrReg, Op1: OffsetCst);
4576
4577 MachineMemOperand *LargeMMO =
4578 MF.getMachineMemOperand(MMO: &MMO, Offset: 0, Size: LargeSplitSize / 8);
4579 MachineMemOperand *SmallMMO =
4580 MF.getMachineMemOperand(MMO: &MMO, Offset: LargeSplitSize / 8, Size: SmallSplitSize / 8);
4581 MIRBuilder.buildStore(Val: ExtVal, Addr: PtrReg, MMO&: *LargeMMO);
4582 MIRBuilder.buildStore(Val: SmallVal, Addr: SmallPtr, MMO&: *SmallMMO);
4583 StoreMI.eraseFromParent();
4584 return Legalized;
4585}
4586
4587LegalizerHelper::LegalizeResult
4588LegalizerHelper::scalarizeVectorBooleanStore(GStore &StoreMI) {
4589 Register SrcReg = StoreMI.getValueReg();
4590 Register PtrReg = StoreMI.getPointerReg();
4591 LLT SrcTy = MRI.getType(Reg: SrcReg);
4592 MachineMemOperand &MMO = **StoreMI.memoperands_begin();
4593 LLT MemTy = MMO.getMemoryType();
4594 LLT MemScalarTy = MemTy.getElementType();
4595 MachineFunction &MF = MIRBuilder.getMF();
4596
4597 assert(SrcTy.isVector() && "Expect a vector store type");
4598
4599 if (!MemScalarTy.isByteSized()) {
4600 // We need to build an integer scalar of the vector bit pattern.
4601 // It's not legal for us to add padding when storing a vector.
4602 unsigned NumBits = MemTy.getSizeInBits();
4603 LLT IntTy = LLT::integer(SizeInBits: NumBits);
4604 auto CurrVal = MIRBuilder.buildConstant(Res: IntTy, Val: 0);
4605 LLT IdxTy = TLI.getVectorIdxLLT(DL: MF.getDataLayout());
4606
4607 for (unsigned I = 0, E = MemTy.getNumElements(); I < E; ++I) {
4608 auto Elt = MIRBuilder.buildExtractVectorElement(
4609 Res: SrcTy.getElementType(), Val: SrcReg, Idx: MIRBuilder.buildConstant(Res: IdxTy, Val: I));
4610 auto Trunc = MIRBuilder.buildTrunc(Res: MemScalarTy, Op: Elt);
4611 auto ZExt = MIRBuilder.buildZExt(Res: IntTy, Op: Trunc);
4612 unsigned ShiftIntoIdx = MF.getDataLayout().isBigEndian()
4613 ? (MemTy.getNumElements() - 1) - I
4614 : I;
4615 auto ShiftAmt = MIRBuilder.buildConstant(
4616 Res: IntTy, Val: ShiftIntoIdx * MemScalarTy.getSizeInBits());
4617 auto Shifted = MIRBuilder.buildShl(Dst: IntTy, Src0: ZExt, Src1: ShiftAmt);
4618 CurrVal = MIRBuilder.buildOr(Dst: IntTy, Src0: CurrVal, Src1: Shifted);
4619 }
4620 auto PtrInfo = MMO.getPointerInfo();
4621 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo, Ty: IntTy);
4622 MIRBuilder.buildStore(Val: CurrVal, Addr: PtrReg, MMO&: *NewMMO);
4623 StoreMI.eraseFromParent();
4624 return Legalized;
4625 }
4626
4627 // TODO: implement simple scalarization.
4628 return UnableToLegalize;
4629}
4630
4631LegalizerHelper::LegalizeResult
4632LegalizerHelper::bitcast(MachineInstr &MI, unsigned TypeIdx, LLT CastTy) {
4633 switch (MI.getOpcode()) {
4634 case TargetOpcode::G_LOAD: {
4635 if (TypeIdx != 0)
4636 return UnableToLegalize;
4637 MachineMemOperand &MMO = **MI.memoperands_begin();
4638
4639 // Not sure how to interpret a bitcast of an extending load.
4640 if (MMO.getMemoryType().getSizeInBits() != CastTy.getSizeInBits())
4641 return UnableToLegalize;
4642
4643 Observer.changingInstr(MI);
4644 bitcastDst(MI, CastTy, OpIdx: 0);
4645 MMO.setType(CastTy);
4646 // The range metadata is no longer valid when reinterpreted as a different
4647 // type.
4648 MMO.clearRanges();
4649 Observer.changedInstr(MI);
4650 return Legalized;
4651 }
4652 case TargetOpcode::G_STORE: {
4653 if (TypeIdx != 0)
4654 return UnableToLegalize;
4655
4656 MachineMemOperand &MMO = **MI.memoperands_begin();
4657
4658 // Not sure how to interpret a bitcast of a truncating store.
4659 if (MMO.getMemoryType().getSizeInBits() != CastTy.getSizeInBits())
4660 return UnableToLegalize;
4661
4662 Observer.changingInstr(MI);
4663 bitcastSrc(MI, CastTy, OpIdx: 0);
4664 MMO.setType(CastTy);
4665 Observer.changedInstr(MI);
4666 return Legalized;
4667 }
4668 case TargetOpcode::G_SELECT: {
4669 if (TypeIdx != 0)
4670 return UnableToLegalize;
4671
4672 if (MRI.getType(Reg: MI.getOperand(i: 1).getReg()).isVector()) {
4673 LLVM_DEBUG(
4674 dbgs() << "bitcast action not implemented for vector select\n");
4675 return UnableToLegalize;
4676 }
4677
4678 Observer.changingInstr(MI);
4679 bitcastSrc(MI, CastTy, OpIdx: 2);
4680 bitcastSrc(MI, CastTy, OpIdx: 3);
4681 bitcastDst(MI, CastTy, OpIdx: 0);
4682 Observer.changedInstr(MI);
4683 return Legalized;
4684 }
4685 case TargetOpcode::G_AND:
4686 case TargetOpcode::G_OR:
4687 case TargetOpcode::G_XOR: {
4688 Observer.changingInstr(MI);
4689 bitcastSrc(MI, CastTy, OpIdx: 1);
4690 bitcastSrc(MI, CastTy, OpIdx: 2);
4691 bitcastDst(MI, CastTy, OpIdx: 0);
4692 Observer.changedInstr(MI);
4693 return Legalized;
4694 }
4695 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
4696 return bitcastExtractVectorElt(MI, TypeIdx, CastTy);
4697 case TargetOpcode::G_INSERT_VECTOR_ELT:
4698 return bitcastInsertVectorElt(MI, TypeIdx, CastTy);
4699 case TargetOpcode::G_CONCAT_VECTORS:
4700 return bitcastConcatVector(MI, TypeIdx, CastTy);
4701 case TargetOpcode::G_SHUFFLE_VECTOR:
4702 return bitcastShuffleVector(MI, TypeIdx, CastTy);
4703 case TargetOpcode::G_EXTRACT_SUBVECTOR:
4704 return bitcastExtractSubvector(MI, TypeIdx, CastTy);
4705 case TargetOpcode::G_INSERT_SUBVECTOR:
4706 return bitcastInsertSubvector(MI, TypeIdx, CastTy);
4707 default:
4708 return UnableToLegalize;
4709 }
4710}
4711
4712// Legalize an instruction by changing the opcode in place.
4713void LegalizerHelper::changeOpcode(MachineInstr &MI, unsigned NewOpcode) {
4714 Observer.changingInstr(MI);
4715 MI.setDesc(MIRBuilder.getTII().get(Opcode: NewOpcode));
4716 Observer.changedInstr(MI);
4717}
4718
4719LegalizerHelper::LegalizeResult
4720LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) {
4721 using namespace TargetOpcode;
4722 switch(MI.getOpcode()) {
4723 default:
4724 return UnableToLegalize;
4725 case TargetOpcode::G_FCONSTANT:
4726 return lowerFConstant(MI);
4727 case TargetOpcode::G_BITCAST:
4728 return lowerBitcast(MI);
4729 case TargetOpcode::G_SREM:
4730 case TargetOpcode::G_UREM: {
4731 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4732 auto Quot =
4733 MIRBuilder.buildInstr(Opc: MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV, DstOps: {Ty},
4734 SrcOps: {MI.getOperand(i: 1), MI.getOperand(i: 2)});
4735
4736 auto Prod = MIRBuilder.buildMul(Dst: Ty, Src0: Quot, Src1: MI.getOperand(i: 2));
4737 MIRBuilder.buildSub(Dst: MI.getOperand(i: 0), Src0: MI.getOperand(i: 1), Src1: Prod);
4738 MI.eraseFromParent();
4739 return Legalized;
4740 }
4741 case TargetOpcode::G_SADDO:
4742 case TargetOpcode::G_SSUBO:
4743 return lowerSADDO_SSUBO(MI);
4744 case TargetOpcode::G_SADDE:
4745 return lowerSADDE(MI);
4746 case TargetOpcode::G_SSUBE:
4747 return lowerSSUBE(MI);
4748 case TargetOpcode::G_UMULH:
4749 case TargetOpcode::G_SMULH:
4750 return lowerSMULH_UMULH(MI);
4751 case TargetOpcode::G_SMULO:
4752 case TargetOpcode::G_UMULO: {
4753 // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the
4754 // result.
4755 auto [Res, Overflow, LHS, RHS] = MI.getFirst4Regs();
4756 LLT Ty = MRI.getType(Reg: Res);
4757
4758 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO
4759 ? TargetOpcode::G_SMULH
4760 : TargetOpcode::G_UMULH;
4761
4762 Observer.changingInstr(MI);
4763 const auto &TII = MIRBuilder.getTII();
4764 MI.setDesc(TII.get(Opcode: TargetOpcode::G_MUL));
4765 MI.removeOperand(OpNo: 1);
4766 Observer.changedInstr(MI);
4767
4768 auto HiPart = MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Ty}, SrcOps: {LHS, RHS});
4769 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
4770
4771 // Move insert point forward so we can use the Res register if needed.
4772 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
4773
4774 // For *signed* multiply, overflow is detected by checking:
4775 // (hi != (lo >> bitwidth-1))
4776 if (Opcode == TargetOpcode::G_SMULH) {
4777 auto ShiftAmt = MIRBuilder.buildConstant(Res: Ty, Val: Ty.getSizeInBits() - 1);
4778 auto Shifted = MIRBuilder.buildAShr(Dst: Ty, Src0: Res, Src1: ShiftAmt);
4779 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: Overflow, Op0: HiPart, Op1: Shifted);
4780 } else {
4781 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: Overflow, Op0: HiPart, Op1: Zero);
4782 }
4783 return Legalized;
4784 }
4785 case TargetOpcode::G_FNEG: {
4786 auto [Res, ResTy, SubByReg, SubByRegTy] = MI.getFirst2RegLLTs();
4787 LLT TyInt =
4788 ResTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: ResTy.getScalarSizeInBits()));
4789 Register CastedSubByReg = SubByReg;
4790
4791 if (!SubByRegTy.getScalarType().isAnyScalar() &&
4792 !SubByRegTy.getScalarType().isInteger()) {
4793 auto BitcastDst = SubByRegTy.changeElementType(
4794 NewEltTy: LLT::integer(SizeInBits: SubByRegTy.getScalarSizeInBits()));
4795 CastedSubByReg = MIRBuilder.buildBitcast(Dst: BitcastDst, Src: SubByReg).getReg(Idx: 0);
4796 }
4797
4798 auto SignMask = MIRBuilder.buildConstant(
4799 Res: TyInt, Val: APInt::getSignMask(BitWidth: TyInt.getScalarSizeInBits()));
4800
4801 if (ResTy != TyInt) {
4802 Register NewDst =
4803 MIRBuilder.buildXor(Dst: TyInt, Src0: CastedSubByReg, Src1: SignMask).getReg(Idx: 0);
4804 MIRBuilder.buildBitcast(Dst: Res, Src: NewDst);
4805 } else
4806 MIRBuilder.buildXor(Dst: Res, Src0: CastedSubByReg, Src1: SignMask).getReg(Idx: 0);
4807
4808 MI.eraseFromParent();
4809 return Legalized;
4810 }
4811 case TargetOpcode::G_FSUB:
4812 case TargetOpcode::G_STRICT_FSUB: {
4813 auto [Res, LHS, RHS] = MI.getFirst3Regs();
4814 LLT Ty = MRI.getType(Reg: Res);
4815
4816 // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)).
4817 auto Neg = MIRBuilder.buildFNeg(Dst: Ty, Src0: RHS);
4818
4819 if (MI.getOpcode() == TargetOpcode::G_STRICT_FSUB)
4820 MIRBuilder.buildStrictFAdd(Dst: Res, Src0: LHS, Src1: Neg, Flags: MI.getFlags());
4821 else
4822 MIRBuilder.buildFAdd(Dst: Res, Src0: LHS, Src1: Neg, Flags: MI.getFlags());
4823
4824 MI.eraseFromParent();
4825 return Legalized;
4826 }
4827 case TargetOpcode::G_FMAD:
4828 return lowerFMad(MI);
4829 case TargetOpcode::G_FFLOOR:
4830 return lowerFFloor(MI);
4831 case TargetOpcode::G_LROUND:
4832 case TargetOpcode::G_LLROUND: {
4833 Register DstReg = MI.getOperand(i: 0).getReg();
4834 Register SrcReg = MI.getOperand(i: 1).getReg();
4835 LLT SrcTy = MRI.getType(Reg: SrcReg);
4836 auto Round = MIRBuilder.buildInstr(Opc: TargetOpcode::G_INTRINSIC_ROUND, DstOps: {SrcTy},
4837 SrcOps: {SrcReg});
4838 MIRBuilder.buildFPTOSI(Dst: DstReg, Src0: Round);
4839 MI.eraseFromParent();
4840 return Legalized;
4841 }
4842 case TargetOpcode::G_INTRINSIC_ROUND:
4843 return lowerIntrinsicRound(MI);
4844 case TargetOpcode::G_FRINT: {
4845 // Since round even is the assumed rounding mode for unconstrained FP
4846 // operations, rint and roundeven are the same operation.
4847 changeOpcode(MI, NewOpcode: TargetOpcode::G_INTRINSIC_ROUNDEVEN);
4848 return Legalized;
4849 }
4850 case TargetOpcode::G_INTRINSIC_LRINT:
4851 case TargetOpcode::G_INTRINSIC_LLRINT: {
4852 Register DstReg = MI.getOperand(i: 0).getReg();
4853 Register SrcReg = MI.getOperand(i: 1).getReg();
4854 LLT SrcTy = MRI.getType(Reg: SrcReg);
4855 auto Round =
4856 MIRBuilder.buildInstr(Opc: TargetOpcode::G_FRINT, DstOps: {SrcTy}, SrcOps: {SrcReg});
4857 MIRBuilder.buildFPTOSI(Dst: DstReg, Src0: Round);
4858 MI.eraseFromParent();
4859 return Legalized;
4860 }
4861 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
4862 auto [OldValRes, SuccessRes, Addr, CmpVal, NewVal] = MI.getFirst5Regs();
4863 Register NewOldValRes = MRI.cloneVirtualRegister(VReg: OldValRes);
4864 MIRBuilder.buildAtomicCmpXchg(OldValRes: NewOldValRes, Addr, CmpVal, NewVal,
4865 MMO&: **MI.memoperands_begin());
4866 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: SuccessRes, Op0: NewOldValRes, Op1: CmpVal);
4867 MIRBuilder.buildCopy(Res: OldValRes, Op: NewOldValRes);
4868 MI.eraseFromParent();
4869 return Legalized;
4870 }
4871 case TargetOpcode::G_LOAD:
4872 case TargetOpcode::G_SEXTLOAD:
4873 case TargetOpcode::G_ZEXTLOAD:
4874 return lowerLoad(LoadMI&: cast<GAnyLoad>(Val&: MI));
4875 case TargetOpcode::G_STORE:
4876 return lowerStore(StoreMI&: cast<GStore>(Val&: MI));
4877 case TargetOpcode::G_CTLZ_ZERO_POISON:
4878 case TargetOpcode::G_CTTZ_ZERO_POISON:
4879 case TargetOpcode::G_CTLZ:
4880 case TargetOpcode::G_CTTZ:
4881 case TargetOpcode::G_CTPOP:
4882 case TargetOpcode::G_CTLS:
4883 return lowerBitCount(MI);
4884 case G_UADDO: {
4885 auto [Res, CarryOut, LHS, RHS] = MI.getFirst4Regs();
4886
4887 Register NewRes = MRI.cloneVirtualRegister(VReg: Res);
4888
4889 MIRBuilder.buildAdd(Dst: NewRes, Src0: LHS, Src1: RHS);
4890 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_ULT, Res: CarryOut, Op0: NewRes, Op1: RHS);
4891
4892 MIRBuilder.buildCopy(Res, Op: NewRes);
4893
4894 MI.eraseFromParent();
4895 return Legalized;
4896 }
4897 case G_UADDE: {
4898 auto [Res, CarryOut, LHS, RHS, CarryIn] = MI.getFirst5Regs();
4899 const LLT CondTy = MRI.getType(Reg: CarryOut);
4900 const LLT Ty = MRI.getType(Reg: Res);
4901
4902 Register NewRes = MRI.cloneVirtualRegister(VReg: Res);
4903
4904 // Initial add of the two operands.
4905 auto TmpRes = MIRBuilder.buildAdd(Dst: Ty, Src0: LHS, Src1: RHS);
4906
4907 // Initial check for carry.
4908 auto Carry = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_ULT, Res: CondTy, Op0: TmpRes, Op1: LHS);
4909
4910 // Add the sum and the carry.
4911 auto ZExtCarryIn = MIRBuilder.buildZExt(Res: Ty, Op: CarryIn);
4912 MIRBuilder.buildAdd(Dst: NewRes, Src0: TmpRes, Src1: ZExtCarryIn);
4913
4914 // Second check for carry. We can only carry if the initial sum is all 1s
4915 // and the carry is set, resulting in a new sum of 0.
4916 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
4917 auto ResEqZero =
4918 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CondTy, Op0: NewRes, Op1: Zero);
4919 auto Carry2 = MIRBuilder.buildAnd(Dst: CondTy, Src0: ResEqZero, Src1: CarryIn);
4920 MIRBuilder.buildOr(Dst: CarryOut, Src0: Carry, Src1: Carry2);
4921
4922 MIRBuilder.buildCopy(Res, Op: NewRes);
4923
4924 MI.eraseFromParent();
4925 return Legalized;
4926 }
4927 case G_USUBO: {
4928 auto [Res, BorrowOut, LHS, RHS] = MI.getFirst4Regs();
4929
4930 MIRBuilder.buildSub(Dst: Res, Src0: LHS, Src1: RHS);
4931 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_ULT, Res: BorrowOut, Op0: LHS, Op1: RHS);
4932
4933 MI.eraseFromParent();
4934 return Legalized;
4935 }
4936 case G_USUBE: {
4937 auto [Res, BorrowOut, LHS, RHS, BorrowIn] = MI.getFirst5Regs();
4938 const LLT CondTy = MRI.getType(Reg: BorrowOut);
4939 const LLT Ty = MRI.getType(Reg: Res);
4940
4941 // Initial subtract of the two operands.
4942 auto TmpRes = MIRBuilder.buildSub(Dst: Ty, Src0: LHS, Src1: RHS);
4943
4944 // Initial check for borrow.
4945 auto Borrow = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_UGT, Res: CondTy, Op0: TmpRes, Op1: LHS);
4946
4947 // Subtract the borrow from the first subtract.
4948 auto ZExtBorrowIn = MIRBuilder.buildZExt(Res: Ty, Op: BorrowIn);
4949 MIRBuilder.buildSub(Dst: Res, Src0: TmpRes, Src1: ZExtBorrowIn);
4950
4951 // Second check for borrow. We can only borrow if the initial difference is
4952 // 0 and the borrow is set, resulting in a new difference of all 1s.
4953 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
4954 auto TmpResEqZero =
4955 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CondTy, Op0: TmpRes, Op1: Zero);
4956 auto Borrow2 = MIRBuilder.buildAnd(Dst: CondTy, Src0: TmpResEqZero, Src1: BorrowIn);
4957 MIRBuilder.buildOr(Dst: BorrowOut, Src0: Borrow, Src1: Borrow2);
4958
4959 MI.eraseFromParent();
4960 return Legalized;
4961 }
4962 case G_UITOFP:
4963 return lowerUITOFP(MI);
4964 case G_SITOFP:
4965 return lowerSITOFP(MI);
4966 case G_FPTOUI:
4967 return lowerFPTOUI(MI);
4968 case G_FPTOSI:
4969 return lowerFPTOSI(MI);
4970 case G_FPTOUI_SAT:
4971 case G_FPTOSI_SAT:
4972 return lowerFPTOINT_SAT(MI);
4973 case G_FPEXT:
4974 return lowerFPEXT(MI);
4975 case G_FPTRUNC:
4976 return lowerFPTRUNC(MI);
4977 case G_FPOWI:
4978 return lowerFPOWI(MI);
4979 case G_FMODF:
4980 return lowerFMODF(MI);
4981 case G_SMIN:
4982 case G_SMAX:
4983 case G_UMIN:
4984 case G_UMAX:
4985 return lowerMinMax(MI);
4986 case G_SCMP:
4987 case G_UCMP:
4988 return lowerThreewayCompare(MI);
4989 case G_FCOPYSIGN:
4990 return lowerFCopySign(MI);
4991 case G_FMINNUM:
4992 case G_FMAXNUM:
4993 case G_FMINIMUMNUM:
4994 case G_FMAXIMUMNUM:
4995 return lowerFMinNumMaxNum(MI);
4996 case G_FMINIMUM:
4997 case G_FMAXIMUM:
4998 return lowerFMinimumMaximum(MI);
4999 case G_MERGE_VALUES:
5000 return lowerMergeValues(MI);
5001 case G_UNMERGE_VALUES:
5002 return lowerUnmergeValues(MI);
5003 case TargetOpcode::G_SEXT_INREG: {
5004 assert(MI.getOperand(2).isImm() && "Expected immediate");
5005 int64_t SizeInBits = MI.getOperand(i: 2).getImm();
5006
5007 auto [DstReg, SrcReg] = MI.getFirst2Regs();
5008 LLT DstTy = MRI.getType(Reg: DstReg);
5009 Register TmpRes = MRI.createGenericVirtualRegister(Ty: DstTy);
5010
5011 auto MIBSz = MIRBuilder.buildConstant(Res: DstTy, Val: DstTy.getScalarSizeInBits() - SizeInBits);
5012 MIRBuilder.buildShl(Dst: TmpRes, Src0: SrcReg, Src1: MIBSz->getOperand(i: 0));
5013 MIRBuilder.buildAShr(Dst: DstReg, Src0: TmpRes, Src1: MIBSz->getOperand(i: 0));
5014 MI.eraseFromParent();
5015 return Legalized;
5016 }
5017 case G_EXTRACT_VECTOR_ELT:
5018 case G_INSERT_VECTOR_ELT:
5019 return lowerExtractInsertVectorElt(MI);
5020 case G_SHUFFLE_VECTOR:
5021 return lowerShuffleVector(MI);
5022 case G_VECTOR_COMPRESS:
5023 return lowerVECTOR_COMPRESS(MI);
5024 case G_DYN_STACKALLOC:
5025 return lowerDynStackAlloc(MI);
5026 case G_INSERT_SUBVECTOR: {
5027 if (MRI.getType(Reg: MI.getOperand(i: 1).getReg()).isScalable() ||
5028 MRI.getType(Reg: MI.getOperand(i: 2).getReg()).isScalable())
5029 return UnableToLegalize;
5030
5031 // Check that subvector is half size of main vector
5032 Register Vector = MI.getOperand(i: 1).getReg();
5033 Register Subvector = MI.getOperand(i: 2).getReg();
5034 auto InsertionPointImm = MI.getOperand(i: 3).getImm();
5035
5036 LLT VectorTy = MRI.getType(Reg: Vector);
5037 LLT DstTy = MRI.getType(Reg: Subvector);
5038 // If so, -> concat(subvector, extract(half of vector))
5039 // (Operands can be either way round depending on insertion point
5040 if (VectorTy.getSizeInBits() == DstTy.getSizeInBits() * 2) {
5041 bool InsertInLowHalf = InsertionPointImm == 0;
5042 auto Extract = MIRBuilder.buildExtractSubvector(
5043 Res: DstTy, Src: Vector,
5044 Index: (uint64_t)(InsertInLowHalf ? VectorTy.getNumElements() / 2 : 0));
5045
5046 auto LowHalf = InsertInLowHalf ? Subvector : Extract.getReg(Idx: 0);
5047 auto HighHalf = InsertInLowHalf ? Extract.getReg(Idx: 0) : Subvector;
5048
5049 MIRBuilder.buildInstr(Opc: TargetOpcode::G_CONCAT_VECTORS, DstOps: {MI.getOperand(i: 0)},
5050 SrcOps: {LowHalf, HighHalf});
5051 MI.eraseFromParent();
5052 return Legalized;
5053 }
5054 // Else -> shuffle(vector, extend(subvector, size(vector)), mask)
5055 else {
5056 // Extend subvector to same size as vector
5057 Register ExtendedSubvector = MRI.createGenericVirtualRegister(Ty: VectorTy);
5058 MIRBuilder.buildPadVectorWithUndefElements(Res: ExtendedSubvector, Op0: Subvector);
5059
5060 // Calculate mask required for this shuffle
5061 SmallVector<int> Mask;
5062 for (int i = 0; i < VectorTy.getNumElements(); i++) {
5063 // If this index is within bounds, put subvector's index into mask
5064 if (i >= InsertionPointImm &&
5065 i < InsertionPointImm + DstTy.getNumElements())
5066 Mask.push_back(Elt: VectorTy.getNumElements() + i - InsertionPointImm);
5067 else
5068 Mask.push_back(Elt: i);
5069 }
5070
5071 // Build shuffle
5072 MIRBuilder.buildShuffleVector(Res: MI.getOperand(i: 0), Src1: Vector, Src2: ExtendedSubvector,
5073 Mask);
5074 MI.eraseFromParent();
5075 return Legalized;
5076 }
5077 }
5078 case G_EXTRACT_SUBVECTOR: {
5079 Register DstReg = MI.getOperand(i: 0).getReg();
5080 Register SrcReg = MI.getOperand(i: 1).getReg();
5081 uint64_t ExtractionPointImm = MI.getOperand(i: 2).getImm();
5082
5083 LLT SrcTy = MRI.getType(Reg: SrcReg);
5084 LLT DstTy = MRI.getType(Reg: DstReg);
5085
5086 if (SrcTy.isScalable())
5087 return UnableToLegalize;
5088
5089 if (SrcTy.getScalarType() != DstTy.getScalarType())
5090 return UnableToLegalize;
5091
5092 // extract_subvector = build_vector(extract_element, extract_element, ...)
5093 SmallVector<Register> ExtractedElements;
5094 for (uint64_t i = 0; i < DstTy.getNumElements(); i++) {
5095 ExtractedElements.push_back(
5096 Elt: MIRBuilder
5097 .buildExtractVectorElementConstant(Res: SrcTy.getScalarType(), Val: SrcReg,
5098 Idx: ExtractionPointImm + i)
5099 .getReg(Idx: 0));
5100 }
5101
5102 MIRBuilder.buildBuildVector(Res: DstReg, Ops: ExtractedElements);
5103 MI.eraseFromParent();
5104 return Legalized;
5105 }
5106 case G_STACKSAVE:
5107 return lowerStackSave(MI);
5108 case G_STACKRESTORE:
5109 return lowerStackRestore(MI);
5110 case G_EXTRACT:
5111 return lowerExtract(MI);
5112 case G_INSERT:
5113 return lowerInsert(MI);
5114 case G_BSWAP:
5115 return lowerBswap(MI);
5116 case G_BITREVERSE:
5117 return lowerBitreverse(MI);
5118 case G_READ_REGISTER:
5119 case G_WRITE_REGISTER:
5120 return lowerReadWriteRegister(MI);
5121 case G_UADDSAT:
5122 case G_USUBSAT: {
5123 // Try to make a reasonable guess about which lowering strategy to use. The
5124 // target can override this with custom lowering and calling the
5125 // implementation functions.
5126 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5127 if (LI.isLegalOrCustom(Query: {G_UMIN, Ty}))
5128 return lowerAddSubSatToMinMax(MI);
5129 return lowerAddSubSatToAddoSubo(MI);
5130 }
5131 case G_SADDSAT:
5132 case G_SSUBSAT: {
5133 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5134
5135 // FIXME: It would probably make more sense to see if G_SADDO is preferred,
5136 // since it's a shorter expansion. However, we would need to figure out the
5137 // preferred boolean type for the carry out for the query.
5138 if (LI.isLegalOrCustom(Query: {G_SMIN, Ty}) && LI.isLegalOrCustom(Query: {G_SMAX, Ty}))
5139 return lowerAddSubSatToMinMax(MI);
5140 return lowerAddSubSatToAddoSubo(MI);
5141 }
5142 case G_SSHLSAT:
5143 case G_USHLSAT:
5144 return lowerShlSat(MI);
5145 case G_TRUNC_SSAT_S:
5146 case G_TRUNC_USAT_U:
5147 case G_TRUNC_SSAT_U:
5148 return lowerTruncSat(MI);
5149 case G_ABS:
5150 return lowerAbsToAddXor(MI);
5151 case G_ABDS:
5152 case G_ABDU: {
5153 bool IsSigned = MI.getOpcode() == G_ABDS;
5154 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5155 if ((IsSigned && LI.isLegal(Query: {G_SMIN, Ty}) && LI.isLegal(Query: {G_SMAX, Ty})) ||
5156 (!IsSigned && LI.isLegal(Query: {G_UMIN, Ty}) && LI.isLegal(Query: {G_UMAX, Ty}))) {
5157 return lowerAbsDiffToMinMax(MI);
5158 }
5159 return lowerAbsDiffToSelect(MI);
5160 }
5161 case G_FABS:
5162 return lowerFAbs(MI);
5163 case G_SELECT:
5164 return lowerSelect(MI);
5165 case G_IS_FPCLASS:
5166 return lowerISFPCLASS(MI);
5167 case G_SDIVREM:
5168 case G_UDIVREM:
5169 return lowerDIVREM(MI);
5170 case G_FSHL:
5171 case G_FSHR:
5172 return lowerFunnelShift(MI);
5173 case G_ROTL:
5174 case G_ROTR:
5175 return lowerRotate(MI);
5176 case G_MEMSET:
5177 case G_MEMCPY:
5178 case G_MEMMOVE:
5179 case G_MEMCPY_INLINE:
5180 case G_MEMSET_INLINE:
5181 return lowerMemCpyFamily(MI);
5182 case G_ZEXT:
5183 case G_SEXT:
5184 case G_ANYEXT:
5185 return lowerEXT(MI);
5186 case G_TRUNC:
5187 return lowerTRUNC(MI);
5188 GISEL_VECREDUCE_CASES_NONSEQ
5189 return lowerVectorReduction(MI);
5190 case G_VAARG:
5191 return lowerVAArg(MI);
5192 case G_ATOMICRMW_SUB: {
5193 auto [Ret, Mem, Val] = MI.getFirst3Regs();
5194 const LLT ValTy = MRI.getType(Reg: Val);
5195 MachineMemOperand *MMO = *MI.memoperands_begin();
5196
5197 auto VNeg = MIRBuilder.buildNeg(Dst: ValTy, Src0: Val);
5198 MIRBuilder.buildAtomicRMW(Opcode: G_ATOMICRMW_ADD, OldValRes: Ret, Addr: Mem, Val: VNeg, MMO&: *MMO);
5199 MI.eraseFromParent();
5200 return Legalized;
5201 }
5202 case G_SMULFIX:
5203 case G_UMULFIX:
5204 case G_SMULFIXSAT:
5205 case G_UMULFIXSAT:
5206 return lowerMulfix(MI);
5207 }
5208}
5209
5210Align LegalizerHelper::getStackTemporaryAlignment(LLT Ty,
5211 Align MinAlign) const {
5212 // FIXME: We're missing a way to go back from LLT to llvm::Type to query the
5213 // datalayout for the preferred alignment. Also there should be a target hook
5214 // for this to allow targets to reduce the alignment and ignore the
5215 // datalayout. e.g. AMDGPU should always use a 4-byte alignment, regardless of
5216 // the type.
5217 return std::max(a: Align(PowerOf2Ceil(A: Ty.getSizeInBytes())), b: MinAlign);
5218}
5219
5220MachineInstrBuilder
5221LegalizerHelper::createStackTemporary(TypeSize Bytes, Align Alignment,
5222 MachinePointerInfo &PtrInfo) {
5223 MachineFunction &MF = MIRBuilder.getMF();
5224 const DataLayout &DL = MIRBuilder.getDataLayout();
5225 int FrameIdx = MF.getFrameInfo().CreateStackObject(Size: Bytes, Alignment, isSpillSlot: false);
5226
5227 unsigned AddrSpace = DL.getAllocaAddrSpace();
5228 LLT FramePtrTy = LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL.getPointerSizeInBits(AS: AddrSpace));
5229
5230 PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIdx);
5231 return MIRBuilder.buildFrameIndex(Res: FramePtrTy, Idx: FrameIdx);
5232}
5233
5234MachineInstrBuilder LegalizerHelper::createStackStoreLoad(const DstOp &Res,
5235 const SrcOp &Val) {
5236 LLT SrcTy = Val.getLLTTy(MRI);
5237 Align StackTypeAlign =
5238 std::max(a: getStackTemporaryAlignment(Ty: SrcTy),
5239 b: getStackTemporaryAlignment(Ty: Res.getLLTTy(MRI)));
5240 MachinePointerInfo PtrInfo;
5241 auto StackTemp =
5242 createStackTemporary(Bytes: SrcTy.getSizeInBytes(), Alignment: StackTypeAlign, PtrInfo);
5243
5244 MIRBuilder.buildStore(Val, Addr: StackTemp, PtrInfo, Alignment: StackTypeAlign);
5245 return MIRBuilder.buildLoad(Res, Addr: StackTemp, PtrInfo, Alignment: StackTypeAlign);
5246}
5247
5248static Register clampVectorIndex(MachineIRBuilder &B, Register IdxReg,
5249 LLT VecTy) {
5250 LLT IdxTy = B.getMRI()->getType(Reg: IdxReg);
5251 unsigned NElts = VecTy.getNumElements();
5252
5253 int64_t IdxVal;
5254 if (mi_match(R: IdxReg, MRI: *B.getMRI(), P: m_ICst(Cst&: IdxVal))) {
5255 if (IdxVal < VecTy.getNumElements())
5256 return IdxReg;
5257 // If a constant index would be out of bounds, clamp it as well.
5258 }
5259
5260 if (isPowerOf2_32(Value: NElts)) {
5261 APInt Imm = APInt::getLowBitsSet(numBits: IdxTy.getSizeInBits(), loBitsSet: Log2_32(Value: NElts));
5262 return B.buildAnd(Dst: IdxTy, Src0: IdxReg, Src1: B.buildConstant(Res: IdxTy, Val: Imm)).getReg(Idx: 0);
5263 }
5264
5265 return B.buildUMin(Dst: IdxTy, Src0: IdxReg, Src1: B.buildConstant(Res: IdxTy, Val: NElts - 1))
5266 .getReg(Idx: 0);
5267}
5268
5269Register LegalizerHelper::getVectorElementPointer(Register VecPtr, LLT VecTy,
5270 Register Index) {
5271 LLT EltTy = VecTy.getElementType();
5272
5273 // Calculate the element offset and add it to the pointer.
5274 unsigned EltSize = EltTy.getSizeInBits() / 8; // FIXME: should be ABI size.
5275 assert(EltSize * 8 == EltTy.getSizeInBits() &&
5276 "Converting bits to bytes lost precision");
5277
5278 Index = clampVectorIndex(B&: MIRBuilder, IdxReg: Index, VecTy);
5279
5280 // Convert index to the correct size for the address space.
5281 const DataLayout &DL = MIRBuilder.getDataLayout();
5282 unsigned AS = MRI.getType(Reg: VecPtr).getAddressSpace();
5283 unsigned IndexSizeInBits = DL.getIndexSize(AS) * 8;
5284 LLT IdxTy = MRI.getType(Reg: Index).changeElementSize(NewEltSize: IndexSizeInBits);
5285 if (IdxTy != MRI.getType(Reg: Index))
5286 Index = MIRBuilder.buildSExtOrTrunc(Res: IdxTy, Op: Index).getReg(Idx: 0);
5287
5288 auto Mul = MIRBuilder.buildMul(Dst: IdxTy, Src0: Index,
5289 Src1: MIRBuilder.buildConstant(Res: IdxTy, Val: EltSize));
5290
5291 LLT PtrTy = MRI.getType(Reg: VecPtr);
5292 return MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: VecPtr, Op1: Mul).getReg(Idx: 0);
5293}
5294
5295#ifndef NDEBUG
5296/// Check that all vector operands have same number of elements. Other operands
5297/// should be listed in NonVecOp.
5298static bool hasSameNumEltsOnAllVectorOperands(
5299 GenericMachineInstr &MI, MachineRegisterInfo &MRI,
5300 std::initializer_list<unsigned> NonVecOpIndices) {
5301 if (MI.getNumMemOperands() != 0)
5302 return false;
5303
5304 LLT VecTy = MRI.getType(MI.getReg(0));
5305 if (!VecTy.isVector())
5306 return false;
5307 unsigned NumElts = VecTy.getNumElements();
5308
5309 for (unsigned OpIdx = 1; OpIdx < MI.getNumOperands(); ++OpIdx) {
5310 MachineOperand &Op = MI.getOperand(OpIdx);
5311 if (!Op.isReg()) {
5312 if (!is_contained(NonVecOpIndices, OpIdx))
5313 return false;
5314 continue;
5315 }
5316
5317 LLT Ty = MRI.getType(Op.getReg());
5318 if (!Ty.isVector()) {
5319 if (!is_contained(NonVecOpIndices, OpIdx))
5320 return false;
5321 continue;
5322 }
5323
5324 if (Ty.getNumElements() != NumElts)
5325 return false;
5326 }
5327
5328 return true;
5329}
5330#endif
5331
5332/// Fill \p DstOps with DstOps that have same number of elements combined as
5333/// the Ty. These DstOps have either scalar type when \p NumElts = 1 or are
5334/// vectors with \p NumElts elements. When Ty.getNumElements() is not multiple
5335/// of \p NumElts last DstOp (leftover) has fewer then \p NumElts elements.
5336static void makeDstOps(SmallVectorImpl<DstOp> &DstOps, LLT Ty,
5337 unsigned NumElts) {
5338 LLT LeftoverTy;
5339 assert(Ty.isVector() && "Expected vector type");
5340 LLT NarrowTy = Ty.changeElementCount(EC: ElementCount::getFixed(MinVal: NumElts));
5341 int NumParts, NumLeftover;
5342 std::tie(args&: NumParts, args&: NumLeftover) =
5343 getNarrowTypeBreakDown(OrigTy: Ty, NarrowTy, LeftoverTy);
5344
5345 assert(NumParts > 0 && "Error in getNarrowTypeBreakDown");
5346 for (int i = 0; i < NumParts; ++i) {
5347 DstOps.push_back(Elt: NarrowTy);
5348 }
5349
5350 if (LeftoverTy.isValid()) {
5351 assert(NumLeftover == 1 && "expected exactly one leftover");
5352 DstOps.push_back(Elt: LeftoverTy);
5353 }
5354}
5355
5356/// Operand \p Op is used on \p N sub-instructions. Fill \p Ops with \p N SrcOps
5357/// made from \p Op depending on operand type.
5358static void broadcastSrcOp(SmallVectorImpl<SrcOp> &Ops, unsigned N,
5359 MachineOperand &Op) {
5360 for (unsigned i = 0; i < N; ++i) {
5361 if (Op.isReg())
5362 Ops.push_back(Elt: Op.getReg());
5363 else if (Op.isImm())
5364 Ops.push_back(Elt: Op.getImm());
5365 else if (Op.isPredicate())
5366 Ops.push_back(Elt: static_cast<CmpInst::Predicate>(Op.getPredicate()));
5367 else
5368 llvm_unreachable("Unsupported type");
5369 }
5370}
5371
5372// Handle splitting vector operations which need to have the same number of
5373// elements in each type index, but each type index may have a different element
5374// type.
5375//
5376// e.g. <4 x s64> = G_SHL <4 x s64>, <4 x s32> ->
5377// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
5378// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
5379//
5380// Also handles some irregular breakdown cases, e.g.
5381// e.g. <3 x s64> = G_SHL <3 x s64>, <3 x s32> ->
5382// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
5383// s64 = G_SHL s64, s32
5384LegalizerHelper::LegalizeResult
5385LegalizerHelper::fewerElementsVectorMultiEltType(
5386 GenericMachineInstr &MI, unsigned NumElts,
5387 std::initializer_list<unsigned> NonVecOpIndices) {
5388 assert(hasSameNumEltsOnAllVectorOperands(MI, MRI, NonVecOpIndices) &&
5389 "Non-compatible opcode or not specified non-vector operands");
5390 unsigned OrigNumElts = MRI.getType(Reg: MI.getReg(Idx: 0)).getNumElements();
5391
5392 unsigned NumInputs = MI.getNumOperands() - MI.getNumDefs();
5393 unsigned NumDefs = MI.getNumDefs();
5394
5395 // Create DstOps (sub-vectors with NumElts elts + Leftover) for each output.
5396 // Build instructions with DstOps to use instruction found by CSE directly.
5397 // CSE copies found instruction into given vreg when building with vreg dest.
5398 SmallVector<SmallVector<DstOp, 8>, 2> OutputOpsPieces(NumDefs);
5399 // Output registers will be taken from created instructions.
5400 SmallVector<SmallVector<Register, 8>, 2> OutputRegs(NumDefs);
5401 for (unsigned i = 0; i < NumDefs; ++i) {
5402 makeDstOps(DstOps&: OutputOpsPieces[i], Ty: MRI.getType(Reg: MI.getReg(Idx: i)), NumElts);
5403 }
5404
5405 // Split vector input operands into sub-vectors with NumElts elts + Leftover.
5406 // Operands listed in NonVecOpIndices will be used as is without splitting;
5407 // examples: compare predicate in icmp and fcmp (op 1), vector select with i1
5408 // scalar condition (op 1), immediate in sext_inreg (op 2).
5409 SmallVector<SmallVector<SrcOp, 8>, 3> InputOpsPieces(NumInputs);
5410 for (unsigned UseIdx = NumDefs, UseNo = 0; UseIdx < MI.getNumOperands();
5411 ++UseIdx, ++UseNo) {
5412 if (is_contained(Set: NonVecOpIndices, Element: UseIdx)) {
5413 broadcastSrcOp(Ops&: InputOpsPieces[UseNo], N: OutputOpsPieces[0].size(),
5414 Op&: MI.getOperand(i: UseIdx));
5415 } else {
5416 SmallVector<Register, 8> SplitPieces;
5417 extractVectorParts(Reg: MI.getReg(Idx: UseIdx), NumElts, VRegs&: SplitPieces, MIRBuilder,
5418 MRI);
5419 llvm::append_range(C&: InputOpsPieces[UseNo], R&: SplitPieces);
5420 }
5421 }
5422
5423 unsigned NumLeftovers = OrigNumElts % NumElts ? 1 : 0;
5424
5425 // Take i-th piece of each input operand split and build sub-vector/scalar
5426 // instruction. Set i-th DstOp(s) from OutputOpsPieces as destination(s).
5427 for (unsigned i = 0; i < OrigNumElts / NumElts + NumLeftovers; ++i) {
5428 SmallVector<DstOp, 2> Defs;
5429 for (unsigned DstNo = 0; DstNo < NumDefs; ++DstNo)
5430 Defs.push_back(Elt: OutputOpsPieces[DstNo][i]);
5431
5432 SmallVector<SrcOp, 3> Uses;
5433 for (unsigned InputNo = 0; InputNo < NumInputs; ++InputNo)
5434 Uses.push_back(Elt: InputOpsPieces[InputNo][i]);
5435
5436 auto I = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: Defs, SrcOps: Uses, Flags: MI.getFlags());
5437 for (unsigned DstNo = 0; DstNo < NumDefs; ++DstNo)
5438 OutputRegs[DstNo].push_back(Elt: I.getReg(Idx: DstNo));
5439 }
5440
5441 // Merge small outputs into MI's output for each def operand.
5442 if (NumLeftovers) {
5443 for (unsigned i = 0; i < NumDefs; ++i)
5444 mergeMixedSubvectors(DstReg: MI.getReg(Idx: i), PartRegs: OutputRegs[i]);
5445 } else {
5446 for (unsigned i = 0; i < NumDefs; ++i)
5447 MIRBuilder.buildMergeLikeInstr(Res: MI.getReg(Idx: i), Ops: OutputRegs[i]);
5448 }
5449
5450 MI.eraseFromParent();
5451 return Legalized;
5452}
5453
5454LegalizerHelper::LegalizeResult
5455LegalizerHelper::fewerElementsVectorPhi(GenericMachineInstr &MI,
5456 unsigned NumElts) {
5457 unsigned OrigNumElts = MRI.getType(Reg: MI.getReg(Idx: 0)).getNumElements();
5458
5459 unsigned NumInputs = MI.getNumOperands() - MI.getNumDefs();
5460 unsigned NumDefs = MI.getNumDefs();
5461
5462 SmallVector<DstOp, 8> OutputOpsPieces;
5463 SmallVector<Register, 8> OutputRegs;
5464 makeDstOps(DstOps&: OutputOpsPieces, Ty: MRI.getType(Reg: MI.getReg(Idx: 0)), NumElts);
5465
5466 // Instructions that perform register split will be inserted in basic block
5467 // where register is defined (basic block is in the next operand).
5468 SmallVector<SmallVector<Register, 8>, 3> InputOpsPieces(NumInputs / 2);
5469 for (unsigned UseIdx = NumDefs, UseNo = 0; UseIdx < MI.getNumOperands();
5470 UseIdx += 2, ++UseNo) {
5471 MachineBasicBlock &OpMBB = *MI.getOperand(i: UseIdx + 1).getMBB();
5472 MIRBuilder.setInsertPt(
5473 MBB&: OpMBB, II: getPHIIncomingInsertPoint(MBB&: OpMBB, Reg: MI.getReg(Idx: UseIdx), MRI));
5474 extractVectorParts(Reg: MI.getReg(Idx: UseIdx), NumElts, VRegs&: InputOpsPieces[UseNo],
5475 MIRBuilder, MRI);
5476 }
5477
5478 // Build PHIs with fewer elements.
5479 unsigned NumLeftovers = OrigNumElts % NumElts ? 1 : 0;
5480 MIRBuilder.setInsertPt(MBB&: *MI.getParent(), II: MI);
5481 for (unsigned i = 0; i < OrigNumElts / NumElts + NumLeftovers; ++i) {
5482 auto Phi = MIRBuilder.buildInstr(Opcode: TargetOpcode::G_PHI);
5483 Phi.addDef(
5484 RegNo: MRI.createGenericVirtualRegister(Ty: OutputOpsPieces[i].getLLTTy(MRI)));
5485 OutputRegs.push_back(Elt: Phi.getReg(Idx: 0));
5486
5487 for (unsigned j = 0; j < NumInputs / 2; ++j) {
5488 Phi.addUse(RegNo: InputOpsPieces[j][i]);
5489 Phi.add(MO: MI.getOperand(i: 1 + j * 2 + 1));
5490 }
5491 }
5492
5493 // Set the insert point after the existing PHIs
5494 MachineBasicBlock &MBB = *MI.getParent();
5495 MIRBuilder.setInsertPt(MBB, II: MBB.getFirstNonPHI());
5496
5497 // Merge small outputs into MI's def.
5498 if (NumLeftovers) {
5499 mergeMixedSubvectors(DstReg: MI.getReg(Idx: 0), PartRegs: OutputRegs);
5500 } else {
5501 MIRBuilder.buildMergeLikeInstr(Res: MI.getReg(Idx: 0), Ops: OutputRegs);
5502 }
5503
5504 MI.eraseFromParent();
5505 return Legalized;
5506}
5507
5508LegalizerHelper::LegalizeResult
5509LegalizerHelper::fewerElementsVectorUnmergeValues(MachineInstr &MI,
5510 unsigned TypeIdx,
5511 LLT NarrowTy) {
5512 const int NumDst = MI.getNumOperands() - 1;
5513 const Register SrcReg = MI.getOperand(i: NumDst).getReg();
5514 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5515 LLT SrcTy = MRI.getType(Reg: SrcReg);
5516
5517 if (TypeIdx != 1 || NarrowTy == DstTy)
5518 return UnableToLegalize;
5519
5520 // Requires compatible types. Otherwise SrcReg should have been defined by
5521 // merge-like instruction that would get artifact combined. Most likely
5522 // instruction that defines SrcReg has to perform more/fewer elements
5523 // legalization compatible with NarrowTy.
5524 assert(SrcTy.isVector() && NarrowTy.isVector() && "Expected vector types");
5525 assert((SrcTy.getScalarType() == NarrowTy.getScalarType()) && "bad type");
5526
5527 if ((SrcTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0) ||
5528 (NarrowTy.getSizeInBits() % DstTy.getSizeInBits() != 0))
5529 return UnableToLegalize;
5530
5531 // This is most likely DstTy (smaller then register size) packed in SrcTy
5532 // (larger then register size) and since unmerge was not combined it will be
5533 // lowered to bit sequence extracts from register. Unpack SrcTy to NarrowTy
5534 // (register size) pieces first. Then unpack each of NarrowTy pieces to DstTy.
5535
5536 // %1:_(DstTy), %2, %3, %4 = G_UNMERGE_VALUES %0:_(SrcTy)
5537 //
5538 // %5:_(NarrowTy), %6 = G_UNMERGE_VALUES %0:_(SrcTy) - reg sequence
5539 // %1:_(DstTy), %2 = G_UNMERGE_VALUES %5:_(NarrowTy) - sequence of bits in reg
5540 // %3:_(DstTy), %4 = G_UNMERGE_VALUES %6:_(NarrowTy)
5541 auto Unmerge = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: SrcReg);
5542 const int NumUnmerge = Unmerge->getNumOperands() - 1;
5543 const int PartsPerUnmerge = NumDst / NumUnmerge;
5544
5545 for (int I = 0; I != NumUnmerge; ++I) {
5546 auto MIB = MIRBuilder.buildInstr(Opcode: TargetOpcode::G_UNMERGE_VALUES);
5547
5548 for (int J = 0; J != PartsPerUnmerge; ++J)
5549 MIB.addDef(RegNo: MI.getOperand(i: I * PartsPerUnmerge + J).getReg());
5550 MIB.addUse(RegNo: Unmerge.getReg(Idx: I));
5551 }
5552
5553 MI.eraseFromParent();
5554 return Legalized;
5555}
5556
5557LegalizerHelper::LegalizeResult
5558LegalizerHelper::fewerElementsVectorMerge(MachineInstr &MI, unsigned TypeIdx,
5559 LLT NarrowTy) {
5560 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
5561 // Requires compatible types. Otherwise user of DstReg did not perform unmerge
5562 // that should have been artifact combined. Most likely instruction that uses
5563 // DstReg has to do more/fewer elements legalization compatible with NarrowTy.
5564 assert(DstTy.isVector() && NarrowTy.isVector() && "Expected vector types");
5565 assert((DstTy.getScalarType() == NarrowTy.getScalarType()) && "bad type");
5566 if (NarrowTy == SrcTy)
5567 return UnableToLegalize;
5568
5569 // This attempts to lower part of LCMTy merge/unmerge sequence. Intended use
5570 // is for old mir tests. Since the changes to more/fewer elements it should no
5571 // longer be possible to generate MIR like this when starting from llvm-ir
5572 // because LCMTy approach was replaced with merge/unmerge to vector elements.
5573 if (TypeIdx == 1) {
5574 assert(SrcTy.isVector() && "Expected vector types");
5575 assert((SrcTy.getScalarType() == NarrowTy.getScalarType()) && "bad type");
5576 if ((DstTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0) ||
5577 (NarrowTy.getNumElements() >= SrcTy.getNumElements()))
5578 return UnableToLegalize;
5579 // %2:_(DstTy) = G_CONCAT_VECTORS %0:_(SrcTy), %1:_(SrcTy)
5580 //
5581 // %3:_(EltTy), %4, %5 = G_UNMERGE_VALUES %0:_(SrcTy)
5582 // %6:_(EltTy), %7, %8 = G_UNMERGE_VALUES %1:_(SrcTy)
5583 // %9:_(NarrowTy) = G_BUILD_VECTOR %3:_(EltTy), %4
5584 // %10:_(NarrowTy) = G_BUILD_VECTOR %5:_(EltTy), %6
5585 // %11:_(NarrowTy) = G_BUILD_VECTOR %7:_(EltTy), %8
5586 // %2:_(DstTy) = G_CONCAT_VECTORS %9:_(NarrowTy), %10, %11
5587
5588 SmallVector<Register, 8> Elts;
5589 LLT EltTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getScalarType();
5590 for (unsigned i = 1; i < MI.getNumOperands(); ++i) {
5591 auto Unmerge = MIRBuilder.buildUnmerge(Res: EltTy, Op: MI.getOperand(i).getReg());
5592 for (unsigned j = 0; j < Unmerge->getNumDefs(); ++j)
5593 Elts.push_back(Elt: Unmerge.getReg(Idx: j));
5594 }
5595
5596 SmallVector<Register, 8> NarrowTyElts;
5597 unsigned NumNarrowTyElts = NarrowTy.getNumElements();
5598 unsigned NumNarrowTyPieces = DstTy.getNumElements() / NumNarrowTyElts;
5599 for (unsigned i = 0, Offset = 0; i < NumNarrowTyPieces;
5600 ++i, Offset += NumNarrowTyElts) {
5601 ArrayRef<Register> Pieces(&Elts[Offset], NumNarrowTyElts);
5602 NarrowTyElts.push_back(
5603 Elt: MIRBuilder.buildMergeLikeInstr(Res: NarrowTy, Ops: Pieces).getReg(Idx: 0));
5604 }
5605
5606 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: NarrowTyElts);
5607 MI.eraseFromParent();
5608 return Legalized;
5609 }
5610
5611 assert(TypeIdx == 0 && "Bad type index");
5612 if ((NarrowTy.getSizeInBits() % SrcTy.getSizeInBits() != 0) ||
5613 (DstTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0))
5614 return UnableToLegalize;
5615
5616 // This is most likely SrcTy (smaller then register size) packed in DstTy
5617 // (larger then register size) and since merge was not combined it will be
5618 // lowered to bit sequence packing into register. Merge SrcTy to NarrowTy
5619 // (register size) pieces first. Then merge each of NarrowTy pieces to DstTy.
5620
5621 // %0:_(DstTy) = G_MERGE_VALUES %1:_(SrcTy), %2, %3, %4
5622 //
5623 // %5:_(NarrowTy) = G_MERGE_VALUES %1:_(SrcTy), %2 - sequence of bits in reg
5624 // %6:_(NarrowTy) = G_MERGE_VALUES %3:_(SrcTy), %4
5625 // %0:_(DstTy) = G_MERGE_VALUES %5:_(NarrowTy), %6 - reg sequence
5626 SmallVector<Register, 8> NarrowTyElts;
5627 unsigned NumParts = DstTy.getNumElements() / NarrowTy.getNumElements();
5628 unsigned NumSrcElts = SrcTy.isVector() ? SrcTy.getNumElements() : 1;
5629 unsigned NumElts = NarrowTy.getNumElements() / NumSrcElts;
5630 for (unsigned i = 0; i < NumParts; ++i) {
5631 SmallVector<Register, 8> Sources;
5632 for (unsigned j = 0; j < NumElts; ++j)
5633 Sources.push_back(Elt: MI.getOperand(i: 1 + i * NumElts + j).getReg());
5634 NarrowTyElts.push_back(
5635 Elt: MIRBuilder.buildMergeLikeInstr(Res: NarrowTy, Ops: Sources).getReg(Idx: 0));
5636 }
5637
5638 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: NarrowTyElts);
5639 MI.eraseFromParent();
5640 return Legalized;
5641}
5642
5643LegalizerHelper::LegalizeResult
5644LegalizerHelper::fewerElementsVectorExtractInsertVectorElt(MachineInstr &MI,
5645 unsigned TypeIdx,
5646 LLT NarrowVecTy) {
5647 auto [DstReg, SrcVec] = MI.getFirst2Regs();
5648 Register InsertVal;
5649 bool IsInsert = MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT;
5650
5651 assert((IsInsert ? TypeIdx == 0 : TypeIdx == 1) && "not a vector type index");
5652 if (IsInsert)
5653 InsertVal = MI.getOperand(i: 2).getReg();
5654
5655 Register Idx = MI.getOperand(i: MI.getNumOperands() - 1).getReg();
5656 LLT VecTy = MRI.getType(Reg: SrcVec);
5657
5658 // If the index is a constant, we can really break this down as you would
5659 // expect, and index into the target size pieces.
5660 auto MaybeCst = getIConstantVRegValWithLookThrough(VReg: Idx, MRI);
5661 if (MaybeCst) {
5662 uint64_t IdxVal = MaybeCst->Value.getZExtValue();
5663 // Avoid out of bounds indexing the pieces.
5664 if (IdxVal >= VecTy.getNumElements()) {
5665 MIRBuilder.buildUndef(Res: DstReg);
5666 MI.eraseFromParent();
5667 return Legalized;
5668 }
5669
5670 if (!NarrowVecTy.isVector()) {
5671 SmallVector<Register, 8> SplitPieces;
5672 extractParts(Reg: MI.getOperand(i: 1).getReg(), Ty: NarrowVecTy,
5673 NumParts: VecTy.getNumElements(), VRegs&: SplitPieces, MIRBuilder, MRI);
5674 if (IsInsert) {
5675 SplitPieces[IdxVal] = InsertVal;
5676 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0).getReg(), Ops: SplitPieces);
5677 } else {
5678 MIRBuilder.buildCopy(Res: MI.getOperand(i: 0).getReg(), Op: SplitPieces[IdxVal]);
5679 }
5680 } else {
5681 SmallVector<Register, 8> VecParts;
5682 LLT GCDTy = extractGCDType(Parts&: VecParts, DstTy: VecTy, NarrowTy: NarrowVecTy, SrcReg: SrcVec);
5683
5684 // Build a sequence of NarrowTy pieces in VecParts for this operand.
5685 LLT LCMTy = buildLCMMergePieces(DstTy: VecTy, NarrowTy: NarrowVecTy, GCDTy, VRegs&: VecParts,
5686 PadStrategy: TargetOpcode::G_ANYEXT);
5687
5688 unsigned NewNumElts = NarrowVecTy.getNumElements();
5689
5690 LLT IdxTy = MRI.getType(Reg: Idx);
5691 int64_t PartIdx = IdxVal / NewNumElts;
5692 auto NewIdx =
5693 MIRBuilder.buildConstant(Res: IdxTy, Val: IdxVal - NewNumElts * PartIdx);
5694
5695 if (IsInsert) {
5696 LLT PartTy = MRI.getType(Reg: VecParts[PartIdx]);
5697
5698 // Use the adjusted index to insert into one of the subvectors.
5699 auto InsertPart = MIRBuilder.buildInsertVectorElement(
5700 Res: PartTy, Val: VecParts[PartIdx], Elt: InsertVal, Idx: NewIdx);
5701 VecParts[PartIdx] = InsertPart.getReg(Idx: 0);
5702
5703 // Recombine the inserted subvector with the others to reform the result
5704 // vector.
5705 buildWidenedRemergeToDst(DstReg, LCMTy, RemergeRegs: VecParts);
5706 } else {
5707 MIRBuilder.buildExtractVectorElement(Res: DstReg, Val: VecParts[PartIdx], Idx: NewIdx);
5708 }
5709 }
5710
5711 MI.eraseFromParent();
5712 return Legalized;
5713 }
5714
5715 // With a variable index, we can't perform the operation in a smaller type, so
5716 // we're forced to expand this.
5717 //
5718 // TODO: We could emit a chain of compare/select to figure out which piece to
5719 // index.
5720 return lowerExtractInsertVectorElt(MI);
5721}
5722
5723LegalizerHelper::LegalizeResult
5724LegalizerHelper::reduceLoadStoreWidth(GLoadStore &LdStMI, unsigned TypeIdx,
5725 LLT NarrowTy) {
5726 // FIXME: Don't know how to handle secondary types yet.
5727 if (TypeIdx != 0)
5728 return UnableToLegalize;
5729
5730 if (!NarrowTy.isByteSized()) {
5731 LLVM_DEBUG(dbgs() << "Can't narrow load/store to non-byte-sized type\n");
5732 return UnableToLegalize;
5733 }
5734
5735 // This implementation doesn't work for atomics. Give up instead of doing
5736 // something invalid.
5737 if (LdStMI.isAtomic())
5738 return UnableToLegalize;
5739
5740 bool IsLoad = isa<GLoad>(Val: LdStMI);
5741 Register ValReg = LdStMI.getReg(Idx: 0);
5742 Register AddrReg = LdStMI.getPointerReg();
5743 LLT ValTy = MRI.getType(Reg: ValReg);
5744
5745 // FIXME: Do we need a distinct NarrowMemory legalize action?
5746 if (ValTy.getSizeInBits() != 8 * LdStMI.getMemSize().getValue()) {
5747 LLVM_DEBUG(dbgs() << "Can't narrow extload/truncstore\n");
5748 return UnableToLegalize;
5749 }
5750
5751 int NumParts = -1;
5752 int NumLeftover = -1;
5753 LLT LeftoverTy;
5754 SmallVector<Register, 8> NarrowRegs, NarrowLeftoverRegs;
5755 if (IsLoad) {
5756 std::tie(args&: NumParts, args&: NumLeftover) = getNarrowTypeBreakDown(OrigTy: ValTy, NarrowTy, LeftoverTy);
5757 } else {
5758 if (extractParts(Reg: ValReg, RegTy: ValTy, MainTy: NarrowTy, LeftoverTy, VRegs&: NarrowRegs,
5759 LeftoverVRegs&: NarrowLeftoverRegs, MIRBuilder, MRI)) {
5760 NumParts = NarrowRegs.size();
5761 NumLeftover = NarrowLeftoverRegs.size();
5762 }
5763 }
5764
5765 if (NumParts == -1)
5766 return UnableToLegalize;
5767
5768 LLT PtrTy = MRI.getType(Reg: AddrReg);
5769 const LLT OffsetTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
5770
5771 unsigned TotalSize = ValTy.getSizeInBits();
5772
5773 // Split the load/store into PartTy sized pieces starting at Offset. If this
5774 // is a load, return the new registers in ValRegs. For a store, each elements
5775 // of ValRegs should be PartTy. Returns the next offset that needs to be
5776 // handled.
5777 bool isBigEndian = MIRBuilder.getDataLayout().isBigEndian();
5778 auto MMO = LdStMI.getMMO();
5779 auto splitTypePieces = [=](LLT PartTy, SmallVectorImpl<Register> &ValRegs,
5780 unsigned NumParts, unsigned Offset) -> unsigned {
5781 MachineFunction &MF = MIRBuilder.getMF();
5782 unsigned PartSize = PartTy.getSizeInBits();
5783 for (unsigned Idx = 0, E = NumParts; Idx != E && Offset < TotalSize;
5784 ++Idx) {
5785 unsigned ByteOffset = Offset / 8;
5786 Register NewAddrReg;
5787
5788 MIRBuilder.materializeObjectPtrOffset(Res&: NewAddrReg, Op0: AddrReg, ValueTy: OffsetTy,
5789 Value: ByteOffset);
5790
5791 MachineMemOperand *NewMMO =
5792 MF.getMachineMemOperand(MMO: &MMO, Offset: ByteOffset, Ty: PartTy);
5793
5794 if (IsLoad) {
5795 Register Dst = MRI.createGenericVirtualRegister(Ty: PartTy);
5796 ValRegs.push_back(Elt: Dst);
5797 MIRBuilder.buildLoad(Res: Dst, Addr: NewAddrReg, MMO&: *NewMMO);
5798 } else {
5799 MIRBuilder.buildStore(Val: ValRegs[Idx], Addr: NewAddrReg, MMO&: *NewMMO);
5800 }
5801 Offset = isBigEndian ? Offset - PartSize : Offset + PartSize;
5802 }
5803
5804 return Offset;
5805 };
5806
5807 unsigned Offset = isBigEndian ? TotalSize - NarrowTy.getSizeInBits() : 0;
5808 unsigned HandledOffset =
5809 splitTypePieces(NarrowTy, NarrowRegs, NumParts, Offset);
5810
5811 // Handle the rest of the register if this isn't an even type breakdown.
5812 if (LeftoverTy.isValid())
5813 splitTypePieces(LeftoverTy, NarrowLeftoverRegs, NumLeftover, HandledOffset);
5814
5815 if (IsLoad) {
5816 insertParts(DstReg: ValReg, ResultTy: ValTy, PartTy: NarrowTy, PartRegs: NarrowRegs,
5817 LeftoverTy, LeftoverRegs: NarrowLeftoverRegs);
5818 }
5819
5820 LdStMI.eraseFromParent();
5821 return Legalized;
5822}
5823
5824LegalizerHelper::LegalizeResult
5825LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
5826 LLT NarrowTy) {
5827 using namespace TargetOpcode;
5828 GenericMachineInstr &GMI = cast<GenericMachineInstr>(Val&: MI);
5829 unsigned NumElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
5830
5831 switch (MI.getOpcode()) {
5832 case G_IMPLICIT_DEF:
5833 case G_TRUNC:
5834 case G_AND:
5835 case G_OR:
5836 case G_XOR:
5837 case G_ADD:
5838 case G_SUB:
5839 case G_MUL:
5840 case G_PTR_ADD:
5841 case G_SMULH:
5842 case G_UMULH:
5843 case G_FADD:
5844 case G_FMUL:
5845 case G_FSUB:
5846 case G_FNEG:
5847 case G_FABS:
5848 case G_FCANONICALIZE:
5849 case G_FDIV:
5850 case G_FREM:
5851 case G_FMA:
5852 case G_FMAD:
5853 case G_FPOW:
5854 case G_FEXP:
5855 case G_FEXP2:
5856 case G_FEXP10:
5857 case G_FLOG:
5858 case G_FLOG2:
5859 case G_FLOG10:
5860 case G_FLDEXP:
5861 case G_FNEARBYINT:
5862 case G_FCEIL:
5863 case G_FFLOOR:
5864 case G_FRINT:
5865 case G_INTRINSIC_LRINT:
5866 case G_INTRINSIC_LLRINT:
5867 case G_INTRINSIC_ROUND:
5868 case G_INTRINSIC_ROUNDEVEN:
5869 case G_LROUND:
5870 case G_LLROUND:
5871 case G_INTRINSIC_TRUNC:
5872 case G_FMODF:
5873 case G_FCOS:
5874 case G_FSIN:
5875 case G_FTAN:
5876 case G_FACOS:
5877 case G_FASIN:
5878 case G_FATAN:
5879 case G_FATAN2:
5880 case G_FCOSH:
5881 case G_FSINH:
5882 case G_FTANH:
5883 case G_FSQRT:
5884 case G_BSWAP:
5885 case G_BITREVERSE:
5886 case G_SDIV:
5887 case G_UDIV:
5888 case G_SREM:
5889 case G_UREM:
5890 case G_SDIVREM:
5891 case G_UDIVREM:
5892 case G_SMIN:
5893 case G_SMAX:
5894 case G_UMIN:
5895 case G_UMAX:
5896 case G_ABS:
5897 case G_FMINNUM:
5898 case G_FMAXNUM:
5899 case G_FMINNUM_IEEE:
5900 case G_FMAXNUM_IEEE:
5901 case G_FMINIMUM:
5902 case G_FMAXIMUM:
5903 case G_FMINIMUMNUM:
5904 case G_FMAXIMUMNUM:
5905 case G_FSHL:
5906 case G_FSHR:
5907 case G_ROTL:
5908 case G_ROTR:
5909 case G_FREEZE:
5910 case G_SADDSAT:
5911 case G_SSUBSAT:
5912 case G_UADDSAT:
5913 case G_USUBSAT:
5914 case G_UMULO:
5915 case G_SMULO:
5916 case G_SHL:
5917 case G_LSHR:
5918 case G_ASHR:
5919 case G_SSHLSAT:
5920 case G_USHLSAT:
5921 case G_CTLZ:
5922 case G_CTLZ_ZERO_POISON:
5923 case G_CTTZ:
5924 case G_CTTZ_ZERO_POISON:
5925 case G_CTPOP:
5926 case G_CTLS:
5927 case G_FCOPYSIGN:
5928 case G_ZEXT:
5929 case G_SEXT:
5930 case G_ANYEXT:
5931 case G_FPEXT:
5932 case G_FPTRUNC:
5933 case G_SITOFP:
5934 case G_UITOFP:
5935 case G_FPTOSI:
5936 case G_FPTOUI:
5937 case G_FPTOSI_SAT:
5938 case G_FPTOUI_SAT:
5939 case G_INTTOPTR:
5940 case G_PTRTOINT:
5941 case G_ADDRSPACE_CAST:
5942 case G_UADDO:
5943 case G_USUBO:
5944 case G_UADDE:
5945 case G_USUBE:
5946 case G_SADDO:
5947 case G_SSUBO:
5948 case G_SADDE:
5949 case G_SSUBE:
5950 case G_STRICT_FADD:
5951 case G_STRICT_FSUB:
5952 case G_STRICT_FMUL:
5953 case G_STRICT_FMA:
5954 case G_STRICT_FLDEXP:
5955 case G_FFREXP:
5956 case G_TRUNC_SSAT_S:
5957 case G_TRUNC_SSAT_U:
5958 case G_TRUNC_USAT_U:
5959 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts);
5960 case G_ICMP:
5961 case G_FCMP:
5962 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {1 /*cpm predicate*/});
5963 case G_IS_FPCLASS:
5964 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {2, 3 /*mask,fpsem*/});
5965 case G_SELECT:
5966 if (MRI.getType(Reg: MI.getOperand(i: 1).getReg()).isVector())
5967 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts);
5968 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {1 /*scalar cond*/});
5969 case G_PHI:
5970 return fewerElementsVectorPhi(MI&: GMI, NumElts);
5971 case G_UNMERGE_VALUES:
5972 return fewerElementsVectorUnmergeValues(MI, TypeIdx, NarrowTy);
5973 case G_BUILD_VECTOR:
5974 assert(TypeIdx == 0 && "not a vector type index");
5975 return fewerElementsVectorMerge(MI, TypeIdx, NarrowTy);
5976 case G_CONCAT_VECTORS:
5977 if (TypeIdx != 1) // TODO: This probably does work as expected already.
5978 return UnableToLegalize;
5979 return fewerElementsVectorMerge(MI, TypeIdx, NarrowTy);
5980 case G_EXTRACT_SUBVECTOR: {
5981 Register DstReg = MI.getOperand(i: 0).getReg();
5982 LLT DstTy = MRI.getType(Reg: DstReg);
5983 Register SrcReg = MI.getOperand(i: 1).getReg();
5984 uint64_t InsertionPointImm = MI.getOperand(i: 2).getImm();
5985
5986 // If Dst > NarrowTy bits, then cannot legalize
5987 if (DstTy.getSizeInBits() > NarrowTy.getSizeInBits())
5988 return UnableToLegalize;
5989
5990 // If DstTy's size is not a multiple of NarrowTy's, then cannot legalize
5991 if (!DstTy.getElementCount().isKnownMultipleOf(RHS: NarrowTy.getElementCount()))
5992 return UnableToLegalize;
5993
5994 auto Unmerge = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: SrcReg);
5995 uint64_t RequiredSubvectorIndex =
5996 InsertionPointImm / NarrowTy.getNumElements();
5997 // If Dst and Narrow are both same size, convert to a copy
5998 if (DstTy.getNumElements() == NarrowTy.getNumElements())
5999 MIRBuilder.buildCopy(Res: DstReg, Op: Unmerge.getReg(Idx: RequiredSubvectorIndex));
6000 else
6001 MIRBuilder.buildExtractSubvector(
6002 Res: DstReg, Src: Unmerge.getReg(Idx: RequiredSubvectorIndex),
6003 Index: InsertionPointImm % NarrowTy.getNumElements());
6004
6005 MI.eraseFromParent();
6006 return Legalized;
6007 }
6008 case G_EXTRACT_VECTOR_ELT:
6009 case G_INSERT_VECTOR_ELT:
6010 return fewerElementsVectorExtractInsertVectorElt(MI, TypeIdx, NarrowVecTy: NarrowTy);
6011 case G_LOAD:
6012 case G_STORE:
6013 return reduceLoadStoreWidth(LdStMI&: cast<GLoadStore>(Val&: MI), TypeIdx, NarrowTy);
6014 case G_SEXT_INREG:
6015 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {2 /*imm*/});
6016 GISEL_VECREDUCE_CASES_NONSEQ
6017 return fewerElementsVectorReductions(MI, TypeIdx, NarrowTy);
6018 case TargetOpcode::G_VECREDUCE_SEQ_FADD:
6019 case TargetOpcode::G_VECREDUCE_SEQ_FMUL:
6020 return fewerElementsVectorSeqReductions(MI, TypeIdx, NarrowTy);
6021 case G_SHUFFLE_VECTOR:
6022 return fewerElementsVectorShuffle(MI, TypeIdx, NarrowTy);
6023 case G_FPOWI:
6024 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {2 /*pow*/});
6025 case G_BITCAST:
6026 return fewerElementsBitcast(MI, TypeIdx, NarrowTy);
6027 case G_INTRINSIC_FPTRUNC_ROUND:
6028 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {2});
6029 default:
6030 return UnableToLegalize;
6031 }
6032}
6033
6034LegalizerHelper::LegalizeResult
6035LegalizerHelper::fewerElementsBitcast(MachineInstr &MI, unsigned int TypeIdx,
6036 LLT NarrowTy) {
6037 assert(MI.getOpcode() == TargetOpcode::G_BITCAST &&
6038 "Not a bitcast operation");
6039
6040 if (TypeIdx != 0)
6041 return UnableToLegalize;
6042
6043 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
6044
6045 unsigned NewElemCount =
6046 NarrowTy.getSizeInBits() / SrcTy.getScalarSizeInBits();
6047 SmallVector<Register> SrcVRegs, BitcastVRegs;
6048 if (NewElemCount == 1) {
6049 LLT SrcNarrowTy = SrcTy.getElementType();
6050
6051 auto Unmerge = MIRBuilder.buildUnmerge(Res: SrcNarrowTy, Op: SrcReg);
6052 getUnmergeResults(Regs&: SrcVRegs, MI: *Unmerge);
6053 } else {
6054 LLT SrcNarrowTy =
6055 SrcTy.changeVectorElementCount(EC: ElementCount::getFixed(MinVal: NewElemCount));
6056
6057 // Split the Src and Dst Reg into smaller registers
6058 if (extractGCDType(Parts&: SrcVRegs, DstTy, NarrowTy: SrcNarrowTy, SrcReg) != SrcNarrowTy)
6059 return UnableToLegalize;
6060 }
6061
6062 // Build new smaller bitcast instructions
6063 // Not supporting Leftover types for now but will have to
6064 for (Register Reg : SrcVRegs)
6065 BitcastVRegs.push_back(Elt: MIRBuilder.buildBitcast(Dst: NarrowTy, Src: Reg).getReg(Idx: 0));
6066
6067 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: BitcastVRegs);
6068 MI.eraseFromParent();
6069 return Legalized;
6070}
6071
6072LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorShuffle(
6073 MachineInstr &MI, unsigned int TypeIdx, LLT NarrowTy) {
6074 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
6075 if (TypeIdx != 0)
6076 return UnableToLegalize;
6077
6078 auto [DstReg, DstTy, Src1Reg, Src1Ty, Src2Reg, Src2Ty] =
6079 MI.getFirst3RegLLTs();
6080 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
6081 // The shuffle should be canonicalized by now.
6082 if (DstTy != Src1Ty)
6083 return UnableToLegalize;
6084 if (DstTy != Src2Ty)
6085 return UnableToLegalize;
6086
6087 if (!isPowerOf2_32(Value: DstTy.getNumElements()))
6088 return UnableToLegalize;
6089
6090 // We only support splitting a shuffle into 2, so adjust NarrowTy accordingly.
6091 // Further legalization attempts will be needed to do split further.
6092 NarrowTy =
6093 DstTy.changeElementCount(EC: DstTy.getElementCount().divideCoefficientBy(RHS: 2));
6094 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
6095
6096 SmallVector<Register> SplitSrc1Regs, SplitSrc2Regs;
6097 extractParts(Reg: Src1Reg, Ty: NarrowTy, NumParts: 2, VRegs&: SplitSrc1Regs, MIRBuilder, MRI);
6098 extractParts(Reg: Src2Reg, Ty: NarrowTy, NumParts: 2, VRegs&: SplitSrc2Regs, MIRBuilder, MRI);
6099 Register Inputs[4] = {SplitSrc1Regs[0], SplitSrc1Regs[1], SplitSrc2Regs[0],
6100 SplitSrc2Regs[1]};
6101
6102 Register Hi, Lo;
6103
6104 // If Lo or Hi uses elements from at most two of the four input vectors, then
6105 // express it as a vector shuffle of those two inputs. Otherwise extract the
6106 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
6107 SmallVector<int, 16> Ops;
6108 for (unsigned High = 0; High < 2; ++High) {
6109 Register &Output = High ? Hi : Lo;
6110
6111 // Build a shuffle mask for the output, discovering on the fly which
6112 // input vectors to use as shuffle operands (recorded in InputUsed).
6113 // If building a suitable shuffle vector proves too hard, then bail
6114 // out with useBuildVector set.
6115 unsigned InputUsed[2] = {-1U, -1U}; // Not yet discovered.
6116 unsigned FirstMaskIdx = High * NewElts;
6117 bool UseBuildVector = false;
6118 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
6119 // The mask element. This indexes into the input.
6120 int Idx = Mask[FirstMaskIdx + MaskOffset];
6121
6122 // The input vector this mask element indexes into.
6123 unsigned Input = (unsigned)Idx / NewElts;
6124
6125 if (Input >= std::size(Inputs)) {
6126 // The mask element does not index into any input vector.
6127 Ops.push_back(Elt: -1);
6128 continue;
6129 }
6130
6131 // Turn the index into an offset from the start of the input vector.
6132 Idx -= Input * NewElts;
6133
6134 // Find or create a shuffle vector operand to hold this input.
6135 unsigned OpNo;
6136 for (OpNo = 0; OpNo < std::size(InputUsed); ++OpNo) {
6137 if (InputUsed[OpNo] == Input) {
6138 // This input vector is already an operand.
6139 break;
6140 } else if (InputUsed[OpNo] == -1U) {
6141 // Create a new operand for this input vector.
6142 InputUsed[OpNo] = Input;
6143 break;
6144 }
6145 }
6146
6147 if (OpNo >= std::size(InputUsed)) {
6148 // More than two input vectors used! Give up on trying to create a
6149 // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
6150 UseBuildVector = true;
6151 break;
6152 }
6153
6154 // Add the mask index for the new shuffle vector.
6155 Ops.push_back(Elt: Idx + OpNo * NewElts);
6156 }
6157
6158 if (UseBuildVector) {
6159 LLT EltTy = NarrowTy.getElementType();
6160 SmallVector<Register, 16> SVOps;
6161
6162 // Extract the input elements by hand.
6163 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
6164 // The mask element. This indexes into the input.
6165 int Idx = Mask[FirstMaskIdx + MaskOffset];
6166
6167 // The input vector this mask element indexes into.
6168 unsigned Input = (unsigned)Idx / NewElts;
6169
6170 if (Input >= std::size(Inputs)) {
6171 // The mask element is "undef" or indexes off the end of the input.
6172 SVOps.push_back(Elt: MIRBuilder.buildUndef(Res: EltTy).getReg(Idx: 0));
6173 continue;
6174 }
6175
6176 // Turn the index into an offset from the start of the input vector.
6177 Idx -= Input * NewElts;
6178
6179 // Extract the vector element by hand.
6180 SVOps.push_back(Elt: MIRBuilder
6181 .buildExtractVectorElement(
6182 Res: EltTy, Val: Inputs[Input],
6183 Idx: MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: 32), Val: Idx))
6184 .getReg(Idx: 0));
6185 }
6186
6187 // Construct the Lo/Hi output using a G_BUILD_VECTOR.
6188 Output = MIRBuilder.buildBuildVector(Res: NarrowTy, Ops: SVOps).getReg(Idx: 0);
6189 } else if (InputUsed[0] == -1U) {
6190 // No input vectors were used! The result is undefined.
6191 Output = MIRBuilder.buildUndef(Res: NarrowTy).getReg(Idx: 0);
6192 } else if (NewElts == 1) {
6193 Output = MIRBuilder.buildCopy(Res: NarrowTy, Op: Inputs[InputUsed[0]]).getReg(Idx: 0);
6194 } else {
6195 Register Op0 = Inputs[InputUsed[0]];
6196 // If only one input was used, use an undefined vector for the other.
6197 Register Op1 = InputUsed[1] == -1U
6198 ? MIRBuilder.buildUndef(Res: NarrowTy).getReg(Idx: 0)
6199 : Inputs[InputUsed[1]];
6200 // At least one input vector was used. Create a new shuffle vector.
6201 Output = MIRBuilder.buildShuffleVector(Res: NarrowTy, Src1: Op0, Src2: Op1, Mask: Ops).getReg(Idx: 0);
6202 }
6203
6204 Ops.clear();
6205 }
6206
6207 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: {Lo, Hi});
6208 MI.eraseFromParent();
6209 return Legalized;
6210}
6211
6212LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorReductions(
6213 MachineInstr &MI, unsigned int TypeIdx, LLT NarrowTy) {
6214 auto &RdxMI = cast<GVecReduce>(Val&: MI);
6215
6216 if (TypeIdx != 1)
6217 return UnableToLegalize;
6218
6219 // The semantics of the normal non-sequential reductions allow us to freely
6220 // re-associate the operation.
6221 auto [DstReg, DstTy, SrcReg, SrcTy] = RdxMI.getFirst2RegLLTs();
6222
6223 if (NarrowTy.isVector() &&
6224 (SrcTy.getNumElements() % NarrowTy.getNumElements() != 0))
6225 return UnableToLegalize;
6226
6227 unsigned ScalarOpc = RdxMI.getScalarOpcForReduction();
6228 SmallVector<Register> SplitSrcs;
6229 // If NarrowTy is a scalar then we're being asked to scalarize.
6230 const unsigned NumParts =
6231 NarrowTy.isVector() ? SrcTy.getNumElements() / NarrowTy.getNumElements()
6232 : SrcTy.getNumElements();
6233
6234 extractParts(Reg: SrcReg, Ty: NarrowTy, NumParts, VRegs&: SplitSrcs, MIRBuilder, MRI);
6235 if (NarrowTy.isScalar()) {
6236 if (DstTy != NarrowTy)
6237 return UnableToLegalize; // FIXME: handle implicit extensions.
6238
6239 if (isPowerOf2_32(Value: NumParts)) {
6240 // Generate a tree of scalar operations to reduce the critical path.
6241 SmallVector<Register> PartialResults;
6242 unsigned NumPartsLeft = NumParts;
6243 while (NumPartsLeft > 1) {
6244 for (unsigned Idx = 0; Idx < NumPartsLeft - 1; Idx += 2) {
6245 PartialResults.emplace_back(
6246 Args: MIRBuilder
6247 .buildInstr(Opc: ScalarOpc, DstOps: {NarrowTy},
6248 SrcOps: {SplitSrcs[Idx], SplitSrcs[Idx + 1]},
6249 Flags: MI.getFlags())
6250 .getReg(Idx: 0));
6251 }
6252 SplitSrcs = PartialResults;
6253 PartialResults.clear();
6254 NumPartsLeft = SplitSrcs.size();
6255 }
6256 assert(SplitSrcs.size() == 1);
6257 MIRBuilder.buildCopy(Res: DstReg, Op: SplitSrcs[0]);
6258 MI.eraseFromParent();
6259 return Legalized;
6260 }
6261 // If we can't generate a tree, then just do sequential operations.
6262 Register Acc = SplitSrcs[0];
6263 for (unsigned Idx = 1; Idx < NumParts; ++Idx)
6264 Acc = MIRBuilder
6265 .buildInstr(Opc: ScalarOpc, DstOps: {NarrowTy}, SrcOps: {Acc, SplitSrcs[Idx]},
6266 Flags: MI.getFlags())
6267 .getReg(Idx: 0);
6268 MIRBuilder.buildCopy(Res: DstReg, Op: Acc);
6269 MI.eraseFromParent();
6270 return Legalized;
6271 }
6272 SmallVector<Register> PartialReductions;
6273 for (unsigned Part = 0; Part < NumParts; ++Part) {
6274 PartialReductions.push_back(Elt: MIRBuilder
6275 .buildInstr(Opc: RdxMI.getOpcode(), DstOps: {DstTy},
6276 SrcOps: {SplitSrcs[Part]},
6277 Flags: MI.getFlags())
6278 .getReg(Idx: 0));
6279 }
6280
6281 // If the types involved are powers of 2, we can generate intermediate vector
6282 // ops, before generating a final reduction operation.
6283 if (isPowerOf2_32(Value: SrcTy.getNumElements()) &&
6284 isPowerOf2_32(Value: NarrowTy.getNumElements())) {
6285 return tryNarrowPow2Reduction(MI, SrcReg, SrcTy, NarrowTy, ScalarOpc);
6286 }
6287
6288 Register Acc = PartialReductions[0];
6289 for (unsigned Part = 1; Part < NumParts; ++Part) {
6290 if (Part == NumParts - 1) {
6291 MIRBuilder.buildInstr(Opc: ScalarOpc, DstOps: {DstReg}, SrcOps: {Acc, PartialReductions[Part]},
6292 Flags: MI.getFlags());
6293 } else {
6294 Acc = MIRBuilder
6295 .buildInstr(Opc: ScalarOpc, DstOps: {DstTy}, SrcOps: {Acc, PartialReductions[Part]},
6296 Flags: MI.getFlags())
6297 .getReg(Idx: 0);
6298 }
6299 }
6300 MI.eraseFromParent();
6301 return Legalized;
6302}
6303
6304LegalizerHelper::LegalizeResult
6305LegalizerHelper::fewerElementsVectorSeqReductions(MachineInstr &MI,
6306 unsigned int TypeIdx,
6307 LLT NarrowTy) {
6308 auto [DstReg, DstTy, ScalarReg, ScalarTy, SrcReg, SrcTy] =
6309 MI.getFirst3RegLLTs();
6310 if (!NarrowTy.isScalar() || TypeIdx != 2 || DstTy != ScalarTy ||
6311 DstTy != NarrowTy)
6312 return UnableToLegalize;
6313
6314 assert((MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FADD ||
6315 MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FMUL) &&
6316 "Unexpected vecreduce opcode");
6317 unsigned ScalarOpc = MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FADD
6318 ? TargetOpcode::G_FADD
6319 : TargetOpcode::G_FMUL;
6320
6321 SmallVector<Register> SplitSrcs;
6322 unsigned NumParts = SrcTy.getNumElements();
6323 extractParts(Reg: SrcReg, Ty: NarrowTy, NumParts, VRegs&: SplitSrcs, MIRBuilder, MRI);
6324 Register Acc = ScalarReg;
6325 for (unsigned i = 0; i < NumParts; i++)
6326 Acc = MIRBuilder
6327 .buildInstr(Opc: ScalarOpc, DstOps: {NarrowTy}, SrcOps: {Acc, SplitSrcs[i]},
6328 Flags: MI.getFlags())
6329 .getReg(Idx: 0);
6330
6331 MIRBuilder.buildCopy(Res: DstReg, Op: Acc);
6332 MI.eraseFromParent();
6333 return Legalized;
6334}
6335
6336LegalizerHelper::LegalizeResult
6337LegalizerHelper::tryNarrowPow2Reduction(MachineInstr &MI, Register SrcReg,
6338 LLT SrcTy, LLT NarrowTy,
6339 unsigned ScalarOpc) {
6340 SmallVector<Register> SplitSrcs;
6341 // Split the sources into NarrowTy size pieces.
6342 extractParts(Reg: SrcReg, Ty: NarrowTy,
6343 NumParts: SrcTy.getNumElements() / NarrowTy.getNumElements(), VRegs&: SplitSrcs,
6344 MIRBuilder, MRI);
6345 // We're going to do a tree reduction using vector operations until we have
6346 // one NarrowTy size value left.
6347 while (SplitSrcs.size() > 1) {
6348 SmallVector<Register> PartialRdxs;
6349 for (unsigned Idx = 0; Idx < SplitSrcs.size()-1; Idx += 2) {
6350 Register LHS = SplitSrcs[Idx];
6351 Register RHS = SplitSrcs[Idx + 1];
6352 // Create the intermediate vector op.
6353 Register Res =
6354 MIRBuilder
6355 .buildInstr(Opc: ScalarOpc, DstOps: {NarrowTy}, SrcOps: {LHS, RHS}, Flags: MI.getFlags())
6356 .getReg(Idx: 0);
6357 PartialRdxs.push_back(Elt: Res);
6358 }
6359 SplitSrcs = std::move(PartialRdxs);
6360 }
6361 // Finally generate the requested NarrowTy based reduction.
6362 Observer.changingInstr(MI);
6363 MI.getOperand(i: 1).setReg(SplitSrcs[0]);
6364 Observer.changedInstr(MI);
6365 return Legalized;
6366}
6367
6368LegalizerHelper::LegalizeResult
6369LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt,
6370 const LLT HalfTy, const LLT AmtTy) {
6371
6372 Register InL = MRI.createGenericVirtualRegister(Ty: HalfTy);
6373 Register InH = MRI.createGenericVirtualRegister(Ty: HalfTy);
6374 MIRBuilder.buildUnmerge(Res: {InL, InH}, Op: MI.getOperand(i: 1));
6375
6376 if (Amt.isZero()) {
6377 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0), Ops: {InL, InH});
6378 MI.eraseFromParent();
6379 return Legalized;
6380 }
6381
6382 LLT NVT = HalfTy;
6383 unsigned NVTBits = HalfTy.getSizeInBits();
6384 unsigned VTBits = 2 * NVTBits;
6385
6386 SrcOp Lo(Register(0)), Hi(Register(0));
6387 if (MI.getOpcode() == TargetOpcode::G_SHL) {
6388 if (Amt.ugt(RHS: VTBits)) {
6389 Lo = Hi = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6390 } else if (Amt.ugt(RHS: NVTBits)) {
6391 Lo = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6392 Hi = MIRBuilder.buildShl(Dst: NVT, Src0: InL,
6393 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt - NVTBits));
6394 } else if (Amt == NVTBits) {
6395 Lo = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6396 Hi = InL;
6397 } else {
6398 Lo = MIRBuilder.buildShl(Dst: NVT, Src0: InL, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt));
6399 auto OrLHS =
6400 MIRBuilder.buildShl(Dst: NVT, Src0: InH, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt));
6401 auto OrRHS = MIRBuilder.buildLShr(
6402 Dst: NVT, Src0: InL, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: -Amt + NVTBits));
6403 Hi = MIRBuilder.buildOr(Dst: NVT, Src0: OrLHS, Src1: OrRHS);
6404 }
6405 } else if (MI.getOpcode() == TargetOpcode::G_LSHR) {
6406 if (Amt.ugt(RHS: VTBits)) {
6407 Lo = Hi = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6408 } else if (Amt.ugt(RHS: NVTBits)) {
6409 Lo = MIRBuilder.buildLShr(Dst: NVT, Src0: InH,
6410 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt - NVTBits));
6411 Hi = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6412 } else if (Amt == NVTBits) {
6413 Lo = InH;
6414 Hi = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6415 } else {
6416 auto ShiftAmtConst = MIRBuilder.buildConstant(Res: AmtTy, Val: Amt);
6417
6418 auto OrLHS = MIRBuilder.buildLShr(Dst: NVT, Src0: InL, Src1: ShiftAmtConst);
6419 auto OrRHS = MIRBuilder.buildShl(
6420 Dst: NVT, Src0: InH, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: -Amt + NVTBits));
6421
6422 Lo = MIRBuilder.buildOr(Dst: NVT, Src0: OrLHS, Src1: OrRHS);
6423 Hi = MIRBuilder.buildLShr(Dst: NVT, Src0: InH, Src1: ShiftAmtConst);
6424 }
6425 } else {
6426 if (Amt.ugt(RHS: VTBits)) {
6427 Hi = Lo = MIRBuilder.buildAShr(
6428 Dst: NVT, Src0: InH, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: NVTBits - 1));
6429 } else if (Amt.ugt(RHS: NVTBits)) {
6430 Lo = MIRBuilder.buildAShr(Dst: NVT, Src0: InH,
6431 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt - NVTBits));
6432 Hi = MIRBuilder.buildAShr(Dst: NVT, Src0: InH,
6433 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: NVTBits - 1));
6434 } else if (Amt == NVTBits) {
6435 Lo = InH;
6436 Hi = MIRBuilder.buildAShr(Dst: NVT, Src0: InH,
6437 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: NVTBits - 1));
6438 } else {
6439 auto ShiftAmtConst = MIRBuilder.buildConstant(Res: AmtTy, Val: Amt);
6440
6441 auto OrLHS = MIRBuilder.buildLShr(Dst: NVT, Src0: InL, Src1: ShiftAmtConst);
6442 auto OrRHS = MIRBuilder.buildShl(
6443 Dst: NVT, Src0: InH, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: -Amt + NVTBits));
6444
6445 Lo = MIRBuilder.buildOr(Dst: NVT, Src0: OrLHS, Src1: OrRHS);
6446 Hi = MIRBuilder.buildAShr(Dst: NVT, Src0: InH, Src1: ShiftAmtConst);
6447 }
6448 }
6449
6450 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0), Ops: {Lo, Hi});
6451 MI.eraseFromParent();
6452
6453 return Legalized;
6454}
6455
6456LegalizerHelper::LegalizeResult
6457LegalizerHelper::narrowScalarShift(MachineInstr &MI, unsigned TypeIdx,
6458 LLT RequestedTy) {
6459 if (TypeIdx == 1) {
6460 Observer.changingInstr(MI);
6461 narrowScalarSrc(MI, NarrowTy: RequestedTy, OpIdx: 2);
6462 Observer.changedInstr(MI);
6463 return Legalized;
6464 }
6465
6466 Register DstReg = MI.getOperand(i: 0).getReg();
6467 LLT DstTy = MRI.getType(Reg: DstReg);
6468 if (DstTy.isVector())
6469 return UnableToLegalize;
6470
6471 Register Amt = MI.getOperand(i: 2).getReg();
6472 LLT ShiftAmtTy = MRI.getType(Reg: Amt);
6473 const unsigned DstEltSize = DstTy.getScalarSizeInBits();
6474 if (DstEltSize % 2 != 0)
6475 return UnableToLegalize;
6476
6477 // Check if we should use multi-way splitting instead of recursive binary
6478 // splitting.
6479 //
6480 // Multi-way splitting directly decomposes wide shifts (e.g., 128-bit ->
6481 // 4×32-bit) in a single legalization step, avoiding the recursive overhead
6482 // and dependency chains created by usual binary splitting approach
6483 // (128->64->32).
6484 //
6485 // The >= 8 parts threshold ensures we only use this optimization when binary
6486 // splitting would require multiple recursive passes, avoiding overhead for
6487 // simple 2-way splits where binary approach is sufficient.
6488 if (RequestedTy.isValid() && RequestedTy.isScalar() &&
6489 DstEltSize % RequestedTy.getSizeInBits() == 0) {
6490 const unsigned NumParts = DstEltSize / RequestedTy.getSizeInBits();
6491 // Use multiway if we have 8 or more parts (i.e., would need 3+ recursive
6492 // steps).
6493 if (NumParts >= 8)
6494 return narrowScalarShiftMultiway(MI, TargetTy: RequestedTy);
6495 }
6496
6497 // Fall back to binary splitting:
6498 // Ignore the input type. We can only go to exactly half the size of the
6499 // input. If that isn't small enough, the resulting pieces will be further
6500 // legalized.
6501 const unsigned NewBitSize = DstEltSize / 2;
6502 const LLT HalfTy = DstTy.getScalarType().changeElementSize(NewEltSize: NewBitSize);
6503 const LLT CondTy = LLT::integer(SizeInBits: 1);
6504
6505 if (auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: Amt, MRI)) {
6506 return narrowScalarShiftByConstant(MI, Amt: VRegAndVal->Value, HalfTy,
6507 AmtTy: ShiftAmtTy);
6508 }
6509
6510 // TODO: Expand with known bits.
6511
6512 // Handle the fully general expansion by an unknown amount.
6513 auto NewBits = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: NewBitSize);
6514
6515 Register InL = MRI.createGenericVirtualRegister(Ty: HalfTy);
6516 Register InH = MRI.createGenericVirtualRegister(Ty: HalfTy);
6517 MIRBuilder.buildUnmerge(Res: {InL, InH}, Op: MI.getOperand(i: 1));
6518
6519 auto AmtExcess = MIRBuilder.buildSub(Dst: ShiftAmtTy, Src0: Amt, Src1: NewBits);
6520 auto AmtLack = MIRBuilder.buildSub(Dst: ShiftAmtTy, Src0: NewBits, Src1: Amt);
6521
6522 auto Zero = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: 0);
6523 auto IsShort = MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_ULT, Res: CondTy, Op0: Amt, Op1: NewBits);
6524 auto IsZero = MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: CondTy, Op0: Amt, Op1: Zero);
6525
6526 Register ResultRegs[2];
6527 switch (MI.getOpcode()) {
6528 case TargetOpcode::G_SHL: {
6529 // Short: ShAmt < NewBitSize
6530 auto LoS = MIRBuilder.buildShl(Dst: HalfTy, Src0: InL, Src1: Amt);
6531
6532 auto LoOr = MIRBuilder.buildLShr(Dst: HalfTy, Src0: InL, Src1: AmtLack);
6533 auto HiOr = MIRBuilder.buildShl(Dst: HalfTy, Src0: InH, Src1: Amt);
6534 auto HiS = MIRBuilder.buildOr(Dst: HalfTy, Src0: LoOr, Src1: HiOr);
6535
6536 // Long: ShAmt >= NewBitSize
6537 auto LoL = MIRBuilder.buildConstant(Res: HalfTy, Val: 0); // Lo part is zero.
6538 auto HiL = MIRBuilder.buildShl(Dst: HalfTy, Src0: InL, Src1: AmtExcess); // Hi from Lo part.
6539
6540 auto Lo = MIRBuilder.buildSelect(Res: HalfTy, Tst: IsShort, Op0: LoS, Op1: LoL);
6541 auto Hi = MIRBuilder.buildSelect(
6542 Res: HalfTy, Tst: IsZero, Op0: InH, Op1: MIRBuilder.buildSelect(Res: HalfTy, Tst: IsShort, Op0: HiS, Op1: HiL));
6543
6544 ResultRegs[0] = Lo.getReg(Idx: 0);
6545 ResultRegs[1] = Hi.getReg(Idx: 0);
6546 break;
6547 }
6548 case TargetOpcode::G_LSHR:
6549 case TargetOpcode::G_ASHR: {
6550 // Short: ShAmt < NewBitSize
6551 auto HiS = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {HalfTy}, SrcOps: {InH, Amt});
6552
6553 auto LoOr = MIRBuilder.buildLShr(Dst: HalfTy, Src0: InL, Src1: Amt);
6554 auto HiOr = MIRBuilder.buildShl(Dst: HalfTy, Src0: InH, Src1: AmtLack);
6555 auto LoS = MIRBuilder.buildOr(Dst: HalfTy, Src0: LoOr, Src1: HiOr);
6556
6557 // Long: ShAmt >= NewBitSize
6558 MachineInstrBuilder HiL;
6559 if (MI.getOpcode() == TargetOpcode::G_LSHR) {
6560 HiL = MIRBuilder.buildConstant(Res: HalfTy, Val: 0); // Hi part is zero.
6561 } else {
6562 auto ShiftAmt = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: NewBitSize - 1);
6563 HiL = MIRBuilder.buildAShr(Dst: HalfTy, Src0: InH, Src1: ShiftAmt); // Sign of Hi part.
6564 }
6565 auto LoL = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {HalfTy},
6566 SrcOps: {InH, AmtExcess}); // Lo from Hi part.
6567
6568 auto Lo = MIRBuilder.buildSelect(
6569 Res: HalfTy, Tst: IsZero, Op0: InL, Op1: MIRBuilder.buildSelect(Res: HalfTy, Tst: IsShort, Op0: LoS, Op1: LoL));
6570
6571 auto Hi = MIRBuilder.buildSelect(Res: HalfTy, Tst: IsShort, Op0: HiS, Op1: HiL);
6572
6573 ResultRegs[0] = Lo.getReg(Idx: 0);
6574 ResultRegs[1] = Hi.getReg(Idx: 0);
6575 break;
6576 }
6577 default:
6578 llvm_unreachable("not a shift");
6579 }
6580
6581 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: ResultRegs);
6582 MI.eraseFromParent();
6583 return Legalized;
6584}
6585
6586Register LegalizerHelper::buildConstantShiftPart(unsigned Opcode,
6587 unsigned PartIdx,
6588 unsigned NumParts,
6589 ArrayRef<Register> SrcParts,
6590 const ShiftParams &Params,
6591 LLT TargetTy, LLT ShiftAmtTy) {
6592 auto WordShiftConst = getIConstantVRegVal(VReg: Params.WordShift, MRI);
6593 auto BitShiftConst = getIConstantVRegVal(VReg: Params.BitShift, MRI);
6594 assert(WordShiftConst && BitShiftConst && "Expected constants");
6595
6596 const unsigned ShiftWords = WordShiftConst->getZExtValue();
6597 const unsigned ShiftBits = BitShiftConst->getZExtValue();
6598 const bool NeedsInterWordShift = ShiftBits != 0;
6599
6600 switch (Opcode) {
6601 case TargetOpcode::G_SHL: {
6602 // Data moves from lower indices to higher indices
6603 // If this part would come from a source beyond our range, it's zero
6604 if (PartIdx < ShiftWords)
6605 return Params.Zero;
6606
6607 unsigned SrcIdx = PartIdx - ShiftWords;
6608 if (!NeedsInterWordShift)
6609 return SrcParts[SrcIdx];
6610
6611 // Combine shifted main part with carry from previous part
6612 auto Hi = MIRBuilder.buildShl(Dst: TargetTy, Src0: SrcParts[SrcIdx], Src1: Params.BitShift);
6613 if (SrcIdx > 0) {
6614 auto Lo = MIRBuilder.buildLShr(Dst: TargetTy, Src0: SrcParts[SrcIdx - 1],
6615 Src1: Params.InvBitShift);
6616 return MIRBuilder.buildOr(Dst: TargetTy, Src0: Hi, Src1: Lo).getReg(Idx: 0);
6617 }
6618 return Hi.getReg(Idx: 0);
6619 }
6620
6621 case TargetOpcode::G_LSHR: {
6622 unsigned SrcIdx = PartIdx + ShiftWords;
6623 if (SrcIdx >= NumParts)
6624 return Params.Zero;
6625 if (!NeedsInterWordShift)
6626 return SrcParts[SrcIdx];
6627
6628 // Combine shifted main part with carry from next part
6629 auto Lo = MIRBuilder.buildLShr(Dst: TargetTy, Src0: SrcParts[SrcIdx], Src1: Params.BitShift);
6630 if (SrcIdx + 1 < NumParts) {
6631 auto Hi = MIRBuilder.buildShl(Dst: TargetTy, Src0: SrcParts[SrcIdx + 1],
6632 Src1: Params.InvBitShift);
6633 return MIRBuilder.buildOr(Dst: TargetTy, Src0: Lo, Src1: Hi).getReg(Idx: 0);
6634 }
6635 return Lo.getReg(Idx: 0);
6636 }
6637
6638 case TargetOpcode::G_ASHR: {
6639 // Like LSHR but preserves sign bit
6640 unsigned SrcIdx = PartIdx + ShiftWords;
6641 if (SrcIdx >= NumParts)
6642 return Params.SignBit;
6643 if (!NeedsInterWordShift)
6644 return SrcParts[SrcIdx];
6645
6646 // Only the original MSB part uses arithmetic shift to preserve sign. All
6647 // other parts use logical shift since they're just moving data bits.
6648 auto Lo =
6649 (SrcIdx == NumParts - 1)
6650 ? MIRBuilder.buildAShr(Dst: TargetTy, Src0: SrcParts[SrcIdx], Src1: Params.BitShift)
6651 : MIRBuilder.buildLShr(Dst: TargetTy, Src0: SrcParts[SrcIdx], Src1: Params.BitShift);
6652 Register HiSrc =
6653 (SrcIdx + 1 < NumParts) ? SrcParts[SrcIdx + 1] : Params.SignBit;
6654 auto Hi = MIRBuilder.buildShl(Dst: TargetTy, Src0: HiSrc, Src1: Params.InvBitShift);
6655 return MIRBuilder.buildOr(Dst: TargetTy, Src0: Lo, Src1: Hi).getReg(Idx: 0);
6656 }
6657
6658 default:
6659 llvm_unreachable("not a shift");
6660 }
6661}
6662
6663Register LegalizerHelper::buildVariableShiftPart(unsigned Opcode,
6664 Register MainOperand,
6665 Register ShiftAmt,
6666 LLT TargetTy,
6667 Register CarryOperand) {
6668 // This helper generates a single output part for variable shifts by combining
6669 // the main operand (shifted by BitShift) with carry bits from an adjacent
6670 // part.
6671
6672 // For G_ASHR, individual parts don't have their own sign bit, only the
6673 // complete value does. So we use LSHR for the main operand shift in ASHR
6674 // context.
6675 unsigned MainOpcode = (Opcode == TargetOpcode::G_ASHR)
6676 ? static_cast<unsigned>(TargetOpcode::G_LSHR)
6677 : Opcode;
6678
6679 // Perform the primary shift on the main operand
6680 Register MainShifted =
6681 MIRBuilder.buildInstr(Opc: MainOpcode, DstOps: {TargetTy}, SrcOps: {MainOperand, ShiftAmt})
6682 .getReg(Idx: 0);
6683
6684 // No carry operand available
6685 if (!CarryOperand.isValid())
6686 return MainShifted;
6687
6688 // If BitShift is 0 (word-aligned shift), no inter-word bit movement occurs,
6689 // so carry bits aren't needed.
6690 LLT ShiftAmtTy = MRI.getType(Reg: ShiftAmt);
6691 auto ZeroConst = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: 0);
6692 LLT BoolTy = LLT::integer(SizeInBits: 1);
6693 auto IsZeroBitShift =
6694 MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: BoolTy, Op0: ShiftAmt, Op1: ZeroConst);
6695
6696 // Extract bits from the adjacent part that will "carry over" into this part.
6697 // The carry direction is opposite to the main shift direction, so we can
6698 // align the two shifted values before combining them with OR.
6699
6700 // Determine the carry shift opcode (opposite direction)
6701 unsigned CarryOpcode = (Opcode == TargetOpcode::G_SHL) ? TargetOpcode::G_LSHR
6702 : TargetOpcode::G_SHL;
6703
6704 // Calculate inverse shift amount: BitWidth - ShiftAmt
6705 auto TargetBitsConst =
6706 MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetTy.getScalarSizeInBits());
6707 auto InvShiftAmt = MIRBuilder.buildSub(Dst: ShiftAmtTy, Src0: TargetBitsConst, Src1: ShiftAmt);
6708
6709 // Shift the carry operand
6710 Register CarryBits =
6711 MIRBuilder
6712 .buildInstr(Opc: CarryOpcode, DstOps: {TargetTy}, SrcOps: {CarryOperand, InvShiftAmt})
6713 .getReg(Idx: 0);
6714
6715 // If BitShift is 0, don't include carry bits (InvShiftAmt would equal
6716 // TargetBits which would be poison for the individual carry shift operation).
6717 auto ZeroReg = MIRBuilder.buildConstant(Res: TargetTy, Val: 0);
6718 Register SafeCarryBits =
6719 MIRBuilder.buildSelect(Res: TargetTy, Tst: IsZeroBitShift, Op0: ZeroReg, Op1: CarryBits)
6720 .getReg(Idx: 0);
6721
6722 // Combine the main shifted part with the carry bits
6723 return MIRBuilder.buildOr(Dst: TargetTy, Src0: MainShifted, Src1: SafeCarryBits).getReg(Idx: 0);
6724}
6725
6726LegalizerHelper::LegalizeResult
6727LegalizerHelper::narrowScalarShiftByConstantMultiway(MachineInstr &MI,
6728 const APInt &Amt,
6729 LLT TargetTy,
6730 LLT ShiftAmtTy) {
6731 // Any wide shift can be decomposed into WordShift + BitShift components.
6732 // When shift amount is known constant, directly compute the decomposition
6733 // values and generate constant registers.
6734 Register DstReg = MI.getOperand(i: 0).getReg();
6735 Register SrcReg = MI.getOperand(i: 1).getReg();
6736 LLT DstTy = MRI.getType(Reg: DstReg);
6737
6738 const unsigned DstBits = DstTy.getScalarSizeInBits();
6739 const unsigned TargetBits = TargetTy.getScalarSizeInBits();
6740 const unsigned NumParts = DstBits / TargetBits;
6741
6742 assert(DstBits % TargetBits == 0 && "Target type must evenly divide source");
6743
6744 // When the shift amount is known at compile time, we just calculate which
6745 // source parts contribute to each output part.
6746
6747 SmallVector<Register, 8> SrcParts;
6748 extractParts(Reg: SrcReg, Ty: TargetTy, NumParts, VRegs&: SrcParts, MIRBuilder, MRI);
6749
6750 if (Amt.isZero()) {
6751 // No shift needed, just copy
6752 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: SrcParts);
6753 MI.eraseFromParent();
6754 return Legalized;
6755 }
6756
6757 ShiftParams Params;
6758 const unsigned ShiftWords = Amt.getZExtValue() / TargetBits;
6759 const unsigned ShiftBits = Amt.getZExtValue() % TargetBits;
6760
6761 // Generate constants and values needed by all shift types
6762 Params.WordShift = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: ShiftWords).getReg(Idx: 0);
6763 Params.BitShift = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: ShiftBits).getReg(Idx: 0);
6764 Params.InvBitShift =
6765 MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBits - ShiftBits).getReg(Idx: 0);
6766 Params.Zero = MIRBuilder.buildConstant(Res: TargetTy, Val: 0).getReg(Idx: 0);
6767
6768 // For ASHR, we need the sign-extended value to fill shifted-out positions
6769 if (MI.getOpcode() == TargetOpcode::G_ASHR)
6770 Params.SignBit =
6771 MIRBuilder
6772 .buildAShr(Dst: TargetTy, Src0: SrcParts[SrcParts.size() - 1],
6773 Src1: MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBits - 1))
6774 .getReg(Idx: 0);
6775
6776 SmallVector<Register, 8> DstParts(NumParts);
6777 for (unsigned I = 0; I < NumParts; ++I)
6778 DstParts[I] = buildConstantShiftPart(Opcode: MI.getOpcode(), PartIdx: I, NumParts, SrcParts,
6779 Params, TargetTy, ShiftAmtTy);
6780
6781 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstParts);
6782 MI.eraseFromParent();
6783 return Legalized;
6784}
6785
6786LegalizerHelper::LegalizeResult
6787LegalizerHelper::narrowScalarShiftMultiway(MachineInstr &MI, LLT TargetTy) {
6788 Register DstReg = MI.getOperand(i: 0).getReg();
6789 Register SrcReg = MI.getOperand(i: 1).getReg();
6790 Register AmtReg = MI.getOperand(i: 2).getReg();
6791 LLT DstTy = MRI.getType(Reg: DstReg);
6792 LLT ShiftAmtTy = MRI.getType(Reg: AmtReg);
6793
6794 const unsigned DstBits = DstTy.getScalarSizeInBits();
6795 const unsigned TargetBits = TargetTy.getScalarSizeInBits();
6796 const unsigned NumParts = DstBits / TargetBits;
6797
6798 assert(DstBits % TargetBits == 0 && "Target type must evenly divide source");
6799 assert(isPowerOf2_32(TargetBits) && "Target bit width must be power of 2");
6800
6801 // If the shift amount is known at compile time, we can use direct indexing
6802 // instead of generating select chains in the general case.
6803 if (auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: AmtReg, MRI))
6804 return narrowScalarShiftByConstantMultiway(MI, Amt: VRegAndVal->Value, TargetTy,
6805 ShiftAmtTy);
6806
6807 // For runtime-variable shift amounts, we must generate a more complex
6808 // sequence that handles all possible shift values using select chains.
6809
6810 // Split the input into target-sized pieces
6811 SmallVector<Register, 8> SrcParts;
6812 extractParts(Reg: SrcReg, Ty: TargetTy, NumParts, VRegs&: SrcParts, MIRBuilder, MRI);
6813
6814 // Shifting by zero should be a no-op.
6815 auto ZeroAmtConst = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: 0);
6816 LLT BoolTy = LLT::integer(SizeInBits: 1);
6817 auto IsZeroShift =
6818 MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: BoolTy, Op0: AmtReg, Op1: ZeroAmtConst);
6819
6820 // Any wide shift can be decomposed into two components:
6821 // 1. WordShift: number of complete target-sized words to shift
6822 // 2. BitShift: number of bits to shift within each word
6823 //
6824 // Example: 128-bit >> 50 with 32-bit target:
6825 // WordShift = 50 / 32 = 1 (shift right by 1 complete word)
6826 // BitShift = 50 % 32 = 18 (shift each word right by 18 bits)
6827 unsigned TargetBitsLog2 = Log2_32(Value: TargetBits);
6828 auto TargetBitsLog2Const =
6829 MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBitsLog2);
6830 auto TargetBitsMask = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBits - 1);
6831
6832 Register WordShift =
6833 MIRBuilder.buildLShr(Dst: ShiftAmtTy, Src0: AmtReg, Src1: TargetBitsLog2Const).getReg(Idx: 0);
6834 Register BitShift =
6835 MIRBuilder.buildAnd(Dst: ShiftAmtTy, Src0: AmtReg, Src1: TargetBitsMask).getReg(Idx: 0);
6836
6837 // Fill values:
6838 // - SHL/LSHR: fill with zeros
6839 // - ASHR: fill with sign-extended MSB
6840 Register ZeroReg = MIRBuilder.buildConstant(Res: TargetTy, Val: 0).getReg(Idx: 0);
6841
6842 Register FillValue;
6843 if (MI.getOpcode() == TargetOpcode::G_ASHR) {
6844 auto TargetBitsMinusOneConst =
6845 MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBits - 1);
6846 FillValue = MIRBuilder
6847 .buildAShr(Dst: TargetTy, Src0: SrcParts[NumParts - 1],
6848 Src1: TargetBitsMinusOneConst)
6849 .getReg(Idx: 0);
6850 } else {
6851 FillValue = ZeroReg;
6852 }
6853
6854 SmallVector<Register, 8> DstParts(NumParts);
6855
6856 // For each output part, generate a select chain that chooses the correct
6857 // result based on the runtime WordShift value. This handles all possible
6858 // word shift amounts by pre-calculating what each would produce.
6859 for (unsigned I = 0; I < NumParts; ++I) {
6860 // Initialize with appropriate default value for this shift type
6861 Register InBoundsResult = FillValue;
6862
6863 // clang-format off
6864 // Build a branchless select chain by pre-computing results for all possible
6865 // WordShift values (0 to NumParts-1). Each iteration nests a new select:
6866 //
6867 // K=0: select(WordShift==0, result0, FillValue)
6868 // K=1: select(WordShift==1, result1, select(WordShift==0, result0, FillValue))
6869 // K=2: select(WordShift==2, result2, select(WordShift==1, result1, select(...)))
6870 // clang-format on
6871 for (unsigned K = 0; K < NumParts; ++K) {
6872 auto WordShiftKConst = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: K);
6873 auto IsWordShiftK = MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: BoolTy,
6874 Op0: WordShift, Op1: WordShiftKConst);
6875
6876 // Calculate source indices for this word shift
6877 //
6878 // For 4-part 128-bit value with K=1 word shift:
6879 // SHL: [3][2][1][0] << K => [2][1][0][Z]
6880 // -> (MainIdx = I-K, CarryIdx = I-K-1)
6881 // LSHR: [3][2][1][0] >> K => [Z][3][2][1]
6882 // -> (MainIdx = I+K, CarryIdx = I+K+1)
6883 int MainSrcIdx;
6884 int CarrySrcIdx; // Index for the word that provides the carried-in bits.
6885
6886 switch (MI.getOpcode()) {
6887 case TargetOpcode::G_SHL:
6888 MainSrcIdx = (int)I - (int)K;
6889 CarrySrcIdx = MainSrcIdx - 1;
6890 break;
6891 case TargetOpcode::G_LSHR:
6892 case TargetOpcode::G_ASHR:
6893 MainSrcIdx = (int)I + (int)K;
6894 CarrySrcIdx = MainSrcIdx + 1;
6895 break;
6896 default:
6897 llvm_unreachable("Not a shift");
6898 }
6899
6900 // Check bounds and build the result for this word shift
6901 Register ResultForK;
6902 if (MainSrcIdx >= 0 && MainSrcIdx < (int)NumParts) {
6903 Register MainOp = SrcParts[MainSrcIdx];
6904 Register CarryOp;
6905
6906 // Determine carry operand with bounds checking
6907 if (CarrySrcIdx >= 0 && CarrySrcIdx < (int)NumParts)
6908 CarryOp = SrcParts[CarrySrcIdx];
6909 else if (MI.getOpcode() == TargetOpcode::G_ASHR &&
6910 CarrySrcIdx >= (int)NumParts)
6911 CarryOp = FillValue; // Use sign extension
6912
6913 ResultForK = buildVariableShiftPart(Opcode: MI.getOpcode(), MainOperand: MainOp, ShiftAmt: BitShift,
6914 TargetTy, CarryOperand: CarryOp);
6915 } else {
6916 // Out of bounds - use fill value for this k
6917 ResultForK = FillValue;
6918 }
6919
6920 // Select this result if WordShift equals k
6921 InBoundsResult =
6922 MIRBuilder
6923 .buildSelect(Res: TargetTy, Tst: IsWordShiftK, Op0: ResultForK, Op1: InBoundsResult)
6924 .getReg(Idx: 0);
6925 }
6926
6927 // Handle zero-shift special case: if shift is 0, use original input
6928 DstParts[I] =
6929 MIRBuilder
6930 .buildSelect(Res: TargetTy, Tst: IsZeroShift, Op0: SrcParts[I], Op1: InBoundsResult)
6931 .getReg(Idx: 0);
6932 }
6933
6934 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstParts);
6935 MI.eraseFromParent();
6936 return Legalized;
6937}
6938
6939LegalizerHelper::LegalizeResult
6940LegalizerHelper::moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
6941 LLT MoreTy) {
6942 assert(TypeIdx == 0 && "Expecting only Idx 0");
6943
6944 Observer.changingInstr(MI);
6945 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
6946 MachineBasicBlock &OpMBB = *MI.getOperand(i: I + 1).getMBB();
6947 MIRBuilder.setInsertPt(MBB&: OpMBB, II: getPHIIncomingInsertPoint(
6948 MBB&: OpMBB, Reg: MI.getOperand(i: I).getReg(), MRI));
6949 moreElementsVectorSrc(MI, MoreTy, OpIdx: I);
6950 }
6951
6952 MachineBasicBlock &MBB = *MI.getParent();
6953 MIRBuilder.setInsertPt(MBB, II: --MBB.getFirstNonPHI());
6954 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
6955 Observer.changedInstr(MI);
6956 return Legalized;
6957}
6958
6959MachineInstrBuilder LegalizerHelper::getNeutralElementForVecReduce(
6960 unsigned Opcode, MachineIRBuilder &MIRBuilder, LLT Ty) {
6961 assert(Ty.isScalar() && "Expected scalar type to make neutral element for");
6962
6963 switch (Opcode) {
6964 default:
6965 llvm_unreachable(
6966 "getNeutralElementForVecReduce called with invalid opcode!");
6967 case TargetOpcode::G_VECREDUCE_ADD:
6968 case TargetOpcode::G_VECREDUCE_OR:
6969 case TargetOpcode::G_VECREDUCE_XOR:
6970 case TargetOpcode::G_VECREDUCE_UMAX:
6971 return MIRBuilder.buildConstant(Res: Ty, Val: 0);
6972 case TargetOpcode::G_VECREDUCE_MUL:
6973 return MIRBuilder.buildConstant(Res: Ty, Val: 1);
6974 case TargetOpcode::G_VECREDUCE_AND:
6975 case TargetOpcode::G_VECREDUCE_UMIN:
6976 return MIRBuilder.buildConstant(
6977 Res: Ty, Val: APInt::getAllOnes(numBits: Ty.getScalarSizeInBits()));
6978 case TargetOpcode::G_VECREDUCE_SMAX:
6979 return MIRBuilder.buildConstant(
6980 Res: Ty, Val: APInt::getSignedMinValue(numBits: Ty.getSizeInBits()));
6981 case TargetOpcode::G_VECREDUCE_SMIN:
6982 return MIRBuilder.buildConstant(
6983 Res: Ty, Val: APInt::getSignedMaxValue(numBits: Ty.getSizeInBits()));
6984 case TargetOpcode::G_VECREDUCE_FADD:
6985 return MIRBuilder.buildFConstant(Res: Ty, Val: -0.0);
6986 case TargetOpcode::G_VECREDUCE_FMUL:
6987 return MIRBuilder.buildFConstant(Res: Ty, Val: 1.0);
6988 case TargetOpcode::G_VECREDUCE_FMINIMUM:
6989 case TargetOpcode::G_VECREDUCE_FMAXIMUM:
6990 assert(false && "getNeutralElementForVecReduce unimplemented for "
6991 "G_VECREDUCE_FMINIMUM and G_VECREDUCE_FMAXIMUM!");
6992 }
6993 llvm_unreachable("switch expected to return!");
6994}
6995
6996LegalizerHelper::LegalizeResult
6997LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
6998 LLT MoreTy) {
6999 unsigned Opc = MI.getOpcode();
7000 switch (Opc) {
7001 case TargetOpcode::G_IMPLICIT_DEF:
7002 case TargetOpcode::G_LOAD: {
7003 if (TypeIdx != 0)
7004 return UnableToLegalize;
7005 Observer.changingInstr(MI);
7006 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7007 Observer.changedInstr(MI);
7008 return Legalized;
7009 }
7010 case TargetOpcode::G_STORE:
7011 if (TypeIdx != 0)
7012 return UnableToLegalize;
7013 Observer.changingInstr(MI);
7014 moreElementsVectorSrc(MI, MoreTy, OpIdx: 0);
7015 Observer.changedInstr(MI);
7016 return Legalized;
7017 case TargetOpcode::G_AND:
7018 case TargetOpcode::G_OR:
7019 case TargetOpcode::G_XOR:
7020 case TargetOpcode::G_ADD:
7021 case TargetOpcode::G_SUB:
7022 case TargetOpcode::G_MUL:
7023 case TargetOpcode::G_FADD:
7024 case TargetOpcode::G_FSUB:
7025 case TargetOpcode::G_FMUL:
7026 case TargetOpcode::G_FDIV:
7027 case TargetOpcode::G_FCOPYSIGN:
7028 case TargetOpcode::G_UADDSAT:
7029 case TargetOpcode::G_USUBSAT:
7030 case TargetOpcode::G_SADDSAT:
7031 case TargetOpcode::G_SSUBSAT:
7032 case TargetOpcode::G_SMIN:
7033 case TargetOpcode::G_SMAX:
7034 case TargetOpcode::G_UMIN:
7035 case TargetOpcode::G_UMAX:
7036 case TargetOpcode::G_FMINNUM:
7037 case TargetOpcode::G_FMAXNUM:
7038 case TargetOpcode::G_FMINNUM_IEEE:
7039 case TargetOpcode::G_FMAXNUM_IEEE:
7040 case TargetOpcode::G_FMINIMUM:
7041 case TargetOpcode::G_FMAXIMUM:
7042 case TargetOpcode::G_FMINIMUMNUM:
7043 case TargetOpcode::G_FMAXIMUMNUM:
7044 case TargetOpcode::G_STRICT_FADD:
7045 case TargetOpcode::G_STRICT_FSUB:
7046 case TargetOpcode::G_STRICT_FMUL: {
7047 Observer.changingInstr(MI);
7048 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
7049 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
7050 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7051 Observer.changedInstr(MI);
7052 return Legalized;
7053 }
7054 case TargetOpcode::G_SHL:
7055 case TargetOpcode::G_ASHR:
7056 case TargetOpcode::G_LSHR: {
7057 Observer.changingInstr(MI);
7058 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
7059 // The shift operand may have a different scalar type from the source and
7060 // destination operands.
7061 LLT ShiftMoreTy = MoreTy.changeElementType(
7062 NewEltTy: MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getElementType());
7063 moreElementsVectorSrc(MI, MoreTy: ShiftMoreTy, OpIdx: 2);
7064 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7065 Observer.changedInstr(MI);
7066 return Legalized;
7067 }
7068 case TargetOpcode::G_FMA:
7069 case TargetOpcode::G_STRICT_FMA:
7070 case TargetOpcode::G_FSHR:
7071 case TargetOpcode::G_FSHL: {
7072 Observer.changingInstr(MI);
7073 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
7074 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
7075 moreElementsVectorSrc(MI, MoreTy, OpIdx: 3);
7076 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7077 Observer.changedInstr(MI);
7078 return Legalized;
7079 }
7080 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
7081 case TargetOpcode::G_EXTRACT:
7082 if (TypeIdx != 1)
7083 return UnableToLegalize;
7084 Observer.changingInstr(MI);
7085 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
7086 Observer.changedInstr(MI);
7087 return Legalized;
7088 case TargetOpcode::G_INSERT:
7089 case TargetOpcode::G_INSERT_VECTOR_ELT:
7090 case TargetOpcode::G_FREEZE:
7091 case TargetOpcode::G_FNEG:
7092 case TargetOpcode::G_FABS:
7093 case TargetOpcode::G_FSQRT:
7094 case TargetOpcode::G_FCEIL:
7095 case TargetOpcode::G_FFLOOR:
7096 case TargetOpcode::G_FNEARBYINT:
7097 case TargetOpcode::G_FRINT:
7098 case TargetOpcode::G_INTRINSIC_ROUND:
7099 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
7100 case TargetOpcode::G_INTRINSIC_TRUNC:
7101 case TargetOpcode::G_BITREVERSE:
7102 case TargetOpcode::G_BSWAP:
7103 case TargetOpcode::G_FCANONICALIZE:
7104 case TargetOpcode::G_SEXT_INREG:
7105 case TargetOpcode::G_ABS:
7106 case TargetOpcode::G_CTLZ:
7107 case TargetOpcode::G_CTPOP:
7108 if (TypeIdx != 0)
7109 return UnableToLegalize;
7110 Observer.changingInstr(MI);
7111 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
7112 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7113 Observer.changedInstr(MI);
7114 return Legalized;
7115 case TargetOpcode::G_SELECT: {
7116 auto [DstReg, DstTy, CondReg, CondTy] = MI.getFirst2RegLLTs();
7117 if (TypeIdx == 1) {
7118 if (!CondTy.isScalar() ||
7119 DstTy.getElementCount() != MoreTy.getElementCount())
7120 return UnableToLegalize;
7121
7122 // This is turning a scalar select of vectors into a vector
7123 // select. Broadcast the select condition.
7124 auto ShufSplat = MIRBuilder.buildShuffleSplat(Res: MoreTy, Src: CondReg);
7125 Observer.changingInstr(MI);
7126 MI.getOperand(i: 1).setReg(ShufSplat.getReg(Idx: 0));
7127 Observer.changedInstr(MI);
7128 return Legalized;
7129 }
7130
7131 if (CondTy.isVector())
7132 return UnableToLegalize;
7133
7134 Observer.changingInstr(MI);
7135 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
7136 moreElementsVectorSrc(MI, MoreTy, OpIdx: 3);
7137 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7138 Observer.changedInstr(MI);
7139 return Legalized;
7140 }
7141 case TargetOpcode::G_UNMERGE_VALUES:
7142 return UnableToLegalize;
7143 case TargetOpcode::G_PHI:
7144 return moreElementsVectorPhi(MI, TypeIdx, MoreTy);
7145 case TargetOpcode::G_SHUFFLE_VECTOR:
7146 return moreElementsVectorShuffle(MI, TypeIdx, MoreTy);
7147 case TargetOpcode::G_BUILD_VECTOR: {
7148 SmallVector<SrcOp, 8> Elts;
7149 for (auto Op : MI.uses()) {
7150 Elts.push_back(Elt: Op.getReg());
7151 }
7152
7153 for (unsigned i = Elts.size(); i < MoreTy.getNumElements(); ++i) {
7154 Elts.push_back(Elt: MIRBuilder.buildUndef(Res: MoreTy.getScalarType()));
7155 }
7156
7157 MIRBuilder.buildDeleteTrailingVectorElements(
7158 Res: MI.getOperand(i: 0).getReg(), Op0: MIRBuilder.buildInstr(Opc, DstOps: {MoreTy}, SrcOps: Elts));
7159 MI.eraseFromParent();
7160 return Legalized;
7161 }
7162 case TargetOpcode::G_SEXT:
7163 case TargetOpcode::G_ZEXT:
7164 case TargetOpcode::G_ANYEXT:
7165 case TargetOpcode::G_TRUNC:
7166 case TargetOpcode::G_FPTRUNC:
7167 case TargetOpcode::G_FPEXT:
7168 case TargetOpcode::G_FPTOSI:
7169 case TargetOpcode::G_FPTOUI:
7170 case TargetOpcode::G_FPTOSI_SAT:
7171 case TargetOpcode::G_FPTOUI_SAT:
7172 case TargetOpcode::G_SITOFP:
7173 case TargetOpcode::G_UITOFP:
7174 case TargetOpcode::G_TRUNC_SSAT_S:
7175 case TargetOpcode::G_TRUNC_SSAT_U:
7176 case TargetOpcode::G_TRUNC_USAT_U: {
7177 Observer.changingInstr(MI);
7178 LLT SrcExtTy;
7179 LLT DstExtTy;
7180 if (TypeIdx == 0) {
7181 DstExtTy = MoreTy;
7182 SrcExtTy = MoreTy.changeElementType(
7183 NewEltTy: MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getElementType());
7184 } else {
7185 DstExtTy = MoreTy.changeElementType(
7186 NewEltTy: MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getElementType());
7187 SrcExtTy = MoreTy;
7188 }
7189 moreElementsVectorSrc(MI, MoreTy: SrcExtTy, OpIdx: 1);
7190 moreElementsVectorDst(MI, WideTy: DstExtTy, OpIdx: 0);
7191 Observer.changedInstr(MI);
7192 return Legalized;
7193 }
7194 case TargetOpcode::G_ICMP:
7195 case TargetOpcode::G_FCMP: {
7196 if (TypeIdx != 1)
7197 return UnableToLegalize;
7198
7199 Observer.changingInstr(MI);
7200 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
7201 moreElementsVectorSrc(MI, MoreTy, OpIdx: 3);
7202 LLT CondTy = MoreTy.changeVectorElementType(
7203 NewEltTy: MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getElementType());
7204 moreElementsVectorDst(MI, WideTy: CondTy, OpIdx: 0);
7205 Observer.changedInstr(MI);
7206 return Legalized;
7207 }
7208 case TargetOpcode::G_BITCAST: {
7209 if (TypeIdx != 0)
7210 return UnableToLegalize;
7211
7212 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
7213 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7214
7215 unsigned coefficient = SrcTy.getNumElements() * MoreTy.getNumElements();
7216 if (coefficient % DstTy.getNumElements() != 0)
7217 return UnableToLegalize;
7218
7219 coefficient = coefficient / DstTy.getNumElements();
7220
7221 LLT NewTy = SrcTy.changeElementCount(
7222 EC: ElementCount::get(MinVal: coefficient, Scalable: MoreTy.isScalable()));
7223 Observer.changingInstr(MI);
7224 moreElementsVectorSrc(MI, MoreTy: NewTy, OpIdx: 1);
7225 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7226 Observer.changedInstr(MI);
7227 return Legalized;
7228 }
7229 case TargetOpcode::G_VECREDUCE_FADD:
7230 case TargetOpcode::G_VECREDUCE_FMUL:
7231 case TargetOpcode::G_VECREDUCE_ADD:
7232 case TargetOpcode::G_VECREDUCE_MUL:
7233 case TargetOpcode::G_VECREDUCE_AND:
7234 case TargetOpcode::G_VECREDUCE_OR:
7235 case TargetOpcode::G_VECREDUCE_XOR:
7236 case TargetOpcode::G_VECREDUCE_SMAX:
7237 case TargetOpcode::G_VECREDUCE_SMIN:
7238 case TargetOpcode::G_VECREDUCE_UMAX:
7239 case TargetOpcode::G_VECREDUCE_UMIN: {
7240 LLT OrigTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
7241 MachineOperand &MO = MI.getOperand(i: 1);
7242 auto NewVec = MIRBuilder.buildPadVectorWithUndefElements(Res: MoreTy, Op0: MO);
7243 auto NeutralElement = getNeutralElementForVecReduce(
7244 Opcode: MI.getOpcode(), MIRBuilder, Ty: MoreTy.getElementType());
7245
7246 LLT IdxTy(TLI.getVectorIdxLLT(DL: MIRBuilder.getDataLayout()));
7247 for (size_t i = OrigTy.getNumElements(), e = MoreTy.getNumElements();
7248 i != e; i++) {
7249 auto Idx = MIRBuilder.buildConstant(Res: IdxTy, Val: i);
7250 NewVec = MIRBuilder.buildInsertVectorElement(Res: MoreTy, Val: NewVec,
7251 Elt: NeutralElement, Idx);
7252 }
7253
7254 Observer.changingInstr(MI);
7255 MO.setReg(NewVec.getReg(Idx: 0));
7256 Observer.changedInstr(MI);
7257 return Legalized;
7258 }
7259
7260 default:
7261 return UnableToLegalize;
7262 }
7263}
7264
7265LegalizerHelper::LegalizeResult
7266LegalizerHelper::equalizeVectorShuffleLengths(MachineInstr &MI) {
7267 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7268 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
7269 unsigned MaskNumElts = Mask.size();
7270 unsigned SrcNumElts = SrcTy.getNumElements();
7271 LLT DestEltTy = DstTy.getElementType();
7272
7273 if (MaskNumElts == SrcNumElts)
7274 return Legalized;
7275
7276 if (MaskNumElts < SrcNumElts) {
7277 // Extend mask to match new destination vector size with
7278 // undef values.
7279 SmallVector<int, 16> NewMask(SrcNumElts, -1);
7280 llvm::copy(Range&: Mask, Out: NewMask.begin());
7281
7282 moreElementsVectorDst(MI, WideTy: SrcTy, OpIdx: 0);
7283 MIRBuilder.setInstrAndDebugLoc(MI);
7284 MIRBuilder.buildShuffleVector(Res: MI.getOperand(i: 0).getReg(),
7285 Src1: MI.getOperand(i: 1).getReg(),
7286 Src2: MI.getOperand(i: 2).getReg(), Mask: NewMask);
7287 MI.eraseFromParent();
7288
7289 return Legalized;
7290 }
7291
7292 unsigned PaddedMaskNumElts = alignTo(Value: MaskNumElts, Align: SrcNumElts);
7293 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
7294 LLT PaddedTy =
7295 DstTy.changeVectorElementCount(EC: ElementCount::getFixed(MinVal: PaddedMaskNumElts));
7296
7297 // Create new source vectors by concatenating the initial
7298 // source vectors with undefined vectors of the same size.
7299 auto Undef = MIRBuilder.buildUndef(Res: SrcTy);
7300 SmallVector<Register, 8> MOps1(NumConcat, Undef.getReg(Idx: 0));
7301 SmallVector<Register, 8> MOps2(NumConcat, Undef.getReg(Idx: 0));
7302 MOps1[0] = MI.getOperand(i: 1).getReg();
7303 MOps2[0] = MI.getOperand(i: 2).getReg();
7304
7305 auto Src1 = MIRBuilder.buildConcatVectors(Res: PaddedTy, Ops: MOps1);
7306 auto Src2 = MIRBuilder.buildConcatVectors(Res: PaddedTy, Ops: MOps2);
7307
7308 // Readjust mask for new input vector length.
7309 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
7310 for (unsigned I = 0; I != MaskNumElts; ++I) {
7311 int Idx = Mask[I];
7312 if (Idx >= static_cast<int>(SrcNumElts))
7313 Idx += PaddedMaskNumElts - SrcNumElts;
7314 MappedOps[I] = Idx;
7315 }
7316
7317 // If we got more elements than required, extract subvector.
7318 if (MaskNumElts != PaddedMaskNumElts) {
7319 auto Shuffle =
7320 MIRBuilder.buildShuffleVector(Res: PaddedTy, Src1, Src2, Mask: MappedOps);
7321
7322 SmallVector<Register, 16> Elts(MaskNumElts);
7323 for (unsigned I = 0; I < MaskNumElts; ++I) {
7324 Elts[I] =
7325 MIRBuilder.buildExtractVectorElementConstant(Res: DestEltTy, Val: Shuffle, Idx: I)
7326 .getReg(Idx: 0);
7327 }
7328 MIRBuilder.buildBuildVector(Res: DstReg, Ops: Elts);
7329 } else {
7330 MIRBuilder.buildShuffleVector(Res: DstReg, Src1, Src2, Mask: MappedOps);
7331 }
7332
7333 MI.eraseFromParent();
7334 return LegalizerHelper::LegalizeResult::Legalized;
7335}
7336
7337LegalizerHelper::LegalizeResult
7338LegalizerHelper::moreElementsVectorShuffle(MachineInstr &MI,
7339 unsigned int TypeIdx, LLT MoreTy) {
7340 auto [DstTy, Src1Ty, Src2Ty] = MI.getFirst3LLTs();
7341 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
7342 unsigned NumElts = DstTy.getNumElements();
7343 unsigned WidenNumElts = MoreTy.getNumElements();
7344
7345 if (DstTy.isVector() && Src1Ty.isVector() &&
7346 DstTy.getNumElements() != Src1Ty.getNumElements()) {
7347 return equalizeVectorShuffleLengths(MI);
7348 }
7349
7350 if (TypeIdx != 0)
7351 return UnableToLegalize;
7352
7353 // Expect a canonicalized shuffle.
7354 if (DstTy != Src1Ty || DstTy != Src2Ty)
7355 return UnableToLegalize;
7356
7357 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
7358 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
7359
7360 // Adjust mask based on new input vector length.
7361 SmallVector<int, 16> NewMask(WidenNumElts, -1);
7362 for (unsigned I = 0; I != NumElts; ++I) {
7363 int Idx = Mask[I];
7364 if (Idx < static_cast<int>(NumElts))
7365 NewMask[I] = Idx;
7366 else
7367 NewMask[I] = Idx - NumElts + WidenNumElts;
7368 }
7369 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7370 MIRBuilder.setInstrAndDebugLoc(MI);
7371 MIRBuilder.buildShuffleVector(Res: MI.getOperand(i: 0).getReg(),
7372 Src1: MI.getOperand(i: 1).getReg(),
7373 Src2: MI.getOperand(i: 2).getReg(), Mask: NewMask);
7374 MI.eraseFromParent();
7375 return Legalized;
7376}
7377
7378void LegalizerHelper::multiplyRegisters(SmallVectorImpl<Register> &DstRegs,
7379 ArrayRef<Register> Src1Regs,
7380 ArrayRef<Register> Src2Regs,
7381 LLT NarrowTy) {
7382 MachineIRBuilder &B = MIRBuilder;
7383 unsigned SrcParts = Src1Regs.size();
7384 unsigned DstParts = DstRegs.size();
7385
7386 unsigned DstIdx = 0; // Low bits of the result.
7387 Register FactorSum =
7388 B.buildMul(Dst: NarrowTy, Src0: Src1Regs[DstIdx], Src1: Src2Regs[DstIdx]).getReg(Idx: 0);
7389 DstRegs[DstIdx] = FactorSum;
7390
7391 Register CarrySumPrevDstIdx;
7392 SmallVector<Register, 4> Factors;
7393
7394 for (DstIdx = 1; DstIdx < DstParts; DstIdx++) {
7395 // Collect high parts of muls from previous DstIdx.
7396 for (unsigned i = DstIdx < SrcParts ? 0 : DstIdx - SrcParts;
7397 i <= std::min(a: DstIdx - 1, b: SrcParts - 1); ++i) {
7398 MachineInstrBuilder Umulh =
7399 B.buildUMulH(Dst: NarrowTy, Src0: Src1Regs[DstIdx - 1 - i], Src1: Src2Regs[i]);
7400 Factors.push_back(Elt: Umulh.getReg(Idx: 0));
7401 }
7402 // Collect low parts of muls for DstIdx. Visit the diagonal starting with
7403 // the low Src1 part, so multiply-add selectors can use it as the first
7404 // accumulated cross product.
7405 unsigned LowStart = DstIdx + 1 < SrcParts ? 0 : DstIdx - SrcParts + 1;
7406 unsigned LowEnd = std::min(a: DstIdx, b: SrcParts - 1);
7407 for (unsigned RevI = LowEnd + 1; RevI != LowStart; --RevI) {
7408 unsigned i = RevI - 1;
7409 MachineInstrBuilder Mul =
7410 B.buildMul(Dst: NarrowTy, Src0: Src1Regs[DstIdx - i], Src1: Src2Regs[i]);
7411 Factors.push_back(Elt: Mul.getReg(Idx: 0));
7412 }
7413 // Add CarrySum from additions calculated for previous DstIdx.
7414 if (DstIdx != 1) {
7415 Factors.push_back(Elt: CarrySumPrevDstIdx);
7416 }
7417
7418 Register CarrySum;
7419 // Add all factors and accumulate all carries into CarrySum.
7420 if (DstIdx != DstParts - 1) {
7421 MachineInstrBuilder Uaddo =
7422 B.buildUAddo(Res: NarrowTy, CarryOut: LLT::integer(SizeInBits: 1), Op0: Factors[0], Op1: Factors[1]);
7423 FactorSum = Uaddo.getReg(Idx: 0);
7424 CarrySum = B.buildZExt(Res: NarrowTy, Op: Uaddo.getReg(Idx: 1)).getReg(Idx: 0);
7425 for (unsigned i = 2; i < Factors.size(); ++i) {
7426 MachineInstrBuilder Uaddo =
7427 B.buildUAddo(Res: NarrowTy, CarryOut: LLT::integer(SizeInBits: 1), Op0: FactorSum, Op1: Factors[i]);
7428 FactorSum = Uaddo.getReg(Idx: 0);
7429 MachineInstrBuilder Carry = B.buildZExt(Res: NarrowTy, Op: Uaddo.getReg(Idx: 1));
7430 CarrySum = B.buildAdd(Dst: NarrowTy, Src0: CarrySum, Src1: Carry).getReg(Idx: 0);
7431 }
7432 } else {
7433 // Since value for the next index is not calculated, neither is CarrySum.
7434 FactorSum = B.buildAdd(Dst: NarrowTy, Src0: Factors[0], Src1: Factors[1]).getReg(Idx: 0);
7435 for (unsigned i = 2; i < Factors.size(); ++i)
7436 FactorSum = B.buildAdd(Dst: NarrowTy, Src0: FactorSum, Src1: Factors[i]).getReg(Idx: 0);
7437 }
7438
7439 CarrySumPrevDstIdx = CarrySum;
7440 DstRegs[DstIdx] = FactorSum;
7441 Factors.clear();
7442 }
7443}
7444
7445LegalizerHelper::LegalizeResult
7446LegalizerHelper::narrowScalarAddSub(MachineInstr &MI, unsigned TypeIdx,
7447 LLT NarrowTy) {
7448 if (TypeIdx != 0)
7449 return UnableToLegalize;
7450
7451 Register DstReg = MI.getOperand(i: 0).getReg();
7452 LLT DstType = MRI.getType(Reg: DstReg);
7453 // FIXME: add support for vector types
7454 if (DstType.isVector())
7455 return UnableToLegalize;
7456
7457 unsigned Opcode = MI.getOpcode();
7458 unsigned OpO, OpE, OpF;
7459 switch (Opcode) {
7460 case TargetOpcode::G_SADDO:
7461 case TargetOpcode::G_SADDE:
7462 case TargetOpcode::G_UADDO:
7463 case TargetOpcode::G_UADDE:
7464 case TargetOpcode::G_ADD:
7465 OpO = TargetOpcode::G_UADDO;
7466 OpE = TargetOpcode::G_UADDE;
7467 OpF = TargetOpcode::G_UADDE;
7468 if (Opcode == TargetOpcode::G_SADDO || Opcode == TargetOpcode::G_SADDE)
7469 OpF = TargetOpcode::G_SADDE;
7470 break;
7471 case TargetOpcode::G_SSUBO:
7472 case TargetOpcode::G_SSUBE:
7473 case TargetOpcode::G_USUBO:
7474 case TargetOpcode::G_USUBE:
7475 case TargetOpcode::G_SUB:
7476 OpO = TargetOpcode::G_USUBO;
7477 OpE = TargetOpcode::G_USUBE;
7478 OpF = TargetOpcode::G_USUBE;
7479 if (Opcode == TargetOpcode::G_SSUBO || Opcode == TargetOpcode::G_SSUBE)
7480 OpF = TargetOpcode::G_SSUBE;
7481 break;
7482 default:
7483 llvm_unreachable("Unexpected add/sub opcode!");
7484 }
7485
7486 // 1 for a plain add/sub, 2 if this is an operation with a carry-out.
7487 unsigned NumDefs = MI.getNumExplicitDefs();
7488 Register Src1 = MI.getOperand(i: NumDefs).getReg();
7489 Register Src2 = MI.getOperand(i: NumDefs + 1).getReg();
7490 Register CarryDst, CarryIn;
7491 if (NumDefs == 2)
7492 CarryDst = MI.getOperand(i: 1).getReg();
7493 if (MI.getNumOperands() == NumDefs + 3)
7494 CarryIn = MI.getOperand(i: NumDefs + 2).getReg();
7495
7496 LLT RegTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7497 LLT LeftoverTy, DummyTy;
7498 SmallVector<Register, 2> Src1Regs, Src2Regs, Src1Left, Src2Left, DstRegs;
7499 extractParts(Reg: Src1, RegTy, MainTy: NarrowTy, LeftoverTy, VRegs&: Src1Regs, LeftoverVRegs&: Src1Left,
7500 MIRBuilder, MRI);
7501 extractParts(Reg: Src2, RegTy, MainTy: NarrowTy, LeftoverTy&: DummyTy, VRegs&: Src2Regs, LeftoverVRegs&: Src2Left, MIRBuilder,
7502 MRI);
7503
7504 int NarrowParts = Src1Regs.size();
7505 Src1Regs.append(RHS: Src1Left);
7506 Src2Regs.append(RHS: Src2Left);
7507 DstRegs.reserve(N: Src1Regs.size());
7508
7509 for (int i = 0, e = Src1Regs.size(); i != e; ++i) {
7510 Register DstReg =
7511 MRI.createGenericVirtualRegister(Ty: MRI.getType(Reg: Src1Regs[i]));
7512 Register CarryOut;
7513 // Forward the final carry-out to the destination register
7514 if (i == e - 1 && CarryDst)
7515 CarryOut = CarryDst;
7516 else
7517 CarryOut = MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: 1));
7518
7519 if (!CarryIn) {
7520 MIRBuilder.buildInstr(Opc: OpO, DstOps: {DstReg, CarryOut},
7521 SrcOps: {Src1Regs[i], Src2Regs[i]});
7522 } else if (i == e - 1) {
7523 MIRBuilder.buildInstr(Opc: OpF, DstOps: {DstReg, CarryOut},
7524 SrcOps: {Src1Regs[i], Src2Regs[i], CarryIn});
7525 } else {
7526 MIRBuilder.buildInstr(Opc: OpE, DstOps: {DstReg, CarryOut},
7527 SrcOps: {Src1Regs[i], Src2Regs[i], CarryIn});
7528 }
7529
7530 DstRegs.push_back(Elt: DstReg);
7531 CarryIn = CarryOut;
7532 }
7533 insertParts(DstReg: MI.getOperand(i: 0).getReg(), ResultTy: RegTy, PartTy: NarrowTy,
7534 PartRegs: ArrayRef(DstRegs).take_front(N: NarrowParts), LeftoverTy,
7535 LeftoverRegs: ArrayRef(DstRegs).drop_front(N: NarrowParts));
7536
7537 MI.eraseFromParent();
7538 return Legalized;
7539}
7540
7541LegalizerHelper::LegalizeResult
7542LegalizerHelper::narrowScalarMul(MachineInstr &MI, LLT NarrowTy) {
7543 auto [DstReg, Src1, Src2] = MI.getFirst3Regs();
7544
7545 LLT Ty = MRI.getType(Reg: DstReg);
7546 if (Ty.isVector())
7547 return UnableToLegalize;
7548
7549 unsigned Size = Ty.getSizeInBits();
7550 unsigned NarrowSize = NarrowTy.getSizeInBits();
7551 if (Size % NarrowSize != 0)
7552 return UnableToLegalize;
7553
7554 unsigned NumParts = Size / NarrowSize;
7555 bool IsMulHigh = MI.getOpcode() == TargetOpcode::G_UMULH ||
7556 MI.getOpcode() == TargetOpcode::G_SMULH;
7557 unsigned DstTmpParts = NumParts * (IsMulHigh ? 2 : 1);
7558
7559 SmallVector<Register, 2> Src1Parts, Src2Parts;
7560 SmallVector<Register, 2> DstTmpRegs(DstTmpParts);
7561 extractParts(Reg: Src1, Ty: NarrowTy, NumParts, VRegs&: Src1Parts, MIRBuilder, MRI);
7562 extractParts(Reg: Src2, Ty: NarrowTy, NumParts, VRegs&: Src2Parts, MIRBuilder, MRI);
7563 multiplyRegisters(DstRegs&: DstTmpRegs, Src1Regs: Src1Parts, Src2Regs: Src2Parts, NarrowTy);
7564
7565 // Take only high half of registers if this is high mul.
7566 ArrayRef<Register> DstRegs(&DstTmpRegs[DstTmpParts - NumParts], NumParts);
7567
7568 if (MI.getOpcode() == TargetOpcode::G_SMULH) {
7569 Register UnsignedHigh = MRI.createGenericVirtualRegister(Ty);
7570 MIRBuilder.buildMergeLikeInstr(Res: UnsignedHigh, Ops: DstRegs);
7571
7572 // Extract the sign from the highest narrow part of each operand.
7573 auto ShiftAmt = MIRBuilder.buildConstant(Res: NarrowTy, Val: NarrowSize - 1);
7574
7575 auto SignAPart =
7576 MIRBuilder.buildAShr(Dst: NarrowTy, Src0: Src1Parts.back(), Src1: ShiftAmt.getReg(Idx: 0));
7577 auto SignBPart =
7578 MIRBuilder.buildAShr(Dst: NarrowTy, Src0: Src2Parts.back(), Src1: ShiftAmt.getReg(Idx: 0));
7579
7580 // Replicate the sign parts to construct full-width sign masks.
7581 SmallVector<Register, 2> SignAParts(NumParts, SignAPart.getReg(Idx: 0));
7582 SmallVector<Register, 2> SignBParts(NumParts, SignBPart.getReg(Idx: 0));
7583
7584 auto SignA = MIRBuilder.buildMergeLikeInstr(Res: Ty, Ops: SignAParts);
7585 auto SignB = MIRBuilder.buildMergeLikeInstr(Res: Ty, Ops: SignBParts);
7586
7587 // Ca = (Src1 < 0) ? Src2 : 0
7588 // Cb = (Src2 < 0) ? Src1 : 0
7589 auto Ca = MIRBuilder.buildAnd(Dst: Ty, Src0: SignA, Src1: Src2);
7590 auto Cb = MIRBuilder.buildAnd(Dst: Ty, Src0: SignB, Src1);
7591
7592 // SignedHigh = UnsignedHigh - Ca - Cb
7593 auto Tmp = MIRBuilder.buildSub(Dst: Ty, Src0: UnsignedHigh, Src1: Ca);
7594 MIRBuilder.buildSub(Dst: DstReg, Src0: Tmp, Src1: Cb);
7595 } else {
7596 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
7597 }
7598
7599 MI.eraseFromParent();
7600 return Legalized;
7601}
7602
7603LegalizerHelper::LegalizeResult
7604LegalizerHelper::narrowScalarFPTOI(MachineInstr &MI, unsigned TypeIdx,
7605 LLT NarrowTy) {
7606 if (TypeIdx != 0)
7607 return UnableToLegalize;
7608
7609 bool IsSigned = MI.getOpcode() == TargetOpcode::G_FPTOSI;
7610
7611 Register Src = MI.getOperand(i: 1).getReg();
7612 LLT SrcTy = MRI.getType(Reg: Src);
7613
7614 // If all finite floats fit into the narrowed integer type, we can just swap
7615 // out the result type. Only IEEE half qualifies: bfloat is also 16 bits wide
7616 // but has float's exponent range. LLT::float16() is equivalent to
7617 // LLT::scalar(16) on targets without extended LLTs.
7618 if (SrcTy.getScalarType() != LLT::float16() ||
7619 NarrowTy.getScalarSizeInBits() < (IsSigned ? 17u : 16u))
7620 return UnableToLegalize;
7621
7622 Observer.changingInstr(MI);
7623 narrowScalarDst(MI, NarrowTy, OpIdx: 0,
7624 ExtOpcode: IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT);
7625 Observer.changedInstr(MI);
7626 return Legalized;
7627}
7628
7629LegalizerHelper::LegalizeResult
7630LegalizerHelper::narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx,
7631 LLT NarrowTy) {
7632 if (TypeIdx != 1)
7633 return UnableToLegalize;
7634
7635 uint64_t NarrowSize = NarrowTy.getSizeInBits();
7636
7637 int64_t SizeOp1 = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
7638 // FIXME: add support for when SizeOp1 isn't an exact multiple of
7639 // NarrowSize.
7640 if (SizeOp1 % NarrowSize != 0)
7641 return UnableToLegalize;
7642 int NumParts = SizeOp1 / NarrowSize;
7643
7644 SmallVector<Register, 2> SrcRegs, DstRegs;
7645 extractParts(Reg: MI.getOperand(i: 1).getReg(), Ty: NarrowTy, NumParts, VRegs&: SrcRegs,
7646 MIRBuilder, MRI);
7647
7648 Register OpReg = MI.getOperand(i: 0).getReg();
7649 uint64_t OpStart = MI.getOperand(i: 2).getImm();
7650 uint64_t OpSize = MRI.getType(Reg: OpReg).getSizeInBits();
7651 for (int i = 0; i < NumParts; ++i) {
7652 unsigned SrcStart = i * NarrowSize;
7653
7654 if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
7655 // No part of the extract uses this subregister, ignore it.
7656 continue;
7657 } else if (SrcStart == OpStart && NarrowTy == MRI.getType(Reg: OpReg)) {
7658 // The entire subregister is extracted, forward the value.
7659 DstRegs.push_back(Elt: SrcRegs[i]);
7660 continue;
7661 }
7662
7663 // OpSegStart is where this destination segment would start in OpReg if it
7664 // extended infinitely in both directions.
7665 int64_t ExtractOffset;
7666 uint64_t SegSize;
7667 if (OpStart < SrcStart) {
7668 ExtractOffset = 0;
7669 SegSize = std::min(a: NarrowSize, b: OpStart + OpSize - SrcStart);
7670 } else {
7671 ExtractOffset = OpStart - SrcStart;
7672 SegSize = std::min(a: SrcStart + NarrowSize - OpStart, b: OpSize);
7673 }
7674
7675 Register SegReg = SrcRegs[i];
7676 if (ExtractOffset != 0 || SegSize != NarrowSize) {
7677 // A genuine extract is needed.
7678 SegReg = MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: SegSize));
7679 MIRBuilder.buildExtract(Res: SegReg, Src: SrcRegs[i], Index: ExtractOffset);
7680 }
7681
7682 DstRegs.push_back(Elt: SegReg);
7683 }
7684
7685 Register DstReg = MI.getOperand(i: 0).getReg();
7686 if (MRI.getType(Reg: DstReg).isVector())
7687 MIRBuilder.buildBuildVector(Res: DstReg, Ops: DstRegs);
7688 else if (DstRegs.size() > 1)
7689 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
7690 else
7691 MIRBuilder.buildCopy(Res: DstReg, Op: DstRegs[0]);
7692 MI.eraseFromParent();
7693 return Legalized;
7694}
7695
7696LegalizerHelper::LegalizeResult
7697LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx,
7698 LLT NarrowTy) {
7699 // FIXME: Don't know how to handle secondary types yet.
7700 if (TypeIdx != 0)
7701 return UnableToLegalize;
7702
7703 SmallVector<Register, 2> SrcRegs, LeftoverRegs, DstRegs;
7704 LLT RegTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7705 LLT LeftoverTy;
7706 extractParts(Reg: MI.getOperand(i: 1).getReg(), RegTy, MainTy: NarrowTy, LeftoverTy, VRegs&: SrcRegs,
7707 LeftoverVRegs&: LeftoverRegs, MIRBuilder, MRI);
7708
7709 SrcRegs.append(RHS: LeftoverRegs);
7710
7711 uint64_t NarrowSize = NarrowTy.getSizeInBits();
7712 Register OpReg = MI.getOperand(i: 2).getReg();
7713 uint64_t OpStart = MI.getOperand(i: 3).getImm();
7714 uint64_t OpSize = MRI.getType(Reg: OpReg).getSizeInBits();
7715 for (int I = 0, E = SrcRegs.size(); I != E; ++I) {
7716 unsigned DstStart = I * NarrowSize;
7717
7718 if (DstStart == OpStart && NarrowTy == MRI.getType(Reg: OpReg)) {
7719 // The entire subregister is defined by this insert, forward the new
7720 // value.
7721 DstRegs.push_back(Elt: OpReg);
7722 continue;
7723 }
7724
7725 Register SrcReg = SrcRegs[I];
7726 if (MRI.getType(Reg: SrcRegs[I]) == LeftoverTy) {
7727 // The leftover reg is smaller than NarrowTy, so we need to extend it.
7728 SrcReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
7729 MIRBuilder.buildAnyExt(Res: SrcReg, Op: SrcRegs[I]);
7730 }
7731
7732 if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
7733 // No part of the insert affects this subregister, forward the original.
7734 DstRegs.push_back(Elt: SrcReg);
7735 continue;
7736 }
7737
7738 // OpSegStart is where this destination segment would start in OpReg if it
7739 // extended infinitely in both directions.
7740 int64_t ExtractOffset, InsertOffset;
7741 uint64_t SegSize;
7742 if (OpStart < DstStart) {
7743 InsertOffset = 0;
7744 ExtractOffset = DstStart - OpStart;
7745 SegSize = std::min(a: NarrowSize, b: OpStart + OpSize - DstStart);
7746 } else {
7747 InsertOffset = OpStart - DstStart;
7748 ExtractOffset = 0;
7749 SegSize = std::min(a: NarrowSize - InsertOffset, b: OpSize);
7750 }
7751
7752 Register SegReg = OpReg;
7753 if (ExtractOffset != 0 || SegSize != OpSize) {
7754 // A genuine extract is needed.
7755 SegReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: SegSize));
7756 MIRBuilder.buildExtract(Res: SegReg, Src: OpReg, Index: ExtractOffset);
7757 }
7758
7759 Register DstReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
7760 MIRBuilder.buildInsert(Res: DstReg, Src: SrcReg, Op: SegReg, Index: InsertOffset);
7761 DstRegs.push_back(Elt: DstReg);
7762 }
7763
7764 uint64_t WideSize = DstRegs.size() * NarrowSize;
7765 Register DstReg = MI.getOperand(i: 0).getReg();
7766 if (WideSize > RegTy.getSizeInBits()) {
7767 Register MergeReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: WideSize));
7768 MIRBuilder.buildMergeLikeInstr(Res: MergeReg, Ops: DstRegs);
7769 MIRBuilder.buildTrunc(Res: DstReg, Op: MergeReg);
7770 } else
7771 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
7772
7773 MI.eraseFromParent();
7774 return Legalized;
7775}
7776
7777LegalizerHelper::LegalizeResult
7778LegalizerHelper::narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx,
7779 LLT NarrowTy) {
7780 Register DstReg = MI.getOperand(i: 0).getReg();
7781 LLT DstTy = MRI.getType(Reg: DstReg);
7782
7783 assert(MI.getNumOperands() == 3 && TypeIdx == 0);
7784
7785 SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
7786 SmallVector<Register, 4> Src0Regs, Src0LeftoverRegs;
7787 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
7788 LLT LeftoverTy;
7789 if (!extractParts(Reg: MI.getOperand(i: 1).getReg(), RegTy: DstTy, MainTy: NarrowTy, LeftoverTy,
7790 VRegs&: Src0Regs, LeftoverVRegs&: Src0LeftoverRegs, MIRBuilder, MRI))
7791 return UnableToLegalize;
7792
7793 LLT Unused;
7794 if (!extractParts(Reg: MI.getOperand(i: 2).getReg(), RegTy: DstTy, MainTy: NarrowTy, LeftoverTy&: Unused,
7795 VRegs&: Src1Regs, LeftoverVRegs&: Src1LeftoverRegs, MIRBuilder, MRI))
7796 llvm_unreachable("inconsistent extractParts result");
7797
7798 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
7799 auto Inst = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {NarrowTy},
7800 SrcOps: {Src0Regs[I], Src1Regs[I]});
7801 DstRegs.push_back(Elt: Inst.getReg(Idx: 0));
7802 }
7803
7804 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
7805 auto Inst = MIRBuilder.buildInstr(
7806 Opc: MI.getOpcode(),
7807 DstOps: {LeftoverTy}, SrcOps: {Src0LeftoverRegs[I], Src1LeftoverRegs[I]});
7808 DstLeftoverRegs.push_back(Elt: Inst.getReg(Idx: 0));
7809 }
7810
7811 insertParts(DstReg, ResultTy: DstTy, PartTy: NarrowTy, PartRegs: DstRegs,
7812 LeftoverTy, LeftoverRegs: DstLeftoverRegs);
7813
7814 MI.eraseFromParent();
7815 return Legalized;
7816}
7817
7818LegalizerHelper::LegalizeResult
7819LegalizerHelper::narrowScalarExt(MachineInstr &MI, unsigned TypeIdx,
7820 LLT NarrowTy) {
7821 if (TypeIdx != 0)
7822 return UnableToLegalize;
7823
7824 auto [DstReg, SrcReg] = MI.getFirst2Regs();
7825
7826 LLT DstTy = MRI.getType(Reg: DstReg);
7827 if (DstTy.isVector())
7828 return UnableToLegalize;
7829
7830 SmallVector<Register, 8> Parts;
7831 LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg);
7832 LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, VRegs&: Parts, PadStrategy: MI.getOpcode());
7833 buildWidenedRemergeToDst(DstReg, LCMTy, RemergeRegs: Parts);
7834
7835 MI.eraseFromParent();
7836 return Legalized;
7837}
7838
7839LegalizerHelper::LegalizeResult
7840LegalizerHelper::narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx,
7841 LLT NarrowTy) {
7842 if (TypeIdx != 0)
7843 return UnableToLegalize;
7844
7845 Register CondReg = MI.getOperand(i: 1).getReg();
7846 LLT CondTy = MRI.getType(Reg: CondReg);
7847 if (CondTy.isVector()) // TODO: Handle vselect
7848 return UnableToLegalize;
7849
7850 Register DstReg = MI.getOperand(i: 0).getReg();
7851 LLT DstTy = MRI.getType(Reg: DstReg);
7852
7853 SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
7854 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
7855 SmallVector<Register, 4> Src2Regs, Src2LeftoverRegs;
7856 LLT LeftoverTy;
7857 if (!extractParts(Reg: MI.getOperand(i: 2).getReg(), RegTy: DstTy, MainTy: NarrowTy, LeftoverTy,
7858 VRegs&: Src1Regs, LeftoverVRegs&: Src1LeftoverRegs, MIRBuilder, MRI))
7859 return UnableToLegalize;
7860
7861 LLT Unused;
7862 if (!extractParts(Reg: MI.getOperand(i: 3).getReg(), RegTy: DstTy, MainTy: NarrowTy, LeftoverTy&: Unused,
7863 VRegs&: Src2Regs, LeftoverVRegs&: Src2LeftoverRegs, MIRBuilder, MRI))
7864 llvm_unreachable("inconsistent extractParts result");
7865
7866 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
7867 auto Select = MIRBuilder.buildSelect(Res: NarrowTy,
7868 Tst: CondReg, Op0: Src1Regs[I], Op1: Src2Regs[I]);
7869 DstRegs.push_back(Elt: Select.getReg(Idx: 0));
7870 }
7871
7872 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
7873 auto Select = MIRBuilder.buildSelect(
7874 Res: LeftoverTy, Tst: CondReg, Op0: Src1LeftoverRegs[I], Op1: Src2LeftoverRegs[I]);
7875 DstLeftoverRegs.push_back(Elt: Select.getReg(Idx: 0));
7876 }
7877
7878 insertParts(DstReg, ResultTy: DstTy, PartTy: NarrowTy, PartRegs: DstRegs,
7879 LeftoverTy, LeftoverRegs: DstLeftoverRegs);
7880
7881 MI.eraseFromParent();
7882 return Legalized;
7883}
7884
7885LegalizerHelper::LegalizeResult
7886LegalizerHelper::narrowScalarCTLZ(MachineInstr &MI, unsigned TypeIdx,
7887 LLT NarrowTy) {
7888 if (TypeIdx != 1)
7889 return UnableToLegalize;
7890
7891 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7892 unsigned NarrowSize = NarrowTy.getSizeInBits();
7893
7894 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
7895 const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_POISON;
7896
7897 MachineIRBuilder &B = MIRBuilder;
7898 auto UnmergeSrc = B.buildUnmerge(Res: NarrowTy, Op: SrcReg);
7899 // ctlz(Hi:Lo) -> Hi == 0 ? (NarrowSize + ctlz(Lo)) : ctlz(Hi)
7900 auto C_0 = B.buildConstant(Res: NarrowTy, Val: 0);
7901 auto HiIsZero = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: LLT::integer(SizeInBits: 1),
7902 Op0: UnmergeSrc.getReg(Idx: 1), Op1: C_0);
7903 auto LoCTLZ = IsUndef ? B.buildCTLZ_ZERO_POISON(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 0))
7904 : B.buildCTLZ(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 0));
7905 auto C_NarrowSize = B.buildConstant(Res: DstTy, Val: NarrowSize);
7906 auto HiIsZeroCTLZ = B.buildAdd(Dst: DstTy, Src0: LoCTLZ, Src1: C_NarrowSize);
7907 auto HiCTLZ = B.buildCTLZ_ZERO_POISON(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 1));
7908 B.buildSelect(Res: DstReg, Tst: HiIsZero, Op0: HiIsZeroCTLZ, Op1: HiCTLZ);
7909
7910 MI.eraseFromParent();
7911 return Legalized;
7912 }
7913
7914 return UnableToLegalize;
7915}
7916
7917LegalizerHelper::LegalizeResult
7918LegalizerHelper::narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx,
7919 LLT NarrowTy) {
7920 if (TypeIdx != 1)
7921 return UnableToLegalize;
7922
7923 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7924 unsigned NarrowSize = NarrowTy.getSizeInBits();
7925
7926 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
7927 const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTTZ_ZERO_POISON;
7928
7929 MachineIRBuilder &B = MIRBuilder;
7930 auto UnmergeSrc = B.buildUnmerge(Res: NarrowTy, Op: SrcReg);
7931 // cttz(Hi:Lo) -> Lo == 0 ? (cttz(Hi) + NarrowSize) : cttz(Lo)
7932 auto C_0 = B.buildConstant(Res: NarrowTy, Val: 0);
7933 auto LoIsZero = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: LLT::integer(SizeInBits: 1),
7934 Op0: UnmergeSrc.getReg(Idx: 0), Op1: C_0);
7935 auto HiCTTZ = IsUndef ? B.buildCTTZ_ZERO_POISON(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 1))
7936 : B.buildCTTZ(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 1));
7937 auto C_NarrowSize = B.buildConstant(Res: DstTy, Val: NarrowSize);
7938 auto LoIsZeroCTTZ = B.buildAdd(Dst: DstTy, Src0: HiCTTZ, Src1: C_NarrowSize);
7939 auto LoCTTZ = B.buildCTTZ_ZERO_POISON(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 0));
7940 B.buildSelect(Res: DstReg, Tst: LoIsZero, Op0: LoIsZeroCTTZ, Op1: LoCTTZ);
7941
7942 MI.eraseFromParent();
7943 return Legalized;
7944 }
7945
7946 return UnableToLegalize;
7947}
7948
7949LegalizerHelper::LegalizeResult
7950LegalizerHelper::narrowScalarCTLS(MachineInstr &MI, unsigned TypeIdx,
7951 LLT NarrowTy) {
7952 if (TypeIdx != 1)
7953 return UnableToLegalize;
7954
7955 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7956 unsigned NarrowSize = NarrowTy.getSizeInBits();
7957
7958 if (!SrcTy.isScalar() || SrcTy.getSizeInBits() != 2 * NarrowSize)
7959 return UnableToLegalize;
7960
7961 MachineIRBuilder &B = MIRBuilder;
7962
7963 auto UnmergeSrc = B.buildUnmerge(Res: NarrowTy, Op: SrcReg);
7964 Register Lo = UnmergeSrc.getReg(Idx: 0);
7965 Register Hi = UnmergeSrc.getReg(Idx: 1);
7966
7967 auto ShAmt = B.buildConstant(Res: NarrowTy, Val: NarrowSize - 1);
7968 auto Sign = B.buildAShr(Dst: NarrowTy, Src0: Hi, Src1: ShAmt);
7969
7970 auto HiIsSign = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: LLT::scalar(SizeInBits: 1), Op0: Hi, Op1: Sign);
7971
7972 // Invert Lo if Hi is negative. Then count the leading zeros. If there are no
7973 // leading zeros, then the MSB of Lo is different than the MSB of Hi.
7974 // Otherwise the leading zeros represent additional sign bits of the original
7975 // value.
7976 auto LoInv = B.buildXor(Dst: DstTy, Src0: Lo, Src1: Sign);
7977 auto LoCTLZ = B.buildCTLZ(Dst: DstTy, Src0: LoInv);
7978
7979 // Add NarrowSize-1 to LoCTLZ. This is the full CTLS if Hi is all sign bits.
7980 auto C_NarrowSizeM1 = B.buildConstant(Res: DstTy, Val: NarrowSize - 1);
7981 auto HiIsSignCTLS = B.buildAdd(Dst: DstTy, Src0: LoCTLZ, Src1: C_NarrowSizeM1);
7982
7983 auto HiCTLS = B.buildCTLS(Dst: DstTy, Src0: Hi);
7984
7985 B.buildSelect(Res: DstReg, Tst: HiIsSign, Op0: HiIsSignCTLS, Op1: HiCTLS);
7986
7987 MI.eraseFromParent();
7988 return Legalized;
7989}
7990
7991LegalizerHelper::LegalizeResult
7992LegalizerHelper::narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx,
7993 LLT NarrowTy) {
7994 if (TypeIdx != 1)
7995 return UnableToLegalize;
7996
7997 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7998 unsigned NarrowSize = NarrowTy.getSizeInBits();
7999
8000 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
8001 auto UnmergeSrc = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: MI.getOperand(i: 1));
8002
8003 auto LoCTPOP = MIRBuilder.buildCTPOP(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 0));
8004 auto HiCTPOP = MIRBuilder.buildCTPOP(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 1));
8005 MIRBuilder.buildAdd(Dst: DstReg, Src0: HiCTPOP, Src1: LoCTPOP);
8006
8007 MI.eraseFromParent();
8008 return Legalized;
8009 }
8010
8011 return UnableToLegalize;
8012}
8013
8014LegalizerHelper::LegalizeResult
8015LegalizerHelper::narrowScalarFLDEXP(MachineInstr &MI, unsigned TypeIdx,
8016 LLT NarrowTy) {
8017 if (TypeIdx != 1)
8018 return UnableToLegalize;
8019
8020 MachineIRBuilder &B = MIRBuilder;
8021 Register ExpReg = MI.getOperand(i: 2).getReg();
8022 LLT ExpTy = MRI.getType(Reg: ExpReg);
8023
8024 unsigned ClampSize = NarrowTy.getScalarSizeInBits();
8025
8026 // Clamp the exponent to the range of the target type.
8027 auto MinExp = B.buildConstant(Res: ExpTy, Val: minIntN(N: ClampSize));
8028 auto ClampMin = B.buildSMax(Dst: ExpTy, Src0: ExpReg, Src1: MinExp);
8029 auto MaxExp = B.buildConstant(Res: ExpTy, Val: maxIntN(N: ClampSize));
8030 auto Clamp = B.buildSMin(Dst: ExpTy, Src0: ClampMin, Src1: MaxExp);
8031
8032 auto Trunc = B.buildTrunc(Res: NarrowTy, Op: Clamp);
8033 Observer.changingInstr(MI);
8034 MI.getOperand(i: 2).setReg(Trunc.getReg(Idx: 0));
8035 Observer.changedInstr(MI);
8036 return Legalized;
8037}
8038
8039LegalizerHelper::LegalizeResult
8040LegalizerHelper::lowerBitCount(MachineInstr &MI) {
8041 unsigned Opc = MI.getOpcode();
8042 const auto &TII = MIRBuilder.getTII();
8043 auto isSupported = [this](const LegalityQuery &Q) {
8044 auto QAction = LI.getAction(Query: Q).Action;
8045 return QAction == Legal || QAction == Libcall || QAction == Custom;
8046 };
8047 switch (Opc) {
8048 default:
8049 return UnableToLegalize;
8050 case TargetOpcode::G_CTLZ_ZERO_POISON: {
8051 // This trivially expands to CTLZ.
8052 Observer.changingInstr(MI);
8053 MI.setDesc(TII.get(Opcode: TargetOpcode::G_CTLZ));
8054 Observer.changedInstr(MI);
8055 return Legalized;
8056 }
8057 case TargetOpcode::G_CTLZ: {
8058 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
8059 unsigned Len = SrcTy.getScalarSizeInBits();
8060
8061 if (isSupported({TargetOpcode::G_CTLZ_ZERO_POISON, {DstTy, SrcTy}})) {
8062 // If CTLZ_ZERO_POISON is supported, emit that and a select for zero.
8063 auto CtlzZU = MIRBuilder.buildCTLZ_ZERO_POISON(Dst: DstTy, Src0: SrcReg);
8064 auto ZeroSrc = MIRBuilder.buildConstant(Res: SrcTy, Val: 0);
8065 auto ICmp = MIRBuilder.buildICmp(
8066 Pred: CmpInst::ICMP_EQ, Res: SrcTy.changeElementSize(NewEltSize: 1), Op0: SrcReg, Op1: ZeroSrc);
8067 auto LenConst = MIRBuilder.buildConstant(Res: DstTy, Val: Len);
8068 MIRBuilder.buildSelect(Res: DstReg, Tst: ICmp, Op0: LenConst, Op1: CtlzZU);
8069 MI.eraseFromParent();
8070 return Legalized;
8071 }
8072 // for now, we do this:
8073 // NewLen = NextPowerOf2(Len);
8074 // x = x | (x >> 1);
8075 // x = x | (x >> 2);
8076 // ...
8077 // x = x | (x >>16);
8078 // x = x | (x >>32); // for 64-bit input
8079 // Upto NewLen/2
8080 // return Len - popcount(x);
8081 //
8082 // Ref: "Hacker's Delight" by Henry Warren
8083 Register Op = SrcReg;
8084 unsigned NewLen = PowerOf2Ceil(A: Len);
8085 for (unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) {
8086 auto MIBShiftAmt = MIRBuilder.buildConstant(Res: SrcTy, Val: 1ULL << i);
8087 auto MIBOp = MIRBuilder.buildOr(
8088 Dst: SrcTy, Src0: Op, Src1: MIRBuilder.buildLShr(Dst: SrcTy, Src0: Op, Src1: MIBShiftAmt));
8089 Op = MIBOp.getReg(Idx: 0);
8090 }
8091 auto MIBPop = MIRBuilder.buildCTPOP(Dst: DstTy, Src0: Op);
8092 MIRBuilder.buildSub(Dst: MI.getOperand(i: 0), Src0: MIRBuilder.buildConstant(Res: DstTy, Val: Len),
8093 Src1: MIBPop);
8094 MI.eraseFromParent();
8095 return Legalized;
8096 }
8097 case TargetOpcode::G_CTTZ_ZERO_POISON: {
8098 // This trivially expands to CTTZ.
8099 Observer.changingInstr(MI);
8100 MI.setDesc(TII.get(Opcode: TargetOpcode::G_CTTZ));
8101 Observer.changedInstr(MI);
8102 return Legalized;
8103 }
8104 case TargetOpcode::G_CTTZ: {
8105 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
8106
8107 unsigned Len = SrcTy.getScalarSizeInBits();
8108 if (isSupported({TargetOpcode::G_CTTZ_ZERO_POISON, {DstTy, SrcTy}})) {
8109 // If CTTZ_ZERO_POISON is legal or custom, emit that and a select with
8110 // zero.
8111 auto CttzZU = MIRBuilder.buildCTTZ_ZERO_POISON(Dst: DstTy, Src0: SrcReg);
8112 auto Zero = MIRBuilder.buildConstant(Res: SrcTy, Val: 0);
8113 auto ICmp = MIRBuilder.buildICmp(
8114 Pred: CmpInst::ICMP_EQ, Res: DstTy.changeElementSize(NewEltSize: 1), Op0: SrcReg, Op1: Zero);
8115 auto LenConst = MIRBuilder.buildConstant(Res: DstTy, Val: Len);
8116 MIRBuilder.buildSelect(Res: DstReg, Tst: ICmp, Op0: LenConst, Op1: CttzZU);
8117 MI.eraseFromParent();
8118 return Legalized;
8119 }
8120 // for now, we use: { return popcount(~x & (x - 1)); }
8121 // unless the target has ctlz but not ctpop, in which case we use:
8122 // { return 32 - nlz(~x & (x-1)); }
8123 // Ref: "Hacker's Delight" by Henry Warren
8124 auto MIBCstNeg1 = MIRBuilder.buildConstant(Res: SrcTy, Val: -1);
8125 auto MIBNot = MIRBuilder.buildXor(Dst: SrcTy, Src0: SrcReg, Src1: MIBCstNeg1);
8126 auto MIBTmp = MIRBuilder.buildAnd(
8127 Dst: SrcTy, Src0: MIBNot, Src1: MIRBuilder.buildAdd(Dst: SrcTy, Src0: SrcReg, Src1: MIBCstNeg1));
8128 if (!isSupported({TargetOpcode::G_CTPOP, {SrcTy, SrcTy}}) &&
8129 isSupported({TargetOpcode::G_CTLZ, {SrcTy, SrcTy}})) {
8130 auto MIBCstLen = MIRBuilder.buildConstant(Res: SrcTy, Val: Len);
8131 MIRBuilder.buildSub(Dst: MI.getOperand(i: 0), Src0: MIBCstLen,
8132 Src1: MIRBuilder.buildCTLZ(Dst: SrcTy, Src0: MIBTmp));
8133 MI.eraseFromParent();
8134 return Legalized;
8135 }
8136 Observer.changingInstr(MI);
8137 MI.setDesc(TII.get(Opcode: TargetOpcode::G_CTPOP));
8138 MI.getOperand(i: 1).setReg(MIBTmp.getReg(Idx: 0));
8139 Observer.changedInstr(MI);
8140 return Legalized;
8141 }
8142 case TargetOpcode::G_CTPOP: {
8143 Register SrcReg = MI.getOperand(i: 1).getReg();
8144 LLT Ty = MRI.getType(Reg: SrcReg);
8145 unsigned Size = Ty.getScalarSizeInBits();
8146 MachineIRBuilder &B = MIRBuilder;
8147
8148 // Bail out on irregular type lengths.
8149 if (Size > 128 || Size % 8 != 0)
8150 return UnableToLegalize;
8151
8152 // Count set bits in blocks of 2 bits. Default approach would be
8153 // B2Count = { val & 0x55555555 } + { (val >> 1) & 0x55555555 }
8154 // We use following formula instead:
8155 // B2Count = val - { (val >> 1) & 0x55555555 }
8156 // since it gives same result in blocks of 2 with one instruction less.
8157 auto C_1 = B.buildConstant(Res: Ty, Val: 1);
8158 auto B2Set1LoTo1Hi = B.buildLShr(Dst: Ty, Src0: SrcReg, Src1: C_1);
8159 APInt B2Mask1HiTo0 = APInt::getSplat(NewLen: Size, V: APInt(8, 0x55));
8160 auto C_B2Mask1HiTo0 = B.buildConstant(Res: Ty, Val: B2Mask1HiTo0);
8161 auto B2Count1Hi = B.buildAnd(Dst: Ty, Src0: B2Set1LoTo1Hi, Src1: C_B2Mask1HiTo0);
8162 auto B2Count = B.buildSub(Dst: Ty, Src0: SrcReg, Src1: B2Count1Hi);
8163
8164 // In order to get count in blocks of 4 add values from adjacent block of 2.
8165 // B4Count = { B2Count & 0x33333333 } + { (B2Count >> 2) & 0x33333333 }
8166 auto C_2 = B.buildConstant(Res: Ty, Val: 2);
8167 auto B4Set2LoTo2Hi = B.buildLShr(Dst: Ty, Src0: B2Count, Src1: C_2);
8168 APInt B4Mask2HiTo0 = APInt::getSplat(NewLen: Size, V: APInt(8, 0x33));
8169 auto C_B4Mask2HiTo0 = B.buildConstant(Res: Ty, Val: B4Mask2HiTo0);
8170 auto B4HiB2Count = B.buildAnd(Dst: Ty, Src0: B4Set2LoTo2Hi, Src1: C_B4Mask2HiTo0);
8171 auto B4LoB2Count = B.buildAnd(Dst: Ty, Src0: B2Count, Src1: C_B4Mask2HiTo0);
8172 auto B4Count = B.buildAdd(Dst: Ty, Src0: B4HiB2Count, Src1: B4LoB2Count);
8173
8174 // For count in blocks of 8 bits we don't have to mask high 4 bits before
8175 // addition since count value sits in range {0,...,8} and 4 bits are enough
8176 // to hold such binary values. After addition high 4 bits still hold count
8177 // of set bits in high 4 bit block, set them to zero and get 8 bit result.
8178 // B8Count = { B4Count + (B4Count >> 4) } & 0x0F0F0F0F
8179 auto C_4 = B.buildConstant(Res: Ty, Val: 4);
8180 auto B8HiB4Count = B.buildLShr(Dst: Ty, Src0: B4Count, Src1: C_4);
8181 auto B8CountDirty4Hi = B.buildAdd(Dst: Ty, Src0: B8HiB4Count, Src1: B4Count);
8182 APInt B8Mask4HiTo0 = APInt::getSplat(NewLen: Size, V: APInt(8, 0x0F));
8183 auto C_B8Mask4HiTo0 = B.buildConstant(Res: Ty, Val: B8Mask4HiTo0);
8184 auto B8Count = B.buildAnd(Dst: Ty, Src0: B8CountDirty4Hi, Src1: C_B8Mask4HiTo0);
8185
8186 assert(Size <= 128 && "Scalar size is too large for CTPOP lower algorithm");
8187
8188 // Avoid the multiply when shift-add is cheaper.
8189 if (Size == 16 && !Ty.isVector()) {
8190 // v = (v + (v >> 8)) & 0xFF;
8191 auto C_8 = B.buildConstant(Res: Ty, Val: 8);
8192 auto HighSum = B.buildLShr(Dst: Ty, Src0: B8Count, Src1: C_8);
8193 auto Res = B.buildAdd(Dst: Ty, Src0: B8Count, Src1: HighSum);
8194 B.buildAnd(Dst: MI.getOperand(i: 0).getReg(), Src0: Res, Src1: B.buildConstant(Res: Ty, Val: 0xFF));
8195 MI.eraseFromParent();
8196 return Legalized;
8197 }
8198
8199 // 8 bits can hold CTPOP result of 128 bit int or smaller. Mul with this
8200 // bitmask will set 8 msb in ResTmp to sum of all B8Counts in 8 bit blocks.
8201 auto MulMask = B.buildConstant(Res: Ty, Val: APInt::getSplat(NewLen: Size, V: APInt(8, 0x01)));
8202
8203 // Shift count result from 8 high bits to low bits.
8204 auto C_SizeM8 = B.buildConstant(Res: Ty, Val: Size - 8);
8205
8206 auto IsMulSupported = [this](const LLT Ty) {
8207 auto Action = LI.getAction(Query: {TargetOpcode::G_MUL, {Ty}}).Action;
8208 return Action == Legal || Action == WidenScalar || Action == Custom;
8209 };
8210 if (IsMulSupported(Ty)) {
8211 auto ResTmp = B.buildMul(Dst: Ty, Src0: B8Count, Src1: MulMask);
8212 B.buildLShr(Dst: MI.getOperand(i: 0).getReg(), Src0: ResTmp, Src1: C_SizeM8);
8213 } else {
8214 auto ResTmp = B8Count;
8215 for (unsigned Shift = 8; Shift < Size; Shift *= 2) {
8216 auto ShiftC = B.buildConstant(Res: Ty, Val: Shift);
8217 auto Shl = B.buildShl(Dst: Ty, Src0: ResTmp, Src1: ShiftC);
8218 ResTmp = B.buildAdd(Dst: Ty, Src0: ResTmp, Src1: Shl);
8219 }
8220 B.buildLShr(Dst: MI.getOperand(i: 0).getReg(), Src0: ResTmp, Src1: C_SizeM8);
8221 }
8222 MI.eraseFromParent();
8223 return Legalized;
8224 }
8225 case TargetOpcode::G_CTLS: {
8226 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
8227
8228 // ctls(x) -> ctlz(x ^ (x >> (N - 1))) - 1
8229 auto SignIdxC =
8230 MIRBuilder.buildConstant(Res: SrcTy, Val: SrcTy.getScalarSizeInBits() - 1);
8231 auto OneC = MIRBuilder.buildConstant(Res: DstTy, Val: 1);
8232
8233 auto Shr = MIRBuilder.buildAShr(Dst: SrcTy, Src0: SrcReg, Src1: SignIdxC);
8234
8235 auto Xor = MIRBuilder.buildXor(Dst: SrcTy, Src0: SrcReg, Src1: Shr);
8236 auto Ctlz = MIRBuilder.buildCTLZ(Dst: DstTy, Src0: Xor);
8237
8238 MIRBuilder.buildSub(Dst: DstReg, Src0: Ctlz, Src1: OneC);
8239 MI.eraseFromParent();
8240 return Legalized;
8241 }
8242 }
8243}
8244
8245// Check that (every element of) Reg is undef or not an exact multiple of BW.
8246static bool isNonZeroModBitWidthOrUndef(const MachineRegisterInfo &MRI,
8247 Register Reg, unsigned BW) {
8248 return matchUnaryPredicate(
8249 MRI, Reg,
8250 Match: [=](const Constant *C) {
8251 // Null constant here means an undef.
8252 const ConstantInt *CI = dyn_cast_or_null<ConstantInt>(Val: C);
8253 return !CI || CI->getValue().urem(RHS: BW) != 0;
8254 },
8255 /*AllowUndefs*/ true);
8256}
8257
8258// Check that (every element of) Reg is a constant that is not an exact multiple
8259// of BW.
8260static bool isNonZeroModBitWidth(const MachineRegisterInfo &MRI, Register Reg,
8261 unsigned BW) {
8262 return matchUnaryPredicate(MRI, Reg, Match: [=](const Constant *C) {
8263 return cast<ConstantInt>(Val: C)->getValue().urem(RHS: BW) != 0;
8264 });
8265}
8266
8267LegalizerHelper::LegalizeResult
8268LegalizerHelper::lowerFunnelShiftWithInverse(MachineInstr &MI) {
8269 auto [Dst, X, Y, Z] = MI.getFirst4Regs();
8270 LLT Ty = MRI.getType(Reg: Dst);
8271 LLT ShTy = MRI.getType(Reg: Z);
8272
8273 unsigned BW = Ty.getScalarSizeInBits();
8274
8275 if (!isPowerOf2_32(Value: BW))
8276 return UnableToLegalize;
8277
8278 const bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL;
8279 unsigned RevOpcode = IsFSHL ? TargetOpcode::G_FSHR : TargetOpcode::G_FSHL;
8280
8281 if (isNonZeroModBitWidthOrUndef(MRI, Reg: Z, BW)) {
8282 // fshl X, Y, Z -> fshr X, Y, -Z
8283 // fshr X, Y, Z -> fshl X, Y, -Z
8284 auto Zero = MIRBuilder.buildConstant(Res: ShTy, Val: 0);
8285 Z = MIRBuilder.buildSub(Dst: Ty, Src0: Zero, Src1: Z).getReg(Idx: 0);
8286 } else {
8287 // fshl X, Y, Z -> fshr (srl X, 1), (fshr X, Y, 1), ~Z
8288 // fshr X, Y, Z -> fshl (fshl X, Y, 1), (shl Y, 1), ~Z
8289 auto One = MIRBuilder.buildConstant(Res: ShTy, Val: 1);
8290 if (IsFSHL) {
8291 Y = MIRBuilder.buildInstr(Opc: RevOpcode, DstOps: {Ty}, SrcOps: {X, Y, One}).getReg(Idx: 0);
8292 X = MIRBuilder.buildLShr(Dst: Ty, Src0: X, Src1: One).getReg(Idx: 0);
8293 } else {
8294 X = MIRBuilder.buildInstr(Opc: RevOpcode, DstOps: {Ty}, SrcOps: {X, Y, One}).getReg(Idx: 0);
8295 Y = MIRBuilder.buildShl(Dst: Ty, Src0: Y, Src1: One).getReg(Idx: 0);
8296 }
8297
8298 Z = MIRBuilder.buildNot(Dst: ShTy, Src0: Z).getReg(Idx: 0);
8299 }
8300
8301 MIRBuilder.buildInstr(Opc: RevOpcode, DstOps: {Dst}, SrcOps: {X, Y, Z});
8302 MI.eraseFromParent();
8303 return Legalized;
8304}
8305
8306LegalizerHelper::LegalizeResult
8307LegalizerHelper::lowerFunnelShiftAsShifts(MachineInstr &MI) {
8308 auto [Dst, X, Y, Z] = MI.getFirst4Regs();
8309 LLT Ty = MRI.getType(Reg: Dst);
8310 LLT ShTy = MRI.getType(Reg: Z);
8311
8312 const unsigned BW = Ty.getScalarSizeInBits();
8313 const bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL;
8314
8315 Register ShX, ShY;
8316 Register ShAmt, InvShAmt;
8317 unsigned OrFlags = MachineInstr::NoFlags;
8318
8319 // FIXME: Emit optimized urem by constant instead of letting it expand later.
8320 if (isNonZeroModBitWidthOrUndef(MRI, Reg: Z, BW)) {
8321 // fshl: X << C | Y >> (BW - C)
8322 // fshr: X << (BW - C) | Y >> C
8323 // where C = Z % BW is not zero
8324 // If Z has undef elements, C may be zero so the or isn't disjoint.
8325 if (isNonZeroModBitWidth(MRI, Reg: Z, BW))
8326 OrFlags |= MachineInstr::Disjoint;
8327 auto BitWidthC = MIRBuilder.buildConstant(Res: ShTy, Val: BW);
8328 ShAmt = MIRBuilder.buildURem(Dst: ShTy, Src0: Z, Src1: BitWidthC).getReg(Idx: 0);
8329 InvShAmt = MIRBuilder.buildSub(Dst: ShTy, Src0: BitWidthC, Src1: ShAmt).getReg(Idx: 0);
8330 ShX = MIRBuilder.buildShl(Dst: Ty, Src0: X, Src1: IsFSHL ? ShAmt : InvShAmt).getReg(Idx: 0);
8331 ShY = MIRBuilder.buildLShr(Dst: Ty, Src0: Y, Src1: IsFSHL ? InvShAmt : ShAmt).getReg(Idx: 0);
8332 } else {
8333 // fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))
8334 // fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)
8335 // The extra shift by 1 guarantees the two sides don't overlap.
8336 OrFlags |= MachineInstr::Disjoint;
8337 auto Mask = MIRBuilder.buildConstant(Res: ShTy, Val: BW - 1);
8338 if (isPowerOf2_32(Value: BW)) {
8339 // Z % BW -> Z & (BW - 1)
8340 ShAmt = MIRBuilder.buildAnd(Dst: ShTy, Src0: Z, Src1: Mask).getReg(Idx: 0);
8341 // (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
8342 auto NotZ = MIRBuilder.buildNot(Dst: ShTy, Src0: Z);
8343 InvShAmt = MIRBuilder.buildAnd(Dst: ShTy, Src0: NotZ, Src1: Mask).getReg(Idx: 0);
8344 } else {
8345 auto BitWidthC = MIRBuilder.buildConstant(Res: ShTy, Val: BW);
8346 ShAmt = MIRBuilder.buildURem(Dst: ShTy, Src0: Z, Src1: BitWidthC).getReg(Idx: 0);
8347 InvShAmt = MIRBuilder.buildSub(Dst: ShTy, Src0: Mask, Src1: ShAmt).getReg(Idx: 0);
8348 }
8349
8350 auto One = MIRBuilder.buildConstant(Res: ShTy, Val: 1);
8351 if (IsFSHL) {
8352 ShX = MIRBuilder.buildShl(Dst: Ty, Src0: X, Src1: ShAmt).getReg(Idx: 0);
8353 auto ShY1 = MIRBuilder.buildLShr(Dst: Ty, Src0: Y, Src1: One);
8354 ShY = MIRBuilder.buildLShr(Dst: Ty, Src0: ShY1, Src1: InvShAmt).getReg(Idx: 0);
8355 } else {
8356 auto ShX1 = MIRBuilder.buildShl(Dst: Ty, Src0: X, Src1: One);
8357 ShX = MIRBuilder.buildShl(Dst: Ty, Src0: ShX1, Src1: InvShAmt).getReg(Idx: 0);
8358 ShY = MIRBuilder.buildLShr(Dst: Ty, Src0: Y, Src1: ShAmt).getReg(Idx: 0);
8359 }
8360 }
8361
8362 MIRBuilder.buildOr(Dst, Src0: ShX, Src1: ShY, Flags: OrFlags);
8363 MI.eraseFromParent();
8364 return Legalized;
8365}
8366
8367LegalizerHelper::LegalizeResult
8368LegalizerHelper::lowerFunnelShift(MachineInstr &MI) {
8369 // These operations approximately do the following (while avoiding undefined
8370 // shifts by BW):
8371 // G_FSHL: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
8372 // G_FSHR: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
8373 Register Dst = MI.getOperand(i: 0).getReg();
8374 LLT Ty = MRI.getType(Reg: Dst);
8375 LLT ShTy = MRI.getType(Reg: MI.getOperand(i: 3).getReg());
8376
8377 bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL;
8378 unsigned RevOpcode = IsFSHL ? TargetOpcode::G_FSHR : TargetOpcode::G_FSHL;
8379
8380 // TODO: Use smarter heuristic that accounts for vector legalization.
8381 if (LI.getAction(Query: {RevOpcode, {Ty, ShTy}}).Action == Lower)
8382 return lowerFunnelShiftAsShifts(MI);
8383
8384 // This only works for powers of 2, fallback to shifts if it fails.
8385 LegalizerHelper::LegalizeResult Result = lowerFunnelShiftWithInverse(MI);
8386 if (Result == UnableToLegalize)
8387 return lowerFunnelShiftAsShifts(MI);
8388 return Result;
8389}
8390
8391LegalizerHelper::LegalizeResult LegalizerHelper::lowerEXT(MachineInstr &MI) {
8392 auto [Dst, Src] = MI.getFirst2Regs();
8393 LLT DstTy = MRI.getType(Reg: Dst);
8394 LLT SrcTy = MRI.getType(Reg: Src);
8395
8396 uint32_t DstTySize = DstTy.getSizeInBits();
8397 uint32_t DstTyScalarSize = DstTy.getScalarSizeInBits();
8398 uint32_t SrcTyScalarSize = SrcTy.getScalarSizeInBits();
8399
8400 if (!isPowerOf2_32(Value: DstTySize) || !isPowerOf2_32(Value: DstTyScalarSize) ||
8401 !isPowerOf2_32(Value: SrcTyScalarSize))
8402 return UnableToLegalize;
8403
8404 // The step between extend is too large, split it by creating an intermediate
8405 // extend instruction
8406 if (SrcTyScalarSize * 2 < DstTyScalarSize) {
8407 LLT MidTy = SrcTy.changeElementSize(NewEltSize: SrcTyScalarSize * 2);
8408 // If the destination type is illegal, split it into multiple statements
8409 // zext x -> zext(merge(zext(unmerge), zext(unmerge)))
8410 auto NewExt = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {MidTy}, SrcOps: {Src});
8411 // Unmerge the vector
8412 LLT EltTy = MidTy.changeElementCount(
8413 EC: MidTy.getElementCount().divideCoefficientBy(RHS: 2));
8414 auto UnmergeSrc = MIRBuilder.buildUnmerge(Res: EltTy, Op: NewExt);
8415
8416 // ZExt the vectors
8417 LLT ZExtResTy = DstTy.changeElementCount(
8418 EC: DstTy.getElementCount().divideCoefficientBy(RHS: 2));
8419 auto ZExtRes1 = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {ZExtResTy},
8420 SrcOps: {UnmergeSrc.getReg(Idx: 0)});
8421 auto ZExtRes2 = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {ZExtResTy},
8422 SrcOps: {UnmergeSrc.getReg(Idx: 1)});
8423
8424 // Merge the ending vectors
8425 MIRBuilder.buildMergeLikeInstr(Res: Dst, Ops: {ZExtRes1, ZExtRes2});
8426
8427 MI.eraseFromParent();
8428 return Legalized;
8429 }
8430 return UnableToLegalize;
8431}
8432
8433LegalizerHelper::LegalizeResult LegalizerHelper::lowerTRUNC(MachineInstr &MI) {
8434 // MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
8435 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
8436 // Similar to how operand splitting is done in SelectiondDAG, we can handle
8437 // %res(v8s8) = G_TRUNC %in(v8s32) by generating:
8438 // %inlo(<4x s32>), %inhi(<4 x s32>) = G_UNMERGE %in(<8 x s32>)
8439 // %lo16(<4 x s16>) = G_TRUNC %inlo
8440 // %hi16(<4 x s16>) = G_TRUNC %inhi
8441 // %in16(<8 x s16>) = G_CONCAT_VECTORS %lo16, %hi16
8442 // %res(<8 x s8>) = G_TRUNC %in16
8443
8444 assert(MI.getOpcode() == TargetOpcode::G_TRUNC);
8445
8446 Register DstReg = MI.getOperand(i: 0).getReg();
8447 Register SrcReg = MI.getOperand(i: 1).getReg();
8448 LLT DstTy = MRI.getType(Reg: DstReg);
8449 LLT SrcTy = MRI.getType(Reg: SrcReg);
8450
8451 if (DstTy.isVector() && isPowerOf2_32(Value: DstTy.getNumElements()) &&
8452 isPowerOf2_32(Value: DstTy.getScalarSizeInBits()) &&
8453 isPowerOf2_32(Value: SrcTy.getNumElements()) &&
8454 isPowerOf2_32(Value: SrcTy.getScalarSizeInBits())) {
8455 // Split input type.
8456 LLT SplitSrcTy = SrcTy.changeElementCount(
8457 EC: SrcTy.getElementCount().divideCoefficientBy(RHS: 2));
8458
8459 // First, split the source into two smaller vectors.
8460 SmallVector<Register, 2> SplitSrcs;
8461 extractParts(Reg: SrcReg, Ty: SplitSrcTy, NumParts: 2, VRegs&: SplitSrcs, MIRBuilder, MRI);
8462
8463 // Truncate the splits into intermediate narrower elements.
8464 LLT InterTy;
8465 if (DstTy.getScalarSizeInBits() * 2 < SrcTy.getScalarSizeInBits())
8466 InterTy = SplitSrcTy.changeElementSize(NewEltSize: DstTy.getScalarSizeInBits() * 2);
8467 else
8468 InterTy = SplitSrcTy.changeElementSize(NewEltSize: DstTy.getScalarSizeInBits());
8469 for (Register &Src : SplitSrcs)
8470 Src = MIRBuilder.buildTrunc(Res: InterTy, Op: Src).getReg(Idx: 0);
8471
8472 // Combine the new truncates into one vector
8473 auto Merge = MIRBuilder.buildMergeLikeInstr(
8474 Res: DstTy.changeElementSize(NewEltSize: InterTy.getScalarSizeInBits()), Ops: SplitSrcs);
8475
8476 // Truncate the new vector to the final result type
8477 if (DstTy.getScalarSizeInBits() * 2 < SrcTy.getScalarSizeInBits())
8478 MIRBuilder.buildTrunc(Res: MI.getOperand(i: 0).getReg(), Op: Merge.getReg(Idx: 0));
8479 else
8480 MIRBuilder.buildCopy(Res: MI.getOperand(i: 0).getReg(), Op: Merge.getReg(Idx: 0));
8481
8482 MI.eraseFromParent();
8483
8484 return Legalized;
8485 }
8486 return UnableToLegalize;
8487}
8488
8489LegalizerHelper::LegalizeResult
8490LegalizerHelper::lowerRotateWithReverseRotate(MachineInstr &MI) {
8491 auto [Dst, DstTy, Src, SrcTy, Amt, AmtTy] = MI.getFirst3RegLLTs();
8492 auto Zero = MIRBuilder.buildConstant(Res: AmtTy, Val: 0);
8493 bool IsLeft = MI.getOpcode() == TargetOpcode::G_ROTL;
8494 unsigned RevRot = IsLeft ? TargetOpcode::G_ROTR : TargetOpcode::G_ROTL;
8495 auto Neg = MIRBuilder.buildSub(Dst: AmtTy, Src0: Zero, Src1: Amt);
8496 MIRBuilder.buildInstr(Opc: RevRot, DstOps: {Dst}, SrcOps: {Src, Neg});
8497 MI.eraseFromParent();
8498 return Legalized;
8499}
8500
8501LegalizerHelper::LegalizeResult LegalizerHelper::lowerRotate(MachineInstr &MI) {
8502 auto [Dst, DstTy, Src, SrcTy, Amt, AmtTy] = MI.getFirst3RegLLTs();
8503
8504 unsigned EltSizeInBits = DstTy.getScalarSizeInBits();
8505 bool IsLeft = MI.getOpcode() == TargetOpcode::G_ROTL;
8506
8507 MIRBuilder.setInstrAndDebugLoc(MI);
8508
8509 // If a rotate in the other direction is supported, use it.
8510 unsigned RevRot = IsLeft ? TargetOpcode::G_ROTR : TargetOpcode::G_ROTL;
8511 if (LI.isLegalOrCustom(Query: {RevRot, {DstTy, SrcTy}}) &&
8512 isPowerOf2_32(Value: EltSizeInBits))
8513 return lowerRotateWithReverseRotate(MI);
8514
8515 // If a funnel shift is supported, use it.
8516 unsigned FShOpc = IsLeft ? TargetOpcode::G_FSHL : TargetOpcode::G_FSHR;
8517 unsigned RevFsh = !IsLeft ? TargetOpcode::G_FSHL : TargetOpcode::G_FSHR;
8518 bool IsFShLegal = false;
8519 if ((IsFShLegal = LI.isLegalOrCustom(Query: {FShOpc, {DstTy, AmtTy}})) ||
8520 LI.isLegalOrCustom(Query: {RevFsh, {DstTy, AmtTy}})) {
8521 auto buildFunnelShift = [&](unsigned Opc, Register R1, Register R2,
8522 Register R3) {
8523 MIRBuilder.buildInstr(Opc, DstOps: {R1}, SrcOps: {R2, R2, R3});
8524 MI.eraseFromParent();
8525 return Legalized;
8526 };
8527 // If a funnel shift in the other direction is supported, use it.
8528 if (IsFShLegal) {
8529 return buildFunnelShift(FShOpc, Dst, Src, Amt);
8530 } else if (isPowerOf2_32(Value: EltSizeInBits)) {
8531 Amt = MIRBuilder.buildNeg(Dst: DstTy, Src0: Amt).getReg(Idx: 0);
8532 return buildFunnelShift(RevFsh, Dst, Src, Amt);
8533 }
8534 }
8535
8536 auto Zero = MIRBuilder.buildConstant(Res: AmtTy, Val: 0);
8537 unsigned ShOpc = IsLeft ? TargetOpcode::G_SHL : TargetOpcode::G_LSHR;
8538 unsigned RevShiftOpc = IsLeft ? TargetOpcode::G_LSHR : TargetOpcode::G_SHL;
8539 auto BitWidthMinusOneC = MIRBuilder.buildConstant(Res: AmtTy, Val: EltSizeInBits - 1);
8540 Register ShVal;
8541 Register RevShiftVal;
8542 unsigned OrFlags = MachineInstr::NoFlags;
8543 if (isPowerOf2_32(Value: EltSizeInBits)) {
8544 // If c may be zero modulo w, both shifts may return x so the or isn't
8545 // disjoint.
8546 if (isNonZeroModBitWidth(MRI, Reg: Amt, BW: EltSizeInBits))
8547 OrFlags |= MachineInstr::Disjoint;
8548 // (rotl x, c) -> x << (c & (w - 1)) | x >> (-c & (w - 1))
8549 // (rotr x, c) -> x >> (c & (w - 1)) | x << (-c & (w - 1))
8550 auto NegAmt = MIRBuilder.buildSub(Dst: AmtTy, Src0: Zero, Src1: Amt);
8551 auto ShAmt = MIRBuilder.buildAnd(Dst: AmtTy, Src0: Amt, Src1: BitWidthMinusOneC);
8552 ShVal = MIRBuilder.buildInstr(Opc: ShOpc, DstOps: {DstTy}, SrcOps: {Src, ShAmt}).getReg(Idx: 0);
8553 auto RevAmt = MIRBuilder.buildAnd(Dst: AmtTy, Src0: NegAmt, Src1: BitWidthMinusOneC);
8554 RevShiftVal =
8555 MIRBuilder.buildInstr(Opc: RevShiftOpc, DstOps: {DstTy}, SrcOps: {Src, RevAmt}).getReg(Idx: 0);
8556 } else {
8557 OrFlags |= MachineInstr::Disjoint;
8558 // (rotl x, c) -> x << (c % w) | x >> 1 >> (w - 1 - (c % w))
8559 // (rotr x, c) -> x >> (c % w) | x << 1 << (w - 1 - (c % w))
8560 auto BitWidthC = MIRBuilder.buildConstant(Res: AmtTy, Val: EltSizeInBits);
8561 auto ShAmt = MIRBuilder.buildURem(Dst: AmtTy, Src0: Amt, Src1: BitWidthC);
8562 ShVal = MIRBuilder.buildInstr(Opc: ShOpc, DstOps: {DstTy}, SrcOps: {Src, ShAmt}).getReg(Idx: 0);
8563 auto RevAmt = MIRBuilder.buildSub(Dst: AmtTy, Src0: BitWidthMinusOneC, Src1: ShAmt);
8564 auto One = MIRBuilder.buildConstant(Res: AmtTy, Val: 1);
8565 auto Inner = MIRBuilder.buildInstr(Opc: RevShiftOpc, DstOps: {DstTy}, SrcOps: {Src, One});
8566 RevShiftVal =
8567 MIRBuilder.buildInstr(Opc: RevShiftOpc, DstOps: {DstTy}, SrcOps: {Inner, RevAmt}).getReg(Idx: 0);
8568 }
8569 MIRBuilder.buildOr(Dst, Src0: ShVal, Src1: RevShiftVal, Flags: OrFlags);
8570 MI.eraseFromParent();
8571 return Legalized;
8572}
8573
8574// Expand s32 = G_UITOFP s64 to an IEEE float representation using bit
8575// operations and G_SITOFP
8576LegalizerHelper::LegalizeResult
8577LegalizerHelper::lowerU64ToF32WithSITOFP(MachineInstr &MI) {
8578 auto [Dst, Src] = MI.getFirst2Regs();
8579 const LLT S64 = LLT::scalar(SizeInBits: 64);
8580 const LLT S32 = LLT::scalar(SizeInBits: 32);
8581 const LLT S1 = LLT::scalar(SizeInBits: 1);
8582
8583 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S32);
8584
8585 // For i64 < INT_MAX we simply reuse SITOFP.
8586 // Otherwise, divide i64 by 2, round result by ORing with the lowest bit
8587 // saved before division, convert to float by SITOFP, multiply the result
8588 // by 2.
8589 auto One = MIRBuilder.buildConstant(Res: S64, Val: 1);
8590 auto Zero = MIRBuilder.buildConstant(Res: S64, Val: 0);
8591 // Result if Src < INT_MAX
8592 auto SmallResult = MIRBuilder.buildSITOFP(Dst: S32, Src0: Src);
8593 // Result if Src >= INT_MAX
8594 auto Halved = MIRBuilder.buildLShr(Dst: S64, Src0: Src, Src1: One);
8595 auto LowerBit = MIRBuilder.buildAnd(Dst: S64, Src0: Src, Src1: One);
8596 auto RoundedHalved = MIRBuilder.buildOr(Dst: S64, Src0: Halved, Src1: LowerBit);
8597 auto HalvedFP = MIRBuilder.buildSITOFP(Dst: S32, Src0: RoundedHalved);
8598 auto LargeResult = MIRBuilder.buildFAdd(Dst: S32, Src0: HalvedFP, Src1: HalvedFP);
8599 // Check if the original value is larger than INT_MAX by comparing with
8600 // zero to pick one of the two conversions.
8601 auto IsLarge =
8602 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_SLT, Res: S1, Op0: Src, Op1: Zero);
8603 MIRBuilder.buildSelect(Res: Dst, Tst: IsLarge, Op0: LargeResult, Op1: SmallResult);
8604
8605 MI.eraseFromParent();
8606 return Legalized;
8607}
8608
8609// Expand s64 = G_UITOFP s64 using bit and float arithmetic operations to an
8610// IEEE double representation.
8611LegalizerHelper::LegalizeResult
8612LegalizerHelper::lowerU64ToF64BitFloatOps(MachineInstr &MI) {
8613 auto [Dst, Src] = MI.getFirst2Regs();
8614 const LLT S64 = LLT::scalar(SizeInBits: 64);
8615 const LLT S32 = LLT::scalar(SizeInBits: 32);
8616
8617 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S64);
8618
8619 // We create double value from 32 bit parts with 32 exponent difference.
8620 // Note that + and - are float operations that adjust the implicit leading
8621 // one, the bases 2^52 and 2^84 are for illustrative purposes.
8622 //
8623 // X = 2^52 * 1.0...LowBits
8624 // Y = 2^84 * 1.0...HighBits
8625 // Scratch = 2^84 * 1.0...HighBits - 2^84 * 1.0 - 2^52 * 1.0
8626 // = - 2^52 * 1.0...HighBits
8627 // Result = - 2^52 * 1.0...HighBits + 2^52 * 1.0...LowBits
8628 auto TwoP52 = MIRBuilder.buildConstant(Res: S64, UINT64_C(0x4330000000000000));
8629 auto TwoP84 = MIRBuilder.buildConstant(Res: S64, UINT64_C(0x4530000000000000));
8630 auto TwoP52P84 = llvm::bit_cast<double>(UINT64_C(0x4530000000100000));
8631 auto TwoP52P84FP = MIRBuilder.buildFConstant(Res: S64, Val: TwoP52P84);
8632 auto HalfWidth = MIRBuilder.buildConstant(Res: S64, Val: 32);
8633
8634 auto LowBits = MIRBuilder.buildTrunc(Res: S32, Op: Src);
8635 LowBits = MIRBuilder.buildZExt(Res: S64, Op: LowBits);
8636 auto LowBitsFP = MIRBuilder.buildOr(Dst: S64, Src0: TwoP52, Src1: LowBits);
8637 auto HighBits = MIRBuilder.buildLShr(Dst: S64, Src0: Src, Src1: HalfWidth);
8638 auto HighBitsFP = MIRBuilder.buildOr(Dst: S64, Src0: TwoP84, Src1: HighBits);
8639 auto Scratch = MIRBuilder.buildFSub(Dst: S64, Src0: HighBitsFP, Src1: TwoP52P84FP);
8640 MIRBuilder.buildFAdd(Dst, Src0: Scratch, Src1: LowBitsFP);
8641
8642 MI.eraseFromParent();
8643 return Legalized;
8644}
8645
8646/// i64->fp16 itofp can be lowered to i64->f64,f64->f32,f32->f16. We cannot
8647/// convert fpround f64->f16 without double-rounding, so we manually perform the
8648/// lowering here where we know it is valid.
8649static LegalizerHelper::LegalizeResult
8650loweri64tof16ITOFP(MachineInstr &MI, Register Dst, LLT DstTy, Register Src,
8651 LLT SrcTy, MachineIRBuilder &MIRBuilder) {
8652 auto DstFpTy =
8653 SrcTy.changeElementType(NewEltTy: LLT::floatIEEE(SizeInBits: SrcTy.getScalarSizeInBits()));
8654 auto M1 = MI.getOpcode() == TargetOpcode::G_UITOFP
8655 ? MIRBuilder.buildUITOFP(Dst: DstFpTy, Src0: Src)
8656 : MIRBuilder.buildSITOFP(Dst: DstFpTy, Src0: Src);
8657 LLT F32Ty = DstFpTy.changeElementSize(NewEltSize: 32);
8658 auto M2 = MIRBuilder.buildFPTrunc(Res: F32Ty, Op: M1);
8659 MIRBuilder.buildFPTrunc(Res: Dst, Op: M2);
8660 MI.eraseFromParent();
8661 return LegalizerHelper::Legalized;
8662}
8663
8664LegalizerHelper::LegalizeResult LegalizerHelper::lowerUITOFP(MachineInstr &MI) {
8665 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8666
8667 if (SrcTy == LLT::scalar(SizeInBits: 1)) {
8668 auto True = MIRBuilder.buildFConstant(Res: DstTy, Val: 1.0);
8669 auto False = MIRBuilder.buildFConstant(Res: DstTy, Val: 0.0);
8670 MIRBuilder.buildSelect(Res: Dst, Tst: Src, Op0: True, Op1: False);
8671 MI.eraseFromParent();
8672 return Legalized;
8673 }
8674
8675 if (DstTy.getScalarSizeInBits() == 16 && SrcTy.getScalarSizeInBits() == 64)
8676 return loweri64tof16ITOFP(MI, Dst, DstTy, Src, SrcTy, MIRBuilder);
8677
8678 if (SrcTy != LLT::scalar(SizeInBits: 64))
8679 return UnableToLegalize;
8680
8681 if (DstTy == LLT::scalar(SizeInBits: 32))
8682 // TODO: SelectionDAG has several alternative expansions to port which may
8683 // be more reasonable depending on the available instructions. We also need
8684 // a more advanced mechanism to choose an optimal version depending on
8685 // target features such as sitofp or CTLZ availability.
8686 return lowerU64ToF32WithSITOFP(MI);
8687
8688 if (DstTy == LLT::scalar(SizeInBits: 64))
8689 return lowerU64ToF64BitFloatOps(MI);
8690
8691 return UnableToLegalize;
8692}
8693
8694LegalizerHelper::LegalizeResult LegalizerHelper::lowerSITOFP(MachineInstr &MI) {
8695 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8696
8697 const LLT I64 = LLT::integer(SizeInBits: 64);
8698 const LLT I32 = LLT::integer(SizeInBits: 32);
8699 const LLT I1 = LLT::integer(SizeInBits: 1);
8700
8701 if (SrcTy == I1) {
8702 auto True = MIRBuilder.buildFConstant(Res: DstTy, Val: -1.0);
8703 auto False = MIRBuilder.buildFConstant(Res: DstTy, Val: 0.0);
8704 MIRBuilder.buildSelect(Res: Dst, Tst: Src, Op0: True, Op1: False);
8705 MI.eraseFromParent();
8706 return Legalized;
8707 }
8708
8709 if (DstTy.getScalarSizeInBits() == 16 && SrcTy.getScalarSizeInBits() == 64)
8710 return loweri64tof16ITOFP(MI, Dst, DstTy, Src, SrcTy, MIRBuilder);
8711
8712 if (SrcTy != I64)
8713 return UnableToLegalize;
8714
8715 if (DstTy.getScalarSizeInBits() == 32) {
8716 // signed cl2f(long l) {
8717 // long s = l >> 63;
8718 // float r = cul2f((l + s) ^ s);
8719 // return s ? -r : r;
8720 // }
8721 Register L = Src;
8722 auto SignBit = MIRBuilder.buildConstant(Res: I64, Val: 63);
8723 auto S = MIRBuilder.buildAShr(Dst: I64, Src0: L, Src1: SignBit);
8724
8725 auto LPlusS = MIRBuilder.buildAdd(Dst: I64, Src0: L, Src1: S);
8726 auto Xor = MIRBuilder.buildXor(Dst: I64, Src0: LPlusS, Src1: S);
8727 auto R = MIRBuilder.buildUITOFP(Dst: I32, Src0: Xor);
8728
8729 auto RNeg = MIRBuilder.buildFNeg(Dst: I32, Src0: R);
8730 auto SignNotZero = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: I1, Op0: S,
8731 Op1: MIRBuilder.buildConstant(Res: I64, Val: 0));
8732 MIRBuilder.buildSelect(Res: Dst, Tst: SignNotZero, Op0: RNeg, Op1: R);
8733 MI.eraseFromParent();
8734 return Legalized;
8735 }
8736
8737 return UnableToLegalize;
8738}
8739
8740LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOUI(MachineInstr &MI) {
8741 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8742 const LLT S64 = LLT::scalar(SizeInBits: 64);
8743 const LLT S32 = LLT::scalar(SizeInBits: 32);
8744
8745 if (SrcTy != S64 && SrcTy != S32)
8746 return UnableToLegalize;
8747 if (DstTy != S32 && DstTy != S64)
8748 return UnableToLegalize;
8749
8750 // FPTOSI gives same result as FPTOUI for positive signed integers.
8751 // FPTOUI needs to deal with fp values that convert to unsigned integers
8752 // greater or equal to 2^31 for float or 2^63 for double. For brevity 2^Exp.
8753
8754 APInt TwoPExpInt = APInt::getSignMask(BitWidth: DstTy.getSizeInBits());
8755 APFloat TwoPExpFP(SrcTy.getSizeInBits() == 32 ? APFloat::IEEEsingle()
8756 : APFloat::IEEEdouble(),
8757 APInt::getZero(numBits: SrcTy.getSizeInBits()));
8758 TwoPExpFP.convertFromAPInt(Input: TwoPExpInt, IsSigned: false, RM: APFloat::rmNearestTiesToEven);
8759
8760 MachineInstrBuilder FPTOSI = MIRBuilder.buildFPTOSI(Dst: DstTy, Src0: Src);
8761
8762 MachineInstrBuilder Threshold = MIRBuilder.buildFConstant(Res: SrcTy, Val: TwoPExpFP);
8763 // For fp Value greater or equal to Threshold(2^Exp), we use FPTOSI on
8764 // (Value - 2^Exp) and add 2^Exp by setting highest bit in result to 1.
8765 MachineInstrBuilder FSub = MIRBuilder.buildFSub(Dst: SrcTy, Src0: Src, Src1: Threshold);
8766 MachineInstrBuilder ResLowBits = MIRBuilder.buildFPTOSI(Dst: DstTy, Src0: FSub);
8767 MachineInstrBuilder ResHighBit = MIRBuilder.buildConstant(Res: DstTy, Val: TwoPExpInt);
8768 MachineInstrBuilder Res = MIRBuilder.buildXor(Dst: DstTy, Src0: ResLowBits, Src1: ResHighBit);
8769
8770 const LLT S1 = LLT::scalar(SizeInBits: 1);
8771
8772 MachineInstrBuilder FCMP =
8773 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_ULT, Res: S1, Op0: Src, Op1: Threshold);
8774 MIRBuilder.buildSelect(Res: Dst, Tst: FCMP, Op0: FPTOSI, Op1: Res);
8775
8776 MI.eraseFromParent();
8777 return Legalized;
8778}
8779
8780LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOSI(MachineInstr &MI) {
8781 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8782 const LLT S64 = LLT::scalar(SizeInBits: 64);
8783 const LLT S32 = LLT::scalar(SizeInBits: 32);
8784
8785 // FIXME: Only f32 to i64 conversions are supported.
8786 if (SrcTy.getScalarType() != S32 || DstTy.getScalarType() != S64)
8787 return UnableToLegalize;
8788
8789 // Expand f32 -> i64 conversion
8790 // This algorithm comes from compiler-rt's implementation of fixsfdi:
8791 // https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/fixsfdi.c
8792
8793 unsigned SrcEltBits = SrcTy.getScalarSizeInBits();
8794
8795 auto ExponentMask = MIRBuilder.buildConstant(Res: SrcTy, Val: 0x7F800000);
8796 auto ExponentLoBit = MIRBuilder.buildConstant(Res: SrcTy, Val: 23);
8797
8798 auto AndExpMask = MIRBuilder.buildAnd(Dst: SrcTy, Src0: Src, Src1: ExponentMask);
8799 auto ExponentBits = MIRBuilder.buildLShr(Dst: SrcTy, Src0: AndExpMask, Src1: ExponentLoBit);
8800
8801 auto SignMask = MIRBuilder.buildConstant(Res: SrcTy,
8802 Val: APInt::getSignMask(BitWidth: SrcEltBits));
8803 auto AndSignMask = MIRBuilder.buildAnd(Dst: SrcTy, Src0: Src, Src1: SignMask);
8804 auto SignLowBit = MIRBuilder.buildConstant(Res: SrcTy, Val: SrcEltBits - 1);
8805 auto Sign = MIRBuilder.buildAShr(Dst: SrcTy, Src0: AndSignMask, Src1: SignLowBit);
8806 Sign = MIRBuilder.buildSExt(Res: DstTy, Op: Sign);
8807
8808 auto MantissaMask = MIRBuilder.buildConstant(Res: SrcTy, Val: 0x007FFFFF);
8809 auto AndMantissaMask = MIRBuilder.buildAnd(Dst: SrcTy, Src0: Src, Src1: MantissaMask);
8810 auto K = MIRBuilder.buildConstant(Res: SrcTy, Val: 0x00800000);
8811
8812 auto R = MIRBuilder.buildOr(Dst: SrcTy, Src0: AndMantissaMask, Src1: K);
8813 R = MIRBuilder.buildZExt(Res: DstTy, Op: R);
8814
8815 auto Bias = MIRBuilder.buildConstant(Res: SrcTy, Val: 127);
8816 auto Exponent = MIRBuilder.buildSub(Dst: SrcTy, Src0: ExponentBits, Src1: Bias);
8817 auto SubExponent = MIRBuilder.buildSub(Dst: SrcTy, Src0: Exponent, Src1: ExponentLoBit);
8818 auto ExponentSub = MIRBuilder.buildSub(Dst: SrcTy, Src0: ExponentLoBit, Src1: Exponent);
8819
8820 auto Shl = MIRBuilder.buildShl(Dst: DstTy, Src0: R, Src1: SubExponent);
8821 auto Srl = MIRBuilder.buildLShr(Dst: DstTy, Src0: R, Src1: ExponentSub);
8822
8823 const LLT S1 = LLT::scalar(SizeInBits: 1);
8824 auto CmpGt = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SGT,
8825 Res: S1, Op0: Exponent, Op1: ExponentLoBit);
8826
8827 R = MIRBuilder.buildSelect(Res: DstTy, Tst: CmpGt, Op0: Shl, Op1: Srl);
8828
8829 auto XorSign = MIRBuilder.buildXor(Dst: DstTy, Src0: R, Src1: Sign);
8830 auto Ret = MIRBuilder.buildSub(Dst: DstTy, Src0: XorSign, Src1: Sign);
8831
8832 auto ZeroSrcTy = MIRBuilder.buildConstant(Res: SrcTy, Val: 0);
8833
8834 auto ExponentLt0 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT,
8835 Res: S1, Op0: Exponent, Op1: ZeroSrcTy);
8836
8837 auto ZeroDstTy = MIRBuilder.buildConstant(Res: DstTy, Val: 0);
8838 MIRBuilder.buildSelect(Res: Dst, Tst: ExponentLt0, Op0: ZeroDstTy, Op1: Ret);
8839
8840 MI.eraseFromParent();
8841 return Legalized;
8842}
8843
8844LegalizerHelper::LegalizeResult
8845LegalizerHelper::lowerFPTOINT_SAT(MachineInstr &MI) {
8846 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8847
8848 bool IsSigned = MI.getOpcode() == TargetOpcode::G_FPTOSI_SAT;
8849 unsigned SatWidth = DstTy.getScalarSizeInBits();
8850
8851 // Determine minimum and maximum integer values and their corresponding
8852 // floating-point values.
8853 APInt MinInt, MaxInt;
8854 if (IsSigned) {
8855 MinInt = APInt::getSignedMinValue(numBits: SatWidth);
8856 MaxInt = APInt::getSignedMaxValue(numBits: SatWidth);
8857 } else {
8858 MinInt = APInt::getMinValue(numBits: SatWidth);
8859 MaxInt = APInt::getMaxValue(numBits: SatWidth);
8860 }
8861
8862 const fltSemantics &Semantics = getFltSemanticForLLT(Ty: SrcTy.getScalarType());
8863 APFloat MinFloat(Semantics);
8864 APFloat MaxFloat(Semantics);
8865
8866 APFloat::opStatus MinStatus =
8867 MinFloat.convertFromAPInt(Input: MinInt, IsSigned, RM: APFloat::rmTowardZero);
8868 APFloat::opStatus MaxStatus =
8869 MaxFloat.convertFromAPInt(Input: MaxInt, IsSigned, RM: APFloat::rmTowardZero);
8870 bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact) &&
8871 !(MaxStatus & APFloat::opStatus::opInexact);
8872
8873 // If the integer bounds are exactly representable as floats, emit a
8874 // min+max+fptoi sequence. Otherwise we have to use a sequence of comparisons
8875 // and selects.
8876 if (AreExactFloatBounds) {
8877 // Clamp Src by MinFloat from below. If Src is NaN the result is MinFloat.
8878 auto MaxC = MIRBuilder.buildFConstant(Res: SrcTy, Val: MinFloat);
8879 auto MaxP =
8880 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OGT, Res: LLT::integer(SizeInBits: 1), Op0: Src, Op1: MaxC);
8881 auto Max = MIRBuilder.buildSelect(Res: SrcTy, Tst: MaxP, Op0: Src, Op1: MaxC);
8882 // Clamp by MaxFloat from above. NaN cannot occur.
8883 auto MinC = MIRBuilder.buildFConstant(Res: SrcTy, Val: MaxFloat);
8884 auto MinP = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OLT, Res: LLT::integer(SizeInBits: 1), Op0: Max,
8885 Op1: MinC, Flags: MachineInstr::FmNoNans);
8886 auto Min =
8887 MIRBuilder.buildSelect(Res: SrcTy, Tst: MinP, Op0: Max, Op1: MinC, Flags: MachineInstr::FmNoNans);
8888 // Convert clamped value to integer. In the unsigned case we're done,
8889 // because we mapped NaN to MinFloat, which will cast to zero.
8890 if (!IsSigned) {
8891 MIRBuilder.buildFPTOUI(Dst, Src0: Min);
8892 MI.eraseFromParent();
8893 return Legalized;
8894 }
8895
8896 // Otherwise, select 0 if Src is NaN.
8897 auto FpToInt = MIRBuilder.buildFPTOSI(Dst: DstTy, Src0: Min);
8898 auto IsZero =
8899 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_UNO, Res: LLT::integer(SizeInBits: 1), Op0: Src, Op1: Src);
8900 MIRBuilder.buildSelect(Res: Dst, Tst: IsZero, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: 0),
8901 Op1: FpToInt);
8902 MI.eraseFromParent();
8903 return Legalized;
8904 }
8905
8906 // Result of direct conversion. The assumption here is that the operation is
8907 // non-trapping and it's fine to apply it to an out-of-range value if we
8908 // select it away later.
8909 auto FpToInt = IsSigned ? MIRBuilder.buildFPTOSI(Dst: DstTy, Src0: Src)
8910 : MIRBuilder.buildFPTOUI(Dst: DstTy, Src0: Src);
8911
8912 // If Src ULT MinFloat, select MinInt. In particular, this also selects
8913 // MinInt if Src is NaN.
8914 auto ULT = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_ULT, Res: LLT::integer(SizeInBits: 1), Op0: Src,
8915 Op1: MIRBuilder.buildFConstant(Res: SrcTy, Val: MinFloat));
8916 auto Max = MIRBuilder.buildSelect(
8917 Res: DstTy, Tst: ULT, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: MinInt), Op1: FpToInt);
8918 // If Src OGT MaxFloat, select MaxInt.
8919 auto OGT = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OGT, Res: LLT::integer(SizeInBits: 1), Op0: Src,
8920 Op1: MIRBuilder.buildFConstant(Res: SrcTy, Val: MaxFloat));
8921
8922 // In the unsigned case we are done, because we mapped NaN to MinInt, which
8923 // is already zero.
8924 if (!IsSigned) {
8925 MIRBuilder.buildSelect(Res: Dst, Tst: OGT, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: MaxInt),
8926 Op1: Max);
8927 MI.eraseFromParent();
8928 return Legalized;
8929 }
8930
8931 // Otherwise, select 0 if Src is NaN.
8932 auto Min = MIRBuilder.buildSelect(
8933 Res: DstTy, Tst: OGT, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: MaxInt), Op1: Max);
8934 auto IsZero =
8935 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_UNO, Res: LLT::integer(SizeInBits: 1), Op0: Src, Op1: Src);
8936 MIRBuilder.buildSelect(Res: Dst, Tst: IsZero, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: 0), Op1: Min);
8937 MI.eraseFromParent();
8938 return Legalized;
8939}
8940
8941// Floating-point conversions using truncating and extending loads and stores.
8942LegalizerHelper::LegalizeResult
8943LegalizerHelper::lowerFPExtAndTruncMem(MachineInstr &MI) {
8944 assert((MI.getOpcode() == TargetOpcode::G_FPEXT ||
8945 MI.getOpcode() == TargetOpcode::G_FPTRUNC) &&
8946 "Only G_FPEXT and G_FPTRUNC are expected");
8947
8948 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
8949 MachinePointerInfo PtrInfo;
8950 unsigned StoreOpc;
8951 unsigned LoadOpc;
8952 LLT StackTy;
8953 if (MI.getOpcode() == TargetOpcode::G_FPEXT) {
8954 StackTy = SrcTy;
8955 StoreOpc = TargetOpcode::G_STORE;
8956 LoadOpc = TargetOpcode::G_FPEXTLOAD;
8957 } else {
8958 StackTy = DstTy;
8959 StoreOpc = TargetOpcode::G_FPTRUNCSTORE;
8960 LoadOpc = TargetOpcode::G_LOAD;
8961 }
8962
8963 Align StackTyAlign = getStackTemporaryAlignment(Ty: StackTy);
8964 auto StackTemp =
8965 createStackTemporary(Bytes: StackTy.getSizeInBytes(), Alignment: StackTyAlign, PtrInfo);
8966
8967 MachineFunction &MF = MIRBuilder.getMF();
8968 auto *StoreMMO = MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
8969 MemTy: StackTy, BaseAlignment: StackTyAlign);
8970 MIRBuilder.buildStoreInstr(Opcode: StoreOpc, Val: SrcReg, Addr: StackTemp, MMO&: *StoreMMO);
8971
8972 auto *LoadMMO = MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOLoad,
8973 MemTy: StackTy, BaseAlignment: StackTyAlign);
8974 MIRBuilder.buildLoadInstr(Opcode: LoadOpc, Res: DstReg, Addr: StackTemp, MMO&: *LoadMMO);
8975
8976 MI.eraseFromParent();
8977 return Legalized;
8978}
8979
8980// Expand a bf16 -> f32/f64 fpext with a shift and bitcast. This is based on the
8981// SDAG ISD::BF16_TO_FP lowering.
8982LegalizerHelper::LegalizeResult
8983LegalizerHelper::lowerFPEXT_BF16(MachineInstr &MI) {
8984 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
8985 assert(SrcTy.getScalarType().isBFloat16() &&
8986 "expected a bf16 source for bf16 fpext lowering");
8987
8988 LLT I16Ty = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 16));
8989 LLT I32Ty = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 32));
8990 LLT F32Ty = SrcTy.changeElementType(NewEltTy: LLT::float32());
8991
8992 auto SrcI =
8993 MIRBuilder.buildAnyExt(Res: I32Ty, Op: MIRBuilder.buildBitcast(Dst: I16Ty, Src: SrcReg));
8994 auto Shl =
8995 MIRBuilder.buildShl(Dst: I32Ty, Src0: SrcI, Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 16));
8996
8997 if (DstTy.getScalarType().isFloat32())
8998 MIRBuilder.buildBitcast(Dst: DstReg, Src: Shl);
8999 else
9000 MIRBuilder.buildFPExt(Res: DstReg, Op: MIRBuilder.buildBitcast(Dst: F32Ty, Src: Shl));
9001
9002 MI.eraseFromParent();
9003 return Legalized;
9004}
9005
9006LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPEXT(MachineInstr &MI) {
9007 auto [DstTy, SrcTy] = MI.getFirst2LLTs();
9008 if (SrcTy.getScalarType().isBFloat16() &&
9009 (DstTy.getScalarType().isFloat32() || DstTy.getScalarType().isFloat64()))
9010 return lowerFPEXT_BF16(MI);
9011
9012 return lowerFPExtAndTruncMem(MI);
9013}
9014
9015// f64 -> f16 conversion using round-to-nearest-even rounding mode.
9016LegalizerHelper::LegalizeResult
9017LegalizerHelper::lowerFPTRUNC_F64_TO_F16(MachineInstr &MI) {
9018 const LLT S1 = LLT::scalar(SizeInBits: 1);
9019 const LLT I32 = LLT::integer(SizeInBits: 32);
9020
9021 auto [Dst, Src] = MI.getFirst2Regs();
9022 assert(MRI.getType(Dst).getScalarType() == LLT::float16() &&
9023 MRI.getType(Src).getScalarType() == LLT::float64());
9024
9025 if (MRI.getType(Reg: Src).isVector()) // TODO: Handle vectors directly.
9026 return UnableToLegalize;
9027
9028 if (MI.getFlag(Flag: MachineInstr::FmAfn)) {
9029 unsigned Flags = MI.getFlags();
9030 auto Src32 = MIRBuilder.buildFPTrunc(Res: LLT::float32(), Op: Src, Flags);
9031 MIRBuilder.buildFPTrunc(Res: Dst, Op: Src32, Flags);
9032 MI.eraseFromParent();
9033 return Legalized;
9034 }
9035
9036 const unsigned ExpMask = 0x7ff;
9037 const unsigned ExpBiasf64 = 1023;
9038 const unsigned ExpBiasf16 = 15;
9039
9040 auto Unmerge = MIRBuilder.buildUnmerge(Res: I32, Op: Src);
9041 Register U = Unmerge.getReg(Idx: 0);
9042 Register UH = Unmerge.getReg(Idx: 1);
9043
9044 auto E = MIRBuilder.buildLShr(Dst: I32, Src0: UH, Src1: MIRBuilder.buildConstant(Res: I32, Val: 20));
9045 E = MIRBuilder.buildAnd(Dst: I32, Src0: E, Src1: MIRBuilder.buildConstant(Res: I32, Val: ExpMask));
9046
9047 // Subtract the fp64 exponent bias (1023) to get the real exponent and
9048 // add the f16 bias (15) to get the biased exponent for the f16 format.
9049 E = MIRBuilder.buildAdd(
9050 Dst: I32, Src0: E, Src1: MIRBuilder.buildConstant(Res: I32, Val: -ExpBiasf64 + ExpBiasf16));
9051
9052 auto M = MIRBuilder.buildLShr(Dst: I32, Src0: UH, Src1: MIRBuilder.buildConstant(Res: I32, Val: 8));
9053 M = MIRBuilder.buildAnd(Dst: I32, Src0: M, Src1: MIRBuilder.buildConstant(Res: I32, Val: 0xffe));
9054
9055 auto MaskedSig =
9056 MIRBuilder.buildAnd(Dst: I32, Src0: UH, Src1: MIRBuilder.buildConstant(Res: I32, Val: 0x1ff));
9057 MaskedSig = MIRBuilder.buildOr(Dst: I32, Src0: MaskedSig, Src1: U);
9058
9059 auto Zero = MIRBuilder.buildConstant(Res: I32, Val: 0);
9060 auto SigCmpNE0 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: S1, Op0: MaskedSig, Op1: Zero);
9061 auto Lo40Set = MIRBuilder.buildZExt(Res: I32, Op: SigCmpNE0);
9062 M = MIRBuilder.buildOr(Dst: I32, Src0: M, Src1: Lo40Set);
9063
9064 // (M != 0 ? 0x0200 : 0) | 0x7c00;
9065 auto Bits0x200 = MIRBuilder.buildConstant(Res: I32, Val: 0x0200);
9066 auto CmpM_NE0 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: S1, Op0: M, Op1: Zero);
9067 auto SelectCC = MIRBuilder.buildSelect(Res: I32, Tst: CmpM_NE0, Op0: Bits0x200, Op1: Zero);
9068
9069 auto Bits0x7c00 = MIRBuilder.buildConstant(Res: I32, Val: 0x7c00);
9070 auto I = MIRBuilder.buildOr(Dst: I32, Src0: SelectCC, Src1: Bits0x7c00);
9071
9072 // N = M | (E << 12);
9073 auto EShl12 = MIRBuilder.buildShl(Dst: I32, Src0: E, Src1: MIRBuilder.buildConstant(Res: I32, Val: 12));
9074 auto N = MIRBuilder.buildOr(Dst: I32, Src0: M, Src1: EShl12);
9075
9076 // B = clamp(1-E, 0, 13);
9077 auto One = MIRBuilder.buildConstant(Res: I32, Val: 1);
9078 auto OneSubExp = MIRBuilder.buildSub(Dst: I32, Src0: One, Src1: E);
9079 auto B = MIRBuilder.buildSMax(Dst: I32, Src0: OneSubExp, Src1: Zero);
9080 B = MIRBuilder.buildSMin(Dst: I32, Src0: B, Src1: MIRBuilder.buildConstant(Res: I32, Val: 13));
9081
9082 auto SigSetHigh =
9083 MIRBuilder.buildOr(Dst: I32, Src0: M, Src1: MIRBuilder.buildConstant(Res: I32, Val: 0x1000));
9084
9085 auto D = MIRBuilder.buildLShr(Dst: I32, Src0: SigSetHigh, Src1: B);
9086 auto D0 = MIRBuilder.buildShl(Dst: I32, Src0: D, Src1: B);
9087
9088 auto D0_NE_SigSetHigh = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: S1,
9089 Op0: D0, Op1: SigSetHigh);
9090 auto D1 = MIRBuilder.buildZExt(Res: I32, Op: D0_NE_SigSetHigh);
9091 D = MIRBuilder.buildOr(Dst: I32, Src0: D, Src1: D1);
9092
9093 auto CmpELtOne = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: S1, Op0: E, Op1: One);
9094 auto V = MIRBuilder.buildSelect(Res: I32, Tst: CmpELtOne, Op0: D, Op1: N);
9095
9096 auto VLow3 = MIRBuilder.buildAnd(Dst: I32, Src0: V, Src1: MIRBuilder.buildConstant(Res: I32, Val: 7));
9097 V = MIRBuilder.buildLShr(Dst: I32, Src0: V, Src1: MIRBuilder.buildConstant(Res: I32, Val: 2));
9098
9099 auto VLow3Eq3 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: S1, Op0: VLow3,
9100 Op1: MIRBuilder.buildConstant(Res: I32, Val: 3));
9101 auto V0 = MIRBuilder.buildZExt(Res: I32, Op: VLow3Eq3);
9102
9103 auto VLow3Gt5 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SGT, Res: S1, Op0: VLow3,
9104 Op1: MIRBuilder.buildConstant(Res: I32, Val: 5));
9105 auto V1 = MIRBuilder.buildZExt(Res: I32, Op: VLow3Gt5);
9106
9107 V1 = MIRBuilder.buildOr(Dst: I32, Src0: V0, Src1: V1);
9108 V = MIRBuilder.buildAdd(Dst: I32, Src0: V, Src1: V1);
9109
9110 auto CmpEGt30 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SGT, Res: S1, Op0: E,
9111 Op1: MIRBuilder.buildConstant(Res: I32, Val: 30));
9112 V = MIRBuilder.buildSelect(Res: I32, Tst: CmpEGt30,
9113 Op0: MIRBuilder.buildConstant(Res: I32, Val: 0x7c00), Op1: V);
9114
9115 auto CmpEGt1039 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: S1, Op0: E,
9116 Op1: MIRBuilder.buildConstant(Res: I32, Val: 1039));
9117 V = MIRBuilder.buildSelect(Res: I32, Tst: CmpEGt1039, Op0: I, Op1: V);
9118
9119 // Extract the sign bit.
9120 auto Sign = MIRBuilder.buildLShr(Dst: I32, Src0: UH, Src1: MIRBuilder.buildConstant(Res: I32, Val: 16));
9121 Sign = MIRBuilder.buildAnd(Dst: I32, Src0: Sign, Src1: MIRBuilder.buildConstant(Res: I32, Val: 0x8000));
9122
9123 // Insert the sign bit
9124 V = MIRBuilder.buildOr(Dst: I32, Src0: Sign, Src1: V);
9125
9126 MIRBuilder.buildTrunc(Res: Dst, Op: V);
9127 MI.eraseFromParent();
9128 return Legalized;
9129}
9130
9131// f32 -> bf16 conversion using round-to-nearest-even rounding mode.
9132LegalizerHelper::LegalizeResult
9133LegalizerHelper::lowerFPTRUNC_F32_TO_BF16(MachineInstr &MI) {
9134 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
9135 assert(DstTy.getScalarType() == LLT::bfloat16() &&
9136 SrcTy.getScalarType() == LLT::float32());
9137
9138 LLT I1Ty = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9139 LLT I16Ty = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 16));
9140 LLT I32Ty = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 32));
9141
9142 auto SrcI = MIRBuilder.buildBitcast(Dst: I32Ty, Src: SrcReg);
9143
9144 // Conversions should set NaN's quiet bit. This also prevents NaNs from
9145 // turning into infinities.
9146 auto NaN = MIRBuilder.buildOr(Dst: I32Ty, Src0: SrcI,
9147 Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 0x400000));
9148
9149 // Factor in the contribution of the low 16 bits.
9150 auto Lsb =
9151 MIRBuilder.buildLShr(Dst: I32Ty, Src0: SrcI, Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 16));
9152 Lsb = MIRBuilder.buildAnd(Dst: I32Ty, Src0: Lsb, Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 1));
9153 auto RoundingBias =
9154 MIRBuilder.buildAdd(Dst: I32Ty, Src0: Lsb, Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 0x7fff));
9155 auto Add = MIRBuilder.buildAdd(Dst: I32Ty, Src0: SrcI, Src1: RoundingBias);
9156
9157 // Don't round if we had a NaN, we don't want to turn 0x7fffffff into
9158 // 0x80000000.
9159 if (!MI.getFlag(Flag: MachineInstr::FmNoNans)) {
9160 auto IsNaN = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_UNO, Res: I1Ty, Op0: SrcReg,
9161 Op1: MIRBuilder.buildFConstant(Res: SrcTy, Val: 0));
9162 Add = MIRBuilder.buildSelect(Res: I32Ty, Tst: IsNaN, Op0: NaN, Op1: Add);
9163 }
9164
9165 // Now that we have rounded, shift the bits into position.
9166 auto Srl =
9167 MIRBuilder.buildLShr(Dst: I32Ty, Src0: Add, Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 16));
9168 auto Trunc = MIRBuilder.buildTrunc(Res: I16Ty, Op: Srl);
9169 MIRBuilder.buildBitcast(Dst: DstReg, Src: Trunc);
9170 MI.eraseFromParent();
9171 return Legalized;
9172}
9173
9174// Round a wide fp value to ResultTy's element size, forcing inexact
9175// results to the odd value so a subsequent narrowing round is correct. This
9176// avoids double-rounding when narrowing e.g. f64 -> f32 -> bf16. See Boldo &
9177// Melquiond, "When double rounding is odd" (2005).
9178Register LegalizerHelper::lowerRoundInexactToOdd(LLT ResultTy, Register Op) {
9179 LLT OperandTy = MRI.getType(Reg: Op);
9180 if (OperandTy.getScalarType() == ResultTy.getScalarType())
9181 return Op;
9182
9183 LLT ResultIntTy =
9184 ResultTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: ResultTy.getScalarSizeInBits()));
9185 LLT ResultCCTy = ResultTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9186 LLT OperandCCTy = OperandTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9187
9188 auto Narrow = MIRBuilder.buildFPTrunc(Res: ResultTy, Op);
9189 auto NarrowAsWide = MIRBuilder.buildFPExt(Res: OperandTy, Op: Narrow);
9190
9191 auto NarrowBits = MIRBuilder.buildBitcast(Dst: ResultIntTy, Src: Narrow);
9192 auto One = MIRBuilder.buildConstant(Res: ResultIntTy, Val: 1);
9193 auto NegativeOne = MIRBuilder.buildConstant(Res: ResultIntTy, Val: -1);
9194 auto Zero = MIRBuilder.buildConstant(Res: ResultIntTy, Val: 0);
9195 auto And = MIRBuilder.buildAnd(Dst: ResultIntTy, Src0: NarrowBits, Src1: One);
9196 // The result is already odd so we don't need to do anything.
9197 auto AlreadyOdd =
9198 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: ResultCCTy, Op0: And, Op1: Zero);
9199
9200 // We keep results which are exact, odd or NaN.
9201 auto KeepNarrow =
9202 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_UEQ, Res: OperandCCTy, Op0: Op, Op1: NarrowAsWide);
9203 KeepNarrow = MIRBuilder.buildOr(Dst: OperandCCTy, Src0: KeepNarrow, Src1: AlreadyOdd);
9204 // We morally performed a round-down if AbsNarrow is smaller than AbsWide.
9205 auto AbsWide = MIRBuilder.buildFAbs(Dst: OperandTy, Src0: Op);
9206 auto AbsNarrowAsWide = MIRBuilder.buildFAbs(Dst: OperandTy, Src0: NarrowAsWide);
9207 auto NarrowIsRd = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OGT, Res: OperandCCTy,
9208 Op0: AbsWide, Op1: AbsNarrowAsWide);
9209 // If narrow is the rounded-down value, pick the rounded-up value as it will
9210 // be odd; otherwise adjust down.
9211 auto Adjust =
9212 MIRBuilder.buildSelect(Res: ResultIntTy, Tst: NarrowIsRd, Op0: One, Op1: NegativeOne);
9213 auto Adjusted = MIRBuilder.buildAdd(Dst: ResultIntTy, Src0: NarrowBits, Src1: Adjust);
9214 auto Res =
9215 MIRBuilder.buildSelect(Res: ResultIntTy, Tst: KeepNarrow, Op0: NarrowBits, Op1: Adjusted);
9216 return MIRBuilder.buildBitcast(Dst: ResultTy, Src: Res).getReg(Idx: 0);
9217}
9218
9219// f64 -> bf16 conversion, correcting for double rounding.
9220LegalizerHelper::LegalizeResult
9221LegalizerHelper::lowerFPTRUNC_F64_TO_BF16(MachineInstr &MI) {
9222 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
9223 assert(DstTy.getScalarType() == LLT::bfloat16() &&
9224 SrcTy.getScalarType() == LLT::float64());
9225
9226 LLT F32Ty = SrcTy.changeElementType(NewEltTy: LLT::float32());
9227 Register OddF32 = lowerRoundInexactToOdd(ResultTy: F32Ty, Op: SrcReg);
9228 MIRBuilder.buildFPTrunc(Res: DstReg, Op: OddF32, Flags: MI.getFlags());
9229 MI.eraseFromParent();
9230 return Legalized;
9231}
9232
9233LegalizerHelper::LegalizeResult
9234LegalizerHelper::lowerFPTRUNC(MachineInstr &MI) {
9235 auto [DstTy, SrcTy] = MI.getFirst2LLTs();
9236 if (DstTy.getScalarType().isFloat16() && SrcTy.getScalarType().isFloat64())
9237 return lowerFPTRUNC_F64_TO_F16(MI);
9238
9239 if (DstTy.getScalarType().isBFloat16() && SrcTy.getScalarType().isFloat32())
9240 return lowerFPTRUNC_F32_TO_BF16(MI);
9241
9242 if (DstTy.getScalarType().isBFloat16() && SrcTy.getScalarType().isFloat64())
9243 return lowerFPTRUNC_F64_TO_BF16(MI);
9244
9245 return lowerFPExtAndTruncMem(MI);
9246}
9247
9248LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPOWI(MachineInstr &MI) {
9249 auto [Dst, Src0, Src1] = MI.getFirst3Regs();
9250 LLT Ty = MRI.getType(Reg: Dst);
9251
9252 auto CvtSrc1 = MIRBuilder.buildSITOFP(Dst: Ty, Src0: Src1);
9253 MIRBuilder.buildFPow(Dst, Src0, Src1: CvtSrc1, Flags: MI.getFlags());
9254 MI.eraseFromParent();
9255 return Legalized;
9256}
9257
9258LegalizerHelper::LegalizeResult LegalizerHelper::lowerFMODF(MachineInstr &MI) {
9259 auto [DstFrac, DstInt, Src] = MI.getFirst3Regs();
9260 LLT Ty = MRI.getType(Reg: Src);
9261 auto Flags = MI.getFlags();
9262 const LLT CondTy = Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9263
9264 auto IntPart = MIRBuilder.buildIntrinsicTrunc(Dst: Ty, Src0: Src, Flags);
9265 auto FracPart = MIRBuilder.buildFSub(Dst: Ty, Src0: Src, Src1: IntPart, Flags);
9266
9267 Register FracToUse;
9268 if (MI.getFlag(Flag: MachineInstr::FmNoInfs)) {
9269 FracToUse = FracPart.getReg(Idx: 0);
9270 } else {
9271 auto Abs = MIRBuilder.buildFAbs(Dst: Ty, Src0: Src, Flags);
9272 const fltSemantics &Semantics = getFltSemanticForLLT(Ty: Ty.getScalarType());
9273 auto Inf = MIRBuilder.buildFConstant(Res: Ty, Val: APFloat::getInf(Sem: Semantics));
9274 auto IsInf = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OEQ, Res: CondTy, Op0: Abs, Op1: Inf);
9275 auto Zero = MIRBuilder.buildFConstant(Res: Ty, Val: 0.0);
9276 auto Select = MIRBuilder.buildSelect(Res: Ty, Tst: IsInf, Op0: Zero, Op1: FracPart);
9277 FracToUse = Select.getReg(Idx: 0);
9278 }
9279
9280 MIRBuilder.buildFCopysign(Dst: DstFrac, Src0: FracToUse, Src1: Src, Flags);
9281 MIRBuilder.buildCopy(Res: DstInt, Op: IntPart.getReg(Idx: 0));
9282
9283 MI.eraseFromParent();
9284 return Legalized;
9285}
9286
9287static CmpInst::Predicate minMaxToCompare(unsigned Opc) {
9288 switch (Opc) {
9289 case TargetOpcode::G_SMIN:
9290 return CmpInst::ICMP_SLT;
9291 case TargetOpcode::G_SMAX:
9292 return CmpInst::ICMP_SGT;
9293 case TargetOpcode::G_UMIN:
9294 return CmpInst::ICMP_ULT;
9295 case TargetOpcode::G_UMAX:
9296 return CmpInst::ICMP_UGT;
9297 default:
9298 llvm_unreachable("not in integer min/max");
9299 }
9300}
9301
9302LegalizerHelper::LegalizeResult LegalizerHelper::lowerMinMax(MachineInstr &MI) {
9303 auto [Dst, Src0, Src1] = MI.getFirst3Regs();
9304
9305 const CmpInst::Predicate Pred = minMaxToCompare(Opc: MI.getOpcode());
9306 LLT CmpType = MRI.getType(Reg: Dst).changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9307
9308 auto Cmp = MIRBuilder.buildICmp(Pred, Res: CmpType, Op0: Src0, Op1: Src1);
9309 MIRBuilder.buildSelect(Res: Dst, Tst: Cmp, Op0: Src0, Op1: Src1);
9310
9311 MI.eraseFromParent();
9312 return Legalized;
9313}
9314
9315LegalizerHelper::LegalizeResult
9316LegalizerHelper::lowerThreewayCompare(MachineInstr &MI) {
9317 GSUCmp *Cmp = cast<GSUCmp>(Val: &MI);
9318
9319 Register Dst = Cmp->getReg(Idx: 0);
9320 LLT DstTy = MRI.getType(Reg: Dst);
9321 LLT SrcTy = MRI.getType(Reg: Cmp->getReg(Idx: 1));
9322 LLT CmpTy = DstTy.changeElementSize(NewEltSize: 1);
9323
9324 CmpInst::Predicate LTPredicate = Cmp->isSigned()
9325 ? CmpInst::Predicate::ICMP_SLT
9326 : CmpInst::Predicate::ICMP_ULT;
9327 CmpInst::Predicate GTPredicate = Cmp->isSigned()
9328 ? CmpInst::Predicate::ICMP_SGT
9329 : CmpInst::Predicate::ICMP_UGT;
9330
9331 auto Zero = MIRBuilder.buildConstant(Res: DstTy, Val: 0);
9332 auto IsGT = MIRBuilder.buildICmp(Pred: GTPredicate, Res: CmpTy, Op0: Cmp->getLHSReg(),
9333 Op1: Cmp->getRHSReg());
9334 auto IsLT = MIRBuilder.buildICmp(Pred: LTPredicate, Res: CmpTy, Op0: Cmp->getLHSReg(),
9335 Op1: Cmp->getRHSReg());
9336
9337 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
9338 auto BC = TLI.getBooleanContents(isVec: DstTy.isVector(), /*isFP=*/isFloat: false);
9339 if (TLI.preferSelectsOverBooleanArithmetic(
9340 VT: getApproximateEVTForLLT(Ty: SrcTy, Ctx)) ||
9341 BC == TargetLowering::UndefinedBooleanContent) {
9342 auto One = MIRBuilder.buildConstant(Res: DstTy, Val: 1);
9343 auto SelectZeroOrOne = MIRBuilder.buildSelect(Res: DstTy, Tst: IsGT, Op0: One, Op1: Zero);
9344
9345 auto MinusOne = MIRBuilder.buildConstant(Res: DstTy, Val: -1);
9346 MIRBuilder.buildSelect(Res: Dst, Tst: IsLT, Op0: MinusOne, Op1: SelectZeroOrOne);
9347 } else {
9348 if (BC == TargetLowering::ZeroOrNegativeOneBooleanContent)
9349 std::swap(a&: IsGT, b&: IsLT);
9350 // Extend boolean results to DstTy, which is at least i2, before subtracting
9351 // them.
9352 unsigned BoolExtOp =
9353 MIRBuilder.getBoolExtOp(IsVec: DstTy.isVector(), /*isFP=*/IsFP: false);
9354 IsGT = MIRBuilder.buildInstr(Opc: BoolExtOp, DstOps: {DstTy}, SrcOps: {IsGT});
9355 IsLT = MIRBuilder.buildInstr(Opc: BoolExtOp, DstOps: {DstTy}, SrcOps: {IsLT});
9356 MIRBuilder.buildSub(Dst, Src0: IsGT, Src1: IsLT);
9357 }
9358
9359 MI.eraseFromParent();
9360 return Legalized;
9361}
9362
9363LegalizerHelper::LegalizeResult
9364LegalizerHelper::lowerFCopySign(MachineInstr &MI) {
9365 auto [Dst, DstTy, Src0, Src0Ty, Src1, Src1Ty] = MI.getFirst3RegLLTs();
9366 const int Src0Size = Src0Ty.getScalarSizeInBits();
9367 const int Src1Size = Src1Ty.getScalarSizeInBits();
9368
9369 LLT DstIntTy =
9370 DstTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: DstTy.getScalarSizeInBits()));
9371 LLT Src0IntTy = Src0Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: Src0Size));
9372 LLT Src1IntTy = Src1Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: Src1Size));
9373
9374 Register Src0Int = Src0;
9375 Register Src1Int = Src1;
9376
9377 if (!(Src0Ty.getScalarType().isAnyScalar() ||
9378 Src0Ty.getScalarType().isInteger()))
9379 Src0Int = MIRBuilder.buildBitcast(Dst: Src0IntTy, Src: Src0).getReg(Idx: 0);
9380
9381 if (!(Src1Ty.getScalarType().isAnyScalar() ||
9382 Src1Ty.getScalarType().isInteger()))
9383 Src1Int = MIRBuilder.buildBitcast(Dst: Src1IntTy, Src: Src1).getReg(Idx: 0);
9384
9385 auto SignBitMask =
9386 MIRBuilder.buildConstant(Res: Src0IntTy, Val: APInt::getSignMask(BitWidth: Src0Size));
9387
9388 auto NotSignBitMask = MIRBuilder.buildConstant(
9389 Res: Src0IntTy, Val: APInt::getLowBitsSet(numBits: Src0Size, loBitsSet: Src0Size - 1));
9390
9391 Register And0 =
9392 MIRBuilder.buildAnd(Dst: Src0IntTy, Src0: Src0Int, Src1: NotSignBitMask).getReg(Idx: 0);
9393 Register And1;
9394 if (Src0Ty == Src1Ty) {
9395 And1 = MIRBuilder.buildAnd(Dst: Src1IntTy, Src0: Src1Int, Src1: SignBitMask).getReg(Idx: 0);
9396 } else if (Src0Size > Src1Size) {
9397 auto ShiftAmt = MIRBuilder.buildConstant(Res: Src0IntTy, Val: Src0Size - Src1Size);
9398 auto Zext = MIRBuilder.buildZExt(Res: Src0IntTy, Op: Src1Int);
9399 auto Shift = MIRBuilder.buildShl(Dst: Src0IntTy, Src0: Zext, Src1: ShiftAmt);
9400 And1 = MIRBuilder.buildAnd(Dst: Src0IntTy, Src0: Shift, Src1: SignBitMask).getReg(Idx: 0);
9401 } else {
9402 auto ShiftAmt = MIRBuilder.buildConstant(Res: Src1IntTy, Val: Src1Size - Src0Size);
9403 auto Shift = MIRBuilder.buildLShr(Dst: Src1IntTy, Src0: Src1Int, Src1: ShiftAmt);
9404 auto Trunc = MIRBuilder.buildTrunc(Res: Src0IntTy, Op: Shift);
9405 And1 = MIRBuilder.buildAnd(Dst: Src0IntTy, Src0: Trunc, Src1: SignBitMask).getReg(Idx: 0);
9406 }
9407
9408 // Be careful about setting nsz/nnan/ninf on every instruction, since the
9409 // constants are a nan and -0.0, but the final result should preserve
9410 // everything.
9411 unsigned Flags = MI.getFlags();
9412
9413 // We masked the sign bit and the not-sign bit, so these are disjoint.
9414 Flags |= MachineInstr::Disjoint;
9415
9416 if (DstTy == DstIntTy)
9417 MIRBuilder.buildOr(Dst, Src0: And0, Src1: And1, Flags).getReg(Idx: 0);
9418 else {
9419 Register NewDst = MIRBuilder.buildOr(Dst: DstIntTy, Src0: And0, Src1: And1, Flags).getReg(Idx: 0);
9420 MIRBuilder.buildBitcast(Dst, Src: NewDst);
9421 }
9422
9423 MI.eraseFromParent();
9424 return Legalized;
9425}
9426
9427LegalizerHelper::LegalizeResult
9428LegalizerHelper::lowerFMinNumMaxNum(MachineInstr &MI) {
9429 // FIXME: fminnum/fmaxnum and fminimumnum/fmaximumnum should not have
9430 // identical handling. fminimumnum/fmaximumnum also need a path that do not
9431 // depend on fminnum/fmaxnum.
9432
9433 unsigned NewOp;
9434 switch (MI.getOpcode()) {
9435 case TargetOpcode::G_FMINNUM:
9436 NewOp = TargetOpcode::G_FMINNUM_IEEE;
9437 break;
9438 case TargetOpcode::G_FMINIMUMNUM:
9439 NewOp = TargetOpcode::G_FMINNUM;
9440 break;
9441 case TargetOpcode::G_FMAXNUM:
9442 NewOp = TargetOpcode::G_FMAXNUM_IEEE;
9443 break;
9444 case TargetOpcode::G_FMAXIMUMNUM:
9445 NewOp = TargetOpcode::G_FMAXNUM;
9446 break;
9447 default:
9448 llvm_unreachable("unexpected min/max opcode");
9449 }
9450
9451 auto [Dst, Src0, Src1] = MI.getFirst3Regs();
9452 LLT Ty = MRI.getType(Reg: Dst);
9453
9454 if (!MI.getFlag(Flag: MachineInstr::FmNoNans)) {
9455 // Insert canonicalizes if it's possible we need to quiet to get correct
9456 // sNaN behavior.
9457
9458 // Note this must be done here, and not as an optimization combine in the
9459 // absence of a dedicate quiet-snan instruction as we're using an
9460 // omni-purpose G_FCANONICALIZE.
9461 if (!VT->isKnownNeverSNaN(Val: Src0))
9462 Src0 = MIRBuilder.buildFCanonicalize(Dst: Ty, Src0, Flags: MI.getFlags()).getReg(Idx: 0);
9463
9464 if (!VT->isKnownNeverSNaN(Val: Src1))
9465 Src1 = MIRBuilder.buildFCanonicalize(Dst: Ty, Src0: Src1, Flags: MI.getFlags()).getReg(Idx: 0);
9466 }
9467
9468 // If there are no nans, it's safe to simply replace this with the non-IEEE
9469 // version.
9470 MIRBuilder.buildInstr(Opc: NewOp, DstOps: {Dst}, SrcOps: {Src0, Src1}, Flags: MI.getFlags());
9471 MI.eraseFromParent();
9472 return Legalized;
9473}
9474
9475LegalizerHelper::LegalizeResult
9476LegalizerHelper::lowerFMinimumMaximum(MachineInstr &MI) {
9477 unsigned Opc = MI.getOpcode();
9478 auto [Dst, Src0, Src1] = MI.getFirst3Regs();
9479 LLT Ty = MRI.getType(Reg: Dst);
9480 const LLT CmpTy = Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9481
9482 bool IsMax = (Opc == TargetOpcode::G_FMAXIMUM);
9483 unsigned OpcIeee =
9484 IsMax ? TargetOpcode::G_FMAXNUM_IEEE : TargetOpcode::G_FMINNUM_IEEE;
9485 unsigned OpcNonIeee =
9486 IsMax ? TargetOpcode::G_FMAXNUM : TargetOpcode::G_FMINNUM;
9487 bool MinMaxMustRespectOrderedZero = false;
9488 Register Res;
9489
9490 // IEEE variants don't need canonicalization
9491 if (LI.isLegalOrCustom(Query: {OpcIeee, Ty})) {
9492 Res = MIRBuilder.buildInstr(Opc: OpcIeee, DstOps: {Ty}, SrcOps: {Src0, Src1}).getReg(Idx: 0);
9493 MinMaxMustRespectOrderedZero = true;
9494 } else if (LI.isLegalOrCustom(Query: {OpcNonIeee, Ty})) {
9495 Res = MIRBuilder.buildInstr(Opc: OpcNonIeee, DstOps: {Ty}, SrcOps: {Src0, Src1}).getReg(Idx: 0);
9496 } else {
9497 auto Compare = MIRBuilder.buildFCmp(
9498 Pred: IsMax ? CmpInst::FCMP_OGT : CmpInst::FCMP_OLT, Res: CmpTy, Op0: Src0, Op1: Src1);
9499 Res = MIRBuilder.buildSelect(Res: Ty, Tst: Compare, Op0: Src0, Op1: Src1).getReg(Idx: 0);
9500 }
9501
9502 // Propagate any NaN of both operands
9503 if (!MI.getFlag(Flag: MachineInstr::FmNoNans) &&
9504 (!VT->isKnownNeverNaN(Val: Src0) || !VT->isKnownNeverNaN(Val: Src1))) {
9505 auto IsOrdered = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_ORD, Res: CmpTy, Op0: Src0, Op1: Src1);
9506
9507 LLT ElementTy = Ty.isScalar() ? Ty : Ty.getElementType();
9508 APFloat NaNValue = APFloat::getNaN(Sem: getFltSemanticForLLT(Ty: ElementTy));
9509 Register NaN = MIRBuilder.buildFConstant(Res: ElementTy, Val: NaNValue).getReg(Idx: 0);
9510 if (Ty.isVector())
9511 NaN = MIRBuilder.buildSplatBuildVector(Res: Ty, Src: NaN).getReg(Idx: 0);
9512
9513 Res = MIRBuilder.buildSelect(Res: Ty, Tst: IsOrdered, Op0: Res, Op1: NaN).getReg(Idx: 0);
9514 }
9515
9516 // fminimum/fmaximum requires -0.0 less than +0.0
9517 if (!MinMaxMustRespectOrderedZero && !MI.getFlag(Flag: MachineInstr::FmNsz)) {
9518 GISelValueTracking VT(MIRBuilder.getMF());
9519 // FIXME: Also demand for subnormal classes depending on the denormal mode.
9520 KnownFPClass Src0Info = VT.computeKnownFPClass(R: Src0, InterestedClasses: fcZero);
9521 KnownFPClass Src1Info = VT.computeKnownFPClass(R: Src1, InterestedClasses: fcZero);
9522
9523 // FIXME: Must use isKnownNeverLogicalZero(Mode)
9524 if (!Src0Info.isKnownNever(Mask: fcZero) && !Src1Info.isKnownNever(Mask: fcZero)) {
9525 const unsigned Flags = MI.getFlags();
9526 Register Zero = MIRBuilder.buildFConstant(Res: Ty, Val: 0.0).getReg(Idx: 0);
9527 auto IsZero = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OEQ, Res: CmpTy, Op0: Res, Op1: Zero);
9528
9529 unsigned TestClass = IsMax ? fcPosZero : fcNegZero;
9530
9531 auto LHSTestZero = MIRBuilder.buildIsFPClass(Res: CmpTy, Src: Src0, Mask: TestClass);
9532 auto LHSSelect =
9533 MIRBuilder.buildSelect(Res: Ty, Tst: LHSTestZero, Op0: Src0, Op1: Res, Flags);
9534
9535 auto RHSTestZero = MIRBuilder.buildIsFPClass(Res: CmpTy, Src: Src1, Mask: TestClass);
9536 auto RHSSelect =
9537 MIRBuilder.buildSelect(Res: Ty, Tst: RHSTestZero, Op0: Src1, Op1: LHSSelect, Flags);
9538
9539 Res = MIRBuilder.buildSelect(Res: Ty, Tst: IsZero, Op0: RHSSelect, Op1: Res, Flags).getReg(Idx: 0);
9540 }
9541 }
9542
9543 MIRBuilder.buildCopy(Res: Dst, Op: Res);
9544 MI.eraseFromParent();
9545 return Legalized;
9546}
9547
9548LegalizerHelper::LegalizeResult LegalizerHelper::lowerFMad(MachineInstr &MI) {
9549 // Expand G_FMAD a, b, c -> G_FADD (G_FMUL a, b), c
9550 Register DstReg = MI.getOperand(i: 0).getReg();
9551 LLT Ty = MRI.getType(Reg: DstReg);
9552 unsigned Flags = MI.getFlags();
9553
9554 auto Mul = MIRBuilder.buildFMul(Dst: Ty, Src0: MI.getOperand(i: 1), Src1: MI.getOperand(i: 2),
9555 Flags);
9556 MIRBuilder.buildFAdd(Dst: DstReg, Src0: Mul, Src1: MI.getOperand(i: 3), Flags);
9557 MI.eraseFromParent();
9558 return Legalized;
9559}
9560
9561LegalizerHelper::LegalizeResult
9562LegalizerHelper::lowerIntrinsicRound(MachineInstr &MI) {
9563 auto [DstReg, X] = MI.getFirst2Regs();
9564 const unsigned Flags = MI.getFlags();
9565 const LLT Ty = MRI.getType(Reg: DstReg);
9566 const LLT CondTy = Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9567
9568 // round(x) =>
9569 // t = trunc(x);
9570 // d = fabs(x - t);
9571 // o = copysign(d >= 0.5 ? 1.0 : 0.0, x);
9572 // return t + o;
9573
9574 auto T = MIRBuilder.buildIntrinsicTrunc(Dst: Ty, Src0: X, Flags);
9575
9576 auto Diff = MIRBuilder.buildFSub(Dst: Ty, Src0: X, Src1: T, Flags);
9577 auto AbsDiff = MIRBuilder.buildFAbs(Dst: Ty, Src0: Diff, Flags);
9578
9579 auto Half = MIRBuilder.buildFConstant(Res: Ty, Val: 0.5);
9580 auto Cmp =
9581 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OGE, Res: CondTy, Op0: AbsDiff, Op1: Half, Flags);
9582
9583 // Could emit G_UITOFP instead
9584 auto One = MIRBuilder.buildFConstant(Res: Ty, Val: 1.0);
9585 auto Zero = MIRBuilder.buildFConstant(Res: Ty, Val: 0.0);
9586 auto BoolFP = MIRBuilder.buildSelect(Res: Ty, Tst: Cmp, Op0: One, Op1: Zero);
9587 auto SignedOffset = MIRBuilder.buildFCopysign(Dst: Ty, Src0: BoolFP, Src1: X);
9588
9589 MIRBuilder.buildFAdd(Dst: DstReg, Src0: T, Src1: SignedOffset, Flags);
9590
9591 MI.eraseFromParent();
9592 return Legalized;
9593}
9594
9595LegalizerHelper::LegalizeResult LegalizerHelper::lowerFFloor(MachineInstr &MI) {
9596 auto [DstReg, SrcReg] = MI.getFirst2Regs();
9597 unsigned Flags = MI.getFlags();
9598 LLT Ty = MRI.getType(Reg: DstReg);
9599 const LLT CondTy = Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9600
9601 // result = trunc(src);
9602 // if (src < 0.0 && src != result)
9603 // result += -1.0.
9604
9605 auto Trunc = MIRBuilder.buildIntrinsicTrunc(Dst: Ty, Src0: SrcReg, Flags);
9606 auto Zero = MIRBuilder.buildFConstant(Res: Ty, Val: 0.0);
9607
9608 auto Lt0 = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OLT, Res: CondTy,
9609 Op0: SrcReg, Op1: Zero, Flags);
9610 auto NeTrunc = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_ONE, Res: CondTy,
9611 Op0: SrcReg, Op1: Trunc, Flags);
9612 auto And = MIRBuilder.buildAnd(Dst: CondTy, Src0: Lt0, Src1: NeTrunc);
9613 auto AddVal = MIRBuilder.buildSITOFP(Dst: Ty, Src0: And);
9614
9615 MIRBuilder.buildFAdd(Dst: DstReg, Src0: Trunc, Src1: AddVal, Flags);
9616 MI.eraseFromParent();
9617 return Legalized;
9618}
9619
9620LegalizerHelper::LegalizeResult
9621LegalizerHelper::lowerMergeValues(MachineInstr &MI) {
9622 const unsigned NumOps = MI.getNumOperands();
9623 auto [DstReg, DstTy, Src0Reg, Src0Ty] = MI.getFirst2RegLLTs();
9624 unsigned PartSize = Src0Ty.getSizeInBits();
9625
9626 LLT WideTy = LLT::integer(SizeInBits: DstTy.getSizeInBits());
9627 Register ResultReg = MIRBuilder.buildZExt(Res: WideTy, Op: Src0Reg).getReg(Idx: 0);
9628
9629 for (unsigned I = 2; I != NumOps; ++I) {
9630 const unsigned Offset = (I - 1) * PartSize;
9631
9632 Register SrcReg = MI.getOperand(i: I).getReg();
9633 auto ZextInput = MIRBuilder.buildZExt(Res: WideTy, Op: SrcReg);
9634
9635 Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg :
9636 MRI.createGenericVirtualRegister(Ty: WideTy);
9637
9638 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: Offset);
9639 auto Shl = MIRBuilder.buildShl(Dst: WideTy, Src0: ZextInput, Src1: ShiftAmt);
9640 MIRBuilder.buildOr(Dst: NextResult, Src0: ResultReg, Src1: Shl);
9641 ResultReg = NextResult;
9642 }
9643
9644 if (DstTy.isPointer()) {
9645 if (MIRBuilder.getDataLayout().isNonIntegralAddressSpace(
9646 AddrSpace: DstTy.getAddressSpace())) {
9647 LLVM_DEBUG(dbgs() << "Not casting nonintegral address space\n");
9648 return UnableToLegalize;
9649 }
9650
9651 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: ResultReg);
9652 } else if (WideTy != DstTy) {
9653 MIRBuilder.buildBitcast(Dst: DstReg, Src: ResultReg);
9654 }
9655
9656 MI.eraseFromParent();
9657 return Legalized;
9658}
9659
9660LegalizerHelper::LegalizeResult
9661LegalizerHelper::lowerUnmergeValues(MachineInstr &MI) {
9662 const unsigned NumDst = MI.getNumOperands() - 1;
9663 Register SrcReg = MI.getOperand(i: NumDst).getReg();
9664 Register Dst0Reg = MI.getOperand(i: 0).getReg();
9665 LLT DstTy = MRI.getType(Reg: Dst0Reg);
9666 if (DstTy.isPointer())
9667 return UnableToLegalize; // TODO
9668
9669 SrcReg = coerceToInteger(Val: SrcReg);
9670 if (!SrcReg)
9671 return UnableToLegalize;
9672
9673 // Expand scalarizing unmerge as bitcast to integer and shift.
9674 LLT IntTy = MRI.getType(Reg: SrcReg);
9675
9676 MIRBuilder.buildTrunc(Res: Dst0Reg, Op: SrcReg);
9677
9678 const unsigned DstSize = DstTy.getSizeInBits();
9679 unsigned Offset = DstSize;
9680 for (unsigned I = 1; I != NumDst; ++I, Offset += DstSize) {
9681 auto ShiftAmt = MIRBuilder.buildConstant(Res: IntTy, Val: Offset);
9682 auto Shift = MIRBuilder.buildLShr(Dst: IntTy, Src0: SrcReg, Src1: ShiftAmt);
9683 MIRBuilder.buildTrunc(Res: MI.getOperand(i: I), Op: Shift);
9684 }
9685
9686 MI.eraseFromParent();
9687 return Legalized;
9688}
9689
9690/// Lower a vector extract or insert by writing the vector to a stack temporary
9691/// and reloading the element or vector.
9692///
9693/// %dst = G_EXTRACT_VECTOR_ELT %vec, %idx
9694/// =>
9695/// %stack_temp = G_FRAME_INDEX
9696/// G_STORE %vec, %stack_temp
9697/// %idx = clamp(%idx, %vec.getNumElements())
9698/// %element_ptr = G_PTR_ADD %stack_temp, %idx
9699/// %dst = G_LOAD %element_ptr
9700LegalizerHelper::LegalizeResult
9701LegalizerHelper::lowerExtractInsertVectorElt(MachineInstr &MI) {
9702 Register DstReg = MI.getOperand(i: 0).getReg();
9703 Register SrcVec = MI.getOperand(i: 1).getReg();
9704 Register InsertVal;
9705 if (MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)
9706 InsertVal = MI.getOperand(i: 2).getReg();
9707
9708 Register Idx = MI.getOperand(i: MI.getNumOperands() - 1).getReg();
9709
9710 LLT VecTy = MRI.getType(Reg: SrcVec);
9711 LLT EltTy = VecTy.getElementType();
9712 unsigned NumElts = VecTy.getNumElements();
9713
9714 int64_t IdxVal;
9715 if (mi_match(R: Idx, MRI, P: m_ICst(Cst&: IdxVal)) && IdxVal <= NumElts) {
9716 SmallVector<Register, 8> SrcRegs;
9717 extractParts(Reg: SrcVec, Ty: EltTy, NumParts: NumElts, VRegs&: SrcRegs, MIRBuilder, MRI);
9718
9719 if (InsertVal) {
9720 SrcRegs[IdxVal] = MI.getOperand(i: 2).getReg();
9721 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: SrcRegs);
9722 } else {
9723 MIRBuilder.buildCopy(Res: DstReg, Op: SrcRegs[IdxVal]);
9724 }
9725
9726 MI.eraseFromParent();
9727 return Legalized;
9728 }
9729
9730 if (!EltTy.isByteSized()) { // Not implemented.
9731 LLVM_DEBUG(dbgs() << "Can't handle non-byte element vectors yet\n");
9732 return UnableToLegalize;
9733 }
9734
9735 unsigned EltBytes = EltTy.getSizeInBytes();
9736 Align VecAlign = getStackTemporaryAlignment(Ty: VecTy);
9737 Align EltAlign;
9738
9739 MachinePointerInfo PtrInfo;
9740 auto StackTemp = createStackTemporary(
9741 Bytes: TypeSize::getFixed(ExactSize: VecTy.getSizeInBytes()), Alignment: VecAlign, PtrInfo);
9742 MIRBuilder.buildStore(Val: SrcVec, Addr: StackTemp, PtrInfo, Alignment: VecAlign);
9743
9744 // Get the pointer to the element, and be sure not to hit undefined behavior
9745 // if the index is out of bounds.
9746 Register EltPtr = getVectorElementPointer(VecPtr: StackTemp.getReg(Idx: 0), VecTy, Index: Idx);
9747
9748 if (mi_match(R: Idx, MRI, P: m_ICst(Cst&: IdxVal))) {
9749 int64_t Offset = IdxVal * EltBytes;
9750 PtrInfo = PtrInfo.getWithOffset(O: Offset);
9751 EltAlign = commonAlignment(A: VecAlign, Offset);
9752 } else {
9753 // We lose information with a variable offset.
9754 EltAlign = getStackTemporaryAlignment(Ty: EltTy);
9755 PtrInfo = MachinePointerInfo(MRI.getType(Reg: EltPtr).getAddressSpace());
9756 }
9757
9758 if (InsertVal) {
9759 // Write the inserted element
9760 MIRBuilder.buildStore(Val: InsertVal, Addr: EltPtr, PtrInfo, Alignment: EltAlign);
9761
9762 // Reload the whole vector.
9763 MIRBuilder.buildLoad(Res: DstReg, Addr: StackTemp, PtrInfo, Alignment: VecAlign);
9764 } else {
9765 MIRBuilder.buildLoad(Res: DstReg, Addr: EltPtr, PtrInfo, Alignment: EltAlign);
9766 }
9767
9768 MI.eraseFromParent();
9769 return Legalized;
9770}
9771
9772LegalizerHelper::LegalizeResult
9773LegalizerHelper::lowerShuffleVector(MachineInstr &MI) {
9774 auto [DstReg, DstTy, Src0Reg, Src0Ty, Src1Reg, Src1Ty] =
9775 MI.getFirst3RegLLTs();
9776 LLT IdxTy = LLT::scalar(SizeInBits: 32);
9777
9778 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
9779 Register Undef;
9780 SmallVector<Register, 32> BuildVec;
9781 LLT EltTy = DstTy.getScalarType();
9782
9783 DenseMap<unsigned, Register> CachedExtract;
9784
9785 for (int Idx : Mask) {
9786 if (Idx < 0) {
9787 if (!Undef.isValid())
9788 Undef = MIRBuilder.buildUndef(Res: EltTy).getReg(Idx: 0);
9789 BuildVec.push_back(Elt: Undef);
9790 continue;
9791 }
9792
9793 assert(!Src0Ty.isScalar() && "Unexpected scalar G_SHUFFLE_VECTOR");
9794
9795 int NumElts = Src0Ty.getNumElements();
9796 Register SrcVec = Idx < NumElts ? Src0Reg : Src1Reg;
9797 int ExtractIdx = Idx < NumElts ? Idx : Idx - NumElts;
9798 auto [It, Inserted] = CachedExtract.try_emplace(Key: Idx);
9799 if (Inserted) {
9800 auto IdxK = MIRBuilder.buildConstant(Res: IdxTy, Val: ExtractIdx);
9801 It->second =
9802 MIRBuilder.buildExtractVectorElement(Res: EltTy, Val: SrcVec, Idx: IdxK).getReg(Idx: 0);
9803 }
9804 BuildVec.push_back(Elt: It->second);
9805 }
9806
9807 assert(DstTy.isVector() && "Unexpected scalar G_SHUFFLE_VECTOR");
9808 MIRBuilder.buildBuildVector(Res: DstReg, Ops: BuildVec);
9809 MI.eraseFromParent();
9810 return Legalized;
9811}
9812
9813LegalizerHelper::LegalizeResult
9814LegalizerHelper::lowerVECTOR_COMPRESS(llvm::MachineInstr &MI) {
9815 auto [Dst, DstTy, Vec, VecTy, Mask, MaskTy, Passthru, PassthruTy] =
9816 MI.getFirst4RegLLTs();
9817
9818 if (VecTy.isScalableVector())
9819 report_fatal_error(reason: "Cannot expand masked_compress for scalable vectors.");
9820
9821 Align VecAlign = getStackTemporaryAlignment(Ty: VecTy);
9822 MachinePointerInfo PtrInfo;
9823 Register StackPtr =
9824 createStackTemporary(Bytes: TypeSize::getFixed(ExactSize: VecTy.getSizeInBytes()), Alignment: VecAlign,
9825 PtrInfo)
9826 .getReg(Idx: 0);
9827 MachinePointerInfo ValPtrInfo =
9828 MachinePointerInfo::getUnknownStack(MF&: *MI.getMF());
9829
9830 LLT IdxTy = LLT::integer(SizeInBits: 32);
9831 LLT ValTy = VecTy.getElementType();
9832 Align ValAlign = getStackTemporaryAlignment(Ty: ValTy);
9833
9834 auto OutPos = MIRBuilder.buildConstant(Res: IdxTy, Val: 0);
9835
9836 bool HasPassthru = !mi_match(R: Passthru, MRI, P: m_GImplicitDef());
9837
9838 if (HasPassthru)
9839 MIRBuilder.buildStore(Val: Passthru, Addr: StackPtr, PtrInfo, Alignment: VecAlign);
9840
9841 Register LastWriteVal;
9842 std::optional<APInt> PassthruSplatVal =
9843 isConstantOrConstantSplatVector(Def: Passthru, MRI);
9844
9845 if (PassthruSplatVal.has_value()) {
9846 LastWriteVal =
9847 MIRBuilder.buildConstant(Res: ValTy, Val: PassthruSplatVal.value()).getReg(Idx: 0);
9848 } else if (HasPassthru) {
9849 auto Popcount = MIRBuilder.buildZExt(Res: MaskTy.changeElementSize(NewEltSize: 32), Op: Mask);
9850 Popcount = MIRBuilder.buildInstr(Opc: TargetOpcode::G_VECREDUCE_ADD,
9851 DstOps: {LLT::integer(SizeInBits: 32)}, SrcOps: {Popcount});
9852
9853 Register LastElmtPtr =
9854 getVectorElementPointer(VecPtr: StackPtr, VecTy, Index: Popcount.getReg(Idx: 0));
9855 LastWriteVal =
9856 MIRBuilder.buildLoad(Res: ValTy, Addr: LastElmtPtr, PtrInfo: ValPtrInfo, Alignment: ValAlign)
9857 .getReg(Idx: 0);
9858 }
9859
9860 unsigned NumElmts = VecTy.getNumElements();
9861 for (unsigned I = 0; I < NumElmts; ++I) {
9862 auto Idx = MIRBuilder.buildConstant(Res: IdxTy, Val: I);
9863 auto Val = MIRBuilder.buildExtractVectorElement(Res: ValTy, Val: Vec, Idx);
9864 Register ElmtPtr =
9865 getVectorElementPointer(VecPtr: StackPtr, VecTy, Index: OutPos.getReg(Idx: 0));
9866 MIRBuilder.buildStore(Val, Addr: ElmtPtr, PtrInfo: ValPtrInfo, Alignment: ValAlign);
9867
9868 LLT MaskITy = MaskTy.getElementType();
9869 auto MaskI = MIRBuilder.buildExtractVectorElement(Res: MaskITy, Val: Mask, Idx);
9870 if (MaskITy.getSizeInBits() > 1)
9871 MaskI = MIRBuilder.buildTrunc(Res: LLT::integer(SizeInBits: 1), Op: MaskI);
9872
9873 MaskI = MIRBuilder.buildZExt(Res: IdxTy, Op: MaskI);
9874 OutPos = MIRBuilder.buildAdd(Dst: IdxTy, Src0: OutPos, Src1: MaskI);
9875
9876 if (HasPassthru && I == NumElmts - 1) {
9877 auto EndOfVector =
9878 MIRBuilder.buildConstant(Res: IdxTy, Val: VecTy.getNumElements() - 1);
9879 auto AllLanesSelected = MIRBuilder.buildICmp(
9880 Pred: CmpInst::ICMP_UGT, Res: LLT::integer(SizeInBits: 1), Op0: OutPos, Op1: EndOfVector);
9881 OutPos = MIRBuilder.buildInstr(Opc: TargetOpcode::G_UMIN, DstOps: {IdxTy},
9882 SrcOps: {OutPos, EndOfVector});
9883 ElmtPtr = getVectorElementPointer(VecPtr: StackPtr, VecTy, Index: OutPos.getReg(Idx: 0));
9884
9885 LastWriteVal =
9886 MIRBuilder.buildSelect(Res: ValTy, Tst: AllLanesSelected, Op0: Val, Op1: LastWriteVal)
9887 .getReg(Idx: 0);
9888 MIRBuilder.buildStore(Val: LastWriteVal, Addr: ElmtPtr, PtrInfo: ValPtrInfo, Alignment: ValAlign);
9889 }
9890 }
9891
9892 // TODO: Use StackPtr's FrameIndex alignment.
9893 MIRBuilder.buildLoad(Res: Dst, Addr: StackPtr, PtrInfo, Alignment: VecAlign);
9894
9895 MI.eraseFromParent();
9896 return Legalized;
9897}
9898
9899Register LegalizerHelper::getDynStackAllocTargetPtr(Register SPReg,
9900 Register AllocSize,
9901 Align Alignment,
9902 LLT PtrTy) {
9903 LLT IntPtrTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
9904
9905 auto SPTmp = MIRBuilder.buildCopy(Res: PtrTy, Op: SPReg);
9906 SPTmp = MIRBuilder.buildCast(Dst: IntPtrTy, Src: SPTmp);
9907
9908 // Subtract the final alloc from the SP. We use G_PTRTOINT here so we don't
9909 // have to generate an extra instruction to negate the alloc and then use
9910 // G_PTR_ADD to add the negative offset.
9911 auto Alloc = MIRBuilder.buildSub(Dst: IntPtrTy, Src0: SPTmp, Src1: AllocSize);
9912 if (Alignment > Align(1)) {
9913 APInt AlignMask(IntPtrTy.getSizeInBits(), Alignment.value(), true);
9914 AlignMask.negate();
9915 auto AlignCst = MIRBuilder.buildConstant(Res: IntPtrTy, Val: AlignMask);
9916 Alloc = MIRBuilder.buildAnd(Dst: IntPtrTy, Src0: Alloc, Src1: AlignCst);
9917 }
9918
9919 return MIRBuilder.buildCast(Dst: PtrTy, Src: Alloc).getReg(Idx: 0);
9920}
9921
9922LegalizerHelper::LegalizeResult
9923LegalizerHelper::lowerDynStackAlloc(MachineInstr &MI) {
9924 const auto &MF = *MI.getMF();
9925 const auto &TFI = *MF.getSubtarget().getFrameLowering();
9926 if (TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp)
9927 return UnableToLegalize;
9928
9929 Register Dst = MI.getOperand(i: 0).getReg();
9930 Register AllocSize = MI.getOperand(i: 1).getReg();
9931 Align Alignment = assumeAligned(Value: MI.getOperand(i: 2).getImm());
9932
9933 LLT PtrTy = MRI.getType(Reg: Dst);
9934 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
9935 Register SPTmp =
9936 getDynStackAllocTargetPtr(SPReg, AllocSize, Alignment, PtrTy);
9937
9938 MIRBuilder.buildCopy(Res: SPReg, Op: SPTmp);
9939 MIRBuilder.buildCopy(Res: Dst, Op: SPTmp);
9940
9941 MI.eraseFromParent();
9942 return Legalized;
9943}
9944
9945LegalizerHelper::LegalizeResult
9946LegalizerHelper::lowerStackSave(MachineInstr &MI) {
9947 Register StackPtr = TLI.getStackPointerRegisterToSaveRestore();
9948 if (!StackPtr)
9949 return UnableToLegalize;
9950
9951 MIRBuilder.buildCopy(Res: MI.getOperand(i: 0), Op: StackPtr);
9952 MI.eraseFromParent();
9953 return Legalized;
9954}
9955
9956LegalizerHelper::LegalizeResult
9957LegalizerHelper::lowerStackRestore(MachineInstr &MI) {
9958 Register StackPtr = TLI.getStackPointerRegisterToSaveRestore();
9959 if (!StackPtr)
9960 return UnableToLegalize;
9961
9962 MIRBuilder.buildCopy(Res: StackPtr, Op: MI.getOperand(i: 0));
9963 MI.eraseFromParent();
9964 return Legalized;
9965}
9966
9967LegalizerHelper::LegalizeResult
9968LegalizerHelper::lowerExtract(MachineInstr &MI) {
9969 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
9970 unsigned Offset = MI.getOperand(i: 2).getImm();
9971
9972 // Extract sub-vector or one element
9973 if (SrcTy.isVector()) {
9974 unsigned SrcEltSize = SrcTy.getElementType().getSizeInBits();
9975 unsigned DstSize = DstTy.getSizeInBits();
9976
9977 if ((Offset % SrcEltSize == 0) && (DstSize % SrcEltSize == 0) &&
9978 (Offset + DstSize <= SrcTy.getSizeInBits())) {
9979 // Unmerge and allow access to each Src element for the artifact combiner.
9980 auto Unmerge = MIRBuilder.buildUnmerge(Res: SrcTy.getElementType(), Op: SrcReg);
9981
9982 // Take element(s) we need to extract and copy it (merge them).
9983 SmallVector<Register, 8> SubVectorElts;
9984 for (unsigned Idx = Offset / SrcEltSize;
9985 Idx < (Offset + DstSize) / SrcEltSize; ++Idx) {
9986 SubVectorElts.push_back(Elt: Unmerge.getReg(Idx));
9987 }
9988 if (SubVectorElts.size() == 1)
9989 MIRBuilder.buildCopy(Res: DstReg, Op: SubVectorElts[0]);
9990 else
9991 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: SubVectorElts);
9992
9993 MI.eraseFromParent();
9994 return Legalized;
9995 }
9996 }
9997
9998 const DataLayout &DL = MIRBuilder.getDataLayout();
9999 if ((SrcTy.isPointer() &&
10000 DL.isNonIntegralAddressSpace(AddrSpace: SrcTy.getAddressSpace())) ||
10001 (DstTy.isPointer() &&
10002 DL.isNonIntegralAddressSpace(AddrSpace: DstTy.getAddressSpace()))) {
10003 LLVM_DEBUG(dbgs() << "Not casting non-integral address space integer\n");
10004 return UnableToLegalize;
10005 }
10006
10007 if ((DstTy.isScalar() || DstTy.isPointer()) &&
10008 (SrcTy.isScalar() || SrcTy.isPointer() ||
10009 (SrcTy.isVector() && DstTy == SrcTy.getElementType()))) {
10010 LLT SrcIntTy = SrcTy;
10011 if (!SrcTy.isScalar()) {
10012 SrcIntTy = LLT::integer(SizeInBits: SrcTy.getSizeInBits());
10013 SrcReg = MIRBuilder.buildCast(Dst: SrcIntTy, Src: SrcReg).getReg(Idx: 0);
10014 }
10015
10016 Register ResultReg = DstReg;
10017 if (DstTy.isPointer())
10018 ResultReg =
10019 MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: DstTy.getSizeInBits()));
10020
10021 if (Offset == 0)
10022 MIRBuilder.buildTrunc(Res: ResultReg, Op: SrcReg);
10023 else {
10024 auto ShiftAmt = MIRBuilder.buildConstant(Res: SrcIntTy, Val: Offset);
10025 auto Shr = MIRBuilder.buildLShr(Dst: SrcIntTy, Src0: SrcReg, Src1: ShiftAmt);
10026 MIRBuilder.buildTrunc(Res: ResultReg, Op: Shr);
10027 }
10028
10029 if (DstTy.isPointer())
10030 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: ResultReg);
10031
10032 MI.eraseFromParent();
10033 return Legalized;
10034 }
10035
10036 return UnableToLegalize;
10037}
10038
10039LegalizerHelper::LegalizeResult LegalizerHelper::lowerInsert(MachineInstr &MI) {
10040 auto [Dst, Src, InsertSrc] = MI.getFirst3Regs();
10041 uint64_t Offset = MI.getOperand(i: 3).getImm();
10042
10043 LLT DstTy = MRI.getType(Reg: Src);
10044 LLT InsertTy = MRI.getType(Reg: InsertSrc);
10045
10046 const DataLayout &DL = MIRBuilder.getDataLayout();
10047 bool IsNonIntegralInsert =
10048 InsertTy.isPointerOrPointerVector() &&
10049 DL.isNonIntegralAddressSpace(AddrSpace: InsertTy.getAddressSpace());
10050 bool IsNonIntegralDst = DstTy.isPointerOrPointerVector() &&
10051 DL.isNonIntegralAddressSpace(AddrSpace: DstTy.getAddressSpace());
10052
10053 // Insert sub-vector or one element
10054 if (DstTy.isVector()) {
10055 LLT EltTy = DstTy.getElementType();
10056
10057 if ((IsNonIntegralInsert || IsNonIntegralDst) && InsertTy != EltTy) {
10058 LLVM_DEBUG(dbgs() << "Not casting non-integral address space integer\n");
10059 return UnableToLegalize;
10060 }
10061
10062 unsigned EltSize = EltTy.getSizeInBits();
10063 unsigned InsertSize = InsertTy.getSizeInBits();
10064
10065 if ((Offset % EltSize == 0) && (InsertSize % EltSize == 0) &&
10066 (Offset + InsertSize <= DstTy.getSizeInBits())) {
10067 auto UnmergeSrc = MIRBuilder.buildUnmerge(Res: EltTy, Op: Src);
10068 SmallVector<Register, 8> DstElts;
10069 unsigned Idx = 0;
10070 // Elements from Src before insert start Offset
10071 for (; Idx < Offset / EltSize; ++Idx) {
10072 DstElts.push_back(Elt: UnmergeSrc.getReg(Idx));
10073 }
10074
10075 // Replace elements in Src with elements from InsertSrc
10076 if (InsertTy.getSizeInBits() > EltSize) {
10077 auto UnmergeInsertSrc = MIRBuilder.buildUnmerge(Res: EltTy, Op: InsertSrc);
10078 for (unsigned i = 0; Idx < (Offset + InsertSize) / EltSize;
10079 ++Idx, ++i) {
10080 DstElts.push_back(Elt: UnmergeInsertSrc.getReg(Idx: i));
10081 }
10082 } else {
10083 if (InsertTy.isPointer() && !EltTy.isPointer())
10084 InsertSrc = MIRBuilder.buildPtrToInt(Dst: EltTy, Src: InsertSrc).getReg(Idx: 0);
10085 else if (!InsertTy.isPointer() && EltTy.isPointer())
10086 InsertSrc = MIRBuilder.buildIntToPtr(Dst: EltTy, Src: InsertSrc).getReg(Idx: 0);
10087 DstElts.push_back(Elt: InsertSrc);
10088 ++Idx;
10089 }
10090
10091 // Remaining elements from Src after insert
10092 for (; Idx < DstTy.getNumElements(); ++Idx) {
10093 DstElts.push_back(Elt: UnmergeSrc.getReg(Idx));
10094 }
10095
10096 MIRBuilder.buildMergeLikeInstr(Res: Dst, Ops: DstElts);
10097 MI.eraseFromParent();
10098 return Legalized;
10099 }
10100 }
10101
10102 if (InsertTy.isVector() ||
10103 (DstTy.isVector() && DstTy.getElementType() != InsertTy))
10104 return UnableToLegalize;
10105
10106 if (IsNonIntegralDst || IsNonIntegralInsert) {
10107 LLVM_DEBUG(dbgs() << "Not casting non-integral address space integer\n");
10108 return UnableToLegalize;
10109 }
10110
10111 LLT IntDstTy = DstTy;
10112
10113 if (!DstTy.isScalar()) {
10114 IntDstTy = LLT::scalar(SizeInBits: DstTy.getSizeInBits());
10115 Src = MIRBuilder.buildCast(Dst: IntDstTy, Src).getReg(Idx: 0);
10116 }
10117
10118 if (!InsertTy.isScalar()) {
10119 const LLT IntInsertTy = LLT::scalar(SizeInBits: InsertTy.getSizeInBits());
10120 InsertSrc = MIRBuilder.buildPtrToInt(Dst: IntInsertTy, Src: InsertSrc).getReg(Idx: 0);
10121 }
10122
10123 Register ExtInsSrc = MIRBuilder.buildZExt(Res: IntDstTy, Op: InsertSrc).getReg(Idx: 0);
10124 if (Offset != 0) {
10125 auto ShiftAmt = MIRBuilder.buildConstant(Res: IntDstTy, Val: Offset);
10126 ExtInsSrc = MIRBuilder.buildShl(Dst: IntDstTy, Src0: ExtInsSrc, Src1: ShiftAmt).getReg(Idx: 0);
10127 }
10128
10129 APInt MaskVal = APInt::getBitsSetWithWrap(
10130 numBits: DstTy.getSizeInBits(), loBit: Offset + InsertTy.getSizeInBits(), hiBit: Offset);
10131
10132 auto Mask = MIRBuilder.buildConstant(Res: IntDstTy, Val: MaskVal);
10133 auto MaskedSrc = MIRBuilder.buildAnd(Dst: IntDstTy, Src0: Src, Src1: Mask);
10134 auto Or = MIRBuilder.buildOr(Dst: IntDstTy, Src0: MaskedSrc, Src1: ExtInsSrc);
10135
10136 MIRBuilder.buildCast(Dst, Src: Or);
10137 MI.eraseFromParent();
10138 return Legalized;
10139}
10140
10141LegalizerHelper::LegalizeResult
10142LegalizerHelper::lowerSADDO_SSUBO(MachineInstr &MI) {
10143 auto [Dst0, Dst0Ty, Dst1, Dst1Ty, LHS, LHSTy, RHS, RHSTy] =
10144 MI.getFirst4RegLLTs();
10145 const bool IsAdd = MI.getOpcode() == TargetOpcode::G_SADDO;
10146
10147 LLT Ty = Dst0Ty;
10148 LLT BoolTy = Dst1Ty;
10149
10150 Register NewDst0 = MRI.cloneVirtualRegister(VReg: Dst0);
10151
10152 if (IsAdd)
10153 MIRBuilder.buildAdd(Dst: NewDst0, Src0: LHS, Src1: RHS);
10154 else
10155 MIRBuilder.buildSub(Dst: NewDst0, Src0: LHS, Src1: RHS);
10156
10157 // TODO: If SADDSAT/SSUBSAT is legal, compare results to detect overflow.
10158
10159 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
10160
10161 if (IsAdd) {
10162 // For an addition, the result should be less than one of the operands (LHS)
10163 // if and only if the other operand (RHS) is negative, otherwise there will
10164 // be overflow.
10165 auto ResultLowerThanLHS =
10166 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: NewDst0, Op1: LHS);
10167 auto RHSNegative =
10168 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: RHS, Op1: Zero);
10169 MIRBuilder.buildXor(Dst: Dst1, Src0: RHSNegative, Src1: ResultLowerThanLHS);
10170 } else {
10171 // For subtraction, overflow occurs when the signed comparison of operands
10172 // doesn't match the sign of the result.
10173 auto LHSLessThanRHS =
10174 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: LHS, Op1: RHS);
10175 auto ResultNegative =
10176 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: NewDst0, Op1: Zero);
10177 MIRBuilder.buildXor(Dst: Dst1, Src0: LHSLessThanRHS, Src1: ResultNegative);
10178 }
10179
10180 MIRBuilder.buildCopy(Res: Dst0, Op: NewDst0);
10181 MI.eraseFromParent();
10182
10183 return Legalized;
10184}
10185
10186LegalizerHelper::LegalizeResult LegalizerHelper::lowerSADDE(MachineInstr &MI) {
10187 auto [Res, OvOut, LHS, RHS, CarryIn] = MI.getFirst5Regs();
10188 const LLT Ty = MRI.getType(Reg: Res);
10189
10190 // sum = LHS + RHS + zext(CarryIn)
10191 auto Tmp = MIRBuilder.buildAdd(Dst: Ty, Src0: LHS, Src1: RHS);
10192 auto CarryZ = MIRBuilder.buildZExt(Res: Ty, Op: CarryIn);
10193 auto Sum = MIRBuilder.buildAdd(Dst: Ty, Src0: Tmp, Src1: CarryZ);
10194 MIRBuilder.buildCopy(Res, Op: Sum);
10195
10196 // OvOut = icmp slt ((sum ^ lhs) & (sum ^ rhs)), 0
10197 auto AX = MIRBuilder.buildXor(Dst: Ty, Src0: Sum, Src1: LHS);
10198 auto BX = MIRBuilder.buildXor(Dst: Ty, Src0: Sum, Src1: RHS);
10199 auto T = MIRBuilder.buildAnd(Dst: Ty, Src0: AX, Src1: BX);
10200
10201 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
10202 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: OvOut, Op0: T, Op1: Zero);
10203
10204 MI.eraseFromParent();
10205 return Legalized;
10206}
10207
10208LegalizerHelper::LegalizeResult LegalizerHelper::lowerSSUBE(MachineInstr &MI) {
10209 auto [Res, OvOut, LHS, RHS, CarryIn] = MI.getFirst5Regs();
10210 const LLT Ty = MRI.getType(Reg: Res);
10211
10212 // Diff = LHS - (RHS + zext(CarryIn))
10213 auto CarryZ = MIRBuilder.buildZExt(Res: Ty, Op: CarryIn);
10214 auto RHSPlusCI = MIRBuilder.buildAdd(Dst: Ty, Src0: RHS, Src1: CarryZ);
10215 auto Diff = MIRBuilder.buildSub(Dst: Ty, Src0: LHS, Src1: RHSPlusCI);
10216 MIRBuilder.buildCopy(Res, Op: Diff);
10217
10218 // ov = msb((LHS ^ RHS) & (LHS ^ Diff))
10219 auto X1 = MIRBuilder.buildXor(Dst: Ty, Src0: LHS, Src1: RHS);
10220 auto X2 = MIRBuilder.buildXor(Dst: Ty, Src0: LHS, Src1: Diff);
10221 auto T = MIRBuilder.buildAnd(Dst: Ty, Src0: X1, Src1: X2);
10222 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
10223 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: OvOut, Op0: T, Op1: Zero);
10224
10225 MI.eraseFromParent();
10226 return Legalized;
10227}
10228
10229LegalizerHelper::LegalizeResult
10230LegalizerHelper::lowerAddSubSatToMinMax(MachineInstr &MI) {
10231 auto [Res, LHS, RHS] = MI.getFirst3Regs();
10232 LLT Ty = MRI.getType(Reg: Res);
10233 bool IsSigned;
10234 bool IsAdd;
10235 unsigned BaseOp;
10236 switch (MI.getOpcode()) {
10237 default:
10238 llvm_unreachable("unexpected addsat/subsat opcode");
10239 case TargetOpcode::G_UADDSAT:
10240 IsSigned = false;
10241 IsAdd = true;
10242 BaseOp = TargetOpcode::G_ADD;
10243 break;
10244 case TargetOpcode::G_SADDSAT:
10245 IsSigned = true;
10246 IsAdd = true;
10247 BaseOp = TargetOpcode::G_ADD;
10248 break;
10249 case TargetOpcode::G_USUBSAT:
10250 IsSigned = false;
10251 IsAdd = false;
10252 BaseOp = TargetOpcode::G_SUB;
10253 break;
10254 case TargetOpcode::G_SSUBSAT:
10255 IsSigned = true;
10256 IsAdd = false;
10257 BaseOp = TargetOpcode::G_SUB;
10258 break;
10259 }
10260
10261 if (IsSigned) {
10262 // sadd.sat(a, b) ->
10263 // hi = 0x7fffffff - smax(a, 0)
10264 // lo = 0x80000000 - smin(a, 0)
10265 // a + smin(smax(lo, b), hi)
10266 // ssub.sat(a, b) ->
10267 // lo = smax(a, -1) - 0x7fffffff
10268 // hi = smin(a, -1) - 0x80000000
10269 // a - smin(smax(lo, b), hi)
10270 // TODO: AMDGPU can use a "median of 3" instruction here:
10271 // a +/- med3(lo, b, hi)
10272 uint64_t NumBits = Ty.getScalarSizeInBits();
10273 auto MaxVal =
10274 MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMaxValue(numBits: NumBits));
10275 auto MinVal =
10276 MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMinValue(numBits: NumBits));
10277 MachineInstrBuilder Hi, Lo;
10278 if (IsAdd) {
10279 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
10280 Hi = MIRBuilder.buildSub(Dst: Ty, Src0: MaxVal, Src1: MIRBuilder.buildSMax(Dst: Ty, Src0: LHS, Src1: Zero));
10281 Lo = MIRBuilder.buildSub(Dst: Ty, Src0: MinVal, Src1: MIRBuilder.buildSMin(Dst: Ty, Src0: LHS, Src1: Zero));
10282 } else {
10283 auto NegOne = MIRBuilder.buildConstant(Res: Ty, Val: -1);
10284 Lo = MIRBuilder.buildSub(Dst: Ty, Src0: MIRBuilder.buildSMax(Dst: Ty, Src0: LHS, Src1: NegOne),
10285 Src1: MaxVal);
10286 Hi = MIRBuilder.buildSub(Dst: Ty, Src0: MIRBuilder.buildSMin(Dst: Ty, Src0: LHS, Src1: NegOne),
10287 Src1: MinVal);
10288 }
10289 auto RHSClamped =
10290 MIRBuilder.buildSMin(Dst: Ty, Src0: MIRBuilder.buildSMax(Dst: Ty, Src0: Lo, Src1: RHS), Src1: Hi);
10291 MIRBuilder.buildInstr(Opc: BaseOp, DstOps: {Res}, SrcOps: {LHS, RHSClamped});
10292 } else {
10293 // uadd.sat(a, b) -> a + umin(~a, b)
10294 // usub.sat(a, b) -> a - umin(a, b)
10295 Register Not = IsAdd ? MIRBuilder.buildNot(Dst: Ty, Src0: LHS).getReg(Idx: 0) : LHS;
10296 auto Min = MIRBuilder.buildUMin(Dst: Ty, Src0: Not, Src1: RHS);
10297 MIRBuilder.buildInstr(Opc: BaseOp, DstOps: {Res}, SrcOps: {LHS, Min});
10298 }
10299
10300 MI.eraseFromParent();
10301 return Legalized;
10302}
10303
10304LegalizerHelper::LegalizeResult
10305LegalizerHelper::lowerAddSubSatToAddoSubo(MachineInstr &MI) {
10306 auto [Res, LHS, RHS] = MI.getFirst3Regs();
10307 LLT Ty = MRI.getType(Reg: Res);
10308 LLT BoolTy = Ty.changeElementSize(NewEltSize: 1);
10309 bool IsSigned;
10310 bool IsAdd;
10311 unsigned OverflowOp;
10312 switch (MI.getOpcode()) {
10313 default:
10314 llvm_unreachable("unexpected addsat/subsat opcode");
10315 case TargetOpcode::G_UADDSAT:
10316 IsSigned = false;
10317 IsAdd = true;
10318 OverflowOp = TargetOpcode::G_UADDO;
10319 break;
10320 case TargetOpcode::G_SADDSAT:
10321 IsSigned = true;
10322 IsAdd = true;
10323 OverflowOp = TargetOpcode::G_SADDO;
10324 break;
10325 case TargetOpcode::G_USUBSAT:
10326 IsSigned = false;
10327 IsAdd = false;
10328 OverflowOp = TargetOpcode::G_USUBO;
10329 break;
10330 case TargetOpcode::G_SSUBSAT:
10331 IsSigned = true;
10332 IsAdd = false;
10333 OverflowOp = TargetOpcode::G_SSUBO;
10334 break;
10335 }
10336
10337 auto OverflowRes =
10338 MIRBuilder.buildInstr(Opc: OverflowOp, DstOps: {Ty, BoolTy}, SrcOps: {LHS, RHS});
10339 Register Tmp = OverflowRes.getReg(Idx: 0);
10340 Register Ov = OverflowRes.getReg(Idx: 1);
10341 MachineInstrBuilder Clamp;
10342 if (IsSigned) {
10343 // sadd.sat(a, b) ->
10344 // {tmp, ov} = saddo(a, b)
10345 // ov ? (tmp >>s 31) + 0x80000000 : r
10346 // ssub.sat(a, b) ->
10347 // {tmp, ov} = ssubo(a, b)
10348 // ov ? (tmp >>s 31) + 0x80000000 : r
10349 uint64_t NumBits = Ty.getScalarSizeInBits();
10350 auto ShiftAmount = MIRBuilder.buildConstant(Res: Ty, Val: NumBits - 1);
10351 auto Sign = MIRBuilder.buildAShr(Dst: Ty, Src0: Tmp, Src1: ShiftAmount);
10352 auto MinVal =
10353 MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMinValue(numBits: NumBits));
10354 Clamp = MIRBuilder.buildAdd(Dst: Ty, Src0: Sign, Src1: MinVal);
10355 } else {
10356 // uadd.sat(a, b) ->
10357 // {tmp, ov} = uaddo(a, b)
10358 // ov ? 0xffffffff : tmp
10359 // usub.sat(a, b) ->
10360 // {tmp, ov} = usubo(a, b)
10361 // ov ? 0 : tmp
10362 Clamp = MIRBuilder.buildConstant(Res: Ty, Val: IsAdd ? -1 : 0);
10363 }
10364 MIRBuilder.buildSelect(Res, Tst: Ov, Op0: Clamp, Op1: Tmp);
10365
10366 MI.eraseFromParent();
10367 return Legalized;
10368}
10369
10370LegalizerHelper::LegalizeResult
10371LegalizerHelper::lowerShlSat(MachineInstr &MI) {
10372 assert((MI.getOpcode() == TargetOpcode::G_SSHLSAT ||
10373 MI.getOpcode() == TargetOpcode::G_USHLSAT) &&
10374 "Expected shlsat opcode!");
10375 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SSHLSAT;
10376 auto [Res, LHS, RHS] = MI.getFirst3Regs();
10377 LLT Ty = MRI.getType(Reg: Res);
10378 LLT BoolTy = Ty.changeElementSize(NewEltSize: 1);
10379
10380 unsigned BW = Ty.getScalarSizeInBits();
10381 auto Result = MIRBuilder.buildShl(Dst: Ty, Src0: LHS, Src1: RHS);
10382 auto Orig = IsSigned ? MIRBuilder.buildAShr(Dst: Ty, Src0: Result, Src1: RHS)
10383 : MIRBuilder.buildLShr(Dst: Ty, Src0: Result, Src1: RHS);
10384
10385 MachineInstrBuilder SatVal;
10386 if (IsSigned) {
10387 auto SatMin = MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMinValue(numBits: BW));
10388 auto SatMax = MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMaxValue(numBits: BW));
10389 auto Cmp = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: LHS,
10390 Op1: MIRBuilder.buildConstant(Res: Ty, Val: 0));
10391 SatVal = MIRBuilder.buildSelect(Res: Ty, Tst: Cmp, Op0: SatMin, Op1: SatMax);
10392 } else {
10393 SatVal = MIRBuilder.buildConstant(Res: Ty, Val: APInt::getMaxValue(numBits: BW));
10394 }
10395 auto Ov = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: BoolTy, Op0: LHS, Op1: Orig);
10396 MIRBuilder.buildSelect(Res, Tst: Ov, Op0: SatVal, Op1: Result);
10397
10398 MI.eraseFromParent();
10399 return Legalized;
10400}
10401
10402LegalizerHelper::LegalizeResult
10403LegalizerHelper::lowerTruncSat(MachineInstr &MI) {
10404 unsigned Opc = MI.getOpcode();
10405 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
10406 unsigned DstSize = DstTy.getScalarSizeInBits();
10407 unsigned SrcSize = SrcTy.getScalarSizeInBits();
10408
10409 if (Opc == TargetOpcode::G_TRUNC_SSAT_S) {
10410 auto Max = MIRBuilder.buildConstant(
10411 Res: SrcTy, Val: APInt::getSignedMaxValue(numBits: DstSize).sext(width: SrcSize));
10412 Src = MIRBuilder.buildSMin(Dst: SrcTy, Src0: Src, Src1: Max).getReg(Idx: 0);
10413 auto Min = MIRBuilder.buildConstant(
10414 Res: SrcTy, Val: APInt::getSignedMinValue(numBits: DstSize).sext(width: SrcSize));
10415 Src = MIRBuilder.buildSMax(Dst: SrcTy, Src0: Src, Src1: Min).getReg(Idx: 0);
10416 } else if (Opc == TargetOpcode::G_TRUNC_USAT_U) {
10417 auto Max = MIRBuilder.buildConstant(
10418 Res: SrcTy, Val: APInt::getAllOnes(numBits: DstSize).zext(width: SrcSize));
10419 Src = MIRBuilder.buildUMin(Dst: SrcTy, Src0: Src, Src1: Max).getReg(Idx: 0);
10420 } else if (Opc == TargetOpcode::G_TRUNC_SSAT_U) {
10421 auto Max = MIRBuilder.buildConstant(
10422 Res: SrcTy, Val: APInt::getAllOnes(numBits: DstSize).zext(width: SrcSize));
10423 Src = MIRBuilder.buildSMin(Dst: SrcTy, Src0: Src, Src1: Max).getReg(Idx: 0);
10424 auto Min = MIRBuilder.buildConstant(Res: SrcTy, Val: APInt::getZero(numBits: SrcSize));
10425 Src = MIRBuilder.buildSMax(Dst: SrcTy, Src0: Src, Src1: Min).getReg(Idx: 0);
10426 } else {
10427 llvm_unreachable("Expected truncsat opcode!");
10428 }
10429
10430 MIRBuilder.buildTrunc(Res: Dst, Op: Src);
10431 MI.eraseFromParent();
10432 return Legalized;
10433}
10434
10435LegalizerHelper::LegalizeResult LegalizerHelper::lowerBswap(MachineInstr &MI) {
10436 auto [Dst, Src] = MI.getFirst2Regs();
10437 const LLT Ty = MRI.getType(Reg: Src);
10438 unsigned SizeInBytes = (Ty.getScalarSizeInBits() + 7) / 8;
10439 unsigned BaseShiftAmt = (SizeInBytes - 1) * 8;
10440
10441 // Swap most and least significant byte, set remaining bytes in Res to zero.
10442 auto ShiftAmt = MIRBuilder.buildConstant(Res: Ty, Val: BaseShiftAmt);
10443 auto LSByteShiftedLeft = MIRBuilder.buildShl(Dst: Ty, Src0: Src, Src1: ShiftAmt);
10444 auto MSByteShiftedRight = MIRBuilder.buildLShr(Dst: Ty, Src0: Src, Src1: ShiftAmt);
10445 auto Res = MIRBuilder.buildOr(Dst: Ty, Src0: MSByteShiftedRight, Src1: LSByteShiftedLeft);
10446
10447 // Set i-th high/low byte in Res to i-th low/high byte from Src.
10448 for (unsigned i = 1; i < SizeInBytes / 2; ++i) {
10449 // AND with Mask leaves byte i unchanged and sets remaining bytes to 0.
10450 APInt APMask = APInt::getBitsSet(numBits: SizeInBytes * 8, loBit: i * 8, hiBit: i * 8 + 8);
10451 auto Mask = MIRBuilder.buildConstant(Res: Ty, Val: APMask);
10452 auto ShiftAmt = MIRBuilder.buildConstant(Res: Ty, Val: BaseShiftAmt - 16 * i);
10453 // Low byte shifted left to place of high byte: (Src & Mask) << ShiftAmt.
10454 auto LoByte = MIRBuilder.buildAnd(Dst: Ty, Src0: Src, Src1: Mask);
10455 auto LoShiftedLeft = MIRBuilder.buildShl(Dst: Ty, Src0: LoByte, Src1: ShiftAmt);
10456 Res = MIRBuilder.buildOr(Dst: Ty, Src0: Res, Src1: LoShiftedLeft);
10457 // High byte shifted right to place of low byte: (Src >> ShiftAmt) & Mask.
10458 auto SrcShiftedRight = MIRBuilder.buildLShr(Dst: Ty, Src0: Src, Src1: ShiftAmt);
10459 auto HiShiftedRight = MIRBuilder.buildAnd(Dst: Ty, Src0: SrcShiftedRight, Src1: Mask);
10460 Res = MIRBuilder.buildOr(Dst: Ty, Src0: Res, Src1: HiShiftedRight);
10461 }
10462 Res.getInstr()->getOperand(i: 0).setReg(Dst);
10463
10464 MI.eraseFromParent();
10465 return Legalized;
10466}
10467
10468//{ (Src & Mask) >> N } | { (Src << N) & Mask }
10469static MachineInstrBuilder SwapN(unsigned N, DstOp Dst, MachineIRBuilder &B,
10470 MachineInstrBuilder Src, const APInt &Mask) {
10471 const LLT Ty = Dst.getLLTTy(MRI: *B.getMRI());
10472 MachineInstrBuilder C_N = B.buildConstant(Res: Ty, Val: N);
10473 MachineInstrBuilder MaskLoNTo0 = B.buildConstant(Res: Ty, Val: Mask);
10474 auto LHS = B.buildLShr(Dst: Ty, Src0: B.buildAnd(Dst: Ty, Src0: Src, Src1: MaskLoNTo0), Src1: C_N);
10475 auto RHS = B.buildAnd(Dst: Ty, Src0: B.buildShl(Dst: Ty, Src0: Src, Src1: C_N), Src1: MaskLoNTo0);
10476 return B.buildOr(Dst, Src0: LHS, Src1: RHS);
10477}
10478
10479LegalizerHelper::LegalizeResult
10480LegalizerHelper::lowerBitreverse(MachineInstr &MI) {
10481 auto [Dst, Src] = MI.getFirst2Regs();
10482 const LLT SrcTy = MRI.getType(Reg: Src);
10483 unsigned Size = SrcTy.getScalarSizeInBits();
10484 unsigned VSize = SrcTy.getSizeInBits();
10485
10486 if (Size >= 8) {
10487 if (SrcTy.isVector() && (VSize % 8 == 0) &&
10488 (LI.isLegal(Query: {TargetOpcode::G_BITREVERSE,
10489 {LLT::fixed_vector(NumElements: VSize / 8, ScalarTy: LLT::integer(SizeInBits: 8)),
10490 LLT::fixed_vector(NumElements: VSize / 8, ScalarTy: LLT::integer(SizeInBits: 8))}}))) {
10491 // If bitreverse is legal for i8 vector of the same size, then cast
10492 // to i8 vector type.
10493 // e.g. v4s32 -> v16s8
10494 LLT VTy = LLT::fixed_vector(NumElements: VSize / 8, ScalarTy: LLT::integer(SizeInBits: 8));
10495 auto BSWAP = MIRBuilder.buildBSwap(Dst: SrcTy, Src0: Src);
10496 auto Cast = MIRBuilder.buildBitcast(Dst: VTy, Src: BSWAP);
10497 auto RBIT = MIRBuilder.buildBitReverse(Dst: VTy, Src: Cast);
10498 MIRBuilder.buildBitcast(Dst, Src: RBIT);
10499 } else {
10500 MachineInstrBuilder BSWAP =
10501 MIRBuilder.buildInstr(Opc: TargetOpcode::G_BSWAP, DstOps: {SrcTy}, SrcOps: {Src});
10502
10503 // swap high and low 4 bits in 8 bit blocks 7654|3210 -> 3210|7654
10504 // [(val & 0xF0F0F0F0) >> 4] | [(val & 0x0F0F0F0F) << 4]
10505 // -> [(val & 0xF0F0F0F0) >> 4] | [(val << 4) & 0xF0F0F0F0]
10506 MachineInstrBuilder Swap4 = SwapN(N: 4, Dst: SrcTy, B&: MIRBuilder, Src: BSWAP,
10507 Mask: APInt::getSplat(NewLen: Size, V: APInt(8, 0xF0)));
10508
10509 // swap high and low 2 bits in 4 bit blocks 32|10 76|54 -> 10|32 54|76
10510 // [(val & 0xCCCCCCCC) >> 2] & [(val & 0x33333333) << 2]
10511 // -> [(val & 0xCCCCCCCC) >> 2] & [(val << 2) & 0xCCCCCCCC]
10512 MachineInstrBuilder Swap2 = SwapN(N: 2, Dst: SrcTy, B&: MIRBuilder, Src: Swap4,
10513 Mask: APInt::getSplat(NewLen: Size, V: APInt(8, 0xCC)));
10514
10515 // swap high and low 1 bit in 2 bit blocks 1|0 3|2 5|4 7|6 -> 0|1 2|3 4|5
10516 // 6|7
10517 // [(val & 0xAAAAAAAA) >> 1] & [(val & 0x55555555) << 1]
10518 // -> [(val & 0xAAAAAAAA) >> 1] & [(val << 1) & 0xAAAAAAAA]
10519 SwapN(N: 1, Dst, B&: MIRBuilder, Src: Swap2, Mask: APInt::getSplat(NewLen: Size, V: APInt(8, 0xAA)));
10520 }
10521 } else {
10522 // Expand bitreverse for types smaller than 8 bits.
10523 MachineInstrBuilder Tmp;
10524 for (unsigned I = 0, J = Size - 1; I < Size; ++I, --J) {
10525 MachineInstrBuilder Tmp2;
10526 if (I < J) {
10527 auto ShAmt = MIRBuilder.buildConstant(Res: SrcTy, Val: J - I);
10528 Tmp2 = MIRBuilder.buildShl(Dst: SrcTy, Src0: Src, Src1: ShAmt);
10529 } else {
10530 auto ShAmt = MIRBuilder.buildConstant(Res: SrcTy, Val: I - J);
10531 Tmp2 = MIRBuilder.buildLShr(Dst: SrcTy, Src0: Src, Src1: ShAmt);
10532 }
10533
10534 auto Mask = MIRBuilder.buildConstant(Res: SrcTy, Val: 1ULL << J);
10535 Tmp2 = MIRBuilder.buildAnd(Dst: SrcTy, Src0: Tmp2, Src1: Mask);
10536 if (I == 0)
10537 Tmp = Tmp2;
10538 else
10539 Tmp = MIRBuilder.buildOr(Dst: SrcTy, Src0: Tmp, Src1: Tmp2);
10540 }
10541 MIRBuilder.buildCopy(Res: Dst, Op: Tmp);
10542 }
10543
10544 MI.eraseFromParent();
10545 return Legalized;
10546}
10547
10548LegalizerHelper::LegalizeResult
10549LegalizerHelper::lowerReadWriteRegister(MachineInstr &MI) {
10550 MachineFunction &MF = MIRBuilder.getMF();
10551
10552 bool IsRead = MI.getOpcode() == TargetOpcode::G_READ_REGISTER;
10553 int NameOpIdx = IsRead ? 1 : 0;
10554 int ValRegIndex = IsRead ? 0 : 1;
10555
10556 Register ValReg = MI.getOperand(i: ValRegIndex).getReg();
10557 const LLT Ty = MRI.getType(Reg: ValReg);
10558 const MDString *RegStr = cast<MDString>(
10559 Val: cast<MDNode>(Val: MI.getOperand(i: NameOpIdx).getMetadata())->getOperand(I: 0));
10560
10561 Register PhysReg = TLI.getRegisterByName(RegName: RegStr->getString().data(), Ty, MF);
10562 if (!PhysReg) {
10563 const Function &Fn = MF.getFunction();
10564 Fn.getContext().diagnose(DI: DiagnosticInfoGenericWithLoc(
10565 "invalid register \"" + Twine(RegStr->getString().data()) + "\" for " +
10566 (IsRead ? "llvm.read_register" : "llvm.write_register"),
10567 Fn, MI.getDebugLoc()));
10568 if (IsRead)
10569 MIRBuilder.buildUndef(Res: ValReg);
10570
10571 MI.eraseFromParent();
10572 return Legalized;
10573 }
10574
10575 if (IsRead)
10576 MIRBuilder.buildCopy(Res: ValReg, Op: PhysReg);
10577 else
10578 MIRBuilder.buildCopy(Res: PhysReg, Op: ValReg);
10579
10580 MI.eraseFromParent();
10581 return Legalized;
10582}
10583
10584LegalizerHelper::LegalizeResult
10585LegalizerHelper::lowerSMULH_UMULH(MachineInstr &MI) {
10586 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SMULH;
10587 unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
10588 Register Result = MI.getOperand(i: 0).getReg();
10589 LLT OrigTy = MRI.getType(Reg: Result);
10590 auto SizeInBits = OrigTy.getScalarSizeInBits();
10591 LLT WideTy = OrigTy.changeElementSize(NewEltSize: SizeInBits * 2);
10592
10593 auto LHS = MIRBuilder.buildInstr(Opc: ExtOp, DstOps: {WideTy}, SrcOps: {MI.getOperand(i: 1)});
10594 auto RHS = MIRBuilder.buildInstr(Opc: ExtOp, DstOps: {WideTy}, SrcOps: {MI.getOperand(i: 2)});
10595 auto Mul = MIRBuilder.buildMul(Dst: WideTy, Src0: LHS, Src1: RHS);
10596 unsigned ShiftOp = IsSigned ? TargetOpcode::G_ASHR : TargetOpcode::G_LSHR;
10597
10598 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: SizeInBits);
10599 auto Shifted = MIRBuilder.buildInstr(Opc: ShiftOp, DstOps: {WideTy}, SrcOps: {Mul, ShiftAmt});
10600 MIRBuilder.buildTrunc(Res: Result, Op: Shifted);
10601
10602 MI.eraseFromParent();
10603 return Legalized;
10604}
10605
10606LegalizerHelper::LegalizeResult
10607LegalizerHelper::lowerISFPCLASS(MachineInstr &MI) {
10608 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
10609 FPClassTest Mask = static_cast<FPClassTest>(MI.getOperand(i: 2).getImm());
10610
10611 if (Mask == fcNone) {
10612 MIRBuilder.buildConstant(Res: DstReg, Val: 0);
10613 MI.eraseFromParent();
10614 return Legalized;
10615 }
10616 if (Mask == fcAllFlags) {
10617 MIRBuilder.buildConstant(Res: DstReg, Val: 1);
10618 MI.eraseFromParent();
10619 return Legalized;
10620 }
10621
10622 // TODO: Try inverting the test with getInvertedFPClassTest like the DAG
10623 // version
10624
10625 unsigned BitSize = SrcTy.getScalarSizeInBits();
10626 const fltSemantics &Semantics = getFltSemanticForLLT(Ty: SrcTy.getScalarType());
10627
10628 LLT IntTy = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: BitSize));
10629 auto AsInt = SrcTy == IntTy ? MIRBuilder.buildCopy(Res: IntTy, Op: SrcReg)
10630 : MIRBuilder.buildBitcast(Dst: IntTy, Src: SrcReg);
10631
10632 // Various masks.
10633 APInt SignBit = APInt::getSignMask(BitWidth: BitSize);
10634 APInt ValueMask = APInt::getSignedMaxValue(numBits: BitSize); // All bits but sign.
10635 APInt Inf = APFloat::getInf(Sem: Semantics).bitcastToAPInt(); // Exp and int bit.
10636 APInt ExpMask = Inf;
10637 APInt AllOneMantissa = APFloat::getLargest(Sem: Semantics).bitcastToAPInt() & ~Inf;
10638 APInt QNaNBitMask =
10639 APInt::getOneBitSet(numBits: BitSize, BitNo: AllOneMantissa.getActiveBits() - 1);
10640 APInt InversionMask = APInt::getAllOnes(numBits: DstTy.getScalarSizeInBits());
10641
10642 auto SignBitC = MIRBuilder.buildConstant(Res: IntTy, Val: SignBit);
10643 auto ValueMaskC = MIRBuilder.buildConstant(Res: IntTy, Val: ValueMask);
10644 auto InfC = MIRBuilder.buildConstant(Res: IntTy, Val: Inf);
10645 auto ExpMaskC = MIRBuilder.buildConstant(Res: IntTy, Val: ExpMask);
10646 auto ZeroC = MIRBuilder.buildConstant(Res: IntTy, Val: 0);
10647
10648 auto Abs = MIRBuilder.buildAnd(Dst: IntTy, Src0: AsInt, Src1: ValueMaskC);
10649 auto Sign =
10650 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_NE, Res: DstTy, Op0: AsInt, Op1: Abs);
10651
10652 auto Res = MIRBuilder.buildConstant(Res: DstTy, Val: 0);
10653 // Clang doesn't support capture of structured bindings:
10654 LLT DstTyCopy = DstTy;
10655 const auto appendToRes = [&](MachineInstrBuilder ToAppend) {
10656 Res = MIRBuilder.buildOr(Dst: DstTyCopy, Src0: Res, Src1: ToAppend);
10657 };
10658
10659 // Tests that involve more than one class should be processed first.
10660 if ((Mask & fcFinite) == fcFinite) {
10661 // finite(V) ==> abs(V) u< exp_mask
10662 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: Abs,
10663 Op1: ExpMaskC));
10664 Mask &= ~fcFinite;
10665 } else if ((Mask & fcFinite) == fcPosFinite) {
10666 // finite(V) && V > 0 ==> V u< exp_mask
10667 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: AsInt,
10668 Op1: ExpMaskC));
10669 Mask &= ~fcPosFinite;
10670 } else if ((Mask & fcFinite) == fcNegFinite) {
10671 // finite(V) && V < 0 ==> abs(V) u< exp_mask && signbit == 1
10672 auto Cmp = MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: Abs,
10673 Op1: ExpMaskC);
10674 auto And = MIRBuilder.buildAnd(Dst: DstTy, Src0: Cmp, Src1: Sign);
10675 appendToRes(And);
10676 Mask &= ~fcNegFinite;
10677 }
10678
10679 if (FPClassTest PartialCheck = Mask & (fcZero | fcSubnormal)) {
10680 // fcZero | fcSubnormal => test all exponent bits are 0
10681 // TODO: Handle sign bit specific cases
10682 // TODO: Handle inverted case
10683 if (PartialCheck == (fcZero | fcSubnormal)) {
10684 auto ExpBits = MIRBuilder.buildAnd(Dst: IntTy, Src0: AsInt, Src1: ExpMaskC);
10685 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10686 Op0: ExpBits, Op1: ZeroC));
10687 Mask &= ~PartialCheck;
10688 }
10689 }
10690
10691 // Check for individual classes.
10692 if (FPClassTest PartialCheck = Mask & fcZero) {
10693 if (PartialCheck == fcPosZero)
10694 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10695 Op0: AsInt, Op1: ZeroC));
10696 else if (PartialCheck == fcZero)
10697 appendToRes(
10698 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy, Op0: Abs, Op1: ZeroC));
10699 else // fcNegZero
10700 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10701 Op0: AsInt, Op1: SignBitC));
10702 }
10703
10704 if (FPClassTest PartialCheck = Mask & fcSubnormal) {
10705 // issubnormal(V) ==> unsigned(abs(V) - 1) u< (all mantissa bits set)
10706 // issubnormal(V) && V>0 ==> unsigned(V - 1) u< (all mantissa bits set)
10707 auto V = (PartialCheck == fcPosSubnormal) ? AsInt : Abs;
10708 auto OneC = MIRBuilder.buildConstant(Res: IntTy, Val: 1);
10709 auto VMinusOne = MIRBuilder.buildSub(Dst: IntTy, Src0: V, Src1: OneC);
10710 auto SubnormalRes =
10711 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: VMinusOne,
10712 Op1: MIRBuilder.buildConstant(Res: IntTy, Val: AllOneMantissa));
10713 if (PartialCheck == fcNegSubnormal)
10714 SubnormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: SubnormalRes, Src1: Sign);
10715 appendToRes(SubnormalRes);
10716 }
10717
10718 if (FPClassTest PartialCheck = Mask & fcInf) {
10719 if (PartialCheck == fcPosInf)
10720 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10721 Op0: AsInt, Op1: InfC));
10722 else if (PartialCheck == fcInf)
10723 appendToRes(
10724 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy, Op0: Abs, Op1: InfC));
10725 else { // fcNegInf
10726 APInt NegInf = APFloat::getInf(Sem: Semantics, Negative: true).bitcastToAPInt();
10727 auto NegInfC = MIRBuilder.buildConstant(Res: IntTy, Val: NegInf);
10728 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10729 Op0: AsInt, Op1: NegInfC));
10730 }
10731 }
10732
10733 if (FPClassTest PartialCheck = Mask & fcNan) {
10734 auto InfWithQnanBitC = MIRBuilder.buildConstant(Res: IntTy, Val: Inf | QNaNBitMask);
10735 if (PartialCheck == fcNan) {
10736 // isnan(V) ==> abs(V) u> int(inf)
10737 appendToRes(
10738 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: DstTy, Op0: Abs, Op1: InfC));
10739 } else if (PartialCheck == fcQNan) {
10740 // isquiet(V) ==> abs(V) u>= (unsigned(Inf) | quiet_bit)
10741 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGE, Res: DstTy, Op0: Abs,
10742 Op1: InfWithQnanBitC));
10743 } else { // fcSNan
10744 // issignaling(V) ==> abs(V) u> unsigned(Inf) &&
10745 // abs(V) u< (unsigned(Inf) | quiet_bit)
10746 auto IsNan =
10747 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: DstTy, Op0: Abs, Op1: InfC);
10748 auto IsNotQnan = MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy,
10749 Op0: Abs, Op1: InfWithQnanBitC);
10750 appendToRes(MIRBuilder.buildAnd(Dst: DstTy, Src0: IsNan, Src1: IsNotQnan));
10751 }
10752 }
10753
10754 if (FPClassTest PartialCheck = Mask & fcNormal) {
10755 // isnormal(V) ==> (0 u< exp u< max_exp) ==> (unsigned(exp-1) u<
10756 // (max_exp-1))
10757 APInt ExpLSB = ExpMask & ~(ExpMask.shl(shiftAmt: 1));
10758 auto ExpMinusOne = MIRBuilder.buildSub(
10759 Dst: IntTy, Src0: Abs, Src1: MIRBuilder.buildConstant(Res: IntTy, Val: ExpLSB));
10760 APInt MaxExpMinusOne = ExpMask - ExpLSB;
10761 auto NormalRes =
10762 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: ExpMinusOne,
10763 Op1: MIRBuilder.buildConstant(Res: IntTy, Val: MaxExpMinusOne));
10764 if (PartialCheck == fcNegNormal)
10765 NormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: NormalRes, Src1: Sign);
10766 else if (PartialCheck == fcPosNormal) {
10767 auto PosSign = MIRBuilder.buildXor(
10768 Dst: DstTy, Src0: Sign, Src1: MIRBuilder.buildConstant(Res: DstTy, Val: InversionMask));
10769 NormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: NormalRes, Src1: PosSign);
10770 }
10771 appendToRes(NormalRes);
10772 }
10773
10774 MIRBuilder.buildCopy(Res: DstReg, Op: Res);
10775 MI.eraseFromParent();
10776 return Legalized;
10777}
10778
10779LegalizerHelper::LegalizeResult LegalizerHelper::lowerSelect(MachineInstr &MI) {
10780 // Implement G_SELECT in terms of XOR, AND, OR.
10781 auto [DstReg, DstTy, MaskReg, MaskTy, Op1Reg, Op1Ty, Op2Reg, Op2Ty] =
10782 MI.getFirst4RegLLTs();
10783
10784 LLT Op1TyInt =
10785 Op1Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: Op1Ty.getScalarSizeInBits()));
10786
10787 bool IsEltPtr = DstTy.isPointerOrPointerVector();
10788 if (IsEltPtr) {
10789 LLT ScalarPtrTy = LLT::integer(SizeInBits: DstTy.getScalarSizeInBits());
10790 LLT NewTy = DstTy.changeElementType(NewEltTy: ScalarPtrTy);
10791 Op1Reg = MIRBuilder.buildPtrToInt(Dst: NewTy, Src: Op1Reg).getReg(Idx: 0);
10792 Op1Ty = MRI.getType(Reg: Op1Reg);
10793 Op2Reg = MIRBuilder.buildPtrToInt(Dst: NewTy, Src: Op2Reg).getReg(Idx: 0);
10794 Op2Ty = MRI.getType(Reg: Op2Reg);
10795 DstTy = NewTy;
10796 }
10797
10798 if (MaskTy.isScalar()) {
10799 // Turn the scalar condition into a vector condition mask if needed.
10800
10801 Register MaskElt = MaskReg;
10802
10803 // The condition was potentially zero extended before, but we want a sign
10804 // extended boolean.
10805 if (MaskTy != LLT::scalar(SizeInBits: 1))
10806 MaskElt = MIRBuilder.buildSExtInReg(Res: MaskTy, Op: MaskElt, ImmOp: 1).getReg(Idx: 0);
10807
10808 // Continue the sign extension (or truncate) to match the data type.
10809 MaskTy = DstTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: DstTy.getScalarSizeInBits()));
10810 MaskElt =
10811 MIRBuilder.buildSExtOrTrunc(Res: MaskTy.getScalarType(), Op: MaskElt).getReg(Idx: 0);
10812
10813 if (DstTy.isVector()) {
10814 // Generate a vector splat idiom.
10815 auto ShufSplat = MIRBuilder.buildShuffleSplat(Res: MaskTy, Src: MaskElt);
10816 MaskReg = ShufSplat.getReg(Idx: 0);
10817 } else {
10818 MaskReg = MaskElt;
10819 }
10820 } else if (!DstTy.isVector()) {
10821 // Cannot handle the case that mask is a vector and dst is a scalar.
10822 return UnableToLegalize;
10823 }
10824
10825 if (MaskTy.getSizeInBits() != DstTy.getSizeInBits()) {
10826 return UnableToLegalize;
10827 }
10828
10829 if (!Op1Ty.getScalarType().isAnyScalar() &&
10830 !Op1Ty.getScalarType().isInteger())
10831 Op1Reg = MIRBuilder.buildBitcast(Dst: Op1TyInt, Src: Op1Reg).getReg(Idx: 0);
10832
10833 if (!Op2Ty.getScalarType().isAnyScalar() &&
10834 !Op2Ty.getScalarType().isInteger()) {
10835 auto Op2TyInt =
10836 Op2Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: Op2Ty.getScalarSizeInBits()));
10837 Op2Reg = MIRBuilder.buildBitcast(Dst: Op2TyInt, Src: Op2Reg).getReg(Idx: 0);
10838 }
10839
10840 auto NotMask = MIRBuilder.buildNot(Dst: MaskTy, Src0: MaskReg);
10841 auto NewOp1 = MIRBuilder.buildAnd(Dst: MaskTy, Src0: Op1Reg, Src1: MaskReg);
10842 auto NewOp2 = MIRBuilder.buildAnd(Dst: MaskTy, Src0: Op2Reg, Src1: NotMask);
10843 if (IsEltPtr) {
10844 auto Or = MIRBuilder.buildOr(Dst: DstTy, Src0: NewOp1, Src1: NewOp2);
10845 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: Or);
10846 } else {
10847 if (DstTy == Op1TyInt)
10848 MIRBuilder.buildOr(Dst: DstReg, Src0: NewOp1, Src1: NewOp2);
10849 else {
10850 auto Or = MIRBuilder.buildOr(Dst: Op1TyInt, Src0: NewOp1, Src1: NewOp2);
10851 MIRBuilder.buildBitcast(Dst: DstReg, Src: Or.getReg(Idx: 0));
10852 }
10853 }
10854 MI.eraseFromParent();
10855 return Legalized;
10856}
10857
10858LegalizerHelper::LegalizeResult LegalizerHelper::lowerDIVREM(MachineInstr &MI) {
10859 // Split DIVREM into individual instructions.
10860 unsigned Opcode = MI.getOpcode();
10861
10862 MIRBuilder.buildInstr(
10863 Opc: Opcode == TargetOpcode::G_SDIVREM ? TargetOpcode::G_SDIV
10864 : TargetOpcode::G_UDIV,
10865 DstOps: {MI.getOperand(i: 0).getReg()}, SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3)});
10866 MIRBuilder.buildInstr(
10867 Opc: Opcode == TargetOpcode::G_SDIVREM ? TargetOpcode::G_SREM
10868 : TargetOpcode::G_UREM,
10869 DstOps: {MI.getOperand(i: 1).getReg()}, SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3)});
10870 MI.eraseFromParent();
10871 return Legalized;
10872}
10873
10874LegalizerHelper::LegalizeResult
10875LegalizerHelper::lowerAbsToAddXor(MachineInstr &MI) {
10876 // Expand %res = G_ABS %a into:
10877 // %v1 = G_ASHR %a, scalar_size-1
10878 // %v2 = G_ADD %a, %v1
10879 // %res = G_XOR %v2, %v1
10880 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
10881 Register OpReg = MI.getOperand(i: 1).getReg();
10882 auto ShiftAmt =
10883 MIRBuilder.buildConstant(Res: DstTy, Val: DstTy.getScalarSizeInBits() - 1);
10884 auto Shift = MIRBuilder.buildAShr(Dst: DstTy, Src0: OpReg, Src1: ShiftAmt);
10885 auto Add = MIRBuilder.buildAdd(Dst: DstTy, Src0: OpReg, Src1: Shift);
10886 MIRBuilder.buildXor(Dst: MI.getOperand(i: 0).getReg(), Src0: Add, Src1: Shift);
10887 MI.eraseFromParent();
10888 return Legalized;
10889}
10890
10891LegalizerHelper::LegalizeResult
10892LegalizerHelper::lowerAbsToMaxNeg(MachineInstr &MI) {
10893 // Expand %res = G_ABS %a into:
10894 // %v1 = G_CONSTANT 0
10895 // %v2 = G_SUB %v1, %a
10896 // %res = G_SMAX %a, %v2
10897 Register SrcReg = MI.getOperand(i: 1).getReg();
10898 LLT Ty = MRI.getType(Reg: SrcReg);
10899 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
10900 auto Sub = MIRBuilder.buildSub(Dst: Ty, Src0: Zero, Src1: SrcReg);
10901 MIRBuilder.buildSMax(Dst: MI.getOperand(i: 0), Src0: SrcReg, Src1: Sub);
10902 MI.eraseFromParent();
10903 return Legalized;
10904}
10905
10906LegalizerHelper::LegalizeResult
10907LegalizerHelper::lowerAbsToCNeg(MachineInstr &MI) {
10908 Register SrcReg = MI.getOperand(i: 1).getReg();
10909 Register DestReg = MI.getOperand(i: 0).getReg();
10910 LLT Ty = MRI.getType(Reg: SrcReg), IType = LLT::integer(SizeInBits: 1);
10911 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0).getReg(Idx: 0);
10912 auto Sub = MIRBuilder.buildSub(Dst: Ty, Src0: Zero, Src1: SrcReg).getReg(Idx: 0);
10913 auto ICmp = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SGT, Res: IType, Op0: SrcReg, Op1: Zero);
10914 MIRBuilder.buildSelect(Res: DestReg, Tst: ICmp, Op0: SrcReg, Op1: Sub);
10915 MI.eraseFromParent();
10916 return Legalized;
10917}
10918
10919LegalizerHelper::LegalizeResult
10920LegalizerHelper::lowerAbsDiffToSelect(MachineInstr &MI) {
10921 assert((MI.getOpcode() == TargetOpcode::G_ABDS ||
10922 MI.getOpcode() == TargetOpcode::G_ABDU) &&
10923 "Expected G_ABDS or G_ABDU instruction");
10924
10925 auto [DstReg, LHS, RHS] = MI.getFirst3Regs();
10926 LLT Ty = MRI.getType(Reg: LHS);
10927
10928 // abds(lhs, rhs) -> select(sgt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
10929 // abdu(lhs, rhs) -> select(ugt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
10930 Register LHSSub = MIRBuilder.buildSub(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10931 Register RHSSub = MIRBuilder.buildSub(Dst: Ty, Src0: RHS, Src1: LHS).getReg(Idx: 0);
10932 CmpInst::Predicate Pred = (MI.getOpcode() == TargetOpcode::G_ABDS)
10933 ? CmpInst::ICMP_SGT
10934 : CmpInst::ICMP_UGT;
10935 auto ICmp = MIRBuilder.buildICmp(Pred, Res: LLT::scalar(SizeInBits: 1), Op0: LHS, Op1: RHS);
10936 MIRBuilder.buildSelect(Res: DstReg, Tst: ICmp, Op0: LHSSub, Op1: RHSSub);
10937
10938 MI.eraseFromParent();
10939 return Legalized;
10940}
10941
10942LegalizerHelper::LegalizeResult
10943LegalizerHelper::lowerAbsDiffToMinMax(MachineInstr &MI) {
10944 assert((MI.getOpcode() == TargetOpcode::G_ABDS ||
10945 MI.getOpcode() == TargetOpcode::G_ABDU) &&
10946 "Expected G_ABDS or G_ABDU instruction");
10947
10948 auto [DstReg, LHS, RHS] = MI.getFirst3Regs();
10949 LLT Ty = MRI.getType(Reg: LHS);
10950
10951 // abds(lhs, rhs) -→ sub(smax(lhs, rhs), smin(lhs, rhs))
10952 // abdu(lhs, rhs) -→ sub(umax(lhs, rhs), umin(lhs, rhs))
10953 Register MaxReg, MinReg;
10954 if (MI.getOpcode() == TargetOpcode::G_ABDS) {
10955 MaxReg = MIRBuilder.buildSMax(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10956 MinReg = MIRBuilder.buildSMin(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10957 } else {
10958 MaxReg = MIRBuilder.buildUMax(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10959 MinReg = MIRBuilder.buildUMin(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10960 }
10961 MIRBuilder.buildSub(Dst: DstReg, Src0: MaxReg, Src1: MinReg);
10962
10963 MI.eraseFromParent();
10964 return Legalized;
10965}
10966
10967LegalizerHelper::LegalizeResult LegalizerHelper::lowerFAbs(MachineInstr &MI) {
10968 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
10969 LLT TyInt =
10970 DstTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: DstTy.getScalarSizeInBits()));
10971 Register CastedSrc = SrcReg;
10972
10973 if (!(SrcTy.getScalarType().isAnyScalar() ||
10974 SrcTy.getScalarType().isInteger())) {
10975 auto SrcTyInt =
10976 SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: SrcTy.getScalarSizeInBits()));
10977 CastedSrc = MIRBuilder.buildBitcast(Dst: SrcTyInt, Src: SrcReg).getReg(Idx: 0);
10978 }
10979
10980 if (MRI.getType(Reg: DstReg) != TyInt) {
10981 // Reset sign bit
10982 Register NewDst =
10983 MIRBuilder
10984 .buildAnd(Dst: TyInt, Src0: CastedSrc,
10985 Src1: MIRBuilder.buildConstant(
10986 Res: TyInt, Val: APInt::getSignedMaxValue(
10987 numBits: DstTy.getScalarSizeInBits())))
10988 .getReg(Idx: 0);
10989
10990 MIRBuilder.buildBitcast(Dst: DstReg, Src: NewDst);
10991 } else
10992 MIRBuilder
10993 .buildAnd(
10994 Dst: DstReg, Src0: CastedSrc,
10995 Src1: MIRBuilder.buildConstant(
10996 Res: TyInt, Val: APInt::getSignedMaxValue(numBits: DstTy.getScalarSizeInBits())))
10997 .getReg(Idx: 0);
10998
10999 MI.eraseFromParent();
11000 return Legalized;
11001}
11002
11003LegalizerHelper::LegalizeResult
11004LegalizerHelper::lowerVectorReduction(MachineInstr &MI) {
11005 Register SrcReg = MI.getOperand(i: 1).getReg();
11006 LLT SrcTy = MRI.getType(Reg: SrcReg);
11007 LLT DstTy = MRI.getType(Reg: SrcReg);
11008
11009 // The source could be a scalar if the IR type was <1 x sN>.
11010 if (SrcTy.isScalar()) {
11011 if (DstTy.getSizeInBits() > SrcTy.getSizeInBits())
11012 return UnableToLegalize; // FIXME: handle extension.
11013 // This can be just a plain copy.
11014 Observer.changingInstr(MI);
11015 MI.setDesc(MIRBuilder.getTII().get(Opcode: TargetOpcode::COPY));
11016 Observer.changedInstr(MI);
11017 return Legalized;
11018 }
11019 return UnableToLegalize;
11020}
11021
11022LegalizerHelper::LegalizeResult LegalizerHelper::lowerVAArg(MachineInstr &MI) {
11023 MachineFunction &MF = *MI.getMF();
11024 const DataLayout &DL = MIRBuilder.getDataLayout();
11025 LLVMContext &Ctx = MF.getFunction().getContext();
11026 Register ListPtr = MI.getOperand(i: 1).getReg();
11027 LLT PtrTy = MRI.getType(Reg: ListPtr);
11028
11029 // LstPtr is a pointer to the head of the list. Get the address
11030 // of the head of the list.
11031 Align PtrAlignment = DL.getABITypeAlign(Ty: getTypeForLLT(Ty: PtrTy, C&: Ctx));
11032 MachineMemOperand *PtrLoadMMO = MF.getMachineMemOperand(
11033 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOLoad, MemTy: PtrTy, BaseAlignment: PtrAlignment);
11034 auto VAList = MIRBuilder.buildLoad(Res: PtrTy, Addr: ListPtr, MMO&: *PtrLoadMMO).getReg(Idx: 0);
11035
11036 const Align A(MI.getOperand(i: 2).getImm());
11037 LLT PtrTyAsScalarTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
11038 if (A > TLI.getMinStackArgumentAlignment()) {
11039 Register AlignAmt =
11040 MIRBuilder.buildConstant(Res: PtrTyAsScalarTy, Val: A.value() - 1).getReg(Idx: 0);
11041 auto AddDst = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: VAList, Op1: AlignAmt);
11042 auto AndDst = MIRBuilder.buildMaskLowPtrBits(Res: PtrTy, Op0: AddDst, NumBits: Log2(A));
11043 VAList = AndDst.getReg(Idx: 0);
11044 }
11045
11046 // Increment the pointer, VAList, to the next vaarg
11047 // The list should be bumped by the size of element in the current head of
11048 // list.
11049 Register Dst = MI.getOperand(i: 0).getReg();
11050 LLT LLTTy = MRI.getType(Reg: Dst);
11051 Type *Ty = getTypeForLLT(Ty: LLTTy, C&: Ctx);
11052 auto IncAmt =
11053 MIRBuilder.buildConstant(Res: PtrTyAsScalarTy, Val: DL.getTypeAllocSize(Ty));
11054 auto Succ = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: VAList, Op1: IncAmt);
11055
11056 // Store the increment VAList to the legalized pointer
11057 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
11058 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOStore, MemTy: PtrTy, BaseAlignment: PtrAlignment);
11059 MIRBuilder.buildStore(Val: Succ, Addr: ListPtr, MMO&: *StoreMMO);
11060 // Load the actual argument out of the pointer VAList
11061 Align EltAlignment = DL.getABITypeAlign(Ty);
11062 MachineMemOperand *EltLoadMMO = MF.getMachineMemOperand(
11063 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOLoad, MemTy: LLTTy, BaseAlignment: EltAlignment);
11064 MIRBuilder.buildLoad(Res: Dst, Addr: VAList, MMO&: *EltLoadMMO);
11065
11066 MI.eraseFromParent();
11067 return Legalized;
11068}
11069
11070LegalizerHelper::LegalizeResult LegalizerHelper::lowerMulfix(MachineInstr &MI) {
11071 unsigned OpCode = MI.getOpcode();
11072 assert((OpCode == TargetOpcode::G_SMULFIX ||
11073 OpCode == TargetOpcode::G_UMULFIX ||
11074 OpCode == TargetOpcode::G_SMULFIXSAT ||
11075 OpCode == TargetOpcode::G_UMULFIXSAT) &&
11076 "Operator must be either G_SMULFIX[SAT] or G_UMULFIX[SAT]!");
11077 auto [Dst, LHS, RHS] = MI.getFirst3Regs();
11078 LLT Ty = MRI.getType(Reg: Dst);
11079 unsigned Scale = MI.getOperand(i: 3).getImm();
11080
11081 bool Saturating = (OpCode == TargetOpcode::G_SMULFIXSAT ||
11082 OpCode == TargetOpcode::G_UMULFIXSAT);
11083 bool IsSigned = (OpCode == TargetOpcode::G_SMULFIX ||
11084 OpCode == TargetOpcode::G_SMULFIXSAT);
11085
11086 if (!Saturating && Scale == 0) {
11087 MIRBuilder.buildMul(Dst, Src0: LHS, Src1: RHS);
11088 MI.eraseFromParent();
11089 return Legalized;
11090 }
11091
11092 // TODO: Port other lowering paths from SelectionDAG.
11093 LLT WideTy = Ty.changeElementSize(NewEltSize: Ty.getScalarSizeInBits() * 2);
11094 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: Scale);
11095 MachineInstrBuilder ExtLHS{}, ExtRHS{}, Shift{};
11096 if (IsSigned) {
11097 ExtLHS = MIRBuilder.buildSExt(Res: WideTy, Op: LHS);
11098 ExtRHS = MIRBuilder.buildSExt(Res: WideTy, Op: RHS);
11099 } else {
11100 ExtLHS = MIRBuilder.buildZExt(Res: WideTy, Op: LHS);
11101 ExtRHS = MIRBuilder.buildZExt(Res: WideTy, Op: RHS);
11102 }
11103
11104 auto Mul = MIRBuilder.buildMul(Dst: WideTy, Src0: ExtLHS, Src1: ExtRHS);
11105 if (IsSigned)
11106 Shift = MIRBuilder.buildAShr(Dst: WideTy, Src0: Mul, Src1: ShiftAmt);
11107 else
11108 Shift = MIRBuilder.buildLShr(Dst: WideTy, Src0: Mul, Src1: ShiftAmt);
11109
11110 if (!Saturating)
11111 MIRBuilder.buildTrunc(Res: Dst, Op: Shift);
11112 else if (IsSigned)
11113 MIRBuilder.buildTruncSSatS(Res: Dst, Op: Shift);
11114 else
11115 MIRBuilder.buildTruncUSatU(Res: Dst, Op: Shift);
11116
11117 MI.eraseFromParent();
11118 return Legalized;
11119}
11120
11121// Get a vectorized representation of the memset value operand, GISel edition.
11122static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB) {
11123 MachineRegisterInfo &MRI = *MIB.getMRI();
11124 unsigned NumBits = Ty.getScalarSizeInBits();
11125 auto ValVRegAndVal = getIConstantVRegValWithLookThrough(VReg: Val, MRI);
11126 if (!Ty.isVector() && ValVRegAndVal) {
11127 APInt Scalar = ValVRegAndVal->Value.trunc(width: 8);
11128 APInt SplatVal = APInt::getSplat(NewLen: NumBits, V: Scalar);
11129 return MIB.buildConstant(Res: Ty, Val: SplatVal).getReg(Idx: 0);
11130 }
11131
11132 // Extend the byte value to the larger type, and then multiply by a magic
11133 // value 0x010101... in order to replicate it across every byte.
11134 // Unless it's zero, in which case just emit a larger G_CONSTANT 0.
11135 if (ValVRegAndVal && ValVRegAndVal->Value == 0) {
11136 return MIB.buildConstant(Res: Ty, Val: 0).getReg(Idx: 0);
11137 }
11138
11139 LLT ExtType = Ty.getScalarType();
11140 auto ZExt = MIB.buildZExtOrTrunc(Res: ExtType, Op: Val);
11141 if (NumBits > 8) {
11142 APInt Magic = APInt::getSplat(NewLen: NumBits, V: APInt(8, 0x01));
11143 auto MagicMI = MIB.buildConstant(Res: ExtType, Val: Magic);
11144 Val = MIB.buildMul(Dst: ExtType, Src0: ZExt, Src1: MagicMI).getReg(Idx: 0);
11145 }
11146
11147 // For vector types create a G_BUILD_VECTOR.
11148 if (Ty.isVector())
11149 Val = MIB.buildSplatBuildVector(Res: Ty, Src: Val).getReg(Idx: 0);
11150
11151 return Val;
11152}
11153
11154LegalizerHelper::LegalizeResult
11155LegalizerHelper::lowerMemset(MachineInstr &MI, Register Dst, Register Val,
11156 uint64_t KnownLen, Align Alignment,
11157 bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
11158 auto &MF = *MI.getParent()->getParent();
11159 const auto &TLI = *MF.getSubtarget().getTargetLowering();
11160 auto &DL = MF.getDataLayout();
11161 LLVMContext &C = MF.getFunction().getContext();
11162
11163 assert(KnownLen != 0 && "Have a zero length memset length!");
11164 assert(!MemOps.empty() && "Expected at least one memory op");
11165
11166 MachineFrameInfo &MFI = MF.getFrameInfo();
11167 MachineInstr *FIDef = getOpcodeDef(Opcode: TargetOpcode::G_FRAME_INDEX, Reg: Dst, MRI);
11168 const auto &DstMMO = **MI.memoperands_begin();
11169
11170 if (DstAlignCanChange) {
11171 // Get an estimate of the type from the LLT.
11172 Type *IRTy = getTypeForLLT(Ty: MemOps[0], C);
11173 Align NewAlign = DL.getABITypeAlign(Ty: IRTy);
11174 if (NewAlign > Alignment) {
11175 Alignment = NewAlign;
11176 unsigned FI = FIDef->getOperand(i: 1).getIndex();
11177 // Give the stack frame object a larger alignment if needed.
11178 if (MFI.getObjectAlign(ObjectIdx: FI) < Alignment)
11179 MFI.setObjectAlignment(ObjectIdx: FI, Alignment);
11180 }
11181 }
11182
11183 MachineIRBuilder MIB(MI);
11184 // Find the largest store and generate the bit pattern for it.
11185 LLT LargestTy = MemOps[0];
11186 for (unsigned i = 1; i < MemOps.size(); i++)
11187 if (MemOps[i].getSizeInBits() > LargestTy.getSizeInBits())
11188 LargestTy = MemOps[i];
11189
11190 // The memset stored value is always defined as an s8, so in order to make it
11191 // work with larger store types we need to repeat the bit pattern across the
11192 // wider type.
11193 Register MemSetValue = getMemsetValue(Val, Ty: LargestTy, MIB);
11194
11195 if (!MemSetValue)
11196 return UnableToLegalize;
11197
11198 // Generate the stores. For each store type in the list, we generate the
11199 // matching store of that type to the destination address.
11200 LLT PtrTy = MRI.getType(Reg: Dst);
11201 unsigned DstOff = 0;
11202 unsigned Size = KnownLen;
11203 for (unsigned I = 0; I < MemOps.size(); I++) {
11204 LLT Ty = MemOps[I];
11205 unsigned TySize = Ty.getSizeInBytes();
11206
11207 if (TySize > Size) {
11208 // Issuing a load / store pair that overlaps with the previous pair.
11209 // Adjust the offset accordingly.
11210 assert(I == MemOps.size() - 1 && I != 0);
11211 DstOff -= TySize - Size;
11212 }
11213
11214 // If this store is smaller than the largest store see whether we can get
11215 // the smaller value for free with a truncate.
11216 Register Value = MemSetValue;
11217 if (Ty.getSizeInBits() < LargestTy.getSizeInBits()) {
11218 MVT VT = getMVTForLLT(Ty);
11219 MVT LargestVT = getMVTForLLT(Ty: LargestTy);
11220 if (!LargestTy.isVector() && !Ty.isVector() &&
11221 TLI.isTruncateFree(FromVT: LargestVT, ToVT: VT))
11222 Value = MIB.buildTrunc(Res: Ty, Op: MemSetValue).getReg(Idx: 0);
11223 else
11224 Value = getMemsetValue(Val, Ty, MIB);
11225 if (!Value)
11226 return UnableToLegalize;
11227 }
11228
11229 auto *StoreMMO = MF.getMachineMemOperand(MMO: &DstMMO, Offset: DstOff, Ty);
11230
11231 Register Ptr = Dst;
11232 if (DstOff != 0) {
11233 auto Offset =
11234 MIB.buildConstant(Res: LLT::integer(SizeInBits: PtrTy.getSizeInBits()), Val: DstOff);
11235 Ptr = MIB.buildObjectPtrOffset(Res: PtrTy, Op0: Dst, Op1: Offset).getReg(Idx: 0);
11236 }
11237
11238 MIB.buildStore(Val: Value, Addr: Ptr, MMO&: *StoreMMO);
11239 DstOff += Ty.getSizeInBytes();
11240 Size -= TySize;
11241 }
11242
11243 MI.eraseFromParent();
11244 return Legalized;
11245}
11246
11247LegalizerHelper::LegalizeResult
11248LegalizerHelper::lowerMemcpy(MachineInstr &MI, Register Dst, Register Src,
11249 uint64_t KnownLen, Align Alignment,
11250 bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
11251 auto &MF = *MI.getParent()->getParent();
11252 auto &DL = MF.getDataLayout();
11253 LLVMContext &C = MF.getFunction().getContext();
11254
11255 assert(KnownLen != 0 && "Have a zero length memcpy length!");
11256 assert(!MemOps.empty() && "Expected at least one memory op");
11257
11258 MachineFrameInfo &MFI = MF.getFrameInfo();
11259 MachineInstr *FIDef = getOpcodeDef(Opcode: TargetOpcode::G_FRAME_INDEX, Reg: Dst, MRI);
11260
11261 // FIXME: infer better src pointer alignment like SelectionDAG does here.
11262 // FIXME: also use the equivalent of isMemSrcFromConstant and alwaysinlining
11263 // if the memcpy is in a tail call position.
11264
11265 const auto &DstMMO = **MI.memoperands_begin();
11266 const auto &SrcMMO = **std::next(x: MI.memoperands_begin());
11267
11268 if (DstAlignCanChange) {
11269 // Get an estimate of the type from the LLT.
11270 Type *IRTy = getTypeForLLT(Ty: MemOps[0], C);
11271 Align NewAlign = DL.getABITypeAlign(Ty: IRTy);
11272
11273 // Don't promote to an alignment that would require dynamic stack
11274 // realignment.
11275 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
11276 if (!TRI->hasStackRealignment(MF))
11277 if (MaybeAlign StackAlign = DL.getStackAlignment())
11278 NewAlign = std::min(a: NewAlign, b: *StackAlign);
11279
11280 if (NewAlign > Alignment) {
11281 Alignment = NewAlign;
11282 unsigned FI = FIDef->getOperand(i: 1).getIndex();
11283 // Give the stack frame object a larger alignment if needed.
11284 if (MFI.getObjectAlign(ObjectIdx: FI) < Alignment)
11285 MFI.setObjectAlignment(ObjectIdx: FI, Alignment);
11286 }
11287 }
11288
11289 LLVM_DEBUG(dbgs() << "Inlining memcpy: " << MI << " into loads & stores\n");
11290
11291 MachineIRBuilder MIB(MI);
11292 // Now we need to emit a pair of load and stores for each of the types we've
11293 // collected. I.e. for each type, generate a load from the source pointer of
11294 // that type width, and then generate a corresponding store to the dest buffer
11295 // of that value loaded. This can result in a sequence of loads and stores
11296 // mixed types, depending on what the target specifies as good types to use.
11297 LLT SrcTy = MRI.getType(Reg: Src);
11298 LLT DstTy = MRI.getType(Reg: Dst);
11299 unsigned SrcIndexSize = DL.getIndexSizeInBits(AS: SrcTy.getAddressSpace());
11300 unsigned DstIndexSize = DL.getIndexSizeInBits(AS: DstTy.getAddressSpace());
11301 unsigned CurrOffset = 0;
11302 unsigned Size = KnownLen;
11303 for (auto CopyTy : MemOps) {
11304 TypeSize TySize = CopyTy.getSizeInBytes();
11305
11306 // Issuing a load / store pair that overlaps with the previous pair. Adjust
11307 // the offset accordingly.
11308 if (TySize > Size) {
11309 unsigned Overlap = TySize - Size;
11310 assert(Overlap < CurrOffset &&
11311 "overlapping memcpy load/store spans the whole region or more");
11312 CurrOffset -= Overlap;
11313 }
11314
11315 // Construct MMOs for the accesses.
11316 auto *LoadMMO = MF.getMachineMemOperand(MMO: &SrcMMO, Offset: CurrOffset, Size: TySize);
11317 auto *StoreMMO = MF.getMachineMemOperand(MMO: &DstMMO, Offset: CurrOffset, Size: TySize);
11318
11319 // Create the load.
11320 Register LoadPtr = Src;
11321 Register Offset;
11322 if (CurrOffset != 0) {
11323 Offset =
11324 MIB.buildConstant(Res: LLT::integer(SizeInBits: SrcIndexSize), Val: CurrOffset).getReg(Idx: 0);
11325 LoadPtr = MIB.buildObjectPtrOffset(Res: SrcTy, Op0: Src, Op1: Offset).getReg(Idx: 0);
11326 }
11327 auto LdVal = MIB.buildLoad(Res: CopyTy, Addr: LoadPtr, MMO&: *LoadMMO);
11328
11329 // Create the store.
11330 Register StorePtr = Dst;
11331 if (CurrOffset != 0) {
11332 if (SrcIndexSize != DstIndexSize) {
11333 Offset =
11334 MIB.buildConstant(Res: LLT::integer(SizeInBits: DstIndexSize), Val: CurrOffset).getReg(Idx: 0);
11335 }
11336 StorePtr = MIB.buildObjectPtrOffset(Res: DstTy, Op0: Dst, Op1: Offset).getReg(Idx: 0);
11337 }
11338 MIB.buildStore(Val: LdVal, Addr: StorePtr, MMO&: *StoreMMO);
11339 CurrOffset += TySize;
11340 Size -= TySize;
11341 }
11342
11343 MI.eraseFromParent();
11344 return Legalized;
11345}
11346
11347LegalizerHelper::LegalizeResult
11348LegalizerHelper::lowerMemmove(MachineInstr &MI, Register Dst, Register Src,
11349 uint64_t KnownLen, Align Alignment,
11350 bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
11351 auto &MF = *MI.getParent()->getParent();
11352 auto &DL = MF.getDataLayout();
11353 LLVMContext &C = MF.getFunction().getContext();
11354
11355 assert(KnownLen != 0 && "Have a zero length memmove length!");
11356 assert(!MemOps.empty() && "Expected at least one memory op");
11357
11358 MachineFrameInfo &MFI = MF.getFrameInfo();
11359 MachineInstr *FIDef = getOpcodeDef(Opcode: TargetOpcode::G_FRAME_INDEX, Reg: Dst, MRI);
11360 const auto &DstMMO = **MI.memoperands_begin();
11361 const auto &SrcMMO = **std::next(x: MI.memoperands_begin());
11362
11363 if (DstAlignCanChange) {
11364 // Get an estimate of the type from the LLT.
11365 Type *IRTy = getTypeForLLT(Ty: MemOps[0], C);
11366 Align NewAlign = DL.getABITypeAlign(Ty: IRTy);
11367
11368 // Don't promote to an alignment that would require dynamic stack
11369 // realignment.
11370 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
11371 if (!TRI->hasStackRealignment(MF))
11372 if (MaybeAlign StackAlign = DL.getStackAlignment())
11373 NewAlign = std::min(a: NewAlign, b: *StackAlign);
11374
11375 if (NewAlign > Alignment) {
11376 Alignment = NewAlign;
11377 unsigned FI = FIDef->getOperand(i: 1).getIndex();
11378 // Give the stack frame object a larger alignment if needed.
11379 if (MFI.getObjectAlign(ObjectIdx: FI) < Alignment)
11380 MFI.setObjectAlignment(ObjectIdx: FI, Alignment);
11381 }
11382 }
11383
11384 LLVM_DEBUG(dbgs() << "Inlining memmove: " << MI << " into loads & stores\n");
11385
11386 MachineIRBuilder MIB(MI);
11387 // Memmove requires that we perform the loads first before issuing the stores.
11388 // Apart from that, this loop is pretty much doing the same thing as the
11389 // memcpy codegen function.
11390 unsigned CurrOffset = 0;
11391 unsigned Size = KnownLen;
11392 SmallVector<Register, 16> LoadVals;
11393 for (auto CopyTy : MemOps) {
11394 TypeSize TySize = CopyTy.getSizeInBytes();
11395
11396 // Issuing a load that overlaps with the previous load. Adjust the offset
11397 // accordingly.
11398 if (TySize > Size) {
11399 unsigned Overlap = TySize - Size;
11400 assert(Overlap < CurrOffset &&
11401 "overlapping memmove load spans the whole region or more");
11402 CurrOffset -= Overlap;
11403 }
11404
11405 // Construct MMO for the load.
11406 auto *LoadMMO = MF.getMachineMemOperand(MMO: &SrcMMO, Offset: CurrOffset, Size: TySize);
11407
11408 // Create the load.
11409 Register LoadPtr = Src;
11410 if (CurrOffset != 0) {
11411 LLT SrcTy = MRI.getType(Reg: Src);
11412 auto Offset =
11413 MIB.buildConstant(Res: LLT::integer(SizeInBits: SrcTy.getSizeInBits()), Val: CurrOffset);
11414 LoadPtr = MIB.buildObjectPtrOffset(Res: SrcTy, Op0: Src, Op1: Offset).getReg(Idx: 0);
11415 }
11416 LoadVals.push_back(Elt: MIB.buildLoad(Res: CopyTy, Addr: LoadPtr, MMO&: *LoadMMO).getReg(Idx: 0));
11417 CurrOffset += TySize;
11418 Size -= TySize;
11419 }
11420
11421 CurrOffset = 0;
11422 Size = KnownLen;
11423 for (unsigned I = 0; I < MemOps.size(); ++I) {
11424 LLT CopyTy = MemOps[I];
11425 TypeSize TySize = CopyTy.getSizeInBytes();
11426
11427 // Issuing a store that overlaps with the previous store. Adjust the offset
11428 // accordingly.
11429 if (TySize > Size) {
11430 unsigned Overlap = TySize - Size;
11431 assert(Overlap < CurrOffset &&
11432 "overlapping memmove store spans the whole region or more");
11433 CurrOffset -= Overlap;
11434 }
11435
11436 // Now store the values loaded.
11437 auto *StoreMMO = MF.getMachineMemOperand(MMO: &DstMMO, Offset: CurrOffset, Size: TySize);
11438
11439 Register StorePtr = Dst;
11440 if (CurrOffset != 0) {
11441 LLT DstTy = MRI.getType(Reg: Dst);
11442 auto Offset =
11443 MIB.buildConstant(Res: LLT::integer(SizeInBits: DstTy.getSizeInBits()), Val: CurrOffset);
11444 StorePtr = MIB.buildObjectPtrOffset(Res: DstTy, Op0: Dst, Op1: Offset).getReg(Idx: 0);
11445 }
11446 MIB.buildStore(Val: LoadVals[I], Addr: StorePtr, MMO&: *StoreMMO);
11447 CurrOffset += TySize;
11448 Size -= TySize;
11449 }
11450 MI.eraseFromParent();
11451 return Legalized;
11452}
11453
11454LegalizerHelper::LegalizeResult LegalizerHelper::lowerMemCpyFamily(
11455 MachineInstr &MI, Register Dst, Register Src, uint64_t KnownLen,
11456 Align Alignment, bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
11457 const unsigned Opc = MI.getOpcode();
11458 assert((Opc == TargetOpcode::G_MEMCPY ||
11459 Opc == TargetOpcode::G_MEMCPY_INLINE ||
11460 Opc == TargetOpcode::G_MEMMOVE || Opc == TargetOpcode::G_MEMSET ||
11461 Opc == TargetOpcode::G_MEMSET_INLINE) &&
11462 "Expected memcpy like instruction");
11463
11464 if (KnownLen == 0) {
11465 MI.eraseFromParent();
11466 return Legalized;
11467 }
11468
11469 if (Opc == TargetOpcode::G_MEMCPY || Opc == TargetOpcode::G_MEMCPY_INLINE) {
11470 return lowerMemcpy(MI, Dst, Src, KnownLen, Alignment, DstAlignCanChange,
11471 MemOps);
11472 }
11473 if (Opc == TargetOpcode::G_MEMMOVE)
11474 return lowerMemmove(MI, Dst, Src, KnownLen, Alignment, DstAlignCanChange,
11475 MemOps);
11476 if (Opc == TargetOpcode::G_MEMSET || Opc == TargetOpcode::G_MEMSET_INLINE)
11477 return lowerMemset(MI, Dst, Val: Src, KnownLen, Alignment, DstAlignCanChange,
11478 MemOps);
11479 return UnableToLegalize;
11480}
11481
11482LegalizerHelper::LegalizeResult
11483LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen) {
11484 Register Dst, Src;
11485 uint64_t KnownLen;
11486 Align Alignment;
11487 bool DstAlignCanChange;
11488 std::vector<LLT> MemOps;
11489 if (!canLowerMemCpyFamily(MI, MRI, MaxLen, Dst, Src, KnownLen, Alignment,
11490 DstAlignCanChange, MemOps))
11491 return UnableToLegalize;
11492 return lowerMemCpyFamily(MI, Dst, Src, KnownLen, Alignment, DstAlignCanChange,
11493 MemOps);
11494}
11495