1//===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file implements the LegalizerHelper class to legalize
10/// individual instructions and the LegalizeMachineIR wrapper pass for the
11/// primary legalization.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
16#include "llvm/CodeGen/GlobalISel/CallLowering.h"
17#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
18#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
19#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
20#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
21#include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h"
22#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
23#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24#include "llvm/CodeGen/GlobalISel/Utils.h"
25#include "llvm/CodeGen/LowLevelTypeUtils.h"
26#include "llvm/CodeGen/MachineConstantPool.h"
27#include "llvm/CodeGen/MachineFrameInfo.h"
28#include "llvm/CodeGen/MachineRegisterInfo.h"
29#include "llvm/CodeGen/RuntimeLibcallUtil.h"
30#include "llvm/CodeGen/TargetFrameLowering.h"
31#include "llvm/CodeGen/TargetInstrInfo.h"
32#include "llvm/CodeGen/TargetLowering.h"
33#include "llvm/CodeGen/TargetOpcodes.h"
34#include "llvm/CodeGen/TargetSubtargetInfo.h"
35#include "llvm/IR/Instructions.h"
36#include "llvm/Support/Debug.h"
37#include "llvm/Support/MathExtras.h"
38#include "llvm/Support/raw_ostream.h"
39#include "llvm/Target/TargetMachine.h"
40#include <cassert>
41#include <numeric>
42#include <optional>
43
44#define DEBUG_TYPE "legalizer"
45
46using namespace llvm;
47using namespace LegalizeActions;
48using namespace MIPatternMatch;
49
50/// Try to break down \p OrigTy into \p NarrowTy sized pieces.
51///
52/// Returns the number of \p NarrowTy elements needed to reconstruct \p OrigTy,
53/// with any leftover piece as type \p LeftoverTy
54///
55/// Returns -1 in the first element of the pair if the breakdown is not
56/// satisfiable.
57static std::pair<int, int>
58getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) {
59 assert(!LeftoverTy.isValid() && "this is an out argument");
60
61 unsigned Size = OrigTy.getSizeInBits();
62 unsigned NarrowSize = NarrowTy.getSizeInBits();
63 unsigned NumParts = Size / NarrowSize;
64 unsigned LeftoverSize = Size - NumParts * NarrowSize;
65 assert(Size > NarrowSize);
66
67 if (LeftoverSize == 0)
68 return {NumParts, 0};
69
70 if (NarrowTy.isVector()) {
71 unsigned EltSize = OrigTy.getScalarSizeInBits();
72 if (LeftoverSize % EltSize != 0)
73 return {-1, -1};
74 LeftoverTy = OrigTy.changeElementCount(
75 EC: ElementCount::getFixed(MinVal: LeftoverSize / EltSize));
76 } else {
77 LeftoverTy = LLT::integer(SizeInBits: LeftoverSize);
78 }
79
80 int NumLeftover = LeftoverSize / LeftoverTy.getSizeInBits();
81 return std::make_pair(x&: NumParts, y&: NumLeftover);
82}
83
84static Type *getFloatTypeForLLT(LLVMContext &Ctx, LLT Ty) {
85
86 if (!Ty.isScalar())
87 return nullptr;
88
89 switch (Ty.getSizeInBits()) {
90 case 16:
91 return Type::getHalfTy(C&: Ctx);
92 case 32:
93 return Type::getFloatTy(C&: Ctx);
94 case 64:
95 return Type::getDoubleTy(C&: Ctx);
96 case 80:
97 return Type::getX86_FP80Ty(C&: Ctx);
98 case 128:
99 return Type::getFP128Ty(C&: Ctx);
100 default:
101 return nullptr;
102 }
103}
104
105LegalizerHelper::LegalizerHelper(MachineFunction &MF,
106 GISelChangeObserver &Observer,
107 MachineIRBuilder &Builder,
108 const LibcallLoweringInfo *Libcalls)
109 : MIRBuilder(Builder), Observer(Observer), MRI(MF.getRegInfo()),
110 LI(*MF.getSubtarget().getLegalizerInfo()),
111 TLI(*MF.getSubtarget().getTargetLowering()), Libcalls(Libcalls) {}
112
113LegalizerHelper::LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI,
114 GISelChangeObserver &Observer,
115 MachineIRBuilder &B,
116 const LibcallLoweringInfo *Libcalls,
117 GISelValueTracking *VT)
118 : MIRBuilder(B), Observer(Observer), MRI(MF.getRegInfo()), LI(LI),
119 TLI(*MF.getSubtarget().getTargetLowering()), Libcalls(Libcalls), VT(VT) {}
120
121LegalizerHelper::LegalizeResult
122LegalizerHelper::legalizeInstrStep(MachineInstr &MI,
123 LostDebugLocObserver &LocObserver) {
124 LLVM_DEBUG(dbgs() << "\nLegalizing: " << MI);
125
126 MIRBuilder.setInstrAndDebugLoc(MI);
127
128 if (isa<GIntrinsic>(Val: MI))
129 return LI.legalizeIntrinsic(Helper&: *this, MI) ? Legalized : UnableToLegalize;
130 auto Step = LI.getAction(MI, MRI);
131 switch (Step.Action) {
132 case Legal:
133 LLVM_DEBUG(dbgs() << ".. Already legal\n");
134 return AlreadyLegal;
135 case Libcall:
136 LLVM_DEBUG(dbgs() << ".. Convert to libcall\n");
137 return libcall(MI, LocObserver);
138 case NarrowScalar:
139 LLVM_DEBUG(dbgs() << ".. Narrow scalar\n");
140 return narrowScalar(MI, TypeIdx: Step.TypeIdx, NarrowTy: Step.NewType);
141 case WidenScalar:
142 LLVM_DEBUG(dbgs() << ".. Widen scalar\n");
143 return widenScalar(MI, TypeIdx: Step.TypeIdx, WideTy: Step.NewType);
144 case Bitcast:
145 LLVM_DEBUG(dbgs() << ".. Bitcast type\n");
146 return bitcast(MI, TypeIdx: Step.TypeIdx, Ty: Step.NewType);
147 case Lower:
148 LLVM_DEBUG(dbgs() << ".. Lower\n");
149 return lower(MI, TypeIdx: Step.TypeIdx, Ty: Step.NewType);
150 case FewerElements:
151 LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n");
152 return fewerElementsVector(MI, TypeIdx: Step.TypeIdx, NarrowTy: Step.NewType);
153 case MoreElements:
154 LLVM_DEBUG(dbgs() << ".. Increase number of elements\n");
155 return moreElementsVector(MI, TypeIdx: Step.TypeIdx, MoreTy: Step.NewType);
156 case Custom:
157 LLVM_DEBUG(dbgs() << ".. Custom legalization\n");
158 return LI.legalizeCustom(Helper&: *this, MI, LocObserver) ? Legalized
159 : UnableToLegalize;
160 default:
161 LLVM_DEBUG(dbgs() << ".. Unable to legalize\n");
162 return UnableToLegalize;
163 }
164}
165
166void LegalizerHelper::insertParts(Register DstReg,
167 LLT ResultTy, LLT PartTy,
168 ArrayRef<Register> PartRegs,
169 LLT LeftoverTy,
170 ArrayRef<Register> LeftoverRegs) {
171 if (!LeftoverTy.isValid()) {
172 assert(LeftoverRegs.empty());
173
174 if (!ResultTy.isVector()) {
175 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: PartRegs);
176 return;
177 }
178
179 if (PartTy.isVector())
180 MIRBuilder.buildConcatVectors(Res: DstReg, Ops: PartRegs);
181 else
182 MIRBuilder.buildBuildVector(Res: DstReg, Ops: PartRegs);
183 return;
184 }
185
186 // Merge sub-vectors with different number of elements and insert into DstReg.
187 if (ResultTy.isVector()) {
188 assert(LeftoverRegs.size() == 1 && "Expected one leftover register");
189 SmallVector<Register, 8> AllRegs(PartRegs);
190 AllRegs.append(in_start: LeftoverRegs.begin(), in_end: LeftoverRegs.end());
191 return mergeMixedSubvectors(DstReg, PartRegs: AllRegs);
192 }
193
194 SmallVector<Register> GCDRegs;
195 LLT GCDTy = getGCDType(OrigTy: getGCDType(OrigTy: ResultTy, TargetTy: LeftoverTy), TargetTy: PartTy);
196 for (auto PartReg : concat<const Register>(Ranges&: PartRegs, Ranges&: LeftoverRegs))
197 extractGCDType(Parts&: GCDRegs, GCDTy, SrcReg: PartReg);
198 LLT ResultLCMTy = buildLCMMergePieces(DstTy: ResultTy, NarrowTy: LeftoverTy, GCDTy, VRegs&: GCDRegs);
199 buildWidenedRemergeToDst(DstReg, LCMTy: ResultLCMTy, RemergeRegs: GCDRegs);
200}
201
202void LegalizerHelper::appendVectorElts(SmallVectorImpl<Register> &Elts,
203 Register Reg) {
204 LLT Ty = MRI.getType(Reg);
205 SmallVector<Register, 8> RegElts;
206 extractParts(Reg, Ty: Ty.getScalarType(), NumParts: Ty.getNumElements(), VRegs&: RegElts,
207 MIRBuilder, MRI);
208 Elts.append(RHS: RegElts);
209}
210
211/// Merge \p PartRegs with different types into \p DstReg.
212void LegalizerHelper::mergeMixedSubvectors(Register DstReg,
213 ArrayRef<Register> PartRegs) {
214 SmallVector<Register, 8> AllElts;
215 for (unsigned i = 0; i < PartRegs.size() - 1; ++i)
216 appendVectorElts(Elts&: AllElts, Reg: PartRegs[i]);
217
218 Register Leftover = PartRegs[PartRegs.size() - 1];
219 if (!MRI.getType(Reg: Leftover).isVector())
220 AllElts.push_back(Elt: Leftover);
221 else
222 appendVectorElts(Elts&: AllElts, Reg: Leftover);
223
224 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: AllElts);
225}
226
227/// Append the result registers of G_UNMERGE_VALUES \p MI to \p Regs.
228static void getUnmergeResults(SmallVectorImpl<Register> &Regs,
229 const MachineInstr &MI) {
230 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
231
232 const int StartIdx = Regs.size();
233 const int NumResults = MI.getNumOperands() - 1;
234 Regs.resize(N: Regs.size() + NumResults);
235 for (int I = 0; I != NumResults; ++I)
236 Regs[StartIdx + I] = MI.getOperand(i: I).getReg();
237}
238
239void LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts,
240 LLT GCDTy, Register SrcReg) {
241 LLT SrcTy = MRI.getType(Reg: SrcReg);
242 if (SrcTy == GCDTy) {
243 // If the source already evenly divides the result type, we don't need to do
244 // anything.
245 Parts.push_back(Elt: SrcReg);
246 } else {
247 // Need to split into common type sized pieces.
248 auto Unmerge = MIRBuilder.buildUnmerge(Res: GCDTy, Op: SrcReg);
249 getUnmergeResults(Regs&: Parts, MI: *Unmerge);
250 }
251}
252
253LLT LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts, LLT DstTy,
254 LLT NarrowTy, Register SrcReg) {
255 LLT SrcTy = MRI.getType(Reg: SrcReg);
256 LLT GCDTy = getGCDType(OrigTy: getGCDType(OrigTy: SrcTy, TargetTy: NarrowTy), TargetTy: DstTy);
257 extractGCDType(Parts, GCDTy, SrcReg);
258 return GCDTy;
259}
260
261LLT LegalizerHelper::buildLCMMergePieces(LLT DstTy, LLT NarrowTy, LLT GCDTy,
262 SmallVectorImpl<Register> &VRegs,
263 unsigned PadStrategy) {
264 LLT LCMTy = getLCMType(OrigTy: DstTy, TargetTy: NarrowTy);
265
266 int NumParts = LCMTy.getSizeInBits() / NarrowTy.getSizeInBits();
267 int NumSubParts = NarrowTy.getSizeInBits() / GCDTy.getSizeInBits();
268 int NumOrigSrc = VRegs.size();
269
270 Register PadReg;
271
272 // Get a value we can use to pad the source value if the sources won't evenly
273 // cover the result type.
274 if (NumOrigSrc < NumParts * NumSubParts) {
275 if (PadStrategy == TargetOpcode::G_ZEXT)
276 PadReg = MIRBuilder.buildConstant(Res: GCDTy, Val: 0).getReg(Idx: 0);
277 else if (PadStrategy == TargetOpcode::G_ANYEXT)
278 PadReg = MIRBuilder.buildUndef(Res: GCDTy).getReg(Idx: 0);
279 else {
280 assert(PadStrategy == TargetOpcode::G_SEXT);
281
282 // Shift the sign bit of the low register through the high register.
283 auto ShiftAmt =
284 MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: GCDTy.getSizeInBits() - 1);
285 PadReg = MIRBuilder.buildAShr(Dst: GCDTy, Src0: VRegs.back(), Src1: ShiftAmt).getReg(Idx: 0);
286 }
287 }
288
289 // Registers for the final merge to be produced.
290 SmallVector<Register, 4> Remerge(NumParts);
291
292 // Registers needed for intermediate merges, which will be merged into a
293 // source for Remerge.
294 SmallVector<Register, 4> SubMerge(NumSubParts);
295
296 // Once we've fully read off the end of the original source bits, we can reuse
297 // the same high bits for remaining padding elements.
298 Register AllPadReg;
299
300 // Build merges to the LCM type to cover the original result type.
301 for (int I = 0; I != NumParts; ++I) {
302 bool AllMergePartsArePadding = true;
303
304 // Build the requested merges to the requested type.
305 for (int J = 0; J != NumSubParts; ++J) {
306 int Idx = I * NumSubParts + J;
307 if (Idx >= NumOrigSrc) {
308 SubMerge[J] = PadReg;
309 continue;
310 }
311
312 SubMerge[J] = VRegs[Idx];
313
314 // There are meaningful bits here we can't reuse later.
315 AllMergePartsArePadding = false;
316 }
317
318 // If we've filled up a complete piece with padding bits, we can directly
319 // emit the natural sized constant if applicable, rather than a merge of
320 // smaller constants.
321 if (AllMergePartsArePadding && !AllPadReg) {
322 if (PadStrategy == TargetOpcode::G_ANYEXT)
323 AllPadReg = MIRBuilder.buildUndef(Res: NarrowTy).getReg(Idx: 0);
324 else if (PadStrategy == TargetOpcode::G_ZEXT)
325 AllPadReg = MIRBuilder.buildConstant(Res: NarrowTy, Val: 0).getReg(Idx: 0);
326
327 // If this is a sign extension, we can't materialize a trivial constant
328 // with the right type and have to produce a merge.
329 }
330
331 if (AllPadReg) {
332 // Avoid creating additional instructions if we're just adding additional
333 // copies of padding bits.
334 Remerge[I] = AllPadReg;
335 continue;
336 }
337
338 if (NumSubParts == 1)
339 Remerge[I] = SubMerge[0];
340 else
341 Remerge[I] = MIRBuilder.buildMergeLikeInstr(Res: NarrowTy, Ops: SubMerge).getReg(Idx: 0);
342
343 // In the sign extend padding case, re-use the first all-signbit merge.
344 if (AllMergePartsArePadding && !AllPadReg)
345 AllPadReg = Remerge[I];
346 }
347
348 VRegs = std::move(Remerge);
349 return LCMTy;
350}
351
352void LegalizerHelper::buildWidenedRemergeToDst(Register DstReg, LLT LCMTy,
353 ArrayRef<Register> RemergeRegs) {
354 LLT DstTy = MRI.getType(Reg: DstReg);
355
356 // Create the merge to the widened source, and extract the relevant bits into
357 // the result.
358
359 if (DstTy == LCMTy) {
360 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: RemergeRegs);
361 return;
362 }
363
364 auto Remerge = MIRBuilder.buildMergeLikeInstr(Res: LCMTy, Ops: RemergeRegs);
365 if (DstTy.isScalar() && LCMTy.isScalar()) {
366 MIRBuilder.buildTrunc(Res: DstReg, Op: Remerge);
367 return;
368 }
369
370 if (LCMTy.isVector()) {
371 unsigned NumDefs = LCMTy.getSizeInBits() / DstTy.getSizeInBits();
372 SmallVector<Register, 8> UnmergeDefs(NumDefs);
373 UnmergeDefs[0] = DstReg;
374 for (unsigned I = 1; I != NumDefs; ++I)
375 UnmergeDefs[I] = MRI.createGenericVirtualRegister(Ty: DstTy);
376
377 MIRBuilder.buildUnmerge(Res: UnmergeDefs,
378 Op: MIRBuilder.buildMergeLikeInstr(Res: LCMTy, Ops: RemergeRegs));
379 return;
380 }
381
382 llvm_unreachable("unhandled case");
383}
384
385static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
386#define RTLIBCASE_INT(LibcallPrefix) \
387 do { \
388 switch (Size) { \
389 case 32: \
390 return RTLIB::LibcallPrefix##32; \
391 case 64: \
392 return RTLIB::LibcallPrefix##64; \
393 case 128: \
394 return RTLIB::LibcallPrefix##128; \
395 default: \
396 llvm_unreachable("unexpected size"); \
397 } \
398 } while (0)
399
400#define RTLIBCASE(LibcallPrefix) \
401 do { \
402 switch (Size) { \
403 case 32: \
404 return RTLIB::LibcallPrefix##32; \
405 case 64: \
406 return RTLIB::LibcallPrefix##64; \
407 case 80: \
408 return RTLIB::LibcallPrefix##80; \
409 case 128: \
410 return RTLIB::LibcallPrefix##128; \
411 default: \
412 llvm_unreachable("unexpected size"); \
413 } \
414 } while (0)
415
416 switch (Opcode) {
417 case TargetOpcode::G_LROUND:
418 RTLIBCASE(LROUND_F);
419 case TargetOpcode::G_LLROUND:
420 RTLIBCASE(LLROUND_F);
421 case TargetOpcode::G_MUL:
422 RTLIBCASE_INT(MUL_I);
423 case TargetOpcode::G_SDIV:
424 RTLIBCASE_INT(SDIV_I);
425 case TargetOpcode::G_UDIV:
426 RTLIBCASE_INT(UDIV_I);
427 case TargetOpcode::G_SREM:
428 RTLIBCASE_INT(SREM_I);
429 case TargetOpcode::G_UREM:
430 RTLIBCASE_INT(UREM_I);
431 case TargetOpcode::G_CTLZ_ZERO_POISON:
432 RTLIBCASE_INT(CTLZ_I);
433 case TargetOpcode::G_FADD:
434 RTLIBCASE(ADD_F);
435 case TargetOpcode::G_FSUB:
436 RTLIBCASE(SUB_F);
437 case TargetOpcode::G_FMUL:
438 RTLIBCASE(MUL_F);
439 case TargetOpcode::G_FDIV:
440 RTLIBCASE(DIV_F);
441 case TargetOpcode::G_FEXP:
442 RTLIBCASE(EXP_F);
443 case TargetOpcode::G_FEXP2:
444 RTLIBCASE(EXP2_F);
445 case TargetOpcode::G_FEXP10:
446 RTLIBCASE(EXP10_F);
447 case TargetOpcode::G_FREM:
448 RTLIBCASE(REM_F);
449 case TargetOpcode::G_FPOW:
450 RTLIBCASE(POW_F);
451 case TargetOpcode::G_FPOWI:
452 RTLIBCASE(POWI_F);
453 case TargetOpcode::G_FMA:
454 RTLIBCASE(FMA_F);
455 case TargetOpcode::G_FSIN:
456 RTLIBCASE(SIN_F);
457 case TargetOpcode::G_FCOS:
458 RTLIBCASE(COS_F);
459 case TargetOpcode::G_FTAN:
460 RTLIBCASE(TAN_F);
461 case TargetOpcode::G_FASIN:
462 RTLIBCASE(ASIN_F);
463 case TargetOpcode::G_FACOS:
464 RTLIBCASE(ACOS_F);
465 case TargetOpcode::G_FATAN:
466 RTLIBCASE(ATAN_F);
467 case TargetOpcode::G_FATAN2:
468 RTLIBCASE(ATAN2_F);
469 case TargetOpcode::G_FSINH:
470 RTLIBCASE(SINH_F);
471 case TargetOpcode::G_FCOSH:
472 RTLIBCASE(COSH_F);
473 case TargetOpcode::G_FTANH:
474 RTLIBCASE(TANH_F);
475 case TargetOpcode::G_FSINCOS:
476 RTLIBCASE(SINCOS_F);
477 case TargetOpcode::G_FMODF:
478 RTLIBCASE(MODF_F);
479 case TargetOpcode::G_FLOG10:
480 RTLIBCASE(LOG10_F);
481 case TargetOpcode::G_FLOG:
482 RTLIBCASE(LOG_F);
483 case TargetOpcode::G_FLOG2:
484 RTLIBCASE(LOG2_F);
485 case TargetOpcode::G_FLDEXP:
486 RTLIBCASE(LDEXP_F);
487 case TargetOpcode::G_FCEIL:
488 RTLIBCASE(CEIL_F);
489 case TargetOpcode::G_FFLOOR:
490 RTLIBCASE(FLOOR_F);
491 case TargetOpcode::G_FMINNUM:
492 RTLIBCASE(FMIN_F);
493 case TargetOpcode::G_FMAXNUM:
494 RTLIBCASE(FMAX_F);
495 case TargetOpcode::G_FMINIMUMNUM:
496 RTLIBCASE(FMINIMUM_NUM_F);
497 case TargetOpcode::G_FMAXIMUMNUM:
498 RTLIBCASE(FMAXIMUM_NUM_F);
499 case TargetOpcode::G_FSQRT:
500 RTLIBCASE(SQRT_F);
501 case TargetOpcode::G_FRINT:
502 RTLIBCASE(RINT_F);
503 case TargetOpcode::G_FNEARBYINT:
504 RTLIBCASE(NEARBYINT_F);
505 case TargetOpcode::G_INTRINSIC_TRUNC:
506 RTLIBCASE(TRUNC_F);
507 case TargetOpcode::G_INTRINSIC_ROUND:
508 RTLIBCASE(ROUND_F);
509 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
510 RTLIBCASE(ROUNDEVEN_F);
511 case TargetOpcode::G_INTRINSIC_LRINT:
512 RTLIBCASE(LRINT_F);
513 case TargetOpcode::G_INTRINSIC_LLRINT:
514 RTLIBCASE(LLRINT_F);
515 }
516 llvm_unreachable("Unknown libcall function");
517#undef RTLIBCASE_INT
518#undef RTLIBCASE
519}
520
521static bool hasSwiftErrorArg(MachineFunction &MF) {
522 const TargetLowering &TLI = *MF.getSubtarget().getTargetLowering();
523 return TLI.supportSwiftError() &&
524 MF.getFunction().getAttributes().hasAttrSomewhere(
525 Kind: Attribute::SwiftError);
526}
527
528/// True if an instruction is in tail position in its caller. Intended for
529/// legalizing libcalls as tail calls when possible.
530static bool isLibCallInTailPosition(const CallLowering::ArgInfo &Result,
531 MachineInstr &MI,
532 const TargetInstrInfo &TII,
533 MachineRegisterInfo &MRI) {
534 MachineBasicBlock &MBB = *MI.getParent();
535 const Function &F = MBB.getParent()->getFunction();
536
537 // Conservatively require the attributes of the call to match those of
538 // the return. Ignore NoAlias and NonNull because they don't affect the
539 // call sequence.
540 AttributeList CallerAttrs = F.getAttributes();
541 if (AttrBuilder(F.getContext(), CallerAttrs.getRetAttrs())
542 .removeAttribute(Val: Attribute::NoAlias)
543 .removeAttribute(Val: Attribute::NonNull)
544 .hasAttributes())
545 return false;
546
547 // It's not safe to eliminate the sign / zero extension of the return value.
548 if (CallerAttrs.hasRetAttr(Kind: Attribute::ZExt) ||
549 CallerAttrs.hasRetAttr(Kind: Attribute::SExt))
550 return false;
551
552 // Check if tail calls have been disabled in this function.
553 if (F.getFnAttribute(Kind: "disable-tail-calls").getValueAsBool())
554 return false;
555
556 // Only tail call if the following instruction is a standard return or if we
557 // have a `thisreturn` callee, and a sequence like:
558 //
559 // G_MEMCPY %0, %1, %2
560 // $x0 = COPY %0
561 // RET_ReallyLR implicit $x0
562 auto Next = next_nodbg(It: MI.getIterator(), End: MBB.instr_end());
563 if (Next != MBB.instr_end() && Next->isCopy()) {
564 if (MI.getOpcode() == TargetOpcode::G_BZERO)
565 return false;
566
567 // For MEMCPY/MOMMOVE/MEMSET these will be the first use (the dst), as the
568 // mempy/etc routines return the same parameter. For other it will be the
569 // returned value.
570 Register VReg = MI.getOperand(i: 0).getReg();
571 if (!VReg.isVirtual() || VReg != Next->getOperand(i: 1).getReg())
572 return false;
573
574 Register PReg = Next->getOperand(i: 0).getReg();
575 if (!PReg.isPhysical())
576 return false;
577
578 auto Ret = next_nodbg(It: Next, End: MBB.instr_end());
579 if (Ret == MBB.instr_end() || !Ret->isReturn())
580 return false;
581
582 if (Ret->getNumImplicitOperands() != 1)
583 return false;
584
585 if (!Ret->getOperand(i: 0).isReg() || PReg != Ret->getOperand(i: 0).getReg())
586 return false;
587
588 // Skip over the COPY that we just validated.
589 Next = Ret;
590 }
591
592 if (Next == MBB.instr_end() || TII.isTailCall(Inst: *Next) || !Next->isReturn())
593 return false;
594
595 return true;
596}
597
598LegalizerHelper::LegalizeResult LegalizerHelper::createLibcall(
599 const char *Name, const CallLowering::ArgInfo &Result,
600 ArrayRef<CallLowering::ArgInfo> Args, const CallingConv::ID CC,
601 LostDebugLocObserver &LocObserver, MachineInstr *MI) const {
602 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
603
604 CallLowering::CallLoweringInfo Info;
605 Info.CallConv = CC;
606 Info.Callee = MachineOperand::CreateES(SymName: Name);
607 Info.OrigRet = Result;
608 if (MI)
609 Info.IsTailCall =
610 (Result.Ty->isVoidTy() ||
611 Result.Ty == MIRBuilder.getMF().getFunction().getReturnType()) &&
612 isLibCallInTailPosition(Result, MI&: *MI, TII: MIRBuilder.getTII(),
613 MRI&: *MIRBuilder.getMRI()) &&
614 // Lowering doesn't support tail calling inside a function with
615 // a swifterror argument yet.
616 !hasSwiftErrorArg(MF&: MIRBuilder.getMF());
617
618 llvm::append_range(C&: Info.OrigArgs, R&: Args);
619 if (!CLI.lowerCall(MIRBuilder, Info))
620 return LegalizerHelper::UnableToLegalize;
621
622 if (MI && Info.LoweredTailCall) {
623 assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?");
624
625 // Check debug locations before removing the return.
626 LocObserver.checkpoint(CheckDebugLocs: true);
627
628 // We must have a return following the call (or debug insts) to get past
629 // isLibCallInTailPosition.
630 do {
631 MachineInstr *Next = MI->getNextNode();
632 assert(Next &&
633 (Next->isCopy() || Next->isReturn() || Next->isDebugInstr()) &&
634 "Expected instr following MI to be return or debug inst?");
635 // We lowered a tail call, so the call is now the return from the block.
636 // Delete the old return.
637 Next->eraseFromParent();
638 } while (MI->getNextNode());
639
640 // We expect to lose the debug location from the return.
641 LocObserver.checkpoint(CheckDebugLocs: false);
642 }
643 return LegalizerHelper::Legalized;
644}
645
646LegalizerHelper::LegalizeResult LegalizerHelper::createLibcall(
647 RTLIB::Libcall Libcall, const CallLowering::ArgInfo &Result,
648 ArrayRef<CallLowering::ArgInfo> Args, LostDebugLocObserver &LocObserver,
649 MachineInstr *MI) const {
650 if (!Libcalls)
651 return LegalizerHelper::UnableToLegalize;
652
653 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: Libcall);
654 if (LibcallImpl == RTLIB::Unsupported)
655 return LegalizerHelper::UnableToLegalize;
656
657 StringRef Name = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: LibcallImpl);
658 const CallingConv::ID CC = Libcalls->getLibcallImplCallingConv(Call: LibcallImpl);
659 return createLibcall(Name: Name.data(), Result, Args, CC, LocObserver, MI);
660}
661
662// Useful for libcalls where all operands have the same type.
663LegalizerHelper::LegalizeResult
664LegalizerHelper::simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder,
665 unsigned Size, Type *OpType,
666 LostDebugLocObserver &LocObserver) const {
667 auto Libcall = getRTLibDesc(Opcode: MI.getOpcode(), Size);
668
669 // FIXME: What does the original arg index mean here?
670 SmallVector<CallLowering::ArgInfo, 3> Args;
671 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands()))
672 Args.push_back(Elt: {MO.getReg(), OpType, 0});
673 return createLibcall(Libcall, Result: {MI.getOperand(i: 0).getReg(), OpType, 0}, Args,
674 LocObserver, MI: &MI);
675}
676
677LegalizerHelper::LegalizeResult LegalizerHelper::emitSincosLibcall(
678 MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size, Type *OpType,
679 LostDebugLocObserver &LocObserver) {
680 MachineFunction &MF = *MI.getMF();
681 MachineRegisterInfo &MRI = MF.getRegInfo();
682
683 Register DstSin = MI.getOperand(i: 0).getReg();
684 Register DstCos = MI.getOperand(i: 1).getReg();
685 Register Src = MI.getOperand(i: 2).getReg();
686 LLT DstTy = MRI.getType(Reg: DstSin);
687
688 int MemSize = DstTy.getSizeInBytes();
689 Align Alignment = getStackTemporaryAlignment(Type: DstTy);
690 const DataLayout &DL = MIRBuilder.getDataLayout();
691 unsigned AddrSpace = DL.getAllocaAddrSpace();
692 MachinePointerInfo PtrInfo;
693
694 Register StackPtrSin =
695 createStackTemporary(Bytes: TypeSize::getFixed(ExactSize: MemSize), Alignment, PtrInfo)
696 .getReg(Idx: 0);
697 Register StackPtrCos =
698 createStackTemporary(Bytes: TypeSize::getFixed(ExactSize: MemSize), Alignment, PtrInfo)
699 .getReg(Idx: 0);
700
701 auto &Ctx = MF.getFunction().getContext();
702 auto LibcallResult = createLibcall(
703 Libcall: getRTLibDesc(Opcode: MI.getOpcode(), Size), Result: {{0}, Type::getVoidTy(C&: Ctx), 0},
704 Args: {{Src, OpType, 0},
705 {StackPtrSin, PointerType::get(C&: Ctx, AddressSpace: AddrSpace), 1},
706 {StackPtrCos, PointerType::get(C&: Ctx, AddressSpace: AddrSpace), 2}},
707 LocObserver, MI: &MI);
708
709 if (LibcallResult != LegalizeResult::Legalized)
710 return LegalizerHelper::UnableToLegalize;
711
712 MachineMemOperand *LoadMMOSin = MF.getMachineMemOperand(
713 PtrInfo, F: MachineMemOperand::MOLoad, Size: MemSize, BaseAlignment: Alignment);
714 MachineMemOperand *LoadMMOCos = MF.getMachineMemOperand(
715 PtrInfo, F: MachineMemOperand::MOLoad, Size: MemSize, BaseAlignment: Alignment);
716
717 MIRBuilder.buildLoad(Res: DstSin, Addr: StackPtrSin, MMO&: *LoadMMOSin);
718 MIRBuilder.buildLoad(Res: DstCos, Addr: StackPtrCos, MMO&: *LoadMMOCos);
719 MI.eraseFromParent();
720
721 return LegalizerHelper::Legalized;
722}
723
724LegalizerHelper::LegalizeResult
725LegalizerHelper::emitModfLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder,
726 unsigned Size, Type *OpType,
727 LostDebugLocObserver &LocObserver) {
728 MachineFunction &MF = MIRBuilder.getMF();
729 MachineRegisterInfo &MRI = MF.getRegInfo();
730
731 Register DstFrac = MI.getOperand(i: 0).getReg();
732 Register DstInt = MI.getOperand(i: 1).getReg();
733 Register Src = MI.getOperand(i: 2).getReg();
734 LLT DstTy = MRI.getType(Reg: DstFrac);
735
736 int MemSize = DstTy.getSizeInBytes();
737 Align Alignment = getStackTemporaryAlignment(Type: DstTy);
738 const DataLayout &DL = MIRBuilder.getDataLayout();
739 unsigned AddrSpace = DL.getAllocaAddrSpace();
740 MachinePointerInfo PtrInfo;
741
742 Register StackPtrInt =
743 createStackTemporary(Bytes: TypeSize::getFixed(ExactSize: MemSize), Alignment, PtrInfo)
744 .getReg(Idx: 0);
745
746 auto &Ctx = MF.getFunction().getContext();
747 auto LibcallResult = createLibcall(
748 Libcall: getRTLibDesc(Opcode: MI.getOpcode(), Size), Result: {DstFrac, OpType, 0},
749 Args: {{Src, OpType, 0}, {StackPtrInt, PointerType::get(C&: Ctx, AddressSpace: AddrSpace), 1}},
750 LocObserver, MI: &MI);
751
752 if (LibcallResult != LegalizeResult::Legalized)
753 return LegalizerHelper::UnableToLegalize;
754
755 MachineMemOperand *LoadMMOInt = MF.getMachineMemOperand(
756 PtrInfo, F: MachineMemOperand::MOLoad, Size: MemSize, BaseAlignment: Alignment);
757
758 MIRBuilder.buildLoad(Res: DstInt, Addr: StackPtrInt, MMO&: *LoadMMOInt);
759 MI.eraseFromParent();
760
761 return LegalizerHelper::Legalized;
762}
763
764static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
765 Type *FromType) {
766 auto ToMVT = MVT::getVT(Ty: ToType);
767 auto FromMVT = MVT::getVT(Ty: FromType);
768
769 switch (Opcode) {
770 case TargetOpcode::G_FPEXT:
771 return RTLIB::getFPEXT(OpVT: FromMVT, RetVT: ToMVT);
772 case TargetOpcode::G_FPTRUNC:
773 return RTLIB::getFPROUND(OpVT: FromMVT, RetVT: ToMVT);
774 case TargetOpcode::G_FPTOSI:
775 return RTLIB::getFPTOSINT(OpVT: FromMVT, RetVT: ToMVT);
776 case TargetOpcode::G_FPTOUI:
777 return RTLIB::getFPTOUINT(OpVT: FromMVT, RetVT: ToMVT);
778 case TargetOpcode::G_SITOFP:
779 return RTLIB::getSINTTOFP(OpVT: FromMVT, RetVT: ToMVT);
780 case TargetOpcode::G_UITOFP:
781 return RTLIB::getUINTTOFP(OpVT: FromMVT, RetVT: ToMVT);
782 }
783 llvm_unreachable("Unsupported libcall function");
784}
785
786LegalizerHelper::LegalizeResult LegalizerHelper::conversionLibcall(
787 MachineInstr &MI, Type *ToType, Type *FromType,
788 LostDebugLocObserver &LocObserver, bool IsSigned) const {
789 CallLowering::ArgInfo Arg = {MI.getOperand(i: 1).getReg(), FromType, 0};
790 if (FromType->isIntegerTy()) {
791 if (TLI.shouldSignExtendTypeInLibCall(Ty: FromType, IsSigned))
792 Arg.Flags[0].setSExt();
793 else
794 Arg.Flags[0].setZExt();
795 }
796
797 RTLIB::Libcall Libcall = getConvRTLibDesc(Opcode: MI.getOpcode(), ToType, FromType);
798 return createLibcall(Libcall, Result: {MI.getOperand(i: 0).getReg(), ToType, 0}, Args: Arg,
799 LocObserver, MI: &MI);
800}
801
802LegalizerHelper::LegalizeResult
803LegalizerHelper::createMemLibcall(MachineRegisterInfo &MRI, MachineInstr &MI,
804 LostDebugLocObserver &LocObserver) const {
805 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
806
807 SmallVector<CallLowering::ArgInfo, 3> Args;
808 // Add all the args, except for the last which is an imm denoting 'tail'.
809 for (unsigned i = 0; i < MI.getNumOperands() - 1; ++i) {
810 Register Reg = MI.getOperand(i).getReg();
811
812 // Need derive an IR type for call lowering.
813 LLT OpLLT = MRI.getType(Reg);
814 Type *OpTy = nullptr;
815 if (OpLLT.isPointer())
816 OpTy = PointerType::get(C&: Ctx, AddressSpace: OpLLT.getAddressSpace());
817 else
818 OpTy = IntegerType::get(C&: Ctx, NumBits: OpLLT.getSizeInBits());
819 Args.push_back(Elt: {Reg, OpTy, 0});
820 }
821
822 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
823 RTLIB::Libcall RTLibcall;
824 unsigned Opc = MI.getOpcode();
825 switch (Opc) {
826 case TargetOpcode::G_BZERO:
827 RTLibcall = RTLIB::BZERO;
828 break;
829 case TargetOpcode::G_MEMCPY:
830 RTLibcall = RTLIB::MEMCPY;
831 Args[0].Flags[0].setReturned();
832 break;
833 case TargetOpcode::G_MEMMOVE:
834 RTLibcall = RTLIB::MEMMOVE;
835 Args[0].Flags[0].setReturned();
836 break;
837 case TargetOpcode::G_MEMSET:
838 RTLibcall = RTLIB::MEMSET;
839 Args[0].Flags[0].setReturned();
840 break;
841 default:
842 llvm_unreachable("unsupported opcode");
843 }
844
845 if (!Libcalls) // FIXME: Should be mandatory
846 return LegalizerHelper::UnableToLegalize;
847
848 RTLIB::LibcallImpl RTLibcallImpl = Libcalls->getLibcallImpl(Call: RTLibcall);
849
850 // Unsupported libcall on the target.
851 if (RTLibcallImpl == RTLIB::Unsupported) {
852 LLVM_DEBUG(dbgs() << ".. .. Could not find libcall name for "
853 << MIRBuilder.getTII().getName(Opc) << "\n");
854 return LegalizerHelper::UnableToLegalize;
855 }
856
857 CallLowering::CallLoweringInfo Info;
858 Info.CallConv = Libcalls->getLibcallImplCallingConv(Call: RTLibcallImpl);
859
860 StringRef LibcallName =
861 RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: RTLibcallImpl);
862 Info.Callee = MachineOperand::CreateES(SymName: LibcallName.data());
863 Info.OrigRet = CallLowering::ArgInfo({0}, Type::getVoidTy(C&: Ctx), 0);
864 Info.IsTailCall =
865 MI.getOperand(i: MI.getNumOperands() - 1).getImm() &&
866 isLibCallInTailPosition(Result: Info.OrigRet, MI, TII: MIRBuilder.getTII(), MRI) &&
867 // Lowering doesn't support tail calling inside a function with
868 // a swifterror argument yet.
869 !hasSwiftErrorArg(MF&: MIRBuilder.getMF());
870
871 llvm::append_range(C&: Info.OrigArgs, R&: Args);
872 if (!CLI.lowerCall(MIRBuilder, Info))
873 return LegalizerHelper::UnableToLegalize;
874
875 if (Info.LoweredTailCall) {
876 assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?");
877
878 // Check debug locations before removing the return.
879 LocObserver.checkpoint(CheckDebugLocs: true);
880
881 // We must have a return following the call (or debug insts) to get past
882 // isLibCallInTailPosition.
883 do {
884 MachineInstr *Next = MI.getNextNode();
885 assert(Next &&
886 (Next->isCopy() || Next->isReturn() || Next->isDebugInstr()) &&
887 "Expected instr following MI to be return or debug inst?");
888 // We lowered a tail call, so the call is now the return from the block.
889 // Delete the old return.
890 Next->eraseFromParent();
891 } while (MI.getNextNode());
892
893 // We expect to lose the debug location from the return.
894 LocObserver.checkpoint(CheckDebugLocs: false);
895 }
896
897 return LegalizerHelper::Legalized;
898}
899
900static RTLIB::Libcall getOutlineAtomicLibcall(MachineInstr &MI) {
901 unsigned Opc = MI.getOpcode();
902 auto &AtomicMI = cast<GMemOperation>(Val&: MI);
903 auto &MMO = AtomicMI.getMMO();
904 auto Ordering = MMO.getMergedOrdering();
905 LLT MemType = MMO.getMemoryType();
906 uint64_t MemSize = MemType.getSizeInBytes();
907 if (MemType.isVector())
908 return RTLIB::UNKNOWN_LIBCALL;
909
910#define LCALLS(A, B) {A##B##_RELAX, A##B##_ACQ, A##B##_REL, A##B##_ACQ_REL}
911#define LCALL5(A) \
912 LCALLS(A, 1), LCALLS(A, 2), LCALLS(A, 4), LCALLS(A, 8), LCALLS(A, 16)
913 switch (Opc) {
914 case TargetOpcode::G_ATOMIC_CMPXCHG:
915 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
916 const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_CAS)};
917 return getOutlineAtomicHelper(LC, Order: Ordering, MemSize);
918 }
919 case TargetOpcode::G_ATOMICRMW_XCHG: {
920 const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_SWP)};
921 return getOutlineAtomicHelper(LC, Order: Ordering, MemSize);
922 }
923 case TargetOpcode::G_ATOMICRMW_ADD:
924 case TargetOpcode::G_ATOMICRMW_SUB: {
925 const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDADD)};
926 return getOutlineAtomicHelper(LC, Order: Ordering, MemSize);
927 }
928 case TargetOpcode::G_ATOMICRMW_AND: {
929 const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDCLR)};
930 return getOutlineAtomicHelper(LC, Order: Ordering, MemSize);
931 }
932 case TargetOpcode::G_ATOMICRMW_OR: {
933 const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDSET)};
934 return getOutlineAtomicHelper(LC, Order: Ordering, MemSize);
935 }
936 case TargetOpcode::G_ATOMICRMW_XOR: {
937 const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDEOR)};
938 return getOutlineAtomicHelper(LC, Order: Ordering, MemSize);
939 }
940 default:
941 return RTLIB::UNKNOWN_LIBCALL;
942 }
943#undef LCALLS
944#undef LCALL5
945}
946
947LegalizerHelper::LegalizeResult
948LegalizerHelper::createAtomicLibcall(MachineInstr &MI) const {
949 auto &Ctx = MIRBuilder.getContext();
950
951 Type *RetTy;
952 SmallVector<Register> RetRegs;
953 SmallVector<CallLowering::ArgInfo, 3> Args;
954 unsigned Opc = MI.getOpcode();
955 switch (Opc) {
956 case TargetOpcode::G_ATOMIC_CMPXCHG:
957 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
958 Register Success;
959 LLT SuccessLLT;
960 auto [Ret, RetLLT, Mem, MemLLT, Cmp, CmpLLT, New, NewLLT] =
961 MI.getFirst4RegLLTs();
962 RetRegs.push_back(Elt: Ret);
963 RetTy = IntegerType::get(C&: Ctx, NumBits: RetLLT.getSizeInBits());
964 if (Opc == TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS) {
965 std::tie(args&: Ret, args&: RetLLT, args&: Success, args&: SuccessLLT, args&: Mem, args&: MemLLT, args&: Cmp, args&: CmpLLT, args&: New,
966 args&: NewLLT) = MI.getFirst5RegLLTs();
967 RetRegs.push_back(Elt: Success);
968 RetTy = StructType::get(
969 Context&: Ctx, Elements: {RetTy, IntegerType::get(C&: Ctx, NumBits: SuccessLLT.getSizeInBits())});
970 }
971 Args.push_back(Elt: {Cmp, IntegerType::get(C&: Ctx, NumBits: CmpLLT.getSizeInBits()), 0});
972 Args.push_back(Elt: {New, IntegerType::get(C&: Ctx, NumBits: NewLLT.getSizeInBits()), 0});
973 Args.push_back(Elt: {Mem, PointerType::get(C&: Ctx, AddressSpace: MemLLT.getAddressSpace()), 0});
974 break;
975 }
976 case TargetOpcode::G_ATOMICRMW_XCHG:
977 case TargetOpcode::G_ATOMICRMW_ADD:
978 case TargetOpcode::G_ATOMICRMW_SUB:
979 case TargetOpcode::G_ATOMICRMW_AND:
980 case TargetOpcode::G_ATOMICRMW_OR:
981 case TargetOpcode::G_ATOMICRMW_XOR: {
982 auto [Ret, RetLLT, Mem, MemLLT, Val, ValLLT] = MI.getFirst3RegLLTs();
983 RetRegs.push_back(Elt: Ret);
984 RetTy = IntegerType::get(C&: Ctx, NumBits: RetLLT.getSizeInBits());
985 if (Opc == TargetOpcode::G_ATOMICRMW_AND)
986 Val =
987 MIRBuilder.buildXor(Dst: ValLLT, Src0: MIRBuilder.buildConstant(Res: ValLLT, Val: -1), Src1: Val)
988 .getReg(Idx: 0);
989 else if (Opc == TargetOpcode::G_ATOMICRMW_SUB)
990 Val =
991 MIRBuilder.buildSub(Dst: ValLLT, Src0: MIRBuilder.buildConstant(Res: ValLLT, Val: 0), Src1: Val)
992 .getReg(Idx: 0);
993 Args.push_back(Elt: {Val, IntegerType::get(C&: Ctx, NumBits: ValLLT.getSizeInBits()), 0});
994 Args.push_back(Elt: {Mem, PointerType::get(C&: Ctx, AddressSpace: MemLLT.getAddressSpace()), 0});
995 break;
996 }
997 default:
998 llvm_unreachable("unsupported opcode");
999 }
1000
1001 if (!Libcalls) // FIXME: Should be mandatory
1002 return LegalizerHelper::UnableToLegalize;
1003
1004 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
1005 RTLIB::Libcall RTLibcall = getOutlineAtomicLibcall(MI);
1006 RTLIB::LibcallImpl RTLibcallImpl = Libcalls->getLibcallImpl(Call: RTLibcall);
1007
1008 // Unsupported libcall on the target.
1009 if (RTLibcallImpl == RTLIB::Unsupported) {
1010 LLVM_DEBUG(dbgs() << ".. .. Could not find libcall name for "
1011 << MIRBuilder.getTII().getName(Opc) << "\n");
1012 return LegalizerHelper::UnableToLegalize;
1013 }
1014
1015 CallLowering::CallLoweringInfo Info;
1016 Info.CallConv = Libcalls->getLibcallImplCallingConv(Call: RTLibcallImpl);
1017
1018 StringRef LibcallName =
1019 RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: RTLibcallImpl);
1020 Info.Callee = MachineOperand::CreateES(SymName: LibcallName.data());
1021 Info.OrigRet = CallLowering::ArgInfo(RetRegs, RetTy, 0);
1022
1023 llvm::append_range(C&: Info.OrigArgs, R&: Args);
1024 if (!CLI.lowerCall(MIRBuilder, Info))
1025 return LegalizerHelper::UnableToLegalize;
1026
1027 return LegalizerHelper::Legalized;
1028}
1029
1030static RTLIB::Libcall
1031getStateLibraryFunctionFor(MachineInstr &MI, const TargetLowering &TLI) {
1032 RTLIB::Libcall RTLibcall;
1033 switch (MI.getOpcode()) {
1034 case TargetOpcode::G_GET_FPENV:
1035 RTLibcall = RTLIB::FEGETENV;
1036 break;
1037 case TargetOpcode::G_SET_FPENV:
1038 case TargetOpcode::G_RESET_FPENV:
1039 RTLibcall = RTLIB::FESETENV;
1040 break;
1041 case TargetOpcode::G_GET_FPMODE:
1042 RTLibcall = RTLIB::FEGETMODE;
1043 break;
1044 case TargetOpcode::G_SET_FPMODE:
1045 case TargetOpcode::G_RESET_FPMODE:
1046 RTLibcall = RTLIB::FESETMODE;
1047 break;
1048 default:
1049 llvm_unreachable("Unexpected opcode");
1050 }
1051 return RTLibcall;
1052}
1053
1054// Some library functions that read FP state (fegetmode, fegetenv) write the
1055// state into a region in memory. IR intrinsics that do the same operations
1056// (get_fpmode, get_fpenv) return the state as integer value. To implement these
1057// intrinsics via the library functions, we need to use temporary variable,
1058// for example:
1059//
1060// %0:_(s32) = G_GET_FPMODE
1061//
1062// is transformed to:
1063//
1064// %1:_(p0) = G_FRAME_INDEX %stack.0
1065// BL &fegetmode
1066// %0:_(s32) = G_LOAD % 1
1067//
1068LegalizerHelper::LegalizeResult
1069LegalizerHelper::createGetStateLibcall(MachineInstr &MI,
1070 LostDebugLocObserver &LocObserver) {
1071 const DataLayout &DL = MIRBuilder.getDataLayout();
1072 auto &MF = MIRBuilder.getMF();
1073 auto &MRI = *MIRBuilder.getMRI();
1074 auto &Ctx = MF.getFunction().getContext();
1075
1076 // Create temporary, where library function will put the read state.
1077 Register Dst = MI.getOperand(i: 0).getReg();
1078 LLT StateTy = MRI.getType(Reg: Dst);
1079 TypeSize StateSize = StateTy.getSizeInBytes();
1080 Align TempAlign = getStackTemporaryAlignment(Type: StateTy);
1081 MachinePointerInfo TempPtrInfo;
1082 auto Temp = createStackTemporary(Bytes: StateSize, Alignment: TempAlign, PtrInfo&: TempPtrInfo);
1083
1084 // Create a call to library function, with the temporary as an argument.
1085 unsigned TempAddrSpace = DL.getAllocaAddrSpace();
1086 Type *StatePtrTy = PointerType::get(C&: Ctx, AddressSpace: TempAddrSpace);
1087 RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI);
1088 auto Res = createLibcall(
1089 Libcall: RTLibcall, Result: CallLowering::ArgInfo({0}, Type::getVoidTy(C&: Ctx), 0),
1090 Args: CallLowering::ArgInfo({Temp.getReg(Idx: 0), StatePtrTy, 0}), LocObserver,
1091 MI: nullptr);
1092 if (Res != LegalizerHelper::Legalized)
1093 return Res;
1094
1095 // Create a load from the temporary.
1096 MachineMemOperand *MMO = MF.getMachineMemOperand(
1097 PtrInfo: TempPtrInfo, F: MachineMemOperand::MOLoad, MemTy: StateTy, BaseAlignment: TempAlign);
1098 MIRBuilder.buildLoadInstr(Opcode: TargetOpcode::G_LOAD, Res: Dst, Addr: Temp, MMO&: *MMO);
1099
1100 return LegalizerHelper::Legalized;
1101}
1102
1103// Similar to `createGetStateLibcall` the function calls a library function
1104// using transient space in stack. In this case the library function reads
1105// content of memory region.
1106LegalizerHelper::LegalizeResult
1107LegalizerHelper::createSetStateLibcall(MachineInstr &MI,
1108 LostDebugLocObserver &LocObserver) {
1109 const DataLayout &DL = MIRBuilder.getDataLayout();
1110 auto &MF = MIRBuilder.getMF();
1111 auto &MRI = *MIRBuilder.getMRI();
1112 auto &Ctx = MF.getFunction().getContext();
1113
1114 // Create temporary, where library function will get the new state.
1115 Register Src = MI.getOperand(i: 0).getReg();
1116 LLT StateTy = MRI.getType(Reg: Src);
1117 TypeSize StateSize = StateTy.getSizeInBytes();
1118 Align TempAlign = getStackTemporaryAlignment(Type: StateTy);
1119 MachinePointerInfo TempPtrInfo;
1120 auto Temp = createStackTemporary(Bytes: StateSize, Alignment: TempAlign, PtrInfo&: TempPtrInfo);
1121
1122 // Put the new state into the temporary.
1123 MachineMemOperand *MMO = MF.getMachineMemOperand(
1124 PtrInfo: TempPtrInfo, F: MachineMemOperand::MOStore, MemTy: StateTy, BaseAlignment: TempAlign);
1125 MIRBuilder.buildStore(Val: Src, Addr: Temp, MMO&: *MMO);
1126
1127 // Create a call to library function, with the temporary as an argument.
1128 unsigned TempAddrSpace = DL.getAllocaAddrSpace();
1129 Type *StatePtrTy = PointerType::get(C&: Ctx, AddressSpace: TempAddrSpace);
1130 RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI);
1131 return createLibcall(Libcall: RTLibcall,
1132 Result: CallLowering::ArgInfo({0}, Type::getVoidTy(C&: Ctx), 0),
1133 Args: CallLowering::ArgInfo({Temp.getReg(Idx: 0), StatePtrTy, 0}),
1134 LocObserver, MI: nullptr);
1135}
1136
1137/// Returns the corresponding libcall for the given Pred and
1138/// the ICMP predicate that should be generated to compare with #0
1139/// after the libcall.
1140static std::pair<RTLIB::Libcall, CmpInst::Predicate>
1141getFCMPLibcallDesc(const CmpInst::Predicate Pred, unsigned Size) {
1142#define RTLIBCASE_CMP(LibcallPrefix, ICmpPred) \
1143 do { \
1144 switch (Size) { \
1145 case 32: \
1146 return {RTLIB::LibcallPrefix##32, ICmpPred}; \
1147 case 64: \
1148 return {RTLIB::LibcallPrefix##64, ICmpPred}; \
1149 case 128: \
1150 return {RTLIB::LibcallPrefix##128, ICmpPred}; \
1151 default: \
1152 llvm_unreachable("unexpected size"); \
1153 } \
1154 } while (0)
1155
1156 // These use the three-way (-1/0/1) compare libcalls, whose result is tested
1157 // against 0 with a signed integer predicate. Unordered (UO) is a boolean.
1158 switch (Pred) {
1159 case CmpInst::FCMP_OEQ:
1160 RTLIBCASE_CMP(FCMP3_PRED_OEQ_F, CmpInst::ICMP_EQ);
1161 case CmpInst::FCMP_UNE:
1162 RTLIBCASE_CMP(FCMP3_PRED_UNE_F, CmpInst::ICMP_NE);
1163 case CmpInst::FCMP_OGE:
1164 RTLIBCASE_CMP(FCMP3_PRED_OGE_F, CmpInst::ICMP_SGE);
1165 case CmpInst::FCMP_OLT:
1166 RTLIBCASE_CMP(FCMP3_PRED_OLT_F, CmpInst::ICMP_SLT);
1167 case CmpInst::FCMP_OLE:
1168 RTLIBCASE_CMP(FCMP3_PRED_OLE_F, CmpInst::ICMP_SLE);
1169 case CmpInst::FCMP_OGT:
1170 RTLIBCASE_CMP(FCMP3_PRED_OGT_F, CmpInst::ICMP_SGT);
1171 case CmpInst::FCMP_UNO:
1172 RTLIBCASE_CMP(UO_F, CmpInst::ICMP_NE);
1173 default:
1174 return {RTLIB::UNKNOWN_LIBCALL, CmpInst::BAD_ICMP_PREDICATE};
1175 }
1176}
1177
1178LegalizerHelper::LegalizeResult
1179LegalizerHelper::createFCMPLibcall(MachineInstr &MI,
1180 LostDebugLocObserver &LocObserver) {
1181 auto &MF = MIRBuilder.getMF();
1182 auto &Ctx = MF.getFunction().getContext();
1183 const GFCmp *Cmp = cast<GFCmp>(Val: &MI);
1184
1185 LLT OpLLT = MRI.getType(Reg: Cmp->getLHSReg());
1186 unsigned Size = OpLLT.getSizeInBits();
1187 if ((Size != 32 && Size != 64 && Size != 128) ||
1188 OpLLT != MRI.getType(Reg: Cmp->getRHSReg()))
1189 return UnableToLegalize;
1190
1191 Type *OpType = getFloatTypeForLLT(Ctx, Ty: OpLLT);
1192
1193 // DstReg type is s32
1194 const Register DstReg = Cmp->getReg(Idx: 0);
1195 LLT DstTy = MRI.getType(Reg: DstReg);
1196 const auto Cond = Cmp->getCond();
1197 Type *RetTy = TLI.getCmpLibcallReturnType(Ctx);
1198
1199 // Reference:
1200 // https://gcc.gnu.org/onlinedocs/gccint/Soft-float-library-routines.html#Comparison-functions-1
1201 // Generates a libcall followed by ICMP.
1202 const auto BuildLibcall = [&](const RTLIB::Libcall Libcall,
1203 const CmpInst::Predicate ICmpPred,
1204 const DstOp &Res) -> Register {
1205 // FCMP libcall returns an integer, and needs an ICMP with #0.
1206 LLT TempLLT = LLT::integer(SizeInBits: RetTy->getIntegerBitWidth());
1207 Register Temp = MRI.createGenericVirtualRegister(Ty: TempLLT);
1208 // Generate libcall, holding result in Temp
1209 const auto Status = createLibcall(
1210 Libcall, Result: {Temp, RetTy, 0},
1211 Args: {{Cmp->getLHSReg(), OpType, 0}, {Cmp->getRHSReg(), OpType, 1}},
1212 LocObserver, MI: &MI);
1213 if (Status != Legalized)
1214 return {};
1215
1216 // Compare temp with #0 to get the final result.
1217 return MIRBuilder
1218 .buildICmp(Pred: ICmpPred, Res, Op0: Temp, Op1: MIRBuilder.buildConstant(Res: TempLLT, Val: 0))
1219 .getReg(Idx: 0);
1220 };
1221
1222 // Simple case if we have a direct mapping from predicate to libcall
1223 if (const auto [Libcall, ICmpPred] = getFCMPLibcallDesc(Pred: Cond, Size);
1224 Libcall != RTLIB::UNKNOWN_LIBCALL &&
1225 ICmpPred != CmpInst::BAD_ICMP_PREDICATE) {
1226 if (BuildLibcall(Libcall, ICmpPred, DstReg)) {
1227 return Legalized;
1228 }
1229 return UnableToLegalize;
1230 }
1231
1232 // No direct mapping found, should be generated as combination of libcalls.
1233
1234 switch (Cond) {
1235 case CmpInst::FCMP_UEQ: {
1236 // FCMP_UEQ: unordered or equal
1237 // Convert into (FCMP_OEQ || FCMP_UNO).
1238
1239 const auto [OeqLibcall, OeqPred] =
1240 getFCMPLibcallDesc(Pred: CmpInst::FCMP_OEQ, Size);
1241 const auto Oeq = BuildLibcall(OeqLibcall, OeqPred, DstTy);
1242
1243 const auto [UnoLibcall, UnoPred] =
1244 getFCMPLibcallDesc(Pred: CmpInst::FCMP_UNO, Size);
1245 const auto Uno = BuildLibcall(UnoLibcall, UnoPred, DstTy);
1246 if (Oeq && Uno)
1247 MIRBuilder.buildOr(Dst: DstReg, Src0: Oeq, Src1: Uno);
1248 else
1249 return UnableToLegalize;
1250
1251 break;
1252 }
1253 case CmpInst::FCMP_ONE: {
1254 // FCMP_ONE: ordered and operands are unequal
1255 // Convert into (!FCMP_OEQ && !FCMP_UNO).
1256
1257 // We inverse the predicate instead of generating a NOT
1258 // to save one instruction.
1259 // On AArch64 isel can even select two cmp into a single ccmp.
1260 const auto [OeqLibcall, OeqPred] =
1261 getFCMPLibcallDesc(Pred: CmpInst::FCMP_OEQ, Size);
1262 const auto NotOeq =
1263 BuildLibcall(OeqLibcall, CmpInst::getInversePredicate(pred: OeqPred), DstTy);
1264
1265 const auto [UnoLibcall, UnoPred] =
1266 getFCMPLibcallDesc(Pred: CmpInst::FCMP_UNO, Size);
1267 const auto NotUno =
1268 BuildLibcall(UnoLibcall, CmpInst::getInversePredicate(pred: UnoPred), DstTy);
1269
1270 if (NotOeq && NotUno)
1271 MIRBuilder.buildAnd(Dst: DstReg, Src0: NotOeq, Src1: NotUno);
1272 else
1273 return UnableToLegalize;
1274
1275 break;
1276 }
1277 case CmpInst::FCMP_ULT:
1278 case CmpInst::FCMP_UGE:
1279 case CmpInst::FCMP_UGT:
1280 case CmpInst::FCMP_ULE:
1281 case CmpInst::FCMP_ORD: {
1282 // Convert into: !(inverse(Pred))
1283 // E.g. FCMP_ULT becomes !FCMP_OGE
1284 // This is equivalent to the following, but saves some instructions.
1285 // MIRBuilder.buildNot(
1286 // PredTy,
1287 // MIRBuilder.buildFCmp(CmpInst::getInversePredicate(Pred), PredTy,
1288 // Op1, Op2));
1289 const auto [InversedLibcall, InversedPred] =
1290 getFCMPLibcallDesc(Pred: CmpInst::getInversePredicate(pred: Cond), Size);
1291 if (!BuildLibcall(InversedLibcall,
1292 CmpInst::getInversePredicate(pred: InversedPred), DstReg))
1293 return UnableToLegalize;
1294 break;
1295 }
1296 default:
1297 return UnableToLegalize;
1298 }
1299
1300 return Legalized;
1301}
1302
1303// The function is used to legalize operations that set default environment
1304// state. In C library a call like `fesetmode(FE_DFL_MODE)` is used for that.
1305// On most targets supported in glibc FE_DFL_MODE is defined as
1306// `((const femode_t *) -1)`. Such assumption is used here. If for some target
1307// it is not true, the target must provide custom lowering.
1308LegalizerHelper::LegalizeResult
1309LegalizerHelper::createResetStateLibcall(MachineInstr &MI,
1310 LostDebugLocObserver &LocObserver) {
1311 const DataLayout &DL = MIRBuilder.getDataLayout();
1312 auto &MF = MIRBuilder.getMF();
1313 auto &Ctx = MF.getFunction().getContext();
1314
1315 // Create an argument for the library function.
1316 unsigned AddrSpace = DL.getDefaultGlobalsAddressSpace();
1317 Type *StatePtrTy = PointerType::get(C&: Ctx, AddressSpace: AddrSpace);
1318 unsigned PtrSize = DL.getPointerSizeInBits(AS: AddrSpace);
1319 LLT MemTy = LLT::pointer(AddressSpace: AddrSpace, SizeInBits: PtrSize);
1320 auto DefValue = MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: PtrSize), Val: -1LL);
1321 DstOp Dest(MRI.createGenericVirtualRegister(Ty: MemTy));
1322 MIRBuilder.buildIntToPtr(Dst: Dest, Src: DefValue);
1323
1324 RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI);
1325 return createLibcall(
1326 Libcall: RTLibcall, Result: CallLowering::ArgInfo({0}, Type::getVoidTy(C&: Ctx), 0),
1327 Args: CallLowering::ArgInfo({Dest.getReg(), StatePtrTy, 0}), LocObserver, MI: &MI);
1328}
1329
1330LegalizerHelper::LegalizeResult
1331LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
1332 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
1333
1334 switch (MI.getOpcode()) {
1335 default:
1336 return UnableToLegalize;
1337 case TargetOpcode::G_MUL:
1338 case TargetOpcode::G_SDIV:
1339 case TargetOpcode::G_UDIV:
1340 case TargetOpcode::G_SREM:
1341 case TargetOpcode::G_UREM:
1342 case TargetOpcode::G_CTLZ_ZERO_POISON: {
1343 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1344 unsigned Size = LLTy.getSizeInBits();
1345 Type *HLTy = IntegerType::get(C&: Ctx, NumBits: Size);
1346 auto Status = simpleLibcall(MI, MIRBuilder, Size, OpType: HLTy, LocObserver);
1347 if (Status != Legalized)
1348 return Status;
1349 break;
1350 }
1351 case TargetOpcode::G_FADD:
1352 case TargetOpcode::G_FSUB:
1353 case TargetOpcode::G_FMUL:
1354 case TargetOpcode::G_FDIV:
1355 case TargetOpcode::G_FMA:
1356 case TargetOpcode::G_FPOW:
1357 case TargetOpcode::G_FREM:
1358 case TargetOpcode::G_FCOS:
1359 case TargetOpcode::G_FSIN:
1360 case TargetOpcode::G_FTAN:
1361 case TargetOpcode::G_FACOS:
1362 case TargetOpcode::G_FASIN:
1363 case TargetOpcode::G_FATAN:
1364 case TargetOpcode::G_FATAN2:
1365 case TargetOpcode::G_FCOSH:
1366 case TargetOpcode::G_FSINH:
1367 case TargetOpcode::G_FTANH:
1368 case TargetOpcode::G_FLOG10:
1369 case TargetOpcode::G_FLOG:
1370 case TargetOpcode::G_FLOG2:
1371 case TargetOpcode::G_FEXP:
1372 case TargetOpcode::G_FEXP2:
1373 case TargetOpcode::G_FEXP10:
1374 case TargetOpcode::G_FCEIL:
1375 case TargetOpcode::G_FFLOOR:
1376 case TargetOpcode::G_FMINNUM:
1377 case TargetOpcode::G_FMAXNUM:
1378 case TargetOpcode::G_FMINIMUMNUM:
1379 case TargetOpcode::G_FMAXIMUMNUM:
1380 case TargetOpcode::G_FSQRT:
1381 case TargetOpcode::G_FRINT:
1382 case TargetOpcode::G_FNEARBYINT:
1383 case TargetOpcode::G_INTRINSIC_TRUNC:
1384 case TargetOpcode::G_INTRINSIC_ROUND:
1385 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: {
1386 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1387 unsigned Size = LLTy.getSizeInBits();
1388 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1389 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1390 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1391 return UnableToLegalize;
1392 }
1393 auto Status = simpleLibcall(MI, MIRBuilder, Size, OpType: HLTy, LocObserver);
1394 if (Status != Legalized)
1395 return Status;
1396 break;
1397 }
1398 case TargetOpcode::G_FSINCOS: {
1399 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1400 unsigned Size = LLTy.getSizeInBits();
1401 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1402 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1403 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1404 return UnableToLegalize;
1405 }
1406 return emitSincosLibcall(MI, MIRBuilder, Size, OpType: HLTy, LocObserver);
1407 }
1408 case TargetOpcode::G_FMODF: {
1409 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1410 unsigned Size = LLTy.getSizeInBits();
1411 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1412 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1413 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1414 return UnableToLegalize;
1415 }
1416 return emitModfLibcall(MI, MIRBuilder, Size, OpType: HLTy, LocObserver);
1417 }
1418 case TargetOpcode::G_LROUND:
1419 case TargetOpcode::G_LLROUND:
1420 case TargetOpcode::G_INTRINSIC_LRINT:
1421 case TargetOpcode::G_INTRINSIC_LLRINT: {
1422 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
1423 unsigned Size = LLTy.getSizeInBits();
1424 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1425 Type *ITy = IntegerType::get(
1426 C&: Ctx, NumBits: MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits());
1427 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1428 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1429 return UnableToLegalize;
1430 }
1431 auto Libcall = getRTLibDesc(Opcode: MI.getOpcode(), Size);
1432 LegalizeResult Status =
1433 createLibcall(Libcall, Result: {MI.getOperand(i: 0).getReg(), ITy, 0},
1434 Args: {{MI.getOperand(i: 1).getReg(), HLTy, 0}}, LocObserver, MI: &MI);
1435 if (Status != Legalized)
1436 return Status;
1437 MI.eraseFromParent();
1438 return Legalized;
1439 }
1440 case TargetOpcode::G_FPOWI:
1441 case TargetOpcode::G_FLDEXP: {
1442 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1443 unsigned Size = LLTy.getSizeInBits();
1444 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1445 Type *ITy = IntegerType::get(
1446 C&: Ctx, NumBits: MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits());
1447 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1448 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1449 return UnableToLegalize;
1450 }
1451 auto Libcall = getRTLibDesc(Opcode: MI.getOpcode(), Size);
1452 SmallVector<CallLowering::ArgInfo, 2> Args = {
1453 {MI.getOperand(i: 1).getReg(), HLTy, 0},
1454 {MI.getOperand(i: 2).getReg(), ITy, 1}};
1455 Args[1].Flags[0].setSExt();
1456 LegalizeResult Status = createLibcall(
1457 Libcall, Result: {MI.getOperand(i: 0).getReg(), HLTy, 0}, Args, LocObserver, MI: &MI);
1458 if (Status != Legalized)
1459 return Status;
1460 break;
1461 }
1462 case TargetOpcode::G_FPEXT:
1463 case TargetOpcode::G_FPTRUNC: {
1464 Type *FromTy = getFloatTypeForLLT(Ctx, Ty: MRI.getType(Reg: MI.getOperand(i: 1).getReg()));
1465 Type *ToTy = getFloatTypeForLLT(Ctx, Ty: MRI.getType(Reg: MI.getOperand(i: 0).getReg()));
1466 if (!FromTy || !ToTy)
1467 return UnableToLegalize;
1468 LegalizeResult Status = conversionLibcall(MI, ToType: ToTy, FromType: FromTy, LocObserver);
1469 if (Status != Legalized)
1470 return Status;
1471 break;
1472 }
1473 case TargetOpcode::G_FCMP: {
1474 LegalizeResult Status = createFCMPLibcall(MI, LocObserver);
1475 if (Status != Legalized)
1476 return Status;
1477 MI.eraseFromParent();
1478 return Status;
1479 }
1480 case TargetOpcode::G_FPTOSI:
1481 case TargetOpcode::G_FPTOUI: {
1482 // FIXME: Support other types
1483 Type *FromTy =
1484 getFloatTypeForLLT(Ctx, Ty: MRI.getType(Reg: MI.getOperand(i: 1).getReg()));
1485 unsigned ToSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
1486 if ((ToSize != 32 && ToSize != 64 && ToSize != 128) || !FromTy)
1487 return UnableToLegalize;
1488 LegalizeResult Status = conversionLibcall(MI, ToType: Type::getIntNTy(C&: Ctx, N: ToSize),
1489 FromType: FromTy, LocObserver);
1490 if (Status != Legalized)
1491 return Status;
1492 break;
1493 }
1494 case TargetOpcode::G_SITOFP:
1495 case TargetOpcode::G_UITOFP: {
1496 unsigned FromSize = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
1497 Type *ToTy =
1498 getFloatTypeForLLT(Ctx, Ty: MRI.getType(Reg: MI.getOperand(i: 0).getReg()));
1499 if ((FromSize != 32 && FromSize != 64 && FromSize != 128) || !ToTy)
1500 return UnableToLegalize;
1501 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SITOFP;
1502 LegalizeResult Status = conversionLibcall(
1503 MI, ToType: ToTy, FromType: Type::getIntNTy(C&: Ctx, N: FromSize), LocObserver, IsSigned);
1504 if (Status != Legalized)
1505 return Status;
1506 break;
1507 }
1508 case TargetOpcode::G_ATOMICRMW_XCHG:
1509 case TargetOpcode::G_ATOMICRMW_ADD:
1510 case TargetOpcode::G_ATOMICRMW_SUB:
1511 case TargetOpcode::G_ATOMICRMW_AND:
1512 case TargetOpcode::G_ATOMICRMW_OR:
1513 case TargetOpcode::G_ATOMICRMW_XOR:
1514 case TargetOpcode::G_ATOMIC_CMPXCHG:
1515 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
1516 auto Status = createAtomicLibcall(MI);
1517 if (Status != Legalized)
1518 return Status;
1519 break;
1520 }
1521 case TargetOpcode::G_BZERO:
1522 case TargetOpcode::G_MEMCPY:
1523 case TargetOpcode::G_MEMMOVE:
1524 case TargetOpcode::G_MEMSET: {
1525 LegalizeResult Result =
1526 createMemLibcall(MRI&: *MIRBuilder.getMRI(), MI, LocObserver);
1527 if (Result != Legalized)
1528 return Result;
1529 MI.eraseFromParent();
1530 return Result;
1531 }
1532 case TargetOpcode::G_GET_FPENV:
1533 case TargetOpcode::G_GET_FPMODE: {
1534 LegalizeResult Result = createGetStateLibcall(MI, LocObserver);
1535 if (Result != Legalized)
1536 return Result;
1537 break;
1538 }
1539 case TargetOpcode::G_SET_FPENV:
1540 case TargetOpcode::G_SET_FPMODE: {
1541 LegalizeResult Result = createSetStateLibcall(MI, LocObserver);
1542 if (Result != Legalized)
1543 return Result;
1544 break;
1545 }
1546 case TargetOpcode::G_RESET_FPENV:
1547 case TargetOpcode::G_RESET_FPMODE: {
1548 LegalizeResult Result = createResetStateLibcall(MI, LocObserver);
1549 if (Result != Legalized)
1550 return Result;
1551 break;
1552 }
1553 }
1554
1555 MI.eraseFromParent();
1556 return Legalized;
1557}
1558
1559LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
1560 unsigned TypeIdx,
1561 LLT NarrowTy) {
1562 uint64_t SizeOp0 = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
1563 uint64_t NarrowSize = NarrowTy.getSizeInBits();
1564
1565 switch (MI.getOpcode()) {
1566 default:
1567 return UnableToLegalize;
1568 case TargetOpcode::G_IMPLICIT_DEF: {
1569 Register DstReg = MI.getOperand(i: 0).getReg();
1570 LLT DstTy = MRI.getType(Reg: DstReg);
1571
1572 // If SizeOp0 is not an exact multiple of NarrowSize, emit
1573 // G_ANYEXT(G_IMPLICIT_DEF). Cast result to vector if needed.
1574 // FIXME: Although this would also be legal for the general case, it causes
1575 // a lot of regressions in the emitted code (superfluous COPYs, artifact
1576 // combines not being hit). This seems to be a problem related to the
1577 // artifact combiner.
1578 if (SizeOp0 % NarrowSize != 0) {
1579 LLT ImplicitTy = DstTy.changeElementType(NewEltTy: NarrowTy);
1580 Register ImplicitReg = MIRBuilder.buildUndef(Res: ImplicitTy).getReg(Idx: 0);
1581 MIRBuilder.buildAnyExt(Res: DstReg, Op: ImplicitReg);
1582
1583 MI.eraseFromParent();
1584 return Legalized;
1585 }
1586
1587 int NumParts = SizeOp0 / NarrowSize;
1588
1589 SmallVector<Register, 2> DstRegs;
1590 for (int i = 0; i < NumParts; ++i)
1591 DstRegs.push_back(Elt: MIRBuilder.buildUndef(Res: NarrowTy).getReg(Idx: 0));
1592
1593 if (DstTy.isVector())
1594 MIRBuilder.buildBuildVector(Res: DstReg, Ops: DstRegs);
1595 else
1596 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
1597 MI.eraseFromParent();
1598 return Legalized;
1599 }
1600 case TargetOpcode::G_CONSTANT: {
1601 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1602 const APInt &Val = MI.getOperand(i: 1).getCImm()->getValue();
1603 unsigned TotalSize = Ty.getSizeInBits();
1604 unsigned NarrowSize = NarrowTy.getSizeInBits();
1605 int NumParts = TotalSize / NarrowSize;
1606
1607 SmallVector<Register, 4> PartRegs;
1608 for (int I = 0; I != NumParts; ++I) {
1609 unsigned Offset = I * NarrowSize;
1610 auto K = MIRBuilder.buildConstant(Res: NarrowTy,
1611 Val: Val.lshr(shiftAmt: Offset).trunc(width: NarrowSize));
1612 PartRegs.push_back(Elt: K.getReg(Idx: 0));
1613 }
1614
1615 LLT LeftoverTy;
1616 unsigned LeftoverBits = TotalSize - NumParts * NarrowSize;
1617 SmallVector<Register, 1> LeftoverRegs;
1618 if (LeftoverBits != 0) {
1619 LeftoverTy = LLT::scalar(SizeInBits: LeftoverBits);
1620 auto K = MIRBuilder.buildConstant(
1621 Res: LeftoverTy,
1622 Val: Val.lshr(shiftAmt: NumParts * NarrowSize).trunc(width: LeftoverBits));
1623 LeftoverRegs.push_back(Elt: K.getReg(Idx: 0));
1624 }
1625
1626 insertParts(DstReg: MI.getOperand(i: 0).getReg(),
1627 ResultTy: Ty, PartTy: NarrowTy, PartRegs, LeftoverTy, LeftoverRegs);
1628
1629 MI.eraseFromParent();
1630 return Legalized;
1631 }
1632 case TargetOpcode::G_SEXT:
1633 case TargetOpcode::G_ZEXT:
1634 case TargetOpcode::G_ANYEXT:
1635 return narrowScalarExt(MI, TypeIdx, Ty: NarrowTy);
1636 case TargetOpcode::G_TRUNC: {
1637 if (TypeIdx != 1)
1638 return UnableToLegalize;
1639
1640 uint64_t SizeOp1 = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
1641 if (NarrowTy.getSizeInBits() * 2 != SizeOp1) {
1642 LLVM_DEBUG(dbgs() << "Can't narrow trunc to type " << NarrowTy << "\n");
1643 return UnableToLegalize;
1644 }
1645
1646 auto Unmerge = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: MI.getOperand(i: 1));
1647 MIRBuilder.buildCopy(Res: MI.getOperand(i: 0), Op: Unmerge.getReg(Idx: 0));
1648 MI.eraseFromParent();
1649 return Legalized;
1650 }
1651 case TargetOpcode::G_CONSTANT_FOLD_BARRIER:
1652 case TargetOpcode::G_FREEZE: {
1653 if (TypeIdx != 0)
1654 return UnableToLegalize;
1655
1656 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1657 // Should widen scalar first
1658 if (Ty.getSizeInBits() % NarrowTy.getSizeInBits() != 0)
1659 return UnableToLegalize;
1660
1661 auto Unmerge = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: MI.getOperand(i: 1).getReg());
1662 SmallVector<Register, 8> Parts;
1663 for (unsigned i = 0; i < Unmerge->getNumDefs(); ++i) {
1664 Parts.push_back(
1665 Elt: MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {NarrowTy}, SrcOps: {Unmerge.getReg(Idx: i)})
1666 .getReg(Idx: 0));
1667 }
1668
1669 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0).getReg(), Ops: Parts);
1670 MI.eraseFromParent();
1671 return Legalized;
1672 }
1673 case TargetOpcode::G_ADD:
1674 case TargetOpcode::G_SUB:
1675 case TargetOpcode::G_SADDO:
1676 case TargetOpcode::G_SSUBO:
1677 case TargetOpcode::G_SADDE:
1678 case TargetOpcode::G_SSUBE:
1679 case TargetOpcode::G_UADDO:
1680 case TargetOpcode::G_USUBO:
1681 case TargetOpcode::G_UADDE:
1682 case TargetOpcode::G_USUBE:
1683 return narrowScalarAddSub(MI, TypeIdx, NarrowTy);
1684 case TargetOpcode::G_MUL:
1685 case TargetOpcode::G_UMULH:
1686 return narrowScalarMul(MI, Ty: NarrowTy);
1687 case TargetOpcode::G_EXTRACT:
1688 return narrowScalarExtract(MI, TypeIdx, Ty: NarrowTy);
1689 case TargetOpcode::G_INSERT:
1690 return narrowScalarInsert(MI, TypeIdx, Ty: NarrowTy);
1691 case TargetOpcode::G_LOAD: {
1692 auto &LoadMI = cast<GLoad>(Val&: MI);
1693 Register DstReg = LoadMI.getDstReg();
1694 LLT DstTy = MRI.getType(Reg: DstReg);
1695 if (DstTy.isVector())
1696 return UnableToLegalize;
1697
1698 if (8 * LoadMI.getMemSize().getValue() != DstTy.getSizeInBits()) {
1699 Register TmpReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
1700 MIRBuilder.buildLoad(Res: TmpReg, Addr: LoadMI.getPointerReg(), MMO&: LoadMI.getMMO());
1701 MIRBuilder.buildAnyExt(Res: DstReg, Op: TmpReg);
1702 LoadMI.eraseFromParent();
1703 return Legalized;
1704 }
1705
1706 return reduceLoadStoreWidth(MI&: LoadMI, TypeIdx, NarrowTy);
1707 }
1708 case TargetOpcode::G_ZEXTLOAD:
1709 case TargetOpcode::G_SEXTLOAD:
1710 case TargetOpcode::G_FPEXTLOAD: {
1711 auto &LoadMI = cast<GExtLoad>(Val&: MI);
1712 Register DstReg = LoadMI.getDstReg();
1713 Register PtrReg = LoadMI.getPointerReg();
1714
1715 Register TmpReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
1716 auto &MMO = LoadMI.getMMO();
1717 unsigned MemSize = MMO.getSizeInBits().getValue();
1718
1719 if (MemSize == NarrowSize) {
1720 MIRBuilder.buildLoad(Res: TmpReg, Addr: PtrReg, MMO);
1721 } else if (MemSize < NarrowSize) {
1722 MIRBuilder.buildLoadInstr(Opcode: LoadMI.getOpcode(), Res: TmpReg, Addr: PtrReg, MMO);
1723 } else if (MemSize > NarrowSize) {
1724 // FIXME: Need to split the load.
1725 return UnableToLegalize;
1726 }
1727
1728 if (isa<GZExtLoad>(Val: LoadMI))
1729 MIRBuilder.buildZExt(Res: DstReg, Op: TmpReg);
1730 else if (isa<GSExtLoad>(Val: LoadMI))
1731 MIRBuilder.buildSExt(Res: DstReg, Op: TmpReg);
1732 else
1733 MIRBuilder.buildFPExt(Res: DstReg, Op: TmpReg);
1734
1735 LoadMI.eraseFromParent();
1736 return Legalized;
1737 }
1738 case TargetOpcode::G_STORE: {
1739 auto &StoreMI = cast<GStore>(Val&: MI);
1740
1741 Register SrcReg = StoreMI.getValueReg();
1742 LLT SrcTy = MRI.getType(Reg: SrcReg);
1743 if (SrcTy.isVector())
1744 return UnableToLegalize;
1745
1746 int NumParts = SizeOp0 / NarrowSize;
1747 unsigned HandledSize = NumParts * NarrowTy.getSizeInBits();
1748 unsigned LeftoverBits = SrcTy.getSizeInBits() - HandledSize;
1749 if (SrcTy.isVector() && LeftoverBits != 0)
1750 return UnableToLegalize;
1751
1752 if (8 * StoreMI.getMemSize().getValue() != SrcTy.getSizeInBits()) {
1753 Register TmpReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
1754 MIRBuilder.buildTrunc(Res: TmpReg, Op: SrcReg);
1755 MIRBuilder.buildStore(Val: TmpReg, Addr: StoreMI.getPointerReg(), MMO&: StoreMI.getMMO());
1756 StoreMI.eraseFromParent();
1757 return Legalized;
1758 }
1759
1760 return reduceLoadStoreWidth(MI&: StoreMI, TypeIdx: 0, NarrowTy);
1761 }
1762 case TargetOpcode::G_FPTRUNCSTORE: {
1763 auto &StoreMI = cast<GFPTruncStore>(Val&: MI);
1764 Register SrcReg = StoreMI.getValueReg();
1765 Register PtrReg = StoreMI.getPointerReg();
1766
1767 auto &MMO = StoreMI.getMMO();
1768 unsigned MemSize = MMO.getSizeInBits().getValue();
1769 if (MemSize > NarrowSize) {
1770 return UnableToLegalize;
1771 }
1772
1773 auto TmpReg = MIRBuilder.buildFPTrunc(Res: NarrowTy, Op: SrcReg);
1774 if (MemSize == NarrowSize) {
1775 MIRBuilder.buildStore(Val: TmpReg, Addr: PtrReg, MMO);
1776 } else if (MemSize < NarrowSize) {
1777 MIRBuilder.buildStoreInstr(Opcode: TargetOpcode::G_FPTRUNCSTORE, Val: TmpReg, Addr: PtrReg,
1778 MMO);
1779 }
1780
1781 StoreMI.eraseFromParent();
1782 return Legalized;
1783 }
1784 case TargetOpcode::G_SELECT:
1785 return narrowScalarSelect(MI, TypeIdx, Ty: NarrowTy);
1786 case TargetOpcode::G_AND:
1787 case TargetOpcode::G_OR:
1788 case TargetOpcode::G_XOR: {
1789 // Legalize bitwise operation:
1790 // A = BinOp<Ty> B, C
1791 // into:
1792 // B1, ..., BN = G_UNMERGE_VALUES B
1793 // C1, ..., CN = G_UNMERGE_VALUES C
1794 // A1 = BinOp<Ty/N> B1, C2
1795 // ...
1796 // AN = BinOp<Ty/N> BN, CN
1797 // A = G_MERGE_VALUES A1, ..., AN
1798 return narrowScalarBasic(MI, TypeIdx, Ty: NarrowTy);
1799 }
1800 case TargetOpcode::G_SHL:
1801 case TargetOpcode::G_LSHR:
1802 case TargetOpcode::G_ASHR:
1803 return narrowScalarShift(MI, TypeIdx, Ty: NarrowTy);
1804 case TargetOpcode::G_CTLZ:
1805 case TargetOpcode::G_CTLZ_ZERO_POISON:
1806 case TargetOpcode::G_CTTZ:
1807 case TargetOpcode::G_CTTZ_ZERO_POISON:
1808 case TargetOpcode::G_CTLS:
1809 case TargetOpcode::G_CTPOP:
1810 if (TypeIdx == 1)
1811 switch (MI.getOpcode()) {
1812 case TargetOpcode::G_CTLZ:
1813 case TargetOpcode::G_CTLZ_ZERO_POISON:
1814 return narrowScalarCTLZ(MI, TypeIdx, Ty: NarrowTy);
1815 case TargetOpcode::G_CTTZ:
1816 case TargetOpcode::G_CTTZ_ZERO_POISON:
1817 return narrowScalarCTTZ(MI, TypeIdx, Ty: NarrowTy);
1818 case TargetOpcode::G_CTPOP:
1819 return narrowScalarCTPOP(MI, TypeIdx, Ty: NarrowTy);
1820 case TargetOpcode::G_CTLS:
1821 return narrowScalarCTLS(MI, TypeIdx, Ty: NarrowTy);
1822 default:
1823 return UnableToLegalize;
1824 }
1825
1826 Observer.changingInstr(MI);
1827 narrowScalarDst(MI, NarrowTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_ZEXT);
1828 Observer.changedInstr(MI);
1829 return Legalized;
1830 case TargetOpcode::G_INTTOPTR:
1831 if (TypeIdx != 1)
1832 return UnableToLegalize;
1833
1834 Observer.changingInstr(MI);
1835 narrowScalarSrc(MI, NarrowTy, OpIdx: 1);
1836 Observer.changedInstr(MI);
1837 return Legalized;
1838 case TargetOpcode::G_PTRTOINT:
1839 if (TypeIdx != 0)
1840 return UnableToLegalize;
1841
1842 Observer.changingInstr(MI);
1843 narrowScalarDst(MI, NarrowTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_ZEXT);
1844 Observer.changedInstr(MI);
1845 return Legalized;
1846 case TargetOpcode::G_PHI: {
1847 // FIXME: add support for when SizeOp0 isn't an exact multiple of
1848 // NarrowSize.
1849 if (SizeOp0 % NarrowSize != 0)
1850 return UnableToLegalize;
1851
1852 unsigned NumParts = SizeOp0 / NarrowSize;
1853 SmallVector<Register, 2> DstRegs(NumParts);
1854 SmallVector<SmallVector<Register, 2>, 2> SrcRegs(MI.getNumOperands() / 2);
1855 Observer.changingInstr(MI);
1856 for (unsigned i = 1; i < MI.getNumOperands(); i += 2) {
1857 MachineBasicBlock &OpMBB = *MI.getOperand(i: i + 1).getMBB();
1858 MIRBuilder.setInsertPt(MBB&: OpMBB, II: OpMBB.getFirstTerminatorForward());
1859 extractParts(Reg: MI.getOperand(i).getReg(), Ty: NarrowTy, NumParts,
1860 VRegs&: SrcRegs[i / 2], MIRBuilder, MRI);
1861 }
1862 MachineBasicBlock &MBB = *MI.getParent();
1863 MIRBuilder.setInsertPt(MBB, II: MI);
1864 for (unsigned i = 0; i < NumParts; ++i) {
1865 DstRegs[i] = MRI.createGenericVirtualRegister(Ty: NarrowTy);
1866 MachineInstrBuilder MIB =
1867 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_PHI).addDef(RegNo: DstRegs[i]);
1868 for (unsigned j = 1; j < MI.getNumOperands(); j += 2)
1869 MIB.addUse(RegNo: SrcRegs[j / 2][i]).add(MO: MI.getOperand(i: j + 1));
1870 }
1871 MIRBuilder.setInsertPt(MBB, II: MBB.getFirstNonPHI());
1872 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0), Ops: DstRegs);
1873 Observer.changedInstr(MI);
1874 MI.eraseFromParent();
1875 return Legalized;
1876 }
1877 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1878 case TargetOpcode::G_INSERT_VECTOR_ELT: {
1879 if (TypeIdx != 2)
1880 return UnableToLegalize;
1881
1882 int OpIdx = MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
1883 Observer.changingInstr(MI);
1884 narrowScalarSrc(MI, NarrowTy, OpIdx);
1885 Observer.changedInstr(MI);
1886 return Legalized;
1887 }
1888 case TargetOpcode::G_ICMP: {
1889 Register LHS = MI.getOperand(i: 2).getReg();
1890 LLT SrcTy = MRI.getType(Reg: LHS);
1891 CmpInst::Predicate Pred =
1892 static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
1893
1894 LLT LeftoverTy; // Example: s88 -> s64 (NarrowTy) + s24 (leftover)
1895 SmallVector<Register, 4> LHSPartRegs, LHSLeftoverRegs;
1896 if (!extractParts(Reg: LHS, RegTy: SrcTy, MainTy: NarrowTy, LeftoverTy, VRegs&: LHSPartRegs,
1897 LeftoverVRegs&: LHSLeftoverRegs, MIRBuilder, MRI))
1898 return UnableToLegalize;
1899
1900 LLT Unused; // Matches LeftoverTy; G_ICMP LHS and RHS are the same type.
1901 SmallVector<Register, 4> RHSPartRegs, RHSLeftoverRegs;
1902 if (!extractParts(Reg: MI.getOperand(i: 3).getReg(), RegTy: SrcTy, MainTy: NarrowTy, LeftoverTy&: Unused,
1903 VRegs&: RHSPartRegs, LeftoverVRegs&: RHSLeftoverRegs, MIRBuilder, MRI))
1904 return UnableToLegalize;
1905
1906 // We now have the LHS and RHS of the compare split into narrow-type
1907 // registers, plus potentially some leftover type.
1908 Register Dst = MI.getOperand(i: 0).getReg();
1909 LLT ResTy = MRI.getType(Reg: Dst);
1910 if (ICmpInst::isEquality(P: Pred)) {
1911 // For each part on the LHS and RHS, keep track of the result of XOR-ing
1912 // them together. For each equal part, the result should be all 0s. For
1913 // each non-equal part, we'll get at least one 1.
1914 auto Zero = MIRBuilder.buildConstant(Res: NarrowTy, Val: 0);
1915 SmallVector<Register, 4> Xors;
1916 for (auto LHSAndRHS : zip(t&: LHSPartRegs, u&: RHSPartRegs)) {
1917 auto LHS = std::get<0>(t&: LHSAndRHS);
1918 auto RHS = std::get<1>(t&: LHSAndRHS);
1919 auto Xor = MIRBuilder.buildXor(Dst: NarrowTy, Src0: LHS, Src1: RHS).getReg(Idx: 0);
1920 Xors.push_back(Elt: Xor);
1921 }
1922
1923 // Build a G_XOR for each leftover register. Each G_XOR must be widened
1924 // to the desired narrow type so that we can OR them together later.
1925 SmallVector<Register, 4> WidenedXors;
1926 for (auto LHSAndRHS : zip(t&: LHSLeftoverRegs, u&: RHSLeftoverRegs)) {
1927 auto LHS = std::get<0>(t&: LHSAndRHS);
1928 auto RHS = std::get<1>(t&: LHSAndRHS);
1929 auto Xor = MIRBuilder.buildXor(Dst: LeftoverTy, Src0: LHS, Src1: RHS).getReg(Idx: 0);
1930 LLT GCDTy = extractGCDType(Parts&: WidenedXors, DstTy: NarrowTy, NarrowTy: LeftoverTy, SrcReg: Xor);
1931 buildLCMMergePieces(DstTy: LeftoverTy, NarrowTy, GCDTy, VRegs&: WidenedXors,
1932 /* PadStrategy = */ TargetOpcode::G_ZEXT);
1933 llvm::append_range(C&: Xors, R&: WidenedXors);
1934 }
1935
1936 // Now, for each part we broke up, we know if they are equal/not equal
1937 // based off the G_XOR. We can OR these all together and compare against
1938 // 0 to get the result.
1939 assert(Xors.size() >= 2 && "Should have gotten at least two Xors?");
1940 auto Or = MIRBuilder.buildOr(Dst: NarrowTy, Src0: Xors[0], Src1: Xors[1]);
1941 for (unsigned I = 2, E = Xors.size(); I < E; ++I)
1942 Or = MIRBuilder.buildOr(Dst: NarrowTy, Src0: Or, Src1: Xors[I]);
1943 MIRBuilder.buildICmp(Pred, Res: Dst, Op0: Or, Op1: Zero);
1944 } else {
1945 Register CmpIn;
1946 for (unsigned I = 0, E = LHSPartRegs.size(); I != E; ++I) {
1947 Register CmpOut;
1948 CmpInst::Predicate PartPred;
1949
1950 if (I == E - 1 && LHSLeftoverRegs.empty()) {
1951 PartPred = Pred;
1952 CmpOut = Dst;
1953 } else {
1954 PartPred = ICmpInst::getUnsignedPredicate(Pred);
1955 CmpOut = MRI.createGenericVirtualRegister(Ty: ResTy);
1956 }
1957
1958 if (!CmpIn) {
1959 MIRBuilder.buildICmp(Pred: PartPred, Res: CmpOut, Op0: LHSPartRegs[I],
1960 Op1: RHSPartRegs[I]);
1961 } else {
1962 auto Cmp = MIRBuilder.buildICmp(Pred: PartPred, Res: ResTy, Op0: LHSPartRegs[I],
1963 Op1: RHSPartRegs[I]);
1964 auto CmpEq = MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: ResTy,
1965 Op0: LHSPartRegs[I], Op1: RHSPartRegs[I]);
1966 MIRBuilder.buildSelect(Res: CmpOut, Tst: CmpEq, Op0: CmpIn, Op1: Cmp);
1967 }
1968
1969 CmpIn = CmpOut;
1970 }
1971
1972 for (unsigned I = 0, E = LHSLeftoverRegs.size(); I != E; ++I) {
1973 Register CmpOut;
1974 CmpInst::Predicate PartPred;
1975
1976 if (I == E - 1) {
1977 PartPred = Pred;
1978 CmpOut = Dst;
1979 } else {
1980 PartPred = ICmpInst::getUnsignedPredicate(Pred);
1981 CmpOut = MRI.createGenericVirtualRegister(Ty: ResTy);
1982 }
1983
1984 if (!CmpIn) {
1985 MIRBuilder.buildICmp(Pred: PartPred, Res: CmpOut, Op0: LHSLeftoverRegs[I],
1986 Op1: RHSLeftoverRegs[I]);
1987 } else {
1988 auto Cmp = MIRBuilder.buildICmp(Pred: PartPred, Res: ResTy, Op0: LHSLeftoverRegs[I],
1989 Op1: RHSLeftoverRegs[I]);
1990 auto CmpEq =
1991 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: ResTy,
1992 Op0: LHSLeftoverRegs[I], Op1: RHSLeftoverRegs[I]);
1993 MIRBuilder.buildSelect(Res: CmpOut, Tst: CmpEq, Op0: CmpIn, Op1: Cmp);
1994 }
1995
1996 CmpIn = CmpOut;
1997 }
1998 }
1999 MI.eraseFromParent();
2000 return Legalized;
2001 }
2002 case TargetOpcode::G_FCMP:
2003 if (TypeIdx != 0)
2004 return UnableToLegalize;
2005
2006 Observer.changingInstr(MI);
2007 narrowScalarDst(MI, NarrowTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_ZEXT);
2008 Observer.changedInstr(MI);
2009 return Legalized;
2010
2011 case TargetOpcode::G_SEXT_INREG: {
2012 if (TypeIdx != 0)
2013 return UnableToLegalize;
2014
2015 int64_t SizeInBits = MI.getOperand(i: 2).getImm();
2016
2017 // So long as the new type has more bits than the bits we're extending we
2018 // don't need to break it apart.
2019 if (NarrowTy.getScalarSizeInBits() > SizeInBits) {
2020 Observer.changingInstr(MI);
2021 // We don't lose any non-extension bits by truncating the src and
2022 // sign-extending the dst.
2023 MachineOperand &MO1 = MI.getOperand(i: 1);
2024 auto TruncMIB = MIRBuilder.buildTrunc(Res: NarrowTy, Op: MO1);
2025 MO1.setReg(TruncMIB.getReg(Idx: 0));
2026
2027 MachineOperand &MO2 = MI.getOperand(i: 0);
2028 Register DstExt = MRI.createGenericVirtualRegister(Ty: NarrowTy);
2029 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2030 MIRBuilder.buildSExt(Res: MO2, Op: DstExt);
2031 MO2.setReg(DstExt);
2032 Observer.changedInstr(MI);
2033 return Legalized;
2034 }
2035
2036 // Break it apart. Components below the extension point are unmodified. The
2037 // component containing the extension point becomes a narrower SEXT_INREG.
2038 // Components above it are ashr'd from the component containing the
2039 // extension point.
2040 if (SizeOp0 % NarrowSize != 0)
2041 return UnableToLegalize;
2042 int NumParts = SizeOp0 / NarrowSize;
2043
2044 // List the registers where the destination will be scattered.
2045 SmallVector<Register, 2> DstRegs;
2046 // List the registers where the source will be split.
2047 SmallVector<Register, 2> SrcRegs;
2048
2049 // Create all the temporary registers.
2050 for (int i = 0; i < NumParts; ++i) {
2051 Register SrcReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
2052
2053 SrcRegs.push_back(Elt: SrcReg);
2054 }
2055
2056 // Explode the big arguments into smaller chunks.
2057 MIRBuilder.buildUnmerge(Res: SrcRegs, Op: MI.getOperand(i: 1));
2058
2059 Register AshrCstReg =
2060 MIRBuilder.buildConstant(Res: NarrowTy, Val: NarrowTy.getScalarSizeInBits() - 1)
2061 .getReg(Idx: 0);
2062 Register FullExtensionReg;
2063 Register PartialExtensionReg;
2064
2065 // Do the operation on each small part.
2066 for (int i = 0; i < NumParts; ++i) {
2067 if ((i + 1) * NarrowTy.getScalarSizeInBits() <= SizeInBits) {
2068 DstRegs.push_back(Elt: SrcRegs[i]);
2069 PartialExtensionReg = DstRegs.back();
2070 } else if (i * NarrowTy.getScalarSizeInBits() >= SizeInBits) {
2071 assert(PartialExtensionReg &&
2072 "Expected to visit partial extension before full");
2073 if (FullExtensionReg) {
2074 DstRegs.push_back(Elt: FullExtensionReg);
2075 continue;
2076 }
2077 DstRegs.push_back(
2078 Elt: MIRBuilder.buildAShr(Dst: NarrowTy, Src0: PartialExtensionReg, Src1: AshrCstReg)
2079 .getReg(Idx: 0));
2080 FullExtensionReg = DstRegs.back();
2081 } else {
2082 DstRegs.push_back(
2083 Elt: MIRBuilder
2084 .buildInstr(
2085 Opc: TargetOpcode::G_SEXT_INREG, DstOps: {NarrowTy},
2086 SrcOps: {SrcRegs[i], SizeInBits % NarrowTy.getScalarSizeInBits()})
2087 .getReg(Idx: 0));
2088 PartialExtensionReg = DstRegs.back();
2089 }
2090 }
2091
2092 // Gather the destination registers into the final destination.
2093 Register DstReg = MI.getOperand(i: 0).getReg();
2094 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
2095 MI.eraseFromParent();
2096 return Legalized;
2097 }
2098 case TargetOpcode::G_BSWAP:
2099 case TargetOpcode::G_BITREVERSE: {
2100 if (SizeOp0 % NarrowSize != 0)
2101 return UnableToLegalize;
2102
2103 Observer.changingInstr(MI);
2104 SmallVector<Register, 2> SrcRegs, DstRegs;
2105 unsigned NumParts = SizeOp0 / NarrowSize;
2106 extractParts(Reg: MI.getOperand(i: 1).getReg(), Ty: NarrowTy, NumParts, VRegs&: SrcRegs,
2107 MIRBuilder, MRI);
2108
2109 for (unsigned i = 0; i < NumParts; ++i) {
2110 auto DstPart = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {NarrowTy},
2111 SrcOps: {SrcRegs[NumParts - 1 - i]});
2112 DstRegs.push_back(Elt: DstPart.getReg(Idx: 0));
2113 }
2114
2115 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0), Ops: DstRegs);
2116
2117 Observer.changedInstr(MI);
2118 MI.eraseFromParent();
2119 return Legalized;
2120 }
2121 case TargetOpcode::G_PTR_ADD:
2122 case TargetOpcode::G_PTRMASK: {
2123 if (TypeIdx != 1)
2124 return UnableToLegalize;
2125 Observer.changingInstr(MI);
2126 narrowScalarSrc(MI, NarrowTy, OpIdx: 2);
2127 Observer.changedInstr(MI);
2128 return Legalized;
2129 }
2130 case TargetOpcode::G_FPTOUI:
2131 case TargetOpcode::G_FPTOSI:
2132 case TargetOpcode::G_FPTOUI_SAT:
2133 case TargetOpcode::G_FPTOSI_SAT:
2134 return narrowScalarFPTOI(MI, TypeIdx, Ty: NarrowTy);
2135 case TargetOpcode::G_FPEXT:
2136 if (TypeIdx != 0)
2137 return UnableToLegalize;
2138 Observer.changingInstr(MI);
2139 narrowScalarDst(MI, NarrowTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_FPEXT);
2140 Observer.changedInstr(MI);
2141 return Legalized;
2142 case TargetOpcode::G_FLDEXP:
2143 case TargetOpcode::G_STRICT_FLDEXP:
2144 return narrowScalarFLDEXP(MI, TypeIdx, Ty: NarrowTy);
2145 case TargetOpcode::G_VSCALE: {
2146 Register Dst = MI.getOperand(i: 0).getReg();
2147 LLT Ty = MRI.getType(Reg: Dst);
2148
2149 // Assume VSCALE(1) fits into a legal integer
2150 const APInt One(NarrowTy.getSizeInBits(), 1);
2151 auto VScaleBase = MIRBuilder.buildVScale(Res: NarrowTy, MinElts: One);
2152 auto ZExt = MIRBuilder.buildZExt(Res: Ty, Op: VScaleBase);
2153 auto C = MIRBuilder.buildConstant(Res: Ty, Val: *MI.getOperand(i: 1).getCImm());
2154 MIRBuilder.buildMul(Dst, Src0: ZExt, Src1: C);
2155
2156 MI.eraseFromParent();
2157 return Legalized;
2158 }
2159 }
2160}
2161
2162Register LegalizerHelper::coerceToInteger(Register Val) {
2163 LLT Ty = MRI.getType(Reg: Val);
2164 if (Ty.isScalar() && !Ty.isFloat())
2165 return Val;
2166
2167 const DataLayout &DL = MIRBuilder.getDataLayout();
2168 LLT NewTy = LLT::integer(SizeInBits: Ty.getSizeInBits());
2169
2170 if (Ty.isFloat())
2171 return MIRBuilder.buildBitcast(Dst: NewTy, Src: Val).getReg(Idx: 0);
2172
2173 if (Ty.isPointer()) {
2174 if (DL.isNonIntegralAddressSpace(AddrSpace: Ty.getAddressSpace()))
2175 return Register();
2176 return MIRBuilder.buildPtrToInt(Dst: NewTy, Src: Val).getReg(Idx: 0);
2177 }
2178
2179 Register NewVal = Val;
2180
2181 assert(Ty.isVector());
2182 if (Ty.isPointerVector())
2183 NewVal = MIRBuilder.buildPtrToInt(Dst: NewTy, Src: NewVal).getReg(Idx: 0);
2184 return MIRBuilder.buildBitcast(Dst: NewTy, Src: NewVal).getReg(Idx: 0);
2185}
2186
2187void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy,
2188 unsigned OpIdx, unsigned ExtOpcode) {
2189 MachineOperand &MO = MI.getOperand(i: OpIdx);
2190 auto ExtB = MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {WideTy}, SrcOps: {MO});
2191 MO.setReg(ExtB.getReg(Idx: 0));
2192}
2193
2194void LegalizerHelper::widenScalarSrcUsingFPExt(MachineInstr &MI, LLT WideTy,
2195 unsigned OpIdx) {
2196 MachineOperand &MO = MI.getOperand(i: OpIdx);
2197 auto ExtB = MIRBuilder.buildInstr(Opc: TargetOpcode::G_FPEXT, DstOps: {WideTy}, SrcOps: {MO},
2198 Flags: MI.getFlags());
2199 MO.setReg(ExtB.getReg(Idx: 0));
2200}
2201
2202void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy,
2203 unsigned OpIdx) {
2204 MachineOperand &MO = MI.getOperand(i: OpIdx);
2205 auto ExtB = MIRBuilder.buildTrunc(Res: NarrowTy, Op: MO);
2206 MO.setReg(ExtB.getReg(Idx: 0));
2207}
2208
2209void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy,
2210 unsigned OpIdx, unsigned TruncOpcode) {
2211 MachineOperand &MO = MI.getOperand(i: OpIdx);
2212 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2213 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2214 MIRBuilder.buildInstr(Opc: TruncOpcode, DstOps: {MO}, SrcOps: {DstExt});
2215 MO.setReg(DstExt);
2216}
2217
2218void LegalizerHelper::widenScalarDstUsingFPTrunc(MachineInstr &MI, LLT WideTy,
2219 unsigned OpIdx) {
2220 MachineOperand &MO = MI.getOperand(i: OpIdx);
2221 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2222 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2223 MIRBuilder.buildInstr(Opc: TargetOpcode::G_FPTRUNC, DstOps: {MO}, SrcOps: {DstExt}, Flags: MI.getFlags());
2224 MO.setReg(DstExt);
2225}
2226
2227void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy,
2228 unsigned OpIdx, unsigned ExtOpcode) {
2229 MachineOperand &MO = MI.getOperand(i: OpIdx);
2230 Register DstTrunc = MRI.createGenericVirtualRegister(Ty: NarrowTy);
2231 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2232 MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {MO}, SrcOps: {DstTrunc});
2233 MO.setReg(DstTrunc);
2234}
2235
2236void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy,
2237 unsigned OpIdx) {
2238 MachineOperand &MO = MI.getOperand(i: OpIdx);
2239 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2240 Register Dst = MO.getReg();
2241 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2242 MO.setReg(DstExt);
2243 MIRBuilder.buildDeleteTrailingVectorElements(Res: Dst, Op0: DstExt);
2244}
2245
2246void LegalizerHelper::moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy,
2247 unsigned OpIdx) {
2248 MachineOperand &MO = MI.getOperand(i: OpIdx);
2249 MO.setReg(MIRBuilder.buildPadVectorWithUndefElements(Res: MoreTy, Op0: MO).getReg(Idx: 0));
2250}
2251
2252void LegalizerHelper::bitcastSrc(MachineInstr &MI, LLT CastTy, unsigned OpIdx) {
2253 MachineOperand &Op = MI.getOperand(i: OpIdx);
2254 Op.setReg(MIRBuilder.buildBitcast(Dst: CastTy, Src: Op).getReg(Idx: 0));
2255}
2256
2257void LegalizerHelper::bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx) {
2258 MachineOperand &MO = MI.getOperand(i: OpIdx);
2259 Register CastDst = MRI.createGenericVirtualRegister(Ty: CastTy);
2260 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2261 MIRBuilder.buildBitcast(Dst: MO, Src: CastDst);
2262 MO.setReg(CastDst);
2263}
2264
2265LegalizerHelper::LegalizeResult
2266LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx,
2267 LLT WideTy) {
2268 if (TypeIdx != 1)
2269 return UnableToLegalize;
2270
2271 auto [DstReg, DstTy, Src1Reg, Src1Ty] = MI.getFirst2RegLLTs();
2272 if (DstTy.isVector())
2273 return UnableToLegalize;
2274
2275 LLT SrcTy = MRI.getType(Reg: Src1Reg);
2276 const int DstSize = DstTy.getSizeInBits();
2277 const int SrcSize = SrcTy.getSizeInBits();
2278 const int WideSize = WideTy.getSizeInBits();
2279 const int NumMerge = (DstSize + WideSize - 1) / WideSize;
2280
2281 unsigned NumOps = MI.getNumOperands();
2282 unsigned NumSrc = MI.getNumOperands() - 1;
2283 unsigned PartSize = DstTy.getSizeInBits() / NumSrc;
2284
2285 if (WideSize >= DstSize) {
2286 // Directly pack the bits in the target type.
2287 Register ResultReg = MIRBuilder.buildZExt(Res: WideTy, Op: Src1Reg).getReg(Idx: 0);
2288
2289 for (unsigned I = 2; I != NumOps; ++I) {
2290 const unsigned Offset = (I - 1) * PartSize;
2291
2292 Register SrcReg = MI.getOperand(i: I).getReg();
2293 assert(MRI.getType(SrcReg) == LLT::scalar(PartSize));
2294
2295 auto ZextInput = MIRBuilder.buildZExt(Res: WideTy, Op: SrcReg);
2296
2297 Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg :
2298 MRI.createGenericVirtualRegister(Ty: WideTy);
2299
2300 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: Offset);
2301 auto Shl = MIRBuilder.buildShl(Dst: WideTy, Src0: ZextInput, Src1: ShiftAmt);
2302 MIRBuilder.buildOr(Dst: NextResult, Src0: ResultReg, Src1: Shl);
2303 ResultReg = NextResult;
2304 }
2305
2306 if (WideSize > DstSize)
2307 MIRBuilder.buildTrunc(Res: DstReg, Op: ResultReg);
2308 else if (DstTy.isPointer())
2309 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: ResultReg);
2310 else if (DstTy != WideTy)
2311 MIRBuilder.buildBitcast(Dst: DstReg, Src: ResultReg);
2312
2313 MI.eraseFromParent();
2314 return Legalized;
2315 }
2316
2317 // Unmerge the original values to the GCD type, and recombine to the next
2318 // multiple greater than the original type.
2319 //
2320 // %3:_(s12) = G_MERGE_VALUES %0:_(s4), %1:_(s4), %2:_(s4) -> s6
2321 // %4:_(s2), %5:_(s2) = G_UNMERGE_VALUES %0
2322 // %6:_(s2), %7:_(s2) = G_UNMERGE_VALUES %1
2323 // %8:_(s2), %9:_(s2) = G_UNMERGE_VALUES %2
2324 // %10:_(s6) = G_MERGE_VALUES %4, %5, %6
2325 // %11:_(s6) = G_MERGE_VALUES %7, %8, %9
2326 // %12:_(s12) = G_MERGE_VALUES %10, %11
2327 //
2328 // Padding with undef if necessary:
2329 //
2330 // %2:_(s8) = G_MERGE_VALUES %0:_(s4), %1:_(s4) -> s6
2331 // %3:_(s2), %4:_(s2) = G_UNMERGE_VALUES %0
2332 // %5:_(s2), %6:_(s2) = G_UNMERGE_VALUES %1
2333 // %7:_(s2) = G_IMPLICIT_DEF
2334 // %8:_(s6) = G_MERGE_VALUES %3, %4, %5
2335 // %9:_(s6) = G_MERGE_VALUES %6, %7, %7
2336 // %10:_(s12) = G_MERGE_VALUES %8, %9
2337
2338 const int GCD = std::gcd(m: SrcSize, n: WideSize);
2339 LLT GCDTy = WideTy.changeElementSize(NewEltSize: GCD);
2340
2341 SmallVector<Register, 8> NewMergeRegs;
2342 SmallVector<Register, 8> Unmerges;
2343 LLT WideDstTy = WideTy.changeElementSize(NewEltSize: NumMerge * WideSize);
2344
2345 // Decompose the original operands if they don't evenly divide.
2346 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands())) {
2347 Register SrcReg = MO.getReg();
2348 if (GCD == SrcSize) {
2349 Unmerges.push_back(Elt: SrcReg);
2350 } else {
2351 auto Unmerge = MIRBuilder.buildUnmerge(Res: GCDTy, Op: SrcReg);
2352 for (int J = 0, JE = Unmerge->getNumOperands() - 1; J != JE; ++J)
2353 Unmerges.push_back(Elt: Unmerge.getReg(Idx: J));
2354 }
2355 }
2356
2357 // Pad with undef to the next size that is a multiple of the requested size.
2358 if (static_cast<int>(Unmerges.size()) != NumMerge * WideSize) {
2359 Register UndefReg = MIRBuilder.buildUndef(Res: GCDTy).getReg(Idx: 0);
2360 for (int I = Unmerges.size(); I != NumMerge * WideSize; ++I)
2361 Unmerges.push_back(Elt: UndefReg);
2362 }
2363
2364 const int PartsPerGCD = WideSize / GCD;
2365
2366 // Build merges of each piece.
2367 ArrayRef<Register> Slicer(Unmerges);
2368 for (int I = 0; I != NumMerge; ++I, Slicer = Slicer.drop_front(N: PartsPerGCD)) {
2369 auto Merge =
2370 MIRBuilder.buildMergeLikeInstr(Res: WideTy, Ops: Slicer.take_front(N: PartsPerGCD));
2371 NewMergeRegs.push_back(Elt: Merge.getReg(Idx: 0));
2372 }
2373
2374 // A truncate may be necessary if the requested type doesn't evenly divide the
2375 // original result type.
2376 if (DstTy.getSizeInBits() == WideDstTy.getSizeInBits()) {
2377 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: NewMergeRegs);
2378 } else {
2379 auto FinalMerge = MIRBuilder.buildMergeLikeInstr(Res: WideDstTy, Ops: NewMergeRegs);
2380 MIRBuilder.buildTrunc(Res: DstReg, Op: FinalMerge.getReg(Idx: 0));
2381 }
2382
2383 MI.eraseFromParent();
2384 return Legalized;
2385}
2386
2387LegalizerHelper::LegalizeResult
2388LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx,
2389 LLT WideTy) {
2390 if (TypeIdx != 0)
2391 return UnableToLegalize;
2392
2393 int NumDst = MI.getNumOperands() - 1;
2394 Register SrcReg = MI.getOperand(i: NumDst).getReg();
2395 LLT SrcTy = MRI.getType(Reg: SrcReg);
2396 if (SrcTy.isVector())
2397 return UnableToLegalize;
2398
2399 Register Dst0Reg = MI.getOperand(i: 0).getReg();
2400 LLT DstTy = MRI.getType(Reg: Dst0Reg);
2401 if (!DstTy.isScalar())
2402 return UnableToLegalize;
2403
2404 if (WideTy.getSizeInBits() >= SrcTy.getSizeInBits()) {
2405 if (SrcTy.isPointer()) {
2406 const DataLayout &DL = MIRBuilder.getDataLayout();
2407 if (DL.isNonIntegralAddressSpace(AddrSpace: SrcTy.getAddressSpace())) {
2408 LLVM_DEBUG(
2409 dbgs() << "Not casting non-integral address space integer\n");
2410 return UnableToLegalize;
2411 }
2412
2413 SrcTy = LLT::scalar(SizeInBits: SrcTy.getSizeInBits());
2414 SrcReg = MIRBuilder.buildPtrToInt(Dst: SrcTy, Src: SrcReg).getReg(Idx: 0);
2415 }
2416
2417 // Widen SrcTy to WideTy. This does not affect the result, but since the
2418 // user requested this size, it is probably better handled than SrcTy and
2419 // should reduce the total number of legalization artifacts.
2420 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
2421 SrcTy = WideTy;
2422 SrcReg = MIRBuilder.buildAnyExt(Res: WideTy, Op: SrcReg).getReg(Idx: 0);
2423 }
2424
2425 // Theres no unmerge type to target. Directly extract the bits from the
2426 // source type
2427 unsigned DstSize = DstTy.getSizeInBits();
2428
2429 if (SrcTy.isFloat()) {
2430 SrcReg = coerceToInteger(Val: SrcReg);
2431 SrcTy = MRI.getType(Reg: SrcReg);
2432 }
2433
2434 MIRBuilder.buildTrunc(Res: Dst0Reg, Op: SrcReg);
2435 for (int I = 1; I != NumDst; ++I) {
2436 auto ShiftAmt = MIRBuilder.buildConstant(Res: SrcTy, Val: DstSize * I);
2437 auto Shr = MIRBuilder.buildLShr(Dst: SrcTy, Src0: SrcReg, Src1: ShiftAmt);
2438 MIRBuilder.buildTrunc(Res: MI.getOperand(i: I), Op: Shr);
2439 }
2440
2441 MI.eraseFromParent();
2442 return Legalized;
2443 }
2444
2445 // Extend the source to a wider type.
2446 LLT LCMTy = getLCMType(OrigTy: SrcTy, TargetTy: WideTy);
2447
2448 Register WideSrc = SrcReg;
2449 if (LCMTy.getSizeInBits() != SrcTy.getSizeInBits()) {
2450 // TODO: If this is an integral address space, cast to integer and anyext.
2451 if (SrcTy.isPointer()) {
2452 LLVM_DEBUG(dbgs() << "Widening pointer source types not implemented\n");
2453 return UnableToLegalize;
2454 }
2455
2456 WideSrc = MIRBuilder.buildAnyExt(Res: LCMTy, Op: WideSrc).getReg(Idx: 0);
2457 }
2458
2459 auto Unmerge = MIRBuilder.buildUnmerge(Res: WideTy, Op: WideSrc);
2460
2461 // Create a sequence of unmerges and merges to the original results. Since we
2462 // may have widened the source, we will need to pad the results with dead defs
2463 // to cover the source register.
2464 // e.g. widen s48 to s64:
2465 // %1:_(s48), %2:_(s48) = G_UNMERGE_VALUES %0:_(s96)
2466 //
2467 // =>
2468 // %4:_(s192) = G_ANYEXT %0:_(s96)
2469 // %5:_(s64), %6, %7 = G_UNMERGE_VALUES %4 ; Requested unmerge
2470 // ; unpack to GCD type, with extra dead defs
2471 // %8:_(s16), %9, %10, %11 = G_UNMERGE_VALUES %5:_(s64)
2472 // %12:_(s16), %13, dead %14, dead %15 = G_UNMERGE_VALUES %6:_(s64)
2473 // dead %16:_(s16), dead %17, dead %18, dead %18 = G_UNMERGE_VALUES %7:_(s64)
2474 // %1:_(s48) = G_MERGE_VALUES %8:_(s16), %9, %10 ; Remerge to destination
2475 // %2:_(s48) = G_MERGE_VALUES %11:_(s16), %12, %13 ; Remerge to destination
2476 const LLT GCDTy = getGCDType(OrigTy: WideTy, TargetTy: DstTy);
2477 const int NumUnmerge = Unmerge->getNumOperands() - 1;
2478 const int PartsPerRemerge = DstTy.getSizeInBits() / GCDTy.getSizeInBits();
2479
2480 // Directly unmerge to the destination without going through a GCD type
2481 // if possible
2482 if (PartsPerRemerge == 1) {
2483 const int PartsPerUnmerge = WideTy.getSizeInBits() / DstTy.getSizeInBits();
2484
2485 for (int I = 0; I != NumUnmerge; ++I) {
2486 auto MIB = MIRBuilder.buildInstr(Opcode: TargetOpcode::G_UNMERGE_VALUES);
2487
2488 for (int J = 0; J != PartsPerUnmerge; ++J) {
2489 int Idx = I * PartsPerUnmerge + J;
2490 if (Idx < NumDst)
2491 MIB.addDef(RegNo: MI.getOperand(i: Idx).getReg());
2492 else {
2493 // Create dead def for excess components.
2494 MIB.addDef(RegNo: MRI.createGenericVirtualRegister(Ty: DstTy));
2495 }
2496 }
2497
2498 MIB.addUse(RegNo: Unmerge.getReg(Idx: I));
2499 }
2500 } else {
2501 SmallVector<Register, 16> Parts;
2502 for (int J = 0; J != NumUnmerge; ++J)
2503 extractGCDType(Parts, GCDTy, SrcReg: Unmerge.getReg(Idx: J));
2504
2505 SmallVector<Register, 8> RemergeParts;
2506 for (int I = 0; I != NumDst; ++I) {
2507 for (int J = 0; J < PartsPerRemerge; ++J) {
2508 const int Idx = I * PartsPerRemerge + J;
2509 RemergeParts.emplace_back(Args&: Parts[Idx]);
2510 }
2511
2512 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: I).getReg(), Ops: RemergeParts);
2513 RemergeParts.clear();
2514 }
2515 }
2516
2517 MI.eraseFromParent();
2518 return Legalized;
2519}
2520
2521LegalizerHelper::LegalizeResult
2522LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx,
2523 LLT WideTy) {
2524 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
2525 unsigned Offset = MI.getOperand(i: 2).getImm();
2526
2527 if (TypeIdx == 0) {
2528 if (SrcTy.isVector() || DstTy.isVector())
2529 return UnableToLegalize;
2530
2531 SrcOp Src(SrcReg);
2532 if (SrcTy.isPointer()) {
2533 // Extracts from pointers can be handled only if they are really just
2534 // simple integers.
2535 const DataLayout &DL = MIRBuilder.getDataLayout();
2536 if (DL.isNonIntegralAddressSpace(AddrSpace: SrcTy.getAddressSpace()))
2537 return UnableToLegalize;
2538
2539 LLT SrcAsIntTy = LLT::scalar(SizeInBits: SrcTy.getSizeInBits());
2540 Src = MIRBuilder.buildPtrToInt(Dst: SrcAsIntTy, Src);
2541 SrcTy = SrcAsIntTy;
2542 }
2543
2544 if (DstTy.isPointer())
2545 return UnableToLegalize;
2546
2547 if (Offset == 0) {
2548 // Avoid a shift in the degenerate case.
2549 MIRBuilder.buildTrunc(Res: DstReg,
2550 Op: MIRBuilder.buildAnyExtOrTrunc(Res: WideTy, Op: Src));
2551 MI.eraseFromParent();
2552 return Legalized;
2553 }
2554
2555 // Do a shift in the source type.
2556 LLT ShiftTy = SrcTy;
2557 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
2558 Src = MIRBuilder.buildAnyExt(Res: WideTy, Op: Src);
2559 ShiftTy = WideTy;
2560 }
2561
2562 auto LShr = MIRBuilder.buildLShr(
2563 Dst: ShiftTy, Src0: Src, Src1: MIRBuilder.buildConstant(Res: ShiftTy, Val: Offset));
2564 MIRBuilder.buildTrunc(Res: DstReg, Op: LShr);
2565 MI.eraseFromParent();
2566 return Legalized;
2567 }
2568
2569 if (SrcTy.isScalar()) {
2570 Observer.changingInstr(MI);
2571 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2572 Observer.changedInstr(MI);
2573 return Legalized;
2574 }
2575
2576 if (!SrcTy.isVector())
2577 return UnableToLegalize;
2578
2579 if (DstTy != SrcTy.getElementType())
2580 return UnableToLegalize;
2581
2582 if (Offset % SrcTy.getScalarSizeInBits() != 0)
2583 return UnableToLegalize;
2584
2585 Observer.changingInstr(MI);
2586 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2587
2588 MI.getOperand(i: 2).setImm((WideTy.getSizeInBits() / SrcTy.getSizeInBits()) *
2589 Offset);
2590 widenScalarDst(MI, WideTy: WideTy.getScalarType(), OpIdx: 0);
2591 Observer.changedInstr(MI);
2592 return Legalized;
2593}
2594
2595LegalizerHelper::LegalizeResult
2596LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx,
2597 LLT WideTy) {
2598 if (TypeIdx != 0 || WideTy.isVector())
2599 return UnableToLegalize;
2600 Observer.changingInstr(MI);
2601 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2602 widenScalarDst(MI, WideTy);
2603 Observer.changedInstr(MI);
2604 return Legalized;
2605}
2606
2607LegalizerHelper::LegalizeResult
2608LegalizerHelper::widenScalarAddSubOverflow(MachineInstr &MI, unsigned TypeIdx,
2609 LLT WideTy) {
2610 unsigned Opcode;
2611 unsigned ExtOpcode;
2612 std::optional<Register> CarryIn;
2613 switch (MI.getOpcode()) {
2614 default:
2615 llvm_unreachable("Unexpected opcode!");
2616 case TargetOpcode::G_SADDO:
2617 Opcode = TargetOpcode::G_ADD;
2618 ExtOpcode = TargetOpcode::G_SEXT;
2619 break;
2620 case TargetOpcode::G_SSUBO:
2621 Opcode = TargetOpcode::G_SUB;
2622 ExtOpcode = TargetOpcode::G_SEXT;
2623 break;
2624 case TargetOpcode::G_UADDO:
2625 Opcode = TargetOpcode::G_ADD;
2626 ExtOpcode = TargetOpcode::G_ZEXT;
2627 break;
2628 case TargetOpcode::G_USUBO:
2629 Opcode = TargetOpcode::G_SUB;
2630 ExtOpcode = TargetOpcode::G_ZEXT;
2631 break;
2632 case TargetOpcode::G_SADDE:
2633 Opcode = TargetOpcode::G_UADDE;
2634 ExtOpcode = TargetOpcode::G_SEXT;
2635 CarryIn = MI.getOperand(i: 4).getReg();
2636 break;
2637 case TargetOpcode::G_SSUBE:
2638 Opcode = TargetOpcode::G_USUBE;
2639 ExtOpcode = TargetOpcode::G_SEXT;
2640 CarryIn = MI.getOperand(i: 4).getReg();
2641 break;
2642 case TargetOpcode::G_UADDE:
2643 Opcode = TargetOpcode::G_UADDE;
2644 ExtOpcode = TargetOpcode::G_ZEXT;
2645 CarryIn = MI.getOperand(i: 4).getReg();
2646 break;
2647 case TargetOpcode::G_USUBE:
2648 Opcode = TargetOpcode::G_USUBE;
2649 ExtOpcode = TargetOpcode::G_ZEXT;
2650 CarryIn = MI.getOperand(i: 4).getReg();
2651 break;
2652 }
2653
2654 if (TypeIdx == 1) {
2655 unsigned BoolExtOp = MIRBuilder.getBoolExtOp(IsVec: WideTy.isVector(), IsFP: false);
2656
2657 Observer.changingInstr(MI);
2658 if (CarryIn)
2659 widenScalarSrc(MI, WideTy, OpIdx: 4, ExtOpcode: BoolExtOp);
2660 widenScalarDst(MI, WideTy, OpIdx: 1);
2661
2662 Observer.changedInstr(MI);
2663 return Legalized;
2664 }
2665
2666 auto LHSExt = MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {WideTy}, SrcOps: {MI.getOperand(i: 2)});
2667 auto RHSExt = MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {WideTy}, SrcOps: {MI.getOperand(i: 3)});
2668 // Do the arithmetic in the larger type.
2669 Register NewOp;
2670 if (CarryIn) {
2671 LLT CarryOutTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
2672 NewOp = MIRBuilder
2673 .buildInstr(Opc: Opcode, DstOps: {WideTy, CarryOutTy},
2674 SrcOps: {LHSExt, RHSExt, *CarryIn})
2675 .getReg(Idx: 0);
2676 } else {
2677 NewOp = MIRBuilder.buildInstr(Opc: Opcode, DstOps: {WideTy}, SrcOps: {LHSExt, RHSExt}).getReg(Idx: 0);
2678 }
2679 LLT OrigTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2680 auto TruncOp = MIRBuilder.buildTrunc(Res: OrigTy, Op: NewOp);
2681 auto ExtOp = MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {WideTy}, SrcOps: {TruncOp});
2682 // There is no overflow if the ExtOp is the same as NewOp.
2683 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: MI.getOperand(i: 1), Op0: NewOp, Op1: ExtOp);
2684 // Now trunc the NewOp to the original result.
2685 MIRBuilder.buildTrunc(Res: MI.getOperand(i: 0), Op: NewOp);
2686 MI.eraseFromParent();
2687 return Legalized;
2688}
2689
2690LegalizerHelper::LegalizeResult
2691LegalizerHelper::widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx,
2692 LLT WideTy) {
2693 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SADDSAT ||
2694 MI.getOpcode() == TargetOpcode::G_SSUBSAT ||
2695 MI.getOpcode() == TargetOpcode::G_SSHLSAT;
2696 bool IsShift = MI.getOpcode() == TargetOpcode::G_SSHLSAT ||
2697 MI.getOpcode() == TargetOpcode::G_USHLSAT;
2698 // We can convert this to:
2699 // 1. Any extend iN to iM
2700 // 2. SHL by M-N
2701 // 3. [US][ADD|SUB|SHL]SAT
2702 // 4. L/ASHR by M-N
2703 //
2704 // It may be more efficient to lower this to a min and a max operation in
2705 // the higher precision arithmetic if the promoted operation isn't legal,
2706 // but this decision is up to the target's lowering request.
2707 Register DstReg = MI.getOperand(i: 0).getReg();
2708
2709 unsigned NewBits = WideTy.getScalarSizeInBits();
2710 unsigned SHLAmount = NewBits - MRI.getType(Reg: DstReg).getScalarSizeInBits();
2711
2712 // Shifts must zero-extend the RHS to preserve the unsigned quantity, and
2713 // must not left shift the RHS to preserve the shift amount.
2714 auto LHS = MIRBuilder.buildAnyExt(Res: WideTy, Op: MI.getOperand(i: 1));
2715 auto RHS = IsShift ? MIRBuilder.buildZExt(Res: WideTy, Op: MI.getOperand(i: 2))
2716 : MIRBuilder.buildAnyExt(Res: WideTy, Op: MI.getOperand(i: 2));
2717 auto ShiftK = MIRBuilder.buildConstant(Res: WideTy, Val: SHLAmount);
2718 auto ShiftL = MIRBuilder.buildShl(Dst: WideTy, Src0: LHS, Src1: ShiftK);
2719 auto ShiftR = IsShift ? RHS : MIRBuilder.buildShl(Dst: WideTy, Src0: RHS, Src1: ShiftK);
2720
2721 auto WideInst = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {WideTy},
2722 SrcOps: {ShiftL, ShiftR}, Flags: MI.getFlags());
2723
2724 // Use a shift that will preserve the number of sign bits when the trunc is
2725 // folded away.
2726 auto Result = IsSigned ? MIRBuilder.buildAShr(Dst: WideTy, Src0: WideInst, Src1: ShiftK)
2727 : MIRBuilder.buildLShr(Dst: WideTy, Src0: WideInst, Src1: ShiftK);
2728
2729 MIRBuilder.buildTrunc(Res: DstReg, Op: Result);
2730 MI.eraseFromParent();
2731 return Legalized;
2732}
2733
2734LegalizerHelper::LegalizeResult
2735LegalizerHelper::widenScalarMulo(MachineInstr &MI, unsigned TypeIdx,
2736 LLT WideTy) {
2737 if (TypeIdx == 1) {
2738 Observer.changingInstr(MI);
2739 widenScalarDst(MI, WideTy, OpIdx: 1);
2740 Observer.changedInstr(MI);
2741 return Legalized;
2742 }
2743
2744 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SMULO;
2745 auto [Result, OriginalOverflow, LHS, RHS] = MI.getFirst4Regs();
2746 LLT SrcTy = MRI.getType(Reg: LHS);
2747 LLT OverflowTy = MRI.getType(Reg: OriginalOverflow);
2748 unsigned SrcBitWidth = SrcTy.getScalarSizeInBits();
2749
2750 // To determine if the result overflowed in the larger type, we extend the
2751 // input to the larger type, do the multiply (checking if it overflows),
2752 // then also check the high bits of the result to see if overflow happened
2753 // there.
2754 unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
2755 auto LeftOperand = MIRBuilder.buildInstr(Opc: ExtOp, DstOps: {WideTy}, SrcOps: {LHS});
2756 auto RightOperand = MIRBuilder.buildInstr(Opc: ExtOp, DstOps: {WideTy}, SrcOps: {RHS});
2757
2758 // Multiplication cannot overflow if the WideTy is >= 2 * original width,
2759 // so we don't need to check the overflow result of larger type Mulo.
2760 bool WideMulCanOverflow = WideTy.getScalarSizeInBits() < 2 * SrcBitWidth;
2761
2762 unsigned MulOpc =
2763 WideMulCanOverflow ? MI.getOpcode() : (unsigned)TargetOpcode::G_MUL;
2764
2765 MachineInstrBuilder Mulo;
2766 if (WideMulCanOverflow)
2767 Mulo = MIRBuilder.buildInstr(Opc: MulOpc, DstOps: {WideTy, OverflowTy},
2768 SrcOps: {LeftOperand, RightOperand});
2769 else
2770 Mulo = MIRBuilder.buildInstr(Opc: MulOpc, DstOps: {WideTy}, SrcOps: {LeftOperand, RightOperand});
2771
2772 auto Mul = Mulo->getOperand(i: 0);
2773 MIRBuilder.buildTrunc(Res: Result, Op: Mul);
2774
2775 MachineInstrBuilder ExtResult;
2776 // Overflow occurred if it occurred in the larger type, or if the high part
2777 // of the result does not zero/sign-extend the low part. Check this second
2778 // possibility first.
2779 if (IsSigned) {
2780 // For signed, overflow occurred when the high part does not sign-extend
2781 // the low part.
2782 ExtResult = MIRBuilder.buildSExtInReg(Res: WideTy, Op: Mul, ImmOp: SrcBitWidth);
2783 } else {
2784 // Unsigned overflow occurred when the high part does not zero-extend the
2785 // low part.
2786 ExtResult = MIRBuilder.buildZExtInReg(Res: WideTy, Op: Mul, ImmOp: SrcBitWidth);
2787 }
2788
2789 if (WideMulCanOverflow) {
2790 auto Overflow =
2791 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: OverflowTy, Op0: Mul, Op1: ExtResult);
2792 // Finally check if the multiplication in the larger type itself overflowed.
2793 MIRBuilder.buildOr(Dst: OriginalOverflow, Src0: Mulo->getOperand(i: 1), Src1: Overflow);
2794 } else {
2795 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: OriginalOverflow, Op0: Mul, Op1: ExtResult);
2796 }
2797 MI.eraseFromParent();
2798 return Legalized;
2799}
2800
2801LegalizerHelper::LegalizeResult
2802LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
2803 unsigned Opcode = MI.getOpcode();
2804 switch (Opcode) {
2805 default:
2806 return UnableToLegalize;
2807 case TargetOpcode::G_ATOMICRMW_XCHG:
2808 case TargetOpcode::G_ATOMICRMW_ADD:
2809 case TargetOpcode::G_ATOMICRMW_SUB:
2810 case TargetOpcode::G_ATOMICRMW_AND:
2811 case TargetOpcode::G_ATOMICRMW_OR:
2812 case TargetOpcode::G_ATOMICRMW_XOR:
2813 case TargetOpcode::G_ATOMICRMW_MIN:
2814 case TargetOpcode::G_ATOMICRMW_MAX:
2815 case TargetOpcode::G_ATOMICRMW_UMIN:
2816 case TargetOpcode::G_ATOMICRMW_UMAX:
2817 assert(TypeIdx == 0 && "atomicrmw with second scalar type");
2818 Observer.changingInstr(MI);
2819 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
2820 widenScalarDst(MI, WideTy, OpIdx: 0);
2821 Observer.changedInstr(MI);
2822 return Legalized;
2823 case TargetOpcode::G_ATOMIC_CMPXCHG:
2824 assert(TypeIdx == 0 && "G_ATOMIC_CMPXCHG with second scalar type");
2825 Observer.changingInstr(MI);
2826 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
2827 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ANYEXT);
2828 widenScalarDst(MI, WideTy, OpIdx: 0);
2829 Observer.changedInstr(MI);
2830 return Legalized;
2831 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS:
2832 if (TypeIdx == 0) {
2833 Observer.changingInstr(MI);
2834 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ANYEXT);
2835 widenScalarSrc(MI, WideTy, OpIdx: 4, ExtOpcode: TargetOpcode::G_ANYEXT);
2836 widenScalarDst(MI, WideTy, OpIdx: 0);
2837 Observer.changedInstr(MI);
2838 return Legalized;
2839 }
2840 assert(TypeIdx == 1 &&
2841 "G_ATOMIC_CMPXCHG_WITH_SUCCESS with third scalar type");
2842 Observer.changingInstr(MI);
2843 widenScalarDst(MI, WideTy, OpIdx: 1);
2844 Observer.changedInstr(MI);
2845 return Legalized;
2846 case TargetOpcode::G_EXTRACT:
2847 return widenScalarExtract(MI, TypeIdx, WideTy);
2848 case TargetOpcode::G_INSERT:
2849 return widenScalarInsert(MI, TypeIdx, WideTy);
2850 case TargetOpcode::G_MERGE_VALUES:
2851 return widenScalarMergeValues(MI, TypeIdx, WideTy);
2852 case TargetOpcode::G_UNMERGE_VALUES:
2853 return widenScalarUnmergeValues(MI, TypeIdx, WideTy);
2854 case TargetOpcode::G_SADDO:
2855 case TargetOpcode::G_SSUBO:
2856 case TargetOpcode::G_UADDO:
2857 case TargetOpcode::G_USUBO:
2858 case TargetOpcode::G_SADDE:
2859 case TargetOpcode::G_SSUBE:
2860 case TargetOpcode::G_UADDE:
2861 case TargetOpcode::G_USUBE:
2862 return widenScalarAddSubOverflow(MI, TypeIdx, WideTy);
2863 case TargetOpcode::G_UMULO:
2864 case TargetOpcode::G_SMULO:
2865 return widenScalarMulo(MI, TypeIdx, WideTy);
2866 case TargetOpcode::G_SADDSAT:
2867 case TargetOpcode::G_SSUBSAT:
2868 case TargetOpcode::G_SSHLSAT:
2869 case TargetOpcode::G_UADDSAT:
2870 case TargetOpcode::G_USUBSAT:
2871 case TargetOpcode::G_USHLSAT:
2872 return widenScalarAddSubShlSat(MI, TypeIdx, WideTy);
2873 case TargetOpcode::G_CTTZ:
2874 case TargetOpcode::G_CTTZ_ZERO_POISON:
2875 case TargetOpcode::G_CTLZ:
2876 case TargetOpcode::G_CTLZ_ZERO_POISON:
2877 case TargetOpcode::G_CTLS:
2878 case TargetOpcode::G_CTPOP: {
2879 if (TypeIdx == 0) {
2880 Observer.changingInstr(MI);
2881 widenScalarDst(MI, WideTy, OpIdx: 0);
2882 Observer.changedInstr(MI);
2883 return Legalized;
2884 }
2885
2886 Register SrcReg = MI.getOperand(i: 1).getReg();
2887
2888 // First extend the input.
2889 unsigned ExtOpc;
2890 switch (Opcode) {
2891 case TargetOpcode::G_CTTZ:
2892 case TargetOpcode::G_CTTZ_ZERO_POISON:
2893 case TargetOpcode::G_CTLZ_ZERO_POISON: // poison shifted out below
2894 ExtOpc = TargetOpcode::G_ANYEXT;
2895 break;
2896 case TargetOpcode::G_CTLS:
2897 ExtOpc = TargetOpcode::G_SEXT;
2898 break;
2899 default:
2900 ExtOpc = TargetOpcode::G_ZEXT;
2901 }
2902
2903 auto MIBSrc = MIRBuilder.buildInstr(Opc: ExtOpc, DstOps: {WideTy}, SrcOps: {SrcReg});
2904 LLT CurTy = MRI.getType(Reg: SrcReg);
2905 unsigned NewOpc = Opcode;
2906 if (NewOpc == TargetOpcode::G_CTTZ) {
2907 // The count is the same in the larger type except if the original
2908 // value was zero. This can be handled by setting the bit just off
2909 // the top of the original type.
2910 auto TopBit = APInt::getOneBitSet(numBits: WideTy.getScalarSizeInBits(),
2911 BitNo: CurTy.getScalarSizeInBits());
2912 MIBSrc = MIRBuilder.buildOr(
2913 Dst: WideTy, Src0: MIBSrc, Src1: MIRBuilder.buildConstant(Res: WideTy, Val: TopBit));
2914 // Now we know the operand is non-zero, use the more relaxed opcode.
2915 NewOpc = TargetOpcode::G_CTTZ_ZERO_POISON;
2916 }
2917
2918 unsigned SizeDiff =
2919 WideTy.getScalarSizeInBits() - CurTy.getScalarSizeInBits();
2920
2921 if (Opcode == TargetOpcode::G_CTLZ_ZERO_POISON) {
2922 // An optimization where the result is the CTLZ after the left shift by
2923 // (Difference in widety and current ty), that is,
2924 // MIBSrc = MIBSrc << (sizeinbits(WideTy) - sizeinbits(CurTy))
2925 // Result = ctlz MIBSrc
2926 MIBSrc = MIRBuilder.buildShl(Dst: WideTy, Src0: MIBSrc,
2927 Src1: MIRBuilder.buildConstant(Res: WideTy, Val: SizeDiff));
2928 }
2929
2930 // Perform the operation at the larger size.
2931 auto MIBNewOp = MIRBuilder.buildInstr(Opc: NewOpc, DstOps: {WideTy}, SrcOps: {MIBSrc});
2932 // This is already the correct result for CTPOP and CTTZs
2933 if (Opcode == TargetOpcode::G_CTLZ || Opcode == TargetOpcode::G_CTLS) {
2934 // The correct result is NewOp - (Difference in widety and current ty).
2935 // At this stage SUB is guaranteed to be positive no-wrap,
2936 // that to be used in further KnownBits optimizations for CTLZ.
2937 MIBNewOp = MIRBuilder.buildSub(
2938 Dst: WideTy, Src0: MIBNewOp, Src1: MIRBuilder.buildConstant(Res: WideTy, Val: SizeDiff),
2939 Flags: Opcode == TargetOpcode::G_CTLZ
2940 ? std::optional<unsigned>(MachineInstr::NoUWrap)
2941 : std::nullopt);
2942 }
2943
2944 MIRBuilder.buildZExtOrTrunc(Res: MI.getOperand(i: 0), Op: MIBNewOp);
2945 MI.eraseFromParent();
2946 return Legalized;
2947 }
2948 case TargetOpcode::G_BSWAP: {
2949 Observer.changingInstr(MI);
2950 Register DstReg = MI.getOperand(i: 0).getReg();
2951
2952 Register ShrReg = MRI.createGenericVirtualRegister(Ty: WideTy);
2953 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2954 Register ShiftAmtReg = MRI.createGenericVirtualRegister(Ty: WideTy);
2955 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2956
2957 MI.getOperand(i: 0).setReg(DstExt);
2958
2959 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2960
2961 LLT Ty = MRI.getType(Reg: DstReg);
2962 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
2963 MIRBuilder.buildConstant(Res: ShiftAmtReg, Val: DiffBits);
2964 MIRBuilder.buildLShr(Dst: ShrReg, Src0: DstExt, Src1: ShiftAmtReg);
2965
2966 MIRBuilder.buildTrunc(Res: DstReg, Op: ShrReg);
2967 Observer.changedInstr(MI);
2968 return Legalized;
2969 }
2970 case TargetOpcode::G_BITREVERSE: {
2971 Observer.changingInstr(MI);
2972
2973 Register DstReg = MI.getOperand(i: 0).getReg();
2974 LLT Ty = MRI.getType(Reg: DstReg);
2975 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
2976
2977 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2978 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2979 MI.getOperand(i: 0).setReg(DstExt);
2980 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2981
2982 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: DiffBits);
2983 auto Shift = MIRBuilder.buildLShr(Dst: WideTy, Src0: DstExt, Src1: ShiftAmt);
2984 MIRBuilder.buildTrunc(Res: DstReg, Op: Shift);
2985 Observer.changedInstr(MI);
2986 return Legalized;
2987 }
2988 case TargetOpcode::G_FREEZE:
2989 case TargetOpcode::G_CONSTANT_FOLD_BARRIER:
2990 Observer.changingInstr(MI);
2991 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2992 widenScalarDst(MI, WideTy);
2993 Observer.changedInstr(MI);
2994 return Legalized;
2995
2996 case TargetOpcode::G_ABS:
2997 Observer.changingInstr(MI);
2998 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_SEXT);
2999 widenScalarDst(MI, WideTy);
3000 Observer.changedInstr(MI);
3001 return Legalized;
3002
3003 case TargetOpcode::G_ADD:
3004 case TargetOpcode::G_AND:
3005 case TargetOpcode::G_MUL:
3006 case TargetOpcode::G_OR:
3007 case TargetOpcode::G_XOR:
3008 case TargetOpcode::G_SUB:
3009 case TargetOpcode::G_SHUFFLE_VECTOR:
3010 // Perform operation at larger width (any extension is fines here, high bits
3011 // don't affect the result) and then truncate the result back to the
3012 // original type.
3013 Observer.changingInstr(MI);
3014 // The G_ANYEXTs below leave the new high bits unconstrained, so no-wrap and
3015 // disjoint claims proved at the narrow width no longer hold. Paths that
3016 // widen with value-preserving G_ZEXT/G_SEXT keep their flags.
3017 MI.clearFlags(flags: MachineInstr::NoUWrap | MachineInstr::NoSWrap |
3018 MachineInstr::Disjoint);
3019 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3020 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
3021 widenScalarDst(MI, WideTy);
3022 Observer.changedInstr(MI);
3023 return Legalized;
3024
3025 case TargetOpcode::G_SBFX:
3026 case TargetOpcode::G_UBFX:
3027 Observer.changingInstr(MI);
3028
3029 if (TypeIdx == 0) {
3030 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3031 widenScalarDst(MI, WideTy);
3032 } else {
3033 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3034 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ZEXT);
3035 }
3036
3037 Observer.changedInstr(MI);
3038 return Legalized;
3039
3040 case TargetOpcode::G_SHL:
3041 Observer.changingInstr(MI);
3042
3043 if (TypeIdx == 0) {
3044 // Widening the result with G_ANYEXT invalidates the no-wrap flags, as in
3045 // the G_ADD/G_SUB/G_MUL case above. TypeIdx 1 widens only the shift
3046 // amount, which is value-preserving, so it keeps them.
3047 MI.clearFlags(flags: MachineInstr::NoUWrap | MachineInstr::NoSWrap);
3048 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3049 widenScalarDst(MI, WideTy);
3050 } else {
3051 assert(TypeIdx == 1);
3052 // The "number of bits to shift" operand must preserve its value as an
3053 // unsigned integer:
3054 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3055 }
3056
3057 Observer.changedInstr(MI);
3058 return Legalized;
3059
3060 case TargetOpcode::G_ROTR:
3061 case TargetOpcode::G_ROTL:
3062 if (TypeIdx != 1)
3063 return UnableToLegalize;
3064
3065 Observer.changingInstr(MI);
3066 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3067 Observer.changedInstr(MI);
3068 return Legalized;
3069
3070 case TargetOpcode::G_SDIV:
3071 case TargetOpcode::G_SREM:
3072 case TargetOpcode::G_SMIN:
3073 case TargetOpcode::G_SMAX:
3074 case TargetOpcode::G_ABDS:
3075 Observer.changingInstr(MI);
3076 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_SEXT);
3077 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_SEXT);
3078 widenScalarDst(MI, WideTy);
3079 Observer.changedInstr(MI);
3080 return Legalized;
3081
3082 case TargetOpcode::G_SDIVREM:
3083 Observer.changingInstr(MI);
3084 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_SEXT);
3085 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_SEXT);
3086 widenScalarDst(MI, WideTy);
3087 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: --MIRBuilder.getInsertPt());
3088 widenScalarDst(MI, WideTy, OpIdx: 1);
3089 Observer.changedInstr(MI);
3090 return Legalized;
3091
3092 case TargetOpcode::G_ASHR:
3093 case TargetOpcode::G_LSHR:
3094 Observer.changingInstr(MI);
3095
3096 if (TypeIdx == 0) {
3097 unsigned CvtOp = Opcode == TargetOpcode::G_ASHR ? TargetOpcode::G_SEXT
3098 : TargetOpcode::G_ZEXT;
3099
3100 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: CvtOp);
3101 widenScalarDst(MI, WideTy);
3102 } else {
3103 assert(TypeIdx == 1);
3104 // The "number of bits to shift" operand must preserve its value as an
3105 // unsigned integer:
3106 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3107 }
3108
3109 Observer.changedInstr(MI);
3110 return Legalized;
3111 case TargetOpcode::G_UDIV:
3112 case TargetOpcode::G_UREM:
3113 case TargetOpcode::G_ABDU:
3114 Observer.changingInstr(MI);
3115 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ZEXT);
3116 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3117 widenScalarDst(MI, WideTy);
3118 Observer.changedInstr(MI);
3119 return Legalized;
3120 case TargetOpcode::G_UDIVREM:
3121 Observer.changingInstr(MI);
3122 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3123 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ZEXT);
3124 widenScalarDst(MI, WideTy);
3125 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: --MIRBuilder.getInsertPt());
3126 widenScalarDst(MI, WideTy, OpIdx: 1);
3127 Observer.changedInstr(MI);
3128 return Legalized;
3129 case TargetOpcode::G_UMIN:
3130 case TargetOpcode::G_UMAX: {
3131 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3132
3133 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
3134 unsigned ExtOpc =
3135 TLI.isSExtCheaperThanZExt(FromTy: getApproximateEVTForLLT(Ty, Ctx),
3136 ToTy: getApproximateEVTForLLT(Ty: WideTy, Ctx))
3137 ? TargetOpcode::G_SEXT
3138 : TargetOpcode::G_ZEXT;
3139
3140 Observer.changingInstr(MI);
3141 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: ExtOpc);
3142 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: ExtOpc);
3143 widenScalarDst(MI, WideTy);
3144 Observer.changedInstr(MI);
3145 return Legalized;
3146 }
3147
3148 case TargetOpcode::G_SELECT:
3149 Observer.changingInstr(MI);
3150 if (TypeIdx == 0) {
3151 // Perform operation at larger width (any extension is fine here, high
3152 // bits don't affect the result) and then truncate the result back to the
3153 // original type.
3154 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
3155 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ANYEXT);
3156 widenScalarDst(MI, WideTy);
3157 } else {
3158 bool IsVec = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).isVector();
3159 // Explicit extension is required here since high bits affect the result.
3160 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: MIRBuilder.getBoolExtOp(IsVec, IsFP: false));
3161 }
3162 Observer.changedInstr(MI);
3163 return Legalized;
3164
3165 case TargetOpcode::G_FPEXT:
3166 if (TypeIdx != 1)
3167 return UnableToLegalize;
3168
3169 Observer.changingInstr(MI);
3170 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3171 Observer.changedInstr(MI);
3172 return Legalized;
3173 case TargetOpcode::G_FPTOSI:
3174 case TargetOpcode::G_FPTOUI:
3175 case TargetOpcode::G_INTRINSIC_LRINT:
3176 case TargetOpcode::G_INTRINSIC_LLRINT:
3177 case TargetOpcode::G_IS_FPCLASS:
3178 Observer.changingInstr(MI);
3179
3180 if (TypeIdx == 0)
3181 widenScalarDst(MI, WideTy);
3182 else
3183 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3184
3185 Observer.changedInstr(MI);
3186 return Legalized;
3187 case TargetOpcode::G_SITOFP:
3188 Observer.changingInstr(MI);
3189
3190 if (TypeIdx == 0)
3191 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3192 else
3193 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_SEXT);
3194
3195 Observer.changedInstr(MI);
3196 return Legalized;
3197 case TargetOpcode::G_UITOFP:
3198 Observer.changingInstr(MI);
3199
3200 if (TypeIdx == 0)
3201 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3202 else
3203 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ZEXT);
3204
3205 Observer.changedInstr(MI);
3206 return Legalized;
3207 case TargetOpcode::G_FPTOSI_SAT:
3208 case TargetOpcode::G_FPTOUI_SAT:
3209 Observer.changingInstr(MI);
3210
3211 if (TypeIdx == 0) {
3212 Register OldDst = MI.getOperand(i: 0).getReg();
3213 LLT Ty = MRI.getType(Reg: OldDst);
3214 Register ExtReg = MRI.createGenericVirtualRegister(Ty: WideTy);
3215 Register NewDst;
3216 MI.getOperand(i: 0).setReg(ExtReg);
3217 uint64_t ShortBits = Ty.getScalarSizeInBits();
3218 uint64_t WideBits = WideTy.getScalarSizeInBits();
3219 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
3220 if (Opcode == TargetOpcode::G_FPTOSI_SAT) {
3221 // z = i16 fptosi_sat(a)
3222 // ->
3223 // x = i32 fptosi_sat(a)
3224 // y = smin(x, 32767)
3225 // z = smax(y, -32768)
3226 auto MaxVal = MIRBuilder.buildConstant(
3227 Res: WideTy, Val: APInt::getSignedMaxValue(numBits: ShortBits).sext(width: WideBits));
3228 auto MinVal = MIRBuilder.buildConstant(
3229 Res: WideTy, Val: APInt::getSignedMinValue(numBits: ShortBits).sext(width: WideBits));
3230 Register MidReg =
3231 MIRBuilder.buildSMin(Dst: WideTy, Src0: ExtReg, Src1: MaxVal).getReg(Idx: 0);
3232 NewDst = MIRBuilder.buildSMax(Dst: WideTy, Src0: MidReg, Src1: MinVal).getReg(Idx: 0);
3233 } else {
3234 // z = i16 fptoui_sat(a)
3235 // ->
3236 // x = i32 fptoui_sat(a)
3237 // y = smin(x, 65535)
3238 auto MaxVal = MIRBuilder.buildConstant(
3239 Res: WideTy, Val: APInt::getAllOnes(numBits: ShortBits).zext(width: WideBits));
3240 NewDst = MIRBuilder.buildUMin(Dst: WideTy, Src0: ExtReg, Src1: MaxVal).getReg(Idx: 0);
3241 }
3242 MIRBuilder.buildTrunc(Res: OldDst, Op: NewDst);
3243 } else
3244 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3245
3246 Observer.changedInstr(MI);
3247 return Legalized;
3248 case TargetOpcode::G_LOAD:
3249 case TargetOpcode::G_SEXTLOAD:
3250 case TargetOpcode::G_ZEXTLOAD:
3251 case TargetOpcode::G_FPEXTLOAD:
3252 Observer.changingInstr(MI);
3253 widenScalarDst(MI, WideTy);
3254 Observer.changedInstr(MI);
3255 return Legalized;
3256
3257 case TargetOpcode::G_STORE: {
3258 if (TypeIdx != 0)
3259 return UnableToLegalize;
3260
3261 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3262 assert(!Ty.isPointerOrPointerVector() && "Can't widen type");
3263 if (!Ty.isScalar()) {
3264 // We need to widen the vector element type.
3265 Observer.changingInstr(MI);
3266 widenScalarSrc(MI, WideTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_ANYEXT);
3267 // We also need to adjust the MMO to turn this into a truncating store.
3268 MachineMemOperand &MMO = **MI.memoperands_begin();
3269 MachineFunction &MF = MIRBuilder.getMF();
3270 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo: MMO.getPointerInfo(), Ty);
3271 MI.setMemRefs(MF, MemRefs: {NewMMO});
3272 Observer.changedInstr(MI);
3273 return Legalized;
3274 }
3275
3276 Observer.changingInstr(MI);
3277
3278 unsigned ExtType = Ty.getScalarSizeInBits() == 1 ?
3279 TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT;
3280 widenScalarSrc(MI, WideTy, OpIdx: 0, ExtOpcode: ExtType);
3281
3282 Observer.changedInstr(MI);
3283 return Legalized;
3284 }
3285 case TargetOpcode::G_FPTRUNCSTORE:
3286 if (TypeIdx != 0)
3287 return UnableToLegalize;
3288 Observer.changingInstr(MI);
3289 widenScalarSrc(MI, WideTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_FPEXT);
3290 Observer.changedInstr(MI);
3291 return Legalized;
3292 case TargetOpcode::G_CONSTANT: {
3293 MachineOperand &SrcMO = MI.getOperand(i: 1);
3294 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
3295 unsigned ExtOpc = LI.getExtOpcodeForWideningConstant(
3296 SmallTy: MRI.getType(Reg: MI.getOperand(i: 0).getReg()));
3297 assert((ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::G_SEXT ||
3298 ExtOpc == TargetOpcode::G_ANYEXT) &&
3299 "Illegal Extend");
3300 const APInt &SrcVal = SrcMO.getCImm()->getValue();
3301 const APInt &Val = (ExtOpc == TargetOpcode::G_SEXT)
3302 ? SrcVal.sext(width: WideTy.getSizeInBits())
3303 : SrcVal.zext(width: WideTy.getSizeInBits());
3304 Observer.changingInstr(MI);
3305 SrcMO.setCImm(ConstantInt::get(Context&: Ctx, V: Val));
3306
3307 widenScalarDst(MI, WideTy);
3308 Observer.changedInstr(MI);
3309 return Legalized;
3310 }
3311 case TargetOpcode::G_FCONSTANT: {
3312 // To avoid changing the bits of the constant due to extension to a larger
3313 // type and then using G_FPTRUNC, we simply convert to a G_CONSTANT.
3314 MachineOperand &SrcMO = MI.getOperand(i: 1);
3315 APInt Val = SrcMO.getFPImm()->getValueAPF().bitcastToAPInt();
3316 MIRBuilder.setInstrAndDebugLoc(MI);
3317 auto IntCst = MIRBuilder.buildConstant(Res: MI.getOperand(i: 0).getReg(), Val);
3318 widenScalarDst(MI&: *IntCst, WideTy, OpIdx: 0, TruncOpcode: TargetOpcode::G_TRUNC);
3319 MI.eraseFromParent();
3320 return Legalized;
3321 }
3322 case TargetOpcode::G_IMPLICIT_DEF: {
3323 Observer.changingInstr(MI);
3324 widenScalarDst(MI, WideTy);
3325 Observer.changedInstr(MI);
3326 return Legalized;
3327 }
3328 case TargetOpcode::G_BRCOND:
3329 Observer.changingInstr(MI);
3330 widenScalarSrc(MI, WideTy, OpIdx: 0, ExtOpcode: MIRBuilder.getBoolExtOp(IsVec: false, IsFP: false));
3331 Observer.changedInstr(MI);
3332 return Legalized;
3333
3334 case TargetOpcode::G_FCMP:
3335 Observer.changingInstr(MI);
3336 if (TypeIdx == 0)
3337 widenScalarDst(MI, WideTy);
3338 else {
3339 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 2);
3340 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 3);
3341 }
3342 Observer.changedInstr(MI);
3343 return Legalized;
3344
3345 case TargetOpcode::G_ICMP:
3346 Observer.changingInstr(MI);
3347 if (TypeIdx == 0)
3348 widenScalarDst(MI, WideTy);
3349 else {
3350 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
3351 CmpInst::Predicate Pred =
3352 static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
3353
3354 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
3355 unsigned ExtOpcode =
3356 (CmpInst::isSigned(Pred) ||
3357 TLI.isSExtCheaperThanZExt(FromTy: getApproximateEVTForLLT(Ty: SrcTy, Ctx),
3358 ToTy: getApproximateEVTForLLT(Ty: WideTy, Ctx)))
3359 ? TargetOpcode::G_SEXT
3360 : TargetOpcode::G_ZEXT;
3361 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode);
3362 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode);
3363 }
3364 Observer.changedInstr(MI);
3365 return Legalized;
3366
3367 case TargetOpcode::G_PTR_ADD:
3368 assert(TypeIdx == 1 && "unable to legalize pointer of G_PTR_ADD");
3369 Observer.changingInstr(MI);
3370 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_SEXT);
3371 Observer.changedInstr(MI);
3372 return Legalized;
3373
3374 case TargetOpcode::G_PHI: {
3375 assert(TypeIdx == 0 && "Expecting only Idx 0");
3376
3377 Observer.changingInstr(MI);
3378 for (unsigned I = 1; I < MI.getNumOperands(); I += 2) {
3379 MachineBasicBlock &OpMBB = *MI.getOperand(i: I + 1).getMBB();
3380 MIRBuilder.setInsertPt(MBB&: OpMBB, II: OpMBB.getFirstTerminatorForward());
3381 widenScalarSrc(MI, WideTy, OpIdx: I, ExtOpcode: TargetOpcode::G_ANYEXT);
3382 }
3383
3384 MachineBasicBlock &MBB = *MI.getParent();
3385 MIRBuilder.setInsertPt(MBB, II: --MBB.getFirstNonPHI());
3386 widenScalarDst(MI, WideTy);
3387 Observer.changedInstr(MI);
3388 return Legalized;
3389 }
3390 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
3391 if (TypeIdx == 0) {
3392 Register VecReg = MI.getOperand(i: 1).getReg();
3393 LLT VecTy = MRI.getType(Reg: VecReg);
3394 Observer.changingInstr(MI);
3395
3396 widenScalarSrc(MI, WideTy: LLT::vector(EC: VecTy.getElementCount(), ScalarTy: WideTy), OpIdx: 1,
3397 ExtOpcode: TargetOpcode::G_ANYEXT);
3398
3399 widenScalarDst(MI, WideTy, OpIdx: 0);
3400 Observer.changedInstr(MI);
3401 return Legalized;
3402 }
3403
3404 if (TypeIdx != 2)
3405 return UnableToLegalize;
3406 Observer.changingInstr(MI);
3407 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3408 Observer.changedInstr(MI);
3409 return Legalized;
3410 }
3411 case TargetOpcode::G_INSERT_VECTOR_ELT: {
3412 if (TypeIdx == 0) {
3413 Observer.changingInstr(MI);
3414 const LLT WideEltTy = WideTy.getElementType();
3415
3416 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3417 widenScalarSrc(MI, WideTy: WideEltTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
3418 widenScalarDst(MI, WideTy, OpIdx: 0);
3419 Observer.changedInstr(MI);
3420 return Legalized;
3421 }
3422
3423 if (TypeIdx == 1) {
3424 Observer.changingInstr(MI);
3425
3426 Register VecReg = MI.getOperand(i: 1).getReg();
3427 LLT VecTy = MRI.getType(Reg: VecReg);
3428 LLT WideVecTy = VecTy.changeVectorElementType(NewEltTy: WideTy);
3429
3430 widenScalarSrc(MI, WideTy: WideVecTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3431 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
3432 widenScalarDst(MI, WideTy: WideVecTy, OpIdx: 0);
3433 Observer.changedInstr(MI);
3434 return Legalized;
3435 }
3436
3437 if (TypeIdx == 2) {
3438 Observer.changingInstr(MI);
3439 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ZEXT);
3440 Observer.changedInstr(MI);
3441 return Legalized;
3442 }
3443
3444 return UnableToLegalize;
3445 }
3446 case TargetOpcode::G_FADD:
3447 case TargetOpcode::G_FMUL:
3448 case TargetOpcode::G_FSUB:
3449 case TargetOpcode::G_FMA:
3450 case TargetOpcode::G_FMAD:
3451 case TargetOpcode::G_FNEG:
3452 case TargetOpcode::G_FABS:
3453 case TargetOpcode::G_FCANONICALIZE:
3454 case TargetOpcode::G_FMINNUM:
3455 case TargetOpcode::G_FMAXNUM:
3456 case TargetOpcode::G_FMINNUM_IEEE:
3457 case TargetOpcode::G_FMAXNUM_IEEE:
3458 case TargetOpcode::G_FMINIMUM:
3459 case TargetOpcode::G_FMAXIMUM:
3460 case TargetOpcode::G_FMINIMUMNUM:
3461 case TargetOpcode::G_FMAXIMUMNUM:
3462 case TargetOpcode::G_FDIV:
3463 case TargetOpcode::G_FREM:
3464 case TargetOpcode::G_FCEIL:
3465 case TargetOpcode::G_FFLOOR:
3466 case TargetOpcode::G_FCOS:
3467 case TargetOpcode::G_FSIN:
3468 case TargetOpcode::G_FTAN:
3469 case TargetOpcode::G_FACOS:
3470 case TargetOpcode::G_FASIN:
3471 case TargetOpcode::G_FATAN:
3472 case TargetOpcode::G_FATAN2:
3473 case TargetOpcode::G_FCOSH:
3474 case TargetOpcode::G_FSINH:
3475 case TargetOpcode::G_FTANH:
3476 case TargetOpcode::G_FLOG10:
3477 case TargetOpcode::G_FLOG:
3478 case TargetOpcode::G_FLOG2:
3479 case TargetOpcode::G_FRINT:
3480 case TargetOpcode::G_FNEARBYINT:
3481 case TargetOpcode::G_FSQRT:
3482 case TargetOpcode::G_FEXP:
3483 case TargetOpcode::G_FEXP2:
3484 case TargetOpcode::G_FEXP10:
3485 case TargetOpcode::G_FPOW:
3486 case TargetOpcode::G_INTRINSIC_TRUNC:
3487 case TargetOpcode::G_INTRINSIC_ROUND:
3488 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
3489 assert(TypeIdx == 0);
3490 Observer.changingInstr(MI);
3491
3492 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I)
3493 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: I);
3494
3495 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3496 Observer.changedInstr(MI);
3497 return Legalized;
3498 case TargetOpcode::G_FMODF: {
3499 Observer.changingInstr(MI);
3500 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 2);
3501
3502 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 1);
3503 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: --MIRBuilder.getInsertPt());
3504 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3505 Observer.changedInstr(MI);
3506 return Legalized;
3507 }
3508 case TargetOpcode::G_FPOWI:
3509 case TargetOpcode::G_FLDEXP:
3510 case TargetOpcode::G_STRICT_FLDEXP: {
3511 if (TypeIdx == 0) {
3512 if (Opcode == TargetOpcode::G_STRICT_FLDEXP)
3513 return UnableToLegalize;
3514
3515 Observer.changingInstr(MI);
3516 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3517 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3518 Observer.changedInstr(MI);
3519 return Legalized;
3520 }
3521
3522 if (TypeIdx == 1) {
3523 // For some reason SelectionDAG tries to promote to a libcall without
3524 // actually changing the integer type for promotion.
3525 Observer.changingInstr(MI);
3526 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_SEXT);
3527 Observer.changedInstr(MI);
3528 return Legalized;
3529 }
3530
3531 return UnableToLegalize;
3532 }
3533 case TargetOpcode::G_FFREXP: {
3534 Observer.changingInstr(MI);
3535
3536 if (TypeIdx == 0) {
3537 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 2);
3538 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3539 } else {
3540 widenScalarDst(MI, WideTy, OpIdx: 1);
3541 }
3542
3543 Observer.changedInstr(MI);
3544 return Legalized;
3545 }
3546 case TargetOpcode::G_LROUND:
3547 case TargetOpcode::G_LLROUND:
3548 Observer.changingInstr(MI);
3549
3550 if (TypeIdx == 0)
3551 widenScalarDst(MI, WideTy);
3552 else
3553 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3554
3555 Observer.changedInstr(MI);
3556 return Legalized;
3557
3558 case TargetOpcode::G_INTTOPTR:
3559 if (TypeIdx != 1)
3560 return UnableToLegalize;
3561
3562 Observer.changingInstr(MI);
3563 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ZEXT);
3564 Observer.changedInstr(MI);
3565 return Legalized;
3566 case TargetOpcode::G_PTRTOINT:
3567 if (TypeIdx != 0)
3568 return UnableToLegalize;
3569
3570 Observer.changingInstr(MI);
3571 widenScalarDst(MI, WideTy, OpIdx: 0);
3572 Observer.changedInstr(MI);
3573 return Legalized;
3574 case TargetOpcode::G_BUILD_VECTOR: {
3575 Observer.changingInstr(MI);
3576
3577 const LLT WideEltTy = TypeIdx == 1 ? WideTy : WideTy.getElementType();
3578 for (int I = 1, E = MI.getNumOperands(); I != E; ++I)
3579 widenScalarSrc(MI, WideTy: WideEltTy, OpIdx: I, ExtOpcode: TargetOpcode::G_ANYEXT);
3580
3581 // Avoid changing the result vector type if the source element type was
3582 // requested.
3583 if (TypeIdx == 1) {
3584 MI.setDesc(MIRBuilder.getTII().get(Opcode: TargetOpcode::G_BUILD_VECTOR_TRUNC));
3585 } else {
3586 widenScalarDst(MI, WideTy, OpIdx: 0);
3587 }
3588
3589 Observer.changedInstr(MI);
3590 return Legalized;
3591 }
3592 case TargetOpcode::G_SEXT_INREG:
3593 if (TypeIdx != 0)
3594 return UnableToLegalize;
3595
3596 Observer.changingInstr(MI);
3597 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3598 widenScalarDst(MI, WideTy, OpIdx: 0, TruncOpcode: TargetOpcode::G_TRUNC);
3599 Observer.changedInstr(MI);
3600 return Legalized;
3601 case TargetOpcode::G_PTRMASK: {
3602 if (TypeIdx != 1)
3603 return UnableToLegalize;
3604 Observer.changingInstr(MI);
3605 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3606 Observer.changedInstr(MI);
3607 return Legalized;
3608 }
3609 case TargetOpcode::G_VECREDUCE_ADD: {
3610 if (TypeIdx != 1)
3611 return UnableToLegalize;
3612 Observer.changingInstr(MI);
3613 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3614 widenScalarDst(MI, WideTy: WideTy.getScalarType(), OpIdx: 0, TruncOpcode: TargetOpcode::G_TRUNC);
3615 Observer.changedInstr(MI);
3616 return Legalized;
3617 }
3618 case TargetOpcode::G_VECREDUCE_FADD:
3619 case TargetOpcode::G_VECREDUCE_FMUL:
3620 case TargetOpcode::G_VECREDUCE_FMIN:
3621 case TargetOpcode::G_VECREDUCE_FMAX:
3622 case TargetOpcode::G_VECREDUCE_FMINIMUM:
3623 case TargetOpcode::G_VECREDUCE_FMAXIMUM: {
3624 if (TypeIdx != 0)
3625 return UnableToLegalize;
3626 Observer.changingInstr(MI);
3627 Register VecReg = MI.getOperand(i: 1).getReg();
3628 LLT VecTy = MRI.getType(Reg: VecReg);
3629 LLT WideVecTy = VecTy.changeElementType(NewEltTy: WideTy);
3630 widenScalarSrcUsingFPExt(MI, WideTy: WideVecTy, OpIdx: 1);
3631 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3632 Observer.changedInstr(MI);
3633 return Legalized;
3634 }
3635 case TargetOpcode::G_VSCALE: {
3636 MachineOperand &SrcMO = MI.getOperand(i: 1);
3637 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
3638 const APInt &SrcVal = SrcMO.getCImm()->getValue();
3639 // The CImm is always a signed value
3640 const APInt Val = SrcVal.sext(width: WideTy.getSizeInBits());
3641 Observer.changingInstr(MI);
3642 SrcMO.setCImm(ConstantInt::get(Context&: Ctx, V: Val));
3643 widenScalarDst(MI, WideTy);
3644 Observer.changedInstr(MI);
3645 return Legalized;
3646 }
3647 case TargetOpcode::G_SPLAT_VECTOR: {
3648 if (TypeIdx != 1)
3649 return UnableToLegalize;
3650
3651 Observer.changingInstr(MI);
3652 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3653 Observer.changedInstr(MI);
3654 return Legalized;
3655 }
3656 case TargetOpcode::G_INSERT_SUBVECTOR: {
3657 if (TypeIdx != 0)
3658 return UnableToLegalize;
3659
3660 GInsertSubvector &IS = cast<GInsertSubvector>(Val&: MI);
3661 Register BigVec = IS.getBigVec();
3662 Register SubVec = IS.getSubVec();
3663
3664 LLT SubVecTy = MRI.getType(Reg: SubVec);
3665 LLT SubVecWideTy = SubVecTy.changeElementType(NewEltTy: WideTy.getElementType());
3666
3667 // Widen the G_INSERT_SUBVECTOR
3668 auto BigZExt = MIRBuilder.buildZExt(Res: WideTy, Op: BigVec);
3669 auto SubZExt = MIRBuilder.buildZExt(Res: SubVecWideTy, Op: SubVec);
3670 auto WideInsert = MIRBuilder.buildInsertSubvector(Res: WideTy, Src0: BigZExt, Src1: SubZExt,
3671 Index: IS.getIndexImm());
3672
3673 // Truncate back down
3674 auto SplatZero = MIRBuilder.buildSplatVector(
3675 Res: WideTy, Val: MIRBuilder.buildConstant(Res: WideTy.getElementType(), Val: 0));
3676 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_NE, Res: IS.getReg(Idx: 0), Op0: WideInsert,
3677 Op1: SplatZero);
3678
3679 MI.eraseFromParent();
3680
3681 return Legalized;
3682 }
3683 case TargetOpcode::G_BITCAST:
3684 if (WideTy.isVector())
3685 return UnableToLegalize;
3686 Observer.changingInstr(MI);
3687 if (TypeIdx == 0)
3688 widenScalarDst(MI, WideTy, OpIdx: 0, TruncOpcode: TargetOpcode::G_TRUNC);
3689 else
3690 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3691 Observer.changedInstr(MI);
3692
3693 Register Dst = MI.getOperand(i: 0).getReg();
3694 Register Src = MI.getOperand(i: 1).getReg();
3695 if (MRI.getType(Reg: Dst) == MRI.getType(Reg: Src)) {
3696 Observer.changingAllUsesOfReg(MRI, Reg: Dst);
3697 MRI.replaceRegWith(FromReg: Dst, ToReg: Src);
3698 Observer.finishedChangingAllUsesOfReg();
3699 MI.eraseFromParent();
3700 }
3701
3702 return Legalized;
3703 }
3704}
3705
3706static void getUnmergePieces(SmallVectorImpl<Register> &Pieces,
3707 MachineIRBuilder &B, Register Src, LLT Ty) {
3708 auto Unmerge = B.buildUnmerge(Res: Ty, Op: Src);
3709 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
3710 Pieces.push_back(Elt: Unmerge.getReg(Idx: I));
3711}
3712
3713static void emitLoadFromConstantPool(Register DstReg, const Constant *ConstVal,
3714 MachineIRBuilder &MIRBuilder) {
3715 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
3716 MachineFunction &MF = MIRBuilder.getMF();
3717 const DataLayout &DL = MIRBuilder.getDataLayout();
3718 unsigned AddrSpace = DL.getDefaultGlobalsAddressSpace();
3719 LLT AddrPtrTy = LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL.getPointerSizeInBits(AS: AddrSpace));
3720 LLT DstLLT = MRI.getType(Reg: DstReg);
3721
3722 Align Alignment(DL.getABITypeAlign(Ty: ConstVal->getType()));
3723
3724 auto Addr = MIRBuilder.buildConstantPool(
3725 Res: AddrPtrTy,
3726 Idx: MF.getConstantPool()->getConstantPoolIndex(C: ConstVal, Alignment));
3727
3728 MachineMemOperand *MMO =
3729 MF.getMachineMemOperand(PtrInfo: MachinePointerInfo::getConstantPool(MF),
3730 F: MachineMemOperand::MOLoad, MemTy: DstLLT, BaseAlignment: Alignment);
3731
3732 MIRBuilder.buildLoadInstr(Opcode: TargetOpcode::G_LOAD, Res: DstReg, Addr, MMO&: *MMO);
3733}
3734
3735LegalizerHelper::LegalizeResult
3736LegalizerHelper::lowerConstant(MachineInstr &MI) {
3737 const MachineOperand &ConstOperand = MI.getOperand(i: 1);
3738 const Constant *ConstantVal = ConstOperand.getCImm();
3739
3740 emitLoadFromConstantPool(DstReg: MI.getOperand(i: 0).getReg(), ConstVal: ConstantVal, MIRBuilder);
3741 MI.eraseFromParent();
3742
3743 return Legalized;
3744}
3745
3746LegalizerHelper::LegalizeResult
3747LegalizerHelper::lowerFConstant(MachineInstr &MI) {
3748 const MachineOperand &ConstOperand = MI.getOperand(i: 1);
3749 const Constant *ConstantVal = ConstOperand.getFPImm();
3750
3751 emitLoadFromConstantPool(DstReg: MI.getOperand(i: 0).getReg(), ConstVal: ConstantVal, MIRBuilder);
3752 MI.eraseFromParent();
3753
3754 return Legalized;
3755}
3756
3757LegalizerHelper::LegalizeResult
3758LegalizerHelper::lowerBitcast(MachineInstr &MI) {
3759 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
3760 if (SrcTy.isVector()) {
3761 LLT SrcEltTy = SrcTy.getElementType();
3762 SmallVector<Register, 8> SrcRegs;
3763
3764 if (DstTy.isVector()) {
3765 LLT DstEltTy = DstTy.getElementType();
3766 ElementCount DstEC = DstTy.getElementCount();
3767 ElementCount SrcEC = SrcTy.getElementCount();
3768
3769 if (!SrcEC.isKnownMultipleOf(RHS: DstEC) && !DstEC.isKnownMultipleOf(RHS: SrcEC)) {
3770 // Split non-integer element ratio bitcast
3771 //
3772 // %1:_(<3 x s16>) = G_BITCAST %0:_(<2 x s24>)
3773 //
3774 // =>
3775 //
3776 // %2:_(<6 x s8>) = G_BITCAST %0:_(<2 x s24>)
3777 // %1:_(<3 x s16>) = G_BITCAST %2:_(<6 x s8>)
3778 unsigned SrcEltSize = SrcEltTy.getScalarSizeInBits();
3779 unsigned PieceSize =
3780 std::gcd(m: SrcEltSize, n: DstEltTy.getScalarSizeInBits());
3781 LLT PieceTy = LLT::integer(SizeInBits: PieceSize);
3782
3783 if (!PieceTy.isByteSized()) {
3784 // Split bitcast whose pieces are not whole bytes through a scalar
3785 //
3786 // %1:_(<3 x s8>) = G_BITCAST %0:_(<2 x s12>)
3787 //
3788 // =>
3789 //
3790 // %2:_(s24) = G_BITCAST %0:_(<2 x s12>)
3791 // %1:_(<3 x s8>) = G_BITCAST %2:_(s24)
3792 LLT ScalarTy = LLT::integer(SizeInBits: SrcTy.getSizeInBits());
3793 Register ScalarReg = MIRBuilder.buildBitcast(Dst: ScalarTy, Src).getReg(Idx: 0);
3794 MIRBuilder.buildBitcast(Dst, Src: ScalarReg);
3795 MI.eraseFromParent();
3796 return Legalized;
3797 }
3798
3799 LLT PiecesVecTy =
3800 LLT::vector(EC: SrcEC * (SrcEltSize / PieceSize), ScalarTy: PieceTy);
3801 Register PiecesReg =
3802 MIRBuilder.buildBitcast(Dst: PiecesVecTy, Src).getReg(Idx: 0);
3803 MIRBuilder.buildBitcast(Dst, Src: PiecesReg);
3804 MI.eraseFromParent();
3805 return Legalized;
3806 }
3807
3808 unsigned NumDstElt = DstEC.getKnownMinValue();
3809 unsigned NumSrcElt = SrcEC.getKnownMinValue();
3810
3811 LLT DstCastTy = DstEltTy; // Intermediate bitcast result type
3812 LLT SrcPartTy = SrcEltTy; // Original unmerge result type.
3813
3814 // If there's an element size mismatch, insert intermediate casts to match
3815 // the result element type.
3816 if (NumSrcElt < NumDstElt) { // Source element type is larger.
3817 // %1:_(<4 x s8>) = G_BITCAST %0:_(<2 x s16>)
3818 //
3819 // =>
3820 //
3821 // %2:_(s16), %3:_(s16) = G_UNMERGE_VALUES %0
3822 // %3:_(<2 x s8>) = G_BITCAST %2
3823 // %4:_(<2 x s8>) = G_BITCAST %3
3824 // %1:_(<4 x s8>) = G_CONCAT_VECTORS %3, %4
3825 DstCastTy = DstTy.changeVectorElementCount(
3826 EC: ElementCount::getFixed(MinVal: NumDstElt / NumSrcElt));
3827 SrcPartTy = SrcEltTy;
3828 } else if (NumSrcElt > NumDstElt) { // Source element type is smaller.
3829 //
3830 // %1:_(<2 x s16>) = G_BITCAST %0:_(<4 x s8>)
3831 //
3832 // =>
3833 //
3834 // %2:_(<2 x s8>), %3:_(<2 x s8>) = G_UNMERGE_VALUES %0
3835 // %3:_(s16) = G_BITCAST %2
3836 // %4:_(s16) = G_BITCAST %3
3837 // %1:_(<2 x s16>) = G_BUILD_VECTOR %3, %4
3838 SrcPartTy = SrcTy.changeVectorElementCount(
3839 EC: ElementCount::getFixed(MinVal: NumSrcElt / NumDstElt));
3840 DstCastTy = DstEltTy;
3841 }
3842
3843 getUnmergePieces(Pieces&: SrcRegs, B&: MIRBuilder, Src, Ty: SrcPartTy);
3844 for (Register &SrcReg : SrcRegs)
3845 SrcReg = MIRBuilder.buildBitcast(Dst: DstCastTy, Src: SrcReg).getReg(Idx: 0);
3846 } else
3847 getUnmergePieces(Pieces&: SrcRegs, B&: MIRBuilder, Src, Ty: SrcEltTy);
3848
3849 MIRBuilder.buildMergeLikeInstr(Res: Dst, Ops: SrcRegs);
3850 MI.eraseFromParent();
3851 return Legalized;
3852 }
3853
3854 if (DstTy.isVector()) {
3855 SmallVector<Register, 8> SrcRegs;
3856 getUnmergePieces(Pieces&: SrcRegs, B&: MIRBuilder, Src, Ty: DstTy.getElementType());
3857 MIRBuilder.buildMergeLikeInstr(Res: Dst, Ops: SrcRegs);
3858 MI.eraseFromParent();
3859 return Legalized;
3860 }
3861
3862 return UnableToLegalize;
3863}
3864
3865/// Figure out the bit offset into a register when coercing a vector index for
3866/// the wide element type. This is only for the case when promoting vector to
3867/// one with larger elements.
3868//
3869///
3870/// %offset_idx = G_AND %idx, ~(-1 << Log2(DstEltSize / SrcEltSize))
3871/// %offset_bits = G_SHL %offset_idx, Log2(SrcEltSize)
3872static Register getBitcastWiderVectorElementOffset(MachineIRBuilder &B,
3873 Register Idx,
3874 unsigned NewEltSize,
3875 unsigned OldEltSize) {
3876 const unsigned Log2EltRatio = Log2_32(Value: NewEltSize / OldEltSize);
3877 LLT IdxTy = B.getMRI()->getType(Reg: Idx);
3878
3879 // Now figure out the amount we need to shift to get the target bits.
3880 auto OffsetMask = B.buildConstant(
3881 Res: IdxTy, Val: ~(APInt::getAllOnes(numBits: IdxTy.getSizeInBits()) << Log2EltRatio));
3882 auto OffsetIdx = B.buildAnd(Dst: IdxTy, Src0: Idx, Src1: OffsetMask);
3883 return B.buildShl(Dst: IdxTy, Src0: OffsetIdx,
3884 Src1: B.buildConstant(Res: IdxTy, Val: Log2_32(Value: OldEltSize))).getReg(Idx: 0);
3885}
3886
3887/// Perform a G_EXTRACT_VECTOR_ELT in a different sized vector element. If this
3888/// is casting to a vector with a smaller element size, perform multiple element
3889/// extracts and merge the results. If this is coercing to a vector with larger
3890/// elements, index the bitcasted vector and extract the target element with bit
3891/// operations. This is intended to force the indexing in the native register
3892/// size for architectures that can dynamically index the register file.
3893LegalizerHelper::LegalizeResult
3894LegalizerHelper::bitcastExtractVectorElt(MachineInstr &MI, unsigned TypeIdx,
3895 LLT CastTy) {
3896 if (TypeIdx != 1)
3897 return UnableToLegalize;
3898
3899 auto [Dst, DstTy, SrcVec, SrcVecTy, Idx, IdxTy] = MI.getFirst3RegLLTs();
3900
3901 LLT SrcEltTy = SrcVecTy.getElementType();
3902 unsigned NewNumElts = CastTy.isVector() ? CastTy.getNumElements() : 1;
3903 unsigned OldNumElts = SrcVecTy.getNumElements();
3904
3905 LLT NewEltTy = CastTy.getScalarType();
3906 Register CastVec = MIRBuilder.buildBitcast(Dst: CastTy, Src: SrcVec).getReg(Idx: 0);
3907
3908 const unsigned NewEltSize = NewEltTy.getSizeInBits();
3909 const unsigned OldEltSize = SrcEltTy.getSizeInBits();
3910 if (NewNumElts > OldNumElts) {
3911 // Decreasing the vector element size
3912 //
3913 // e.g. i64 = extract_vector_elt x:v2i64, y:i32
3914 // =>
3915 // v4i32:castx = bitcast x:v2i64
3916 //
3917 // i64 = bitcast
3918 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
3919 // (i32 (extract_vector_elt castx, (2 * y + 1)))
3920 //
3921 if (NewNumElts % OldNumElts != 0)
3922 return UnableToLegalize;
3923
3924 // Type of the intermediate result vector.
3925 const unsigned NewEltsPerOldElt = NewNumElts / OldNumElts;
3926 LLT MidTy =
3927 CastTy.changeElementCount(EC: ElementCount::getFixed(MinVal: NewEltsPerOldElt));
3928
3929 auto NewEltsPerOldEltK = MIRBuilder.buildConstant(Res: IdxTy, Val: NewEltsPerOldElt);
3930
3931 SmallVector<Register, 8> NewOps(NewEltsPerOldElt);
3932 auto NewBaseIdx = MIRBuilder.buildMul(Dst: IdxTy, Src0: Idx, Src1: NewEltsPerOldEltK);
3933
3934 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
3935 auto IdxOffset = MIRBuilder.buildConstant(Res: IdxTy, Val: I);
3936 auto TmpIdx = MIRBuilder.buildAdd(Dst: IdxTy, Src0: NewBaseIdx, Src1: IdxOffset);
3937 auto Elt = MIRBuilder.buildExtractVectorElement(Res: NewEltTy, Val: CastVec, Idx: TmpIdx);
3938 NewOps[I] = Elt.getReg(Idx: 0);
3939 }
3940
3941 auto NewVec = MIRBuilder.buildBuildVector(Res: MidTy, Ops: NewOps);
3942 MIRBuilder.buildBitcast(Dst, Src: NewVec);
3943 MI.eraseFromParent();
3944 return Legalized;
3945 }
3946
3947 if (NewNumElts < OldNumElts) {
3948 if (NewEltSize % OldEltSize != 0)
3949 return UnableToLegalize;
3950
3951 // This only depends on powers of 2 because we use bit tricks to figure out
3952 // the bit offset we need to shift to get the target element. A general
3953 // expansion could emit division/multiply.
3954 if (!isPowerOf2_32(Value: NewEltSize / OldEltSize))
3955 return UnableToLegalize;
3956
3957 // Increasing the vector element size.
3958 // %elt:_(small_elt) = G_EXTRACT_VECTOR_ELT %vec:_(<N x small_elt>), %idx
3959 //
3960 // =>
3961 //
3962 // %cast = G_BITCAST %vec
3963 // %scaled_idx = G_LSHR %idx, Log2(DstEltSize / SrcEltSize)
3964 // %wide_elt = G_EXTRACT_VECTOR_ELT %cast, %scaled_idx
3965 // %offset_idx = G_AND %idx, ~(-1 << Log2(DstEltSize / SrcEltSize))
3966 // %offset_bits = G_SHL %offset_idx, Log2(SrcEltSize)
3967 // %elt_bits = G_LSHR %wide_elt, %offset_bits
3968 // %elt = G_TRUNC %elt_bits
3969
3970 const unsigned Log2EltRatio = Log2_32(Value: NewEltSize / OldEltSize);
3971 auto Log2Ratio = MIRBuilder.buildConstant(Res: IdxTy, Val: Log2EltRatio);
3972
3973 // Divide to get the index in the wider element type.
3974 auto ScaledIdx = MIRBuilder.buildLShr(Dst: IdxTy, Src0: Idx, Src1: Log2Ratio);
3975
3976 Register WideElt = CastVec;
3977 if (CastTy.isVector()) {
3978 WideElt = MIRBuilder.buildExtractVectorElement(Res: NewEltTy, Val: CastVec,
3979 Idx: ScaledIdx).getReg(Idx: 0);
3980 }
3981
3982 // Compute the bit offset into the register of the target element.
3983 Register OffsetBits = getBitcastWiderVectorElementOffset(
3984 B&: MIRBuilder, Idx, NewEltSize, OldEltSize);
3985
3986 // Shift the wide element to get the target element.
3987 auto ExtractedBits = MIRBuilder.buildLShr(Dst: NewEltTy, Src0: WideElt, Src1: OffsetBits);
3988 MIRBuilder.buildTrunc(Res: Dst, Op: ExtractedBits);
3989 MI.eraseFromParent();
3990 return Legalized;
3991 }
3992
3993 return UnableToLegalize;
3994}
3995
3996/// Emit code to insert \p InsertReg into \p TargetRet at \p OffsetBits in \p
3997/// TargetReg, while preserving other bits in \p TargetReg.
3998///
3999/// (InsertReg << Offset) | (TargetReg & ~(-1 >> InsertReg.size()) << Offset)
4000static Register buildBitFieldInsert(MachineIRBuilder &B,
4001 Register TargetReg, Register InsertReg,
4002 Register OffsetBits) {
4003 LLT TargetTy = B.getMRI()->getType(Reg: TargetReg);
4004 LLT InsertTy = B.getMRI()->getType(Reg: InsertReg);
4005 auto ZextVal = B.buildZExt(Res: TargetTy, Op: InsertReg);
4006 auto ShiftedInsertVal = B.buildShl(Dst: TargetTy, Src0: ZextVal, Src1: OffsetBits);
4007
4008 // Produce a bitmask of the value to insert
4009 auto EltMask = B.buildConstant(
4010 Res: TargetTy, Val: APInt::getLowBitsSet(numBits: TargetTy.getSizeInBits(),
4011 loBitsSet: InsertTy.getSizeInBits()));
4012 // Shift it into position
4013 auto ShiftedMask = B.buildShl(Dst: TargetTy, Src0: EltMask, Src1: OffsetBits);
4014 auto InvShiftedMask = B.buildNot(Dst: TargetTy, Src0: ShiftedMask);
4015
4016 // Clear out the bits in the wide element
4017 auto MaskedOldElt = B.buildAnd(Dst: TargetTy, Src0: TargetReg, Src1: InvShiftedMask);
4018
4019 // The value to insert has all zeros already, so stick it into the masked
4020 // wide element.
4021 return B.buildOr(Dst: TargetTy, Src0: MaskedOldElt, Src1: ShiftedInsertVal).getReg(Idx: 0);
4022}
4023
4024/// Perform a G_INSERT_VECTOR_ELT in a different sized vector element. If this
4025/// is increasing the element size, perform the indexing in the target element
4026/// type, and use bit operations to insert at the element position. This is
4027/// intended for architectures that can dynamically index the register file and
4028/// want to force indexing in the native register size.
4029LegalizerHelper::LegalizeResult
4030LegalizerHelper::bitcastInsertVectorElt(MachineInstr &MI, unsigned TypeIdx,
4031 LLT CastTy) {
4032 if (TypeIdx != 0)
4033 return UnableToLegalize;
4034
4035 auto [Dst, DstTy, SrcVec, SrcVecTy, Val, ValTy, Idx, IdxTy] =
4036 MI.getFirst4RegLLTs();
4037 LLT VecTy = DstTy;
4038
4039 LLT VecEltTy = VecTy.getElementType();
4040 LLT NewEltTy = CastTy.isVector() ? CastTy.getElementType() : CastTy;
4041 const unsigned NewEltSize = NewEltTy.getSizeInBits();
4042 const unsigned OldEltSize = VecEltTy.getSizeInBits();
4043
4044 unsigned NewNumElts = CastTy.isVector() ? CastTy.getNumElements() : 1;
4045 unsigned OldNumElts = VecTy.getNumElements();
4046
4047 Register CastVec = MIRBuilder.buildBitcast(Dst: CastTy, Src: SrcVec).getReg(Idx: 0);
4048 if (NewNumElts < OldNumElts) {
4049 if (NewEltSize % OldEltSize != 0)
4050 return UnableToLegalize;
4051
4052 // This only depends on powers of 2 because we use bit tricks to figure out
4053 // the bit offset we need to shift to get the target element. A general
4054 // expansion could emit division/multiply.
4055 if (!isPowerOf2_32(Value: NewEltSize / OldEltSize))
4056 return UnableToLegalize;
4057
4058 const unsigned Log2EltRatio = Log2_32(Value: NewEltSize / OldEltSize);
4059 auto Log2Ratio = MIRBuilder.buildConstant(Res: IdxTy, Val: Log2EltRatio);
4060
4061 // Divide to get the index in the wider element type.
4062 auto ScaledIdx = MIRBuilder.buildLShr(Dst: IdxTy, Src0: Idx, Src1: Log2Ratio);
4063
4064 Register ExtractedElt = CastVec;
4065 if (CastTy.isVector()) {
4066 ExtractedElt = MIRBuilder.buildExtractVectorElement(Res: NewEltTy, Val: CastVec,
4067 Idx: ScaledIdx).getReg(Idx: 0);
4068 }
4069
4070 // Compute the bit offset into the register of the target element.
4071 Register OffsetBits = getBitcastWiderVectorElementOffset(
4072 B&: MIRBuilder, Idx, NewEltSize, OldEltSize);
4073
4074 Register InsertedElt = buildBitFieldInsert(B&: MIRBuilder, TargetReg: ExtractedElt,
4075 InsertReg: Val, OffsetBits);
4076 if (CastTy.isVector()) {
4077 InsertedElt = MIRBuilder.buildInsertVectorElement(
4078 Res: CastTy, Val: CastVec, Elt: InsertedElt, Idx: ScaledIdx).getReg(Idx: 0);
4079 }
4080
4081 MIRBuilder.buildBitcast(Dst, Src: InsertedElt);
4082 MI.eraseFromParent();
4083 return Legalized;
4084 }
4085
4086 return UnableToLegalize;
4087}
4088
4089// This attempts to handle G_CONCAT_VECTORS with illegal operands, particularly
4090// those that have smaller than legal operands.
4091//
4092// <16 x s8> = G_CONCAT_VECTORS <4 x s8>, <4 x s8>, <4 x s8>, <4 x s8>
4093//
4094// ===>
4095//
4096// s32 = G_BITCAST <4 x s8>
4097// s32 = G_BITCAST <4 x s8>
4098// s32 = G_BITCAST <4 x s8>
4099// s32 = G_BITCAST <4 x s8>
4100// <4 x s32> = G_BUILD_VECTOR s32, s32, s32, s32
4101// <16 x s8> = G_BITCAST <4 x s32>
4102LegalizerHelper::LegalizeResult
4103LegalizerHelper::bitcastConcatVector(MachineInstr &MI, unsigned TypeIdx,
4104 LLT CastTy) {
4105 // Convert it to CONCAT instruction
4106 auto ConcatMI = dyn_cast<GConcatVectors>(Val: &MI);
4107 if (!ConcatMI) {
4108 return UnableToLegalize;
4109 }
4110
4111 // Check if bitcast is Legal
4112 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
4113 LLT SrcScalTy = CastTy.getScalarType();
4114
4115 // Check if the build vector is Legal
4116 if (!LI.isLegal(Query: {TargetOpcode::G_BUILD_VECTOR, {CastTy, SrcScalTy}})) {
4117 return UnableToLegalize;
4118 }
4119
4120 // Bitcast the sources
4121 SmallVector<Register> BitcastRegs;
4122 for (unsigned i = 0; i < ConcatMI->getNumSources(); i++) {
4123 BitcastRegs.push_back(
4124 Elt: MIRBuilder.buildBitcast(Dst: SrcScalTy, Src: ConcatMI->getSourceReg(I: i))
4125 .getReg(Idx: 0));
4126 }
4127
4128 // Build the scalar values into a vector
4129 Register BuildReg =
4130 MIRBuilder.buildBuildVector(Res: CastTy, Ops: BitcastRegs).getReg(Idx: 0);
4131 MIRBuilder.buildBitcast(Dst: DstReg, Src: BuildReg);
4132
4133 MI.eraseFromParent();
4134 return Legalized;
4135}
4136
4137// This bitcasts a shuffle vector to a different type currently of the same
4138// element size. Mostly used to legalize ptr vectors, where ptrtoint/inttoptr
4139// will be used instead.
4140//
4141// <16 x p0> = G_CONCAT_VECTORS <4 x p0>, <4 x p0>, mask
4142// ===>
4143// <4 x s64> = G_PTRTOINT <4 x p0>
4144// <4 x s64> = G_PTRTOINT <4 x p0>
4145// <16 x s64> = G_CONCAT_VECTORS <4 x s64>, <4 x s64>, mask
4146// <16 x p0> = G_INTTOPTR <16 x s64>
4147LegalizerHelper::LegalizeResult
4148LegalizerHelper::bitcastShuffleVector(MachineInstr &MI, unsigned TypeIdx,
4149 LLT CastTy) {
4150 auto ShuffleMI = cast<GShuffleVector>(Val: &MI);
4151 LLT DstTy = MRI.getType(Reg: ShuffleMI->getReg(Idx: 0));
4152 LLT SrcTy = MRI.getType(Reg: ShuffleMI->getReg(Idx: 1));
4153
4154 // We currently only handle vectors of the same size.
4155 if (TypeIdx != 0 ||
4156 CastTy.getScalarSizeInBits() != DstTy.getScalarSizeInBits() ||
4157 CastTy.getElementCount() != DstTy.getElementCount())
4158 return UnableToLegalize;
4159
4160 LLT NewSrcTy = SrcTy.changeElementType(NewEltTy: CastTy.getScalarType());
4161
4162 auto Inp1 = MIRBuilder.buildCast(Dst: NewSrcTy, Src: ShuffleMI->getReg(Idx: 1));
4163 auto Inp2 = MIRBuilder.buildCast(Dst: NewSrcTy, Src: ShuffleMI->getReg(Idx: 2));
4164 auto Shuf =
4165 MIRBuilder.buildShuffleVector(Res: CastTy, Src1: Inp1, Src2: Inp2, Mask: ShuffleMI->getMask());
4166 MIRBuilder.buildCast(Dst: ShuffleMI->getReg(Idx: 0), Src: Shuf);
4167
4168 MI.eraseFromParent();
4169 return Legalized;
4170}
4171
4172/// This attempts to bitcast G_EXTRACT_SUBVECTOR to CastTy.
4173///
4174/// <vscale x 8 x i1> = G_EXTRACT_SUBVECTOR <vscale x 16 x i1>, N
4175///
4176/// ===>
4177///
4178/// <vscale x 2 x i1> = G_BITCAST <vscale x 16 x i1>
4179/// <vscale x 1 x i8> = G_EXTRACT_SUBVECTOR <vscale x 2 x i1>, N / 8
4180/// <vscale x 8 x i1> = G_BITCAST <vscale x 1 x i8>
4181LegalizerHelper::LegalizeResult
4182LegalizerHelper::bitcastExtractSubvector(MachineInstr &MI, unsigned TypeIdx,
4183 LLT CastTy) {
4184 auto ES = cast<GExtractSubvector>(Val: &MI);
4185
4186 if (!CastTy.isVector())
4187 return UnableToLegalize;
4188
4189 if (TypeIdx != 0)
4190 return UnableToLegalize;
4191
4192 Register Dst = ES->getReg(Idx: 0);
4193 Register Src = ES->getSrcVec();
4194 uint64_t Idx = ES->getIndexImm();
4195
4196 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4197
4198 LLT DstTy = MRI.getType(Reg: Dst);
4199 LLT SrcTy = MRI.getType(Reg: Src);
4200 ElementCount DstTyEC = DstTy.getElementCount();
4201 ElementCount SrcTyEC = SrcTy.getElementCount();
4202 auto DstTyMinElts = DstTyEC.getKnownMinValue();
4203 auto SrcTyMinElts = SrcTyEC.getKnownMinValue();
4204
4205 if (DstTy == CastTy)
4206 return Legalized;
4207
4208 if (DstTy.getSizeInBits() != CastTy.getSizeInBits())
4209 return UnableToLegalize;
4210
4211 unsigned CastEltSize = CastTy.getElementType().getSizeInBits();
4212 unsigned DstEltSize = DstTy.getElementType().getSizeInBits();
4213 if (CastEltSize < DstEltSize)
4214 return UnableToLegalize;
4215
4216 auto AdjustAmt = CastEltSize / DstEltSize;
4217 if (Idx % AdjustAmt != 0 || DstTyMinElts % AdjustAmt != 0 ||
4218 SrcTyMinElts % AdjustAmt != 0)
4219 return UnableToLegalize;
4220
4221 Idx /= AdjustAmt;
4222 SrcTy = LLT::vector(EC: SrcTyEC.divideCoefficientBy(RHS: AdjustAmt), ScalarSizeInBits: AdjustAmt);
4223 auto CastVec = MIRBuilder.buildBitcast(Dst: SrcTy, Src);
4224 auto PromotedES = MIRBuilder.buildExtractSubvector(Res: CastTy, Src: CastVec, Index: Idx);
4225 MIRBuilder.buildBitcast(Dst, Src: PromotedES);
4226
4227 ES->eraseFromParent();
4228 return Legalized;
4229}
4230
4231/// This attempts to bitcast G_INSERT_SUBVECTOR to CastTy.
4232///
4233/// <vscale x 16 x i1> = G_INSERT_SUBVECTOR <vscale x 16 x i1>,
4234/// <vscale x 8 x i1>,
4235/// N
4236///
4237/// ===>
4238///
4239/// <vscale x 2 x i8> = G_BITCAST <vscale x 16 x i1>
4240/// <vscale x 1 x i8> = G_BITCAST <vscale x 8 x i1>
4241/// <vscale x 2 x i8> = G_INSERT_SUBVECTOR <vscale x 2 x i8>,
4242/// <vscale x 1 x i8>, N / 8
4243/// <vscale x 16 x i1> = G_BITCAST <vscale x 2 x i8>
4244LegalizerHelper::LegalizeResult
4245LegalizerHelper::bitcastInsertSubvector(MachineInstr &MI, unsigned TypeIdx,
4246 LLT CastTy) {
4247 auto ES = cast<GInsertSubvector>(Val: &MI);
4248
4249 if (!CastTy.isVector())
4250 return UnableToLegalize;
4251
4252 if (TypeIdx != 0)
4253 return UnableToLegalize;
4254
4255 Register Dst = ES->getReg(Idx: 0);
4256 Register BigVec = ES->getBigVec();
4257 Register SubVec = ES->getSubVec();
4258 uint64_t Idx = ES->getIndexImm();
4259
4260 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4261
4262 LLT DstTy = MRI.getType(Reg: Dst);
4263 LLT BigVecTy = MRI.getType(Reg: BigVec);
4264 LLT SubVecTy = MRI.getType(Reg: SubVec);
4265
4266 if (DstTy == CastTy)
4267 return Legalized;
4268
4269 if (DstTy.getSizeInBits() != CastTy.getSizeInBits())
4270 return UnableToLegalize;
4271
4272 ElementCount DstTyEC = DstTy.getElementCount();
4273 ElementCount BigVecTyEC = BigVecTy.getElementCount();
4274 ElementCount SubVecTyEC = SubVecTy.getElementCount();
4275 auto DstTyMinElts = DstTyEC.getKnownMinValue();
4276 auto BigVecTyMinElts = BigVecTyEC.getKnownMinValue();
4277 auto SubVecTyMinElts = SubVecTyEC.getKnownMinValue();
4278
4279 unsigned CastEltSize = CastTy.getElementType().getSizeInBits();
4280 unsigned DstEltSize = DstTy.getElementType().getSizeInBits();
4281 if (CastEltSize < DstEltSize)
4282 return UnableToLegalize;
4283
4284 auto AdjustAmt = CastEltSize / DstEltSize;
4285 if (Idx % AdjustAmt != 0 || DstTyMinElts % AdjustAmt != 0 ||
4286 BigVecTyMinElts % AdjustAmt != 0 || SubVecTyMinElts % AdjustAmt != 0)
4287 return UnableToLegalize;
4288
4289 Idx /= AdjustAmt;
4290 BigVecTy = LLT::vector(EC: BigVecTyEC.divideCoefficientBy(RHS: AdjustAmt), ScalarSizeInBits: AdjustAmt);
4291 SubVecTy = LLT::vector(EC: SubVecTyEC.divideCoefficientBy(RHS: AdjustAmt), ScalarSizeInBits: AdjustAmt);
4292 auto CastBigVec = MIRBuilder.buildBitcast(Dst: BigVecTy, Src: BigVec);
4293 auto CastSubVec = MIRBuilder.buildBitcast(Dst: SubVecTy, Src: SubVec);
4294 auto PromotedIS =
4295 MIRBuilder.buildInsertSubvector(Res: CastTy, Src0: CastBigVec, Src1: CastSubVec, Index: Idx);
4296 MIRBuilder.buildBitcast(Dst, Src: PromotedIS);
4297
4298 ES->eraseFromParent();
4299 return Legalized;
4300}
4301
4302LegalizerHelper::LegalizeResult LegalizerHelper::lowerLoad(GAnyLoad &LoadMI) {
4303 // Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT
4304 Register DstReg = LoadMI.getDstReg();
4305 Register PtrReg = LoadMI.getPointerReg();
4306 LLT DstTy = MRI.getType(Reg: DstReg);
4307 MachineMemOperand &MMO = LoadMI.getMMO();
4308 LLT MemTy = MMO.getMemoryType();
4309 MachineFunction &MF = MIRBuilder.getMF();
4310
4311 LLT EltTy = MemTy.getScalarType();
4312
4313 unsigned MemSizeInBits = MemTy.getSizeInBits();
4314 unsigned MemStoreSizeInBits = 8 * MemTy.getSizeInBytes();
4315
4316 if (MemSizeInBits != MemStoreSizeInBits) {
4317 if (MemTy.isVector())
4318 return UnableToLegalize;
4319
4320 // Promote to a byte-sized load if not loading an integral number of
4321 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
4322 LLT WideMemTy = EltTy.changeElementSize(NewEltSize: MemStoreSizeInBits);
4323 MachineMemOperand *NewMMO =
4324 MF.getMachineMemOperand(MMO: &MMO, PtrInfo: MMO.getPointerInfo(), Ty: WideMemTy);
4325
4326 Register LoadReg = DstReg;
4327 LLT LoadTy = DstTy;
4328
4329 // If this wasn't already an extending load, we need to widen the result
4330 // register to avoid creating a load with a narrower result than the source.
4331 if (MemStoreSizeInBits > DstTy.getSizeInBits()) {
4332 LoadTy = WideMemTy;
4333 LoadReg = MRI.createGenericVirtualRegister(Ty: WideMemTy);
4334 }
4335
4336 if (isa<GSExtLoad>(Val: LoadMI)) {
4337 auto NewLoad = MIRBuilder.buildLoad(Res: LoadTy, Addr: PtrReg, MMO&: *NewMMO);
4338 MIRBuilder.buildSExtInReg(Res: LoadReg, Op: NewLoad, ImmOp: MemSizeInBits);
4339 } else if (isa<GZExtLoad>(Val: LoadMI) || WideMemTy == LoadTy) {
4340 auto NewLoad = MIRBuilder.buildLoad(Res: LoadTy, Addr: PtrReg, MMO&: *NewMMO);
4341 // The extra bits are guaranteed to be zero, since we stored them that
4342 // way. A zext load from Wide thus automatically gives zext from MemVT.
4343 MIRBuilder.buildAssertZExt(Res: LoadReg, Op: NewLoad, Size: MemSizeInBits);
4344 } else {
4345 MIRBuilder.buildLoad(Res: LoadReg, Addr: PtrReg, MMO&: *NewMMO);
4346 }
4347
4348 if (DstTy != LoadTy)
4349 MIRBuilder.buildTrunc(Res: DstReg, Op: LoadReg);
4350
4351 LoadMI.eraseFromParent();
4352 return Legalized;
4353 }
4354
4355 // Big endian lowering not implemented.
4356 if (MIRBuilder.getDataLayout().isBigEndian())
4357 return UnableToLegalize;
4358
4359 // This load needs splitting into power of 2 sized loads.
4360 //
4361 // Our strategy here is to generate anyextending loads for the smaller
4362 // types up to next power-2 result type, and then combine the two larger
4363 // result values together, before truncating back down to the non-pow-2
4364 // type.
4365 // E.g. v1 = i24 load =>
4366 // v2 = i32 zextload (2 byte)
4367 // v3 = i32 load (1 byte)
4368 // v4 = i32 shl v3, 16
4369 // v5 = i32 or v4, v2
4370 // v1 = i24 trunc v5
4371 // By doing this we generate the correct truncate which should get
4372 // combined away as an artifact with a matching extend.
4373
4374 uint64_t LargeSplitSize, SmallSplitSize;
4375
4376 if (!isPowerOf2_32(Value: MemSizeInBits)) {
4377 // This load needs splitting into power of 2 sized loads.
4378 LargeSplitSize = llvm::bit_floor(Value: MemSizeInBits);
4379 SmallSplitSize = MemSizeInBits - LargeSplitSize;
4380 } else {
4381 // This is already a power of 2, but we still need to split this in half.
4382 //
4383 // Assume we're being asked to decompose an unaligned load.
4384 // TODO: If this requires multiple splits, handle them all at once.
4385 auto &Ctx = MF.getFunction().getContext();
4386 if (TLI.allowsMemoryAccess(Context&: Ctx, DL: MIRBuilder.getDataLayout(), Ty: MemTy, MMO))
4387 return UnableToLegalize;
4388
4389 SmallSplitSize = LargeSplitSize = MemSizeInBits / 2;
4390 }
4391
4392 if (MemTy.isVector()) {
4393 // TODO: Handle vector extloads
4394 if (MemTy != DstTy)
4395 return UnableToLegalize;
4396
4397 Align Alignment = LoadMI.getAlign();
4398 // Given an alignment larger than the size of the memory, we can increase
4399 // the size of the load without needing to scalarize it.
4400 if (Alignment.value() * 8 > MemSizeInBits &&
4401 isPowerOf2_64(Value: DstTy.getScalarSizeInBits())) {
4402 LLT MoreTy = DstTy.changeVectorElementCount(
4403 EC: ElementCount::getFixed(MinVal: NextPowerOf2(A: DstTy.getNumElements())));
4404 MachineMemOperand *NewMMO = MF.getMachineMemOperand(MMO: &MMO, Offset: 0, Ty: MoreTy);
4405 auto NewLoad = MIRBuilder.buildLoad(Res: MoreTy, Addr: PtrReg, MMO&: *NewMMO);
4406 MIRBuilder.buildDeleteTrailingVectorElements(Res: LoadMI.getReg(Idx: 0),
4407 Op0: NewLoad.getReg(Idx: 0));
4408 LoadMI.eraseFromParent();
4409 return Legalized;
4410 }
4411
4412 // TODO: We can do better than scalarizing the vector and at least split it
4413 // in half.
4414 return reduceLoadStoreWidth(MI&: LoadMI, TypeIdx: 0, NarrowTy: DstTy.getElementType());
4415 }
4416
4417 MachineMemOperand *LargeMMO =
4418 MF.getMachineMemOperand(MMO: &MMO, Offset: 0, Size: LargeSplitSize / 8);
4419 MachineMemOperand *SmallMMO =
4420 MF.getMachineMemOperand(MMO: &MMO, Offset: LargeSplitSize / 8, Size: SmallSplitSize / 8);
4421
4422 LLT PtrTy = MRI.getType(Reg: PtrReg);
4423 unsigned AnyExtSize = PowerOf2Ceil(A: DstTy.getSizeInBits());
4424
4425 LLT AnyExtTy = LLT::integer(SizeInBits: AnyExtSize);
4426 LLT OffsetCstRes = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
4427
4428 auto LargeLoad = MIRBuilder.buildLoadInstr(Opcode: TargetOpcode::G_ZEXTLOAD, Res: AnyExtTy,
4429 Addr: PtrReg, MMO&: *LargeMMO);
4430
4431 auto OffsetCst = MIRBuilder.buildConstant(Res: OffsetCstRes, Val: LargeSplitSize / 8);
4432 Register PtrAddReg = MRI.createGenericVirtualRegister(Ty: PtrTy);
4433 auto SmallPtr = MIRBuilder.buildObjectPtrOffset(Res: PtrAddReg, Op0: PtrReg, Op1: OffsetCst);
4434 auto SmallLoad = MIRBuilder.buildLoadInstr(Opcode: LoadMI.getOpcode(), Res: AnyExtTy,
4435 Addr: SmallPtr, MMO&: *SmallMMO);
4436
4437 auto ShiftAmt = MIRBuilder.buildConstant(Res: AnyExtTy, Val: LargeSplitSize);
4438 auto Shift = MIRBuilder.buildShl(Dst: AnyExtTy, Src0: SmallLoad, Src1: ShiftAmt);
4439
4440 if (AnyExtTy == DstTy)
4441 MIRBuilder.buildOr(Dst: DstReg, Src0: Shift, Src1: LargeLoad);
4442 else if (AnyExtTy.getSizeInBits() != DstTy.getSizeInBits()) {
4443 auto Or = MIRBuilder.buildOr(Dst: AnyExtTy, Src0: Shift, Src1: LargeLoad);
4444 LLT IntDstTy = DstTy.changeToInteger();
4445 if (IntDstTy == DstTy) {
4446 MIRBuilder.buildTrunc(Res: DstReg, Op: {Or});
4447 } else {
4448 auto Trunc = MIRBuilder.buildTrunc(Res: IntDstTy, Op: Or);
4449 MIRBuilder.buildBitcast(Dst: DstReg, Src: Trunc);
4450 }
4451 } else if (DstTy.isPointer()) {
4452 auto Or = MIRBuilder.buildOr(Dst: AnyExtTy, Src0: Shift, Src1: LargeLoad);
4453
4454 // FIXME: We currently consider this to be illegal for non-integral address
4455 // spaces, but we need still need a way to reinterpret the bits.
4456 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: Or);
4457 } else {
4458 auto Or = MIRBuilder.buildOr(Dst: AnyExtTy, Src0: Shift, Src1: LargeLoad);
4459 MIRBuilder.buildBitcast(Dst: DstReg, Src: Or);
4460 }
4461
4462 LoadMI.eraseFromParent();
4463 return Legalized;
4464}
4465
4466LegalizerHelper::LegalizeResult LegalizerHelper::lowerStore(GStore &StoreMI) {
4467 // Lower a non-power of 2 store into multiple pow-2 stores.
4468 // E.g. split an i24 store into an i16 store + i8 store.
4469 // We do this by first extending the stored value to the next largest power
4470 // of 2 type, and then using truncating stores to store the components.
4471 // By doing this, likewise with G_LOAD, generate an extend that can be
4472 // artifact-combined away instead of leaving behind extracts.
4473 Register SrcReg = StoreMI.getValueReg();
4474 Register PtrReg = StoreMI.getPointerReg();
4475 LLT SrcTy = MRI.getType(Reg: SrcReg);
4476 MachineFunction &MF = MIRBuilder.getMF();
4477 MachineMemOperand &MMO = **StoreMI.memoperands_begin();
4478 LLT MemTy = MMO.getMemoryType();
4479
4480 unsigned StoreWidth = MemTy.getSizeInBits();
4481 unsigned StoreSizeInBits = 8 * MemTy.getSizeInBytes();
4482
4483 if (StoreWidth != StoreSizeInBits && !SrcTy.isVector()) {
4484 // Promote to a byte-sized store with upper bits zero if not
4485 // storing an integral number of bytes. For example, promote
4486 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
4487 LLT WideTy = LLT::integer(SizeInBits: StoreSizeInBits);
4488
4489 if (StoreSizeInBits > SrcTy.getSizeInBits()) {
4490 // Avoid creating a store with a narrower source than result.
4491 SrcReg = MIRBuilder.buildAnyExt(Res: WideTy, Op: SrcReg).getReg(Idx: 0);
4492 SrcTy = WideTy;
4493 }
4494
4495 auto ZextInReg = MIRBuilder.buildZExtInReg(Res: SrcTy, Op: SrcReg, ImmOp: StoreWidth);
4496
4497 MachineMemOperand *NewMMO =
4498 MF.getMachineMemOperand(MMO: &MMO, PtrInfo: MMO.getPointerInfo(), Ty: WideTy);
4499 MIRBuilder.buildStore(Val: ZextInReg, Addr: PtrReg, MMO&: *NewMMO);
4500 StoreMI.eraseFromParent();
4501 return Legalized;
4502 }
4503
4504 if (MemTy.isVector()) {
4505 if (MemTy != SrcTy)
4506 return scalarizeVectorBooleanStore(MI&: StoreMI);
4507
4508 // TODO: We can do better than scalarizing the vector and at least split it
4509 // in half.
4510 return reduceLoadStoreWidth(MI&: StoreMI, TypeIdx: 0, NarrowTy: SrcTy.getElementType());
4511 }
4512
4513 unsigned MemSizeInBits = MemTy.getSizeInBits();
4514 uint64_t LargeSplitSize, SmallSplitSize;
4515
4516 if (!isPowerOf2_32(Value: MemSizeInBits)) {
4517 LargeSplitSize = llvm::bit_floor<uint64_t>(Value: MemTy.getSizeInBits());
4518 SmallSplitSize = MemTy.getSizeInBits() - LargeSplitSize;
4519 } else {
4520 auto &Ctx = MF.getFunction().getContext();
4521 if (TLI.allowsMemoryAccess(Context&: Ctx, DL: MIRBuilder.getDataLayout(), Ty: MemTy, MMO))
4522 return UnableToLegalize; // Don't know what we're being asked to do.
4523
4524 SmallSplitSize = LargeSplitSize = MemSizeInBits / 2;
4525 }
4526
4527 // Extend to the next pow-2. If this store was itself the result of lowering,
4528 // e.g. an s56 store being broken into s32 + s24, we might have a stored type
4529 // that's wider than the stored size.
4530 unsigned AnyExtSize = PowerOf2Ceil(A: MemTy.getSizeInBits());
4531 const LLT NewSrcTy = LLT::integer(SizeInBits: AnyExtSize);
4532
4533 if (SrcTy.isPointer()) {
4534 const LLT IntPtrTy = LLT::integer(SizeInBits: SrcTy.getSizeInBits());
4535 SrcReg = MIRBuilder.buildPtrToInt(Dst: IntPtrTy, Src: SrcReg).getReg(Idx: 0);
4536 }
4537
4538 auto ExtVal = MIRBuilder.buildAnyExtOrTrunc(Res: NewSrcTy, Op: SrcReg);
4539
4540 // Obtain the smaller value by shifting away the larger value.
4541 auto ShiftAmt = MIRBuilder.buildConstant(Res: NewSrcTy, Val: LargeSplitSize);
4542 auto SmallVal = MIRBuilder.buildLShr(Dst: NewSrcTy, Src0: ExtVal, Src1: ShiftAmt);
4543
4544 // Generate the PtrAdd and truncating stores.
4545 LLT PtrTy = MRI.getType(Reg: PtrReg);
4546 auto OffsetCst = MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: PtrTy.getSizeInBits()),
4547 Val: LargeSplitSize / 8);
4548 auto SmallPtr = MIRBuilder.buildObjectPtrOffset(Res: PtrTy, Op0: PtrReg, Op1: OffsetCst);
4549
4550 MachineMemOperand *LargeMMO =
4551 MF.getMachineMemOperand(MMO: &MMO, Offset: 0, Size: LargeSplitSize / 8);
4552 MachineMemOperand *SmallMMO =
4553 MF.getMachineMemOperand(MMO: &MMO, Offset: LargeSplitSize / 8, Size: SmallSplitSize / 8);
4554 MIRBuilder.buildStore(Val: ExtVal, Addr: PtrReg, MMO&: *LargeMMO);
4555 MIRBuilder.buildStore(Val: SmallVal, Addr: SmallPtr, MMO&: *SmallMMO);
4556 StoreMI.eraseFromParent();
4557 return Legalized;
4558}
4559
4560LegalizerHelper::LegalizeResult
4561LegalizerHelper::scalarizeVectorBooleanStore(GStore &StoreMI) {
4562 Register SrcReg = StoreMI.getValueReg();
4563 Register PtrReg = StoreMI.getPointerReg();
4564 LLT SrcTy = MRI.getType(Reg: SrcReg);
4565 MachineMemOperand &MMO = **StoreMI.memoperands_begin();
4566 LLT MemTy = MMO.getMemoryType();
4567 LLT MemScalarTy = MemTy.getElementType();
4568 MachineFunction &MF = MIRBuilder.getMF();
4569
4570 assert(SrcTy.isVector() && "Expect a vector store type");
4571
4572 if (!MemScalarTy.isByteSized()) {
4573 // We need to build an integer scalar of the vector bit pattern.
4574 // It's not legal for us to add padding when storing a vector.
4575 unsigned NumBits = MemTy.getSizeInBits();
4576 LLT IntTy = LLT::integer(SizeInBits: NumBits);
4577 auto CurrVal = MIRBuilder.buildConstant(Res: IntTy, Val: 0);
4578 LLT IdxTy = TLI.getVectorIdxLLT(DL: MF.getDataLayout());
4579
4580 for (unsigned I = 0, E = MemTy.getNumElements(); I < E; ++I) {
4581 auto Elt = MIRBuilder.buildExtractVectorElement(
4582 Res: SrcTy.getElementType(), Val: SrcReg, Idx: MIRBuilder.buildConstant(Res: IdxTy, Val: I));
4583 auto Trunc = MIRBuilder.buildTrunc(Res: MemScalarTy, Op: Elt);
4584 auto ZExt = MIRBuilder.buildZExt(Res: IntTy, Op: Trunc);
4585 unsigned ShiftIntoIdx = MF.getDataLayout().isBigEndian()
4586 ? (MemTy.getNumElements() - 1) - I
4587 : I;
4588 auto ShiftAmt = MIRBuilder.buildConstant(
4589 Res: IntTy, Val: ShiftIntoIdx * MemScalarTy.getSizeInBits());
4590 auto Shifted = MIRBuilder.buildShl(Dst: IntTy, Src0: ZExt, Src1: ShiftAmt);
4591 CurrVal = MIRBuilder.buildOr(Dst: IntTy, Src0: CurrVal, Src1: Shifted);
4592 }
4593 auto PtrInfo = MMO.getPointerInfo();
4594 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo, Ty: IntTy);
4595 MIRBuilder.buildStore(Val: CurrVal, Addr: PtrReg, MMO&: *NewMMO);
4596 StoreMI.eraseFromParent();
4597 return Legalized;
4598 }
4599
4600 // TODO: implement simple scalarization.
4601 return UnableToLegalize;
4602}
4603
4604LegalizerHelper::LegalizeResult
4605LegalizerHelper::bitcast(MachineInstr &MI, unsigned TypeIdx, LLT CastTy) {
4606 switch (MI.getOpcode()) {
4607 case TargetOpcode::G_LOAD: {
4608 if (TypeIdx != 0)
4609 return UnableToLegalize;
4610 MachineMemOperand &MMO = **MI.memoperands_begin();
4611
4612 // Not sure how to interpret a bitcast of an extending load.
4613 if (MMO.getMemoryType().getSizeInBits() != CastTy.getSizeInBits())
4614 return UnableToLegalize;
4615
4616 Observer.changingInstr(MI);
4617 bitcastDst(MI, CastTy, OpIdx: 0);
4618 MMO.setType(CastTy);
4619 // The range metadata is no longer valid when reinterpreted as a different
4620 // type.
4621 MMO.clearRanges();
4622 Observer.changedInstr(MI);
4623 return Legalized;
4624 }
4625 case TargetOpcode::G_STORE: {
4626 if (TypeIdx != 0)
4627 return UnableToLegalize;
4628
4629 MachineMemOperand &MMO = **MI.memoperands_begin();
4630
4631 // Not sure how to interpret a bitcast of a truncating store.
4632 if (MMO.getMemoryType().getSizeInBits() != CastTy.getSizeInBits())
4633 return UnableToLegalize;
4634
4635 Observer.changingInstr(MI);
4636 bitcastSrc(MI, CastTy, OpIdx: 0);
4637 MMO.setType(CastTy);
4638 Observer.changedInstr(MI);
4639 return Legalized;
4640 }
4641 case TargetOpcode::G_SELECT: {
4642 if (TypeIdx != 0)
4643 return UnableToLegalize;
4644
4645 if (MRI.getType(Reg: MI.getOperand(i: 1).getReg()).isVector()) {
4646 LLVM_DEBUG(
4647 dbgs() << "bitcast action not implemented for vector select\n");
4648 return UnableToLegalize;
4649 }
4650
4651 Observer.changingInstr(MI);
4652 bitcastSrc(MI, CastTy, OpIdx: 2);
4653 bitcastSrc(MI, CastTy, OpIdx: 3);
4654 bitcastDst(MI, CastTy, OpIdx: 0);
4655 Observer.changedInstr(MI);
4656 return Legalized;
4657 }
4658 case TargetOpcode::G_AND:
4659 case TargetOpcode::G_OR:
4660 case TargetOpcode::G_XOR: {
4661 Observer.changingInstr(MI);
4662 bitcastSrc(MI, CastTy, OpIdx: 1);
4663 bitcastSrc(MI, CastTy, OpIdx: 2);
4664 bitcastDst(MI, CastTy, OpIdx: 0);
4665 Observer.changedInstr(MI);
4666 return Legalized;
4667 }
4668 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
4669 return bitcastExtractVectorElt(MI, TypeIdx, CastTy);
4670 case TargetOpcode::G_INSERT_VECTOR_ELT:
4671 return bitcastInsertVectorElt(MI, TypeIdx, CastTy);
4672 case TargetOpcode::G_CONCAT_VECTORS:
4673 return bitcastConcatVector(MI, TypeIdx, CastTy);
4674 case TargetOpcode::G_SHUFFLE_VECTOR:
4675 return bitcastShuffleVector(MI, TypeIdx, CastTy);
4676 case TargetOpcode::G_EXTRACT_SUBVECTOR:
4677 return bitcastExtractSubvector(MI, TypeIdx, CastTy);
4678 case TargetOpcode::G_INSERT_SUBVECTOR:
4679 return bitcastInsertSubvector(MI, TypeIdx, CastTy);
4680 default:
4681 return UnableToLegalize;
4682 }
4683}
4684
4685// Legalize an instruction by changing the opcode in place.
4686void LegalizerHelper::changeOpcode(MachineInstr &MI, unsigned NewOpcode) {
4687 Observer.changingInstr(MI);
4688 MI.setDesc(MIRBuilder.getTII().get(Opcode: NewOpcode));
4689 Observer.changedInstr(MI);
4690}
4691
4692LegalizerHelper::LegalizeResult
4693LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) {
4694 using namespace TargetOpcode;
4695 switch(MI.getOpcode()) {
4696 default:
4697 return UnableToLegalize;
4698 case TargetOpcode::G_FCONSTANT:
4699 return lowerFConstant(MI);
4700 case TargetOpcode::G_BITCAST:
4701 return lowerBitcast(MI);
4702 case TargetOpcode::G_SREM:
4703 case TargetOpcode::G_UREM: {
4704 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4705 auto Quot =
4706 MIRBuilder.buildInstr(Opc: MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV, DstOps: {Ty},
4707 SrcOps: {MI.getOperand(i: 1), MI.getOperand(i: 2)});
4708
4709 auto Prod = MIRBuilder.buildMul(Dst: Ty, Src0: Quot, Src1: MI.getOperand(i: 2));
4710 MIRBuilder.buildSub(Dst: MI.getOperand(i: 0), Src0: MI.getOperand(i: 1), Src1: Prod);
4711 MI.eraseFromParent();
4712 return Legalized;
4713 }
4714 case TargetOpcode::G_SADDO:
4715 case TargetOpcode::G_SSUBO:
4716 return lowerSADDO_SSUBO(MI);
4717 case TargetOpcode::G_SADDE:
4718 return lowerSADDE(MI);
4719 case TargetOpcode::G_SSUBE:
4720 return lowerSSUBE(MI);
4721 case TargetOpcode::G_UMULH:
4722 case TargetOpcode::G_SMULH:
4723 return lowerSMULH_UMULH(MI);
4724 case TargetOpcode::G_SMULO:
4725 case TargetOpcode::G_UMULO: {
4726 // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the
4727 // result.
4728 auto [Res, Overflow, LHS, RHS] = MI.getFirst4Regs();
4729 LLT Ty = MRI.getType(Reg: Res);
4730
4731 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO
4732 ? TargetOpcode::G_SMULH
4733 : TargetOpcode::G_UMULH;
4734
4735 Observer.changingInstr(MI);
4736 const auto &TII = MIRBuilder.getTII();
4737 MI.setDesc(TII.get(Opcode: TargetOpcode::G_MUL));
4738 MI.removeOperand(OpNo: 1);
4739 Observer.changedInstr(MI);
4740
4741 auto HiPart = MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Ty}, SrcOps: {LHS, RHS});
4742 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
4743
4744 // Move insert point forward so we can use the Res register if needed.
4745 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
4746
4747 // For *signed* multiply, overflow is detected by checking:
4748 // (hi != (lo >> bitwidth-1))
4749 if (Opcode == TargetOpcode::G_SMULH) {
4750 auto ShiftAmt = MIRBuilder.buildConstant(Res: Ty, Val: Ty.getSizeInBits() - 1);
4751 auto Shifted = MIRBuilder.buildAShr(Dst: Ty, Src0: Res, Src1: ShiftAmt);
4752 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: Overflow, Op0: HiPart, Op1: Shifted);
4753 } else {
4754 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: Overflow, Op0: HiPart, Op1: Zero);
4755 }
4756 return Legalized;
4757 }
4758 case TargetOpcode::G_FNEG: {
4759 auto [Res, ResTy, SubByReg, SubByRegTy] = MI.getFirst2RegLLTs();
4760 LLT TyInt =
4761 ResTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: ResTy.getScalarSizeInBits()));
4762 Register CastedSubByReg = SubByReg;
4763
4764 if (!SubByRegTy.getScalarType().isAnyScalar() &&
4765 !SubByRegTy.getScalarType().isInteger()) {
4766 auto BitcastDst = SubByRegTy.changeElementType(
4767 NewEltTy: LLT::integer(SizeInBits: SubByRegTy.getScalarSizeInBits()));
4768 CastedSubByReg = MIRBuilder.buildBitcast(Dst: BitcastDst, Src: SubByReg).getReg(Idx: 0);
4769 }
4770
4771 auto SignMask = MIRBuilder.buildConstant(
4772 Res: TyInt, Val: APInt::getSignMask(BitWidth: TyInt.getScalarSizeInBits()));
4773
4774 if (ResTy != TyInt) {
4775 Register NewDst =
4776 MIRBuilder.buildXor(Dst: TyInt, Src0: CastedSubByReg, Src1: SignMask).getReg(Idx: 0);
4777 MIRBuilder.buildBitcast(Dst: Res, Src: NewDst);
4778 } else
4779 MIRBuilder.buildXor(Dst: Res, Src0: CastedSubByReg, Src1: SignMask).getReg(Idx: 0);
4780
4781 MI.eraseFromParent();
4782 return Legalized;
4783 }
4784 case TargetOpcode::G_FSUB:
4785 case TargetOpcode::G_STRICT_FSUB: {
4786 auto [Res, LHS, RHS] = MI.getFirst3Regs();
4787 LLT Ty = MRI.getType(Reg: Res);
4788
4789 // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)).
4790 auto Neg = MIRBuilder.buildFNeg(Dst: Ty, Src0: RHS);
4791
4792 if (MI.getOpcode() == TargetOpcode::G_STRICT_FSUB)
4793 MIRBuilder.buildStrictFAdd(Dst: Res, Src0: LHS, Src1: Neg, Flags: MI.getFlags());
4794 else
4795 MIRBuilder.buildFAdd(Dst: Res, Src0: LHS, Src1: Neg, Flags: MI.getFlags());
4796
4797 MI.eraseFromParent();
4798 return Legalized;
4799 }
4800 case TargetOpcode::G_FMAD:
4801 return lowerFMad(MI);
4802 case TargetOpcode::G_FFLOOR:
4803 return lowerFFloor(MI);
4804 case TargetOpcode::G_LROUND:
4805 case TargetOpcode::G_LLROUND: {
4806 Register DstReg = MI.getOperand(i: 0).getReg();
4807 Register SrcReg = MI.getOperand(i: 1).getReg();
4808 LLT SrcTy = MRI.getType(Reg: SrcReg);
4809 auto Round = MIRBuilder.buildInstr(Opc: TargetOpcode::G_INTRINSIC_ROUND, DstOps: {SrcTy},
4810 SrcOps: {SrcReg});
4811 MIRBuilder.buildFPTOSI(Dst: DstReg, Src0: Round);
4812 MI.eraseFromParent();
4813 return Legalized;
4814 }
4815 case TargetOpcode::G_INTRINSIC_ROUND:
4816 return lowerIntrinsicRound(MI);
4817 case TargetOpcode::G_FRINT: {
4818 // Since round even is the assumed rounding mode for unconstrained FP
4819 // operations, rint and roundeven are the same operation.
4820 changeOpcode(MI, NewOpcode: TargetOpcode::G_INTRINSIC_ROUNDEVEN);
4821 return Legalized;
4822 }
4823 case TargetOpcode::G_INTRINSIC_LRINT:
4824 case TargetOpcode::G_INTRINSIC_LLRINT: {
4825 Register DstReg = MI.getOperand(i: 0).getReg();
4826 Register SrcReg = MI.getOperand(i: 1).getReg();
4827 LLT SrcTy = MRI.getType(Reg: SrcReg);
4828 auto Round =
4829 MIRBuilder.buildInstr(Opc: TargetOpcode::G_FRINT, DstOps: {SrcTy}, SrcOps: {SrcReg});
4830 MIRBuilder.buildFPTOSI(Dst: DstReg, Src0: Round);
4831 MI.eraseFromParent();
4832 return Legalized;
4833 }
4834 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
4835 auto [OldValRes, SuccessRes, Addr, CmpVal, NewVal] = MI.getFirst5Regs();
4836 Register NewOldValRes = MRI.cloneVirtualRegister(VReg: OldValRes);
4837 MIRBuilder.buildAtomicCmpXchg(OldValRes: NewOldValRes, Addr, CmpVal, NewVal,
4838 MMO&: **MI.memoperands_begin());
4839 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: SuccessRes, Op0: NewOldValRes, Op1: CmpVal);
4840 MIRBuilder.buildCopy(Res: OldValRes, Op: NewOldValRes);
4841 MI.eraseFromParent();
4842 return Legalized;
4843 }
4844 case TargetOpcode::G_LOAD:
4845 case TargetOpcode::G_SEXTLOAD:
4846 case TargetOpcode::G_ZEXTLOAD:
4847 return lowerLoad(LoadMI&: cast<GAnyLoad>(Val&: MI));
4848 case TargetOpcode::G_STORE:
4849 return lowerStore(StoreMI&: cast<GStore>(Val&: MI));
4850 case TargetOpcode::G_CTLZ_ZERO_POISON:
4851 case TargetOpcode::G_CTTZ_ZERO_POISON:
4852 case TargetOpcode::G_CTLZ:
4853 case TargetOpcode::G_CTTZ:
4854 case TargetOpcode::G_CTPOP:
4855 case TargetOpcode::G_CTLS:
4856 return lowerBitCount(MI);
4857 case G_UADDO: {
4858 auto [Res, CarryOut, LHS, RHS] = MI.getFirst4Regs();
4859
4860 Register NewRes = MRI.cloneVirtualRegister(VReg: Res);
4861
4862 MIRBuilder.buildAdd(Dst: NewRes, Src0: LHS, Src1: RHS);
4863 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_ULT, Res: CarryOut, Op0: NewRes, Op1: RHS);
4864
4865 MIRBuilder.buildCopy(Res, Op: NewRes);
4866
4867 MI.eraseFromParent();
4868 return Legalized;
4869 }
4870 case G_UADDE: {
4871 auto [Res, CarryOut, LHS, RHS, CarryIn] = MI.getFirst5Regs();
4872 const LLT CondTy = MRI.getType(Reg: CarryOut);
4873 const LLT Ty = MRI.getType(Reg: Res);
4874
4875 Register NewRes = MRI.cloneVirtualRegister(VReg: Res);
4876
4877 // Initial add of the two operands.
4878 auto TmpRes = MIRBuilder.buildAdd(Dst: Ty, Src0: LHS, Src1: RHS);
4879
4880 // Initial check for carry.
4881 auto Carry = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_ULT, Res: CondTy, Op0: TmpRes, Op1: LHS);
4882
4883 // Add the sum and the carry.
4884 auto ZExtCarryIn = MIRBuilder.buildZExt(Res: Ty, Op: CarryIn);
4885 MIRBuilder.buildAdd(Dst: NewRes, Src0: TmpRes, Src1: ZExtCarryIn);
4886
4887 // Second check for carry. We can only carry if the initial sum is all 1s
4888 // and the carry is set, resulting in a new sum of 0.
4889 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
4890 auto ResEqZero =
4891 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CondTy, Op0: NewRes, Op1: Zero);
4892 auto Carry2 = MIRBuilder.buildAnd(Dst: CondTy, Src0: ResEqZero, Src1: CarryIn);
4893 MIRBuilder.buildOr(Dst: CarryOut, Src0: Carry, Src1: Carry2);
4894
4895 MIRBuilder.buildCopy(Res, Op: NewRes);
4896
4897 MI.eraseFromParent();
4898 return Legalized;
4899 }
4900 case G_USUBO: {
4901 auto [Res, BorrowOut, LHS, RHS] = MI.getFirst4Regs();
4902
4903 MIRBuilder.buildSub(Dst: Res, Src0: LHS, Src1: RHS);
4904 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_ULT, Res: BorrowOut, Op0: LHS, Op1: RHS);
4905
4906 MI.eraseFromParent();
4907 return Legalized;
4908 }
4909 case G_USUBE: {
4910 auto [Res, BorrowOut, LHS, RHS, BorrowIn] = MI.getFirst5Regs();
4911 const LLT CondTy = MRI.getType(Reg: BorrowOut);
4912 const LLT Ty = MRI.getType(Reg: Res);
4913
4914 // Initial subtract of the two operands.
4915 auto TmpRes = MIRBuilder.buildSub(Dst: Ty, Src0: LHS, Src1: RHS);
4916
4917 // Initial check for borrow.
4918 auto Borrow = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_UGT, Res: CondTy, Op0: TmpRes, Op1: LHS);
4919
4920 // Subtract the borrow from the first subtract.
4921 auto ZExtBorrowIn = MIRBuilder.buildZExt(Res: Ty, Op: BorrowIn);
4922 MIRBuilder.buildSub(Dst: Res, Src0: TmpRes, Src1: ZExtBorrowIn);
4923
4924 // Second check for borrow. We can only borrow if the initial difference is
4925 // 0 and the borrow is set, resulting in a new difference of all 1s.
4926 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
4927 auto TmpResEqZero =
4928 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CondTy, Op0: TmpRes, Op1: Zero);
4929 auto Borrow2 = MIRBuilder.buildAnd(Dst: CondTy, Src0: TmpResEqZero, Src1: BorrowIn);
4930 MIRBuilder.buildOr(Dst: BorrowOut, Src0: Borrow, Src1: Borrow2);
4931
4932 MI.eraseFromParent();
4933 return Legalized;
4934 }
4935 case G_UITOFP:
4936 return lowerUITOFP(MI);
4937 case G_SITOFP:
4938 return lowerSITOFP(MI);
4939 case G_FPTOUI:
4940 return lowerFPTOUI(MI);
4941 case G_FPTOSI:
4942 return lowerFPTOSI(MI);
4943 case G_FPTOUI_SAT:
4944 case G_FPTOSI_SAT:
4945 return lowerFPTOINT_SAT(MI);
4946 case G_FPEXT:
4947 return lowerFPEXT(MI);
4948 case G_FPTRUNC:
4949 return lowerFPTRUNC(MI);
4950 case G_FPOWI:
4951 return lowerFPOWI(MI);
4952 case G_FMODF:
4953 return lowerFMODF(MI);
4954 case G_SMIN:
4955 case G_SMAX:
4956 case G_UMIN:
4957 case G_UMAX:
4958 return lowerMinMax(MI);
4959 case G_SCMP:
4960 case G_UCMP:
4961 return lowerThreewayCompare(MI);
4962 case G_FCOPYSIGN:
4963 return lowerFCopySign(MI);
4964 case G_FMINNUM:
4965 case G_FMAXNUM:
4966 case G_FMINIMUMNUM:
4967 case G_FMAXIMUMNUM:
4968 return lowerFMinNumMaxNum(MI);
4969 case G_FMINIMUM:
4970 case G_FMAXIMUM:
4971 return lowerFMinimumMaximum(MI);
4972 case G_MERGE_VALUES:
4973 return lowerMergeValues(MI);
4974 case G_UNMERGE_VALUES:
4975 return lowerUnmergeValues(MI);
4976 case TargetOpcode::G_SEXT_INREG: {
4977 assert(MI.getOperand(2).isImm() && "Expected immediate");
4978 int64_t SizeInBits = MI.getOperand(i: 2).getImm();
4979
4980 auto [DstReg, SrcReg] = MI.getFirst2Regs();
4981 LLT DstTy = MRI.getType(Reg: DstReg);
4982 Register TmpRes = MRI.createGenericVirtualRegister(Ty: DstTy);
4983
4984 auto MIBSz = MIRBuilder.buildConstant(Res: DstTy, Val: DstTy.getScalarSizeInBits() - SizeInBits);
4985 MIRBuilder.buildShl(Dst: TmpRes, Src0: SrcReg, Src1: MIBSz->getOperand(i: 0));
4986 MIRBuilder.buildAShr(Dst: DstReg, Src0: TmpRes, Src1: MIBSz->getOperand(i: 0));
4987 MI.eraseFromParent();
4988 return Legalized;
4989 }
4990 case G_EXTRACT_VECTOR_ELT:
4991 case G_INSERT_VECTOR_ELT:
4992 return lowerExtractInsertVectorElt(MI);
4993 case G_SHUFFLE_VECTOR:
4994 return lowerShuffleVector(MI);
4995 case G_VECTOR_COMPRESS:
4996 return lowerVECTOR_COMPRESS(MI);
4997 case G_DYN_STACKALLOC:
4998 return lowerDynStackAlloc(MI);
4999 case G_INSERT_SUBVECTOR: {
5000 if (MRI.getType(Reg: MI.getOperand(i: 1).getReg()).isScalable() ||
5001 MRI.getType(Reg: MI.getOperand(i: 2).getReg()).isScalable())
5002 return UnableToLegalize;
5003
5004 // Check that subvector is half size of main vector
5005 Register Vector = MI.getOperand(i: 1).getReg();
5006 Register Subvector = MI.getOperand(i: 2).getReg();
5007 auto InsertionPointImm = MI.getOperand(i: 3).getImm();
5008
5009 LLT VectorTy = MRI.getType(Reg: Vector);
5010 LLT DstTy = MRI.getType(Reg: Subvector);
5011 // If so, -> concat(subvector, extract(half of vector))
5012 // (Operands can be either way round depending on insertion point
5013 if (VectorTy.getSizeInBits() == DstTy.getSizeInBits() * 2) {
5014 bool InsertInLowHalf = InsertionPointImm == 0;
5015 auto Extract = MIRBuilder.buildExtractSubvector(
5016 Res: DstTy, Src: Vector,
5017 Index: (uint64_t)(InsertInLowHalf ? VectorTy.getNumElements() / 2 : 0));
5018
5019 auto LowHalf = InsertInLowHalf ? Subvector : Extract.getReg(Idx: 0);
5020 auto HighHalf = InsertInLowHalf ? Extract.getReg(Idx: 0) : Subvector;
5021
5022 MIRBuilder.buildInstr(Opc: TargetOpcode::G_CONCAT_VECTORS, DstOps: {MI.getOperand(i: 0)},
5023 SrcOps: {LowHalf, HighHalf});
5024 MI.eraseFromParent();
5025 return Legalized;
5026 }
5027 // Else -> shuffle(vector, extend(subvector, size(vector)), mask)
5028 else {
5029 // Extend subvector to same size as vector
5030 Register ExtendedSubvector = MRI.createGenericVirtualRegister(Ty: VectorTy);
5031 MIRBuilder.buildPadVectorWithUndefElements(Res: ExtendedSubvector, Op0: Subvector);
5032
5033 // Calculate mask required for this shuffle
5034 SmallVector<int> Mask;
5035 for (int i = 0; i < VectorTy.getNumElements(); i++) {
5036 // If this index is within bounds, put subvector's index into mask
5037 if (i >= InsertionPointImm &&
5038 i < InsertionPointImm + DstTy.getNumElements())
5039 Mask.push_back(Elt: VectorTy.getNumElements() + i - InsertionPointImm);
5040 else
5041 Mask.push_back(Elt: i);
5042 }
5043
5044 // Build shuffle
5045 MIRBuilder.buildShuffleVector(Res: MI.getOperand(i: 0), Src1: Vector, Src2: ExtendedSubvector,
5046 Mask);
5047 MI.eraseFromParent();
5048 return Legalized;
5049 }
5050 }
5051 case G_EXTRACT_SUBVECTOR: {
5052 Register DstReg = MI.getOperand(i: 0).getReg();
5053 Register SrcReg = MI.getOperand(i: 1).getReg();
5054 uint64_t ExtractionPointImm = MI.getOperand(i: 2).getImm();
5055
5056 LLT SrcTy = MRI.getType(Reg: SrcReg);
5057 LLT DstTy = MRI.getType(Reg: DstReg);
5058
5059 if (SrcTy.isScalable())
5060 return UnableToLegalize;
5061
5062 if (SrcTy.getScalarType() != DstTy.getScalarType())
5063 return UnableToLegalize;
5064
5065 // extract_subvector = build_vector(extract_element, extract_element, ...)
5066 SmallVector<Register> ExtractedElements;
5067 for (uint64_t i = 0; i < DstTy.getNumElements(); i++) {
5068 ExtractedElements.push_back(
5069 Elt: MIRBuilder
5070 .buildExtractVectorElementConstant(Res: SrcTy.getScalarType(), Val: SrcReg,
5071 Idx: ExtractionPointImm + i)
5072 .getReg(Idx: 0));
5073 }
5074
5075 MIRBuilder.buildBuildVector(Res: DstReg, Ops: ExtractedElements);
5076 MI.eraseFromParent();
5077 return Legalized;
5078 }
5079 case G_STACKSAVE:
5080 return lowerStackSave(MI);
5081 case G_STACKRESTORE:
5082 return lowerStackRestore(MI);
5083 case G_EXTRACT:
5084 return lowerExtract(MI);
5085 case G_INSERT:
5086 return lowerInsert(MI);
5087 case G_BSWAP:
5088 return lowerBswap(MI);
5089 case G_BITREVERSE:
5090 return lowerBitreverse(MI);
5091 case G_READ_REGISTER:
5092 case G_WRITE_REGISTER:
5093 return lowerReadWriteRegister(MI);
5094 case G_UADDSAT:
5095 case G_USUBSAT: {
5096 // Try to make a reasonable guess about which lowering strategy to use. The
5097 // target can override this with custom lowering and calling the
5098 // implementation functions.
5099 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5100 if (LI.isLegalOrCustom(Query: {G_UMIN, Ty}))
5101 return lowerAddSubSatToMinMax(MI);
5102 return lowerAddSubSatToAddoSubo(MI);
5103 }
5104 case G_SADDSAT:
5105 case G_SSUBSAT: {
5106 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5107
5108 // FIXME: It would probably make more sense to see if G_SADDO is preferred,
5109 // since it's a shorter expansion. However, we would need to figure out the
5110 // preferred boolean type for the carry out for the query.
5111 if (LI.isLegalOrCustom(Query: {G_SMIN, Ty}) && LI.isLegalOrCustom(Query: {G_SMAX, Ty}))
5112 return lowerAddSubSatToMinMax(MI);
5113 return lowerAddSubSatToAddoSubo(MI);
5114 }
5115 case G_SSHLSAT:
5116 case G_USHLSAT:
5117 return lowerShlSat(MI);
5118 case G_TRUNC_SSAT_S:
5119 case G_TRUNC_USAT_U:
5120 case G_TRUNC_SSAT_U:
5121 return lowerTruncSat(MI);
5122 case G_ABS:
5123 return lowerAbsToAddXor(MI);
5124 case G_ABDS:
5125 case G_ABDU: {
5126 bool IsSigned = MI.getOpcode() == G_ABDS;
5127 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5128 if ((IsSigned && LI.isLegal(Query: {G_SMIN, Ty}) && LI.isLegal(Query: {G_SMAX, Ty})) ||
5129 (!IsSigned && LI.isLegal(Query: {G_UMIN, Ty}) && LI.isLegal(Query: {G_UMAX, Ty}))) {
5130 return lowerAbsDiffToMinMax(MI);
5131 }
5132 return lowerAbsDiffToSelect(MI);
5133 }
5134 case G_FABS:
5135 return lowerFAbs(MI);
5136 case G_SELECT:
5137 return lowerSelect(MI);
5138 case G_IS_FPCLASS:
5139 return lowerISFPCLASS(MI);
5140 case G_SDIVREM:
5141 case G_UDIVREM:
5142 return lowerDIVREM(MI);
5143 case G_FSHL:
5144 case G_FSHR:
5145 return lowerFunnelShift(MI);
5146 case G_ROTL:
5147 case G_ROTR:
5148 return lowerRotate(MI);
5149 case G_MEMSET:
5150 case G_MEMCPY:
5151 case G_MEMMOVE:
5152 case G_MEMCPY_INLINE:
5153 case G_MEMSET_INLINE:
5154 return lowerMemCpyFamily(MI);
5155 case G_ZEXT:
5156 case G_SEXT:
5157 case G_ANYEXT:
5158 return lowerEXT(MI);
5159 case G_TRUNC:
5160 return lowerTRUNC(MI);
5161 GISEL_VECREDUCE_CASES_NONSEQ
5162 return lowerVectorReduction(MI);
5163 case G_VAARG:
5164 return lowerVAArg(MI);
5165 case G_ATOMICRMW_SUB: {
5166 auto [Ret, Mem, Val] = MI.getFirst3Regs();
5167 const LLT ValTy = MRI.getType(Reg: Val);
5168 MachineMemOperand *MMO = *MI.memoperands_begin();
5169
5170 auto VNeg = MIRBuilder.buildNeg(Dst: ValTy, Src0: Val);
5171 MIRBuilder.buildAtomicRMW(Opcode: G_ATOMICRMW_ADD, OldValRes: Ret, Addr: Mem, Val: VNeg, MMO&: *MMO);
5172 MI.eraseFromParent();
5173 return Legalized;
5174 }
5175 case G_SMULFIX:
5176 case G_UMULFIX:
5177 case G_SMULFIXSAT:
5178 case G_UMULFIXSAT:
5179 return lowerMulfix(MI);
5180 }
5181}
5182
5183Align LegalizerHelper::getStackTemporaryAlignment(LLT Ty,
5184 Align MinAlign) const {
5185 // FIXME: We're missing a way to go back from LLT to llvm::Type to query the
5186 // datalayout for the preferred alignment. Also there should be a target hook
5187 // for this to allow targets to reduce the alignment and ignore the
5188 // datalayout. e.g. AMDGPU should always use a 4-byte alignment, regardless of
5189 // the type.
5190 return std::max(a: Align(PowerOf2Ceil(A: Ty.getSizeInBytes())), b: MinAlign);
5191}
5192
5193MachineInstrBuilder
5194LegalizerHelper::createStackTemporary(TypeSize Bytes, Align Alignment,
5195 MachinePointerInfo &PtrInfo) {
5196 MachineFunction &MF = MIRBuilder.getMF();
5197 const DataLayout &DL = MIRBuilder.getDataLayout();
5198 int FrameIdx = MF.getFrameInfo().CreateStackObject(Size: Bytes, Alignment, isSpillSlot: false);
5199
5200 unsigned AddrSpace = DL.getAllocaAddrSpace();
5201 LLT FramePtrTy = LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL.getPointerSizeInBits(AS: AddrSpace));
5202
5203 PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIdx);
5204 return MIRBuilder.buildFrameIndex(Res: FramePtrTy, Idx: FrameIdx);
5205}
5206
5207MachineInstrBuilder LegalizerHelper::createStackStoreLoad(const DstOp &Res,
5208 const SrcOp &Val) {
5209 LLT SrcTy = Val.getLLTTy(MRI);
5210 Align StackTypeAlign =
5211 std::max(a: getStackTemporaryAlignment(Ty: SrcTy),
5212 b: getStackTemporaryAlignment(Ty: Res.getLLTTy(MRI)));
5213 MachinePointerInfo PtrInfo;
5214 auto StackTemp =
5215 createStackTemporary(Bytes: SrcTy.getSizeInBytes(), Alignment: StackTypeAlign, PtrInfo);
5216
5217 MIRBuilder.buildStore(Val, Addr: StackTemp, PtrInfo, Alignment: StackTypeAlign);
5218 return MIRBuilder.buildLoad(Res, Addr: StackTemp, PtrInfo, Alignment: StackTypeAlign);
5219}
5220
5221static Register clampVectorIndex(MachineIRBuilder &B, Register IdxReg,
5222 LLT VecTy) {
5223 LLT IdxTy = B.getMRI()->getType(Reg: IdxReg);
5224 unsigned NElts = VecTy.getNumElements();
5225
5226 int64_t IdxVal;
5227 if (mi_match(R: IdxReg, MRI: *B.getMRI(), P: m_ICst(Cst&: IdxVal))) {
5228 if (IdxVal < VecTy.getNumElements())
5229 return IdxReg;
5230 // If a constant index would be out of bounds, clamp it as well.
5231 }
5232
5233 if (isPowerOf2_32(Value: NElts)) {
5234 APInt Imm = APInt::getLowBitsSet(numBits: IdxTy.getSizeInBits(), loBitsSet: Log2_32(Value: NElts));
5235 return B.buildAnd(Dst: IdxTy, Src0: IdxReg, Src1: B.buildConstant(Res: IdxTy, Val: Imm)).getReg(Idx: 0);
5236 }
5237
5238 return B.buildUMin(Dst: IdxTy, Src0: IdxReg, Src1: B.buildConstant(Res: IdxTy, Val: NElts - 1))
5239 .getReg(Idx: 0);
5240}
5241
5242Register LegalizerHelper::getVectorElementPointer(Register VecPtr, LLT VecTy,
5243 Register Index) {
5244 LLT EltTy = VecTy.getElementType();
5245
5246 // Calculate the element offset and add it to the pointer.
5247 unsigned EltSize = EltTy.getSizeInBits() / 8; // FIXME: should be ABI size.
5248 assert(EltSize * 8 == EltTy.getSizeInBits() &&
5249 "Converting bits to bytes lost precision");
5250
5251 Index = clampVectorIndex(B&: MIRBuilder, IdxReg: Index, VecTy);
5252
5253 // Convert index to the correct size for the address space.
5254 const DataLayout &DL = MIRBuilder.getDataLayout();
5255 unsigned AS = MRI.getType(Reg: VecPtr).getAddressSpace();
5256 unsigned IndexSizeInBits = DL.getIndexSize(AS) * 8;
5257 LLT IdxTy = MRI.getType(Reg: Index).changeElementSize(NewEltSize: IndexSizeInBits);
5258 if (IdxTy != MRI.getType(Reg: Index))
5259 Index = MIRBuilder.buildSExtOrTrunc(Res: IdxTy, Op: Index).getReg(Idx: 0);
5260
5261 auto Mul = MIRBuilder.buildMul(Dst: IdxTy, Src0: Index,
5262 Src1: MIRBuilder.buildConstant(Res: IdxTy, Val: EltSize));
5263
5264 LLT PtrTy = MRI.getType(Reg: VecPtr);
5265 return MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: VecPtr, Op1: Mul).getReg(Idx: 0);
5266}
5267
5268#ifndef NDEBUG
5269/// Check that all vector operands have same number of elements. Other operands
5270/// should be listed in NonVecOp.
5271static bool hasSameNumEltsOnAllVectorOperands(
5272 GenericMachineInstr &MI, MachineRegisterInfo &MRI,
5273 std::initializer_list<unsigned> NonVecOpIndices) {
5274 if (MI.getNumMemOperands() != 0)
5275 return false;
5276
5277 LLT VecTy = MRI.getType(MI.getReg(0));
5278 if (!VecTy.isVector())
5279 return false;
5280 unsigned NumElts = VecTy.getNumElements();
5281
5282 for (unsigned OpIdx = 1; OpIdx < MI.getNumOperands(); ++OpIdx) {
5283 MachineOperand &Op = MI.getOperand(OpIdx);
5284 if (!Op.isReg()) {
5285 if (!is_contained(NonVecOpIndices, OpIdx))
5286 return false;
5287 continue;
5288 }
5289
5290 LLT Ty = MRI.getType(Op.getReg());
5291 if (!Ty.isVector()) {
5292 if (!is_contained(NonVecOpIndices, OpIdx))
5293 return false;
5294 continue;
5295 }
5296
5297 if (Ty.getNumElements() != NumElts)
5298 return false;
5299 }
5300
5301 return true;
5302}
5303#endif
5304
5305/// Fill \p DstOps with DstOps that have same number of elements combined as
5306/// the Ty. These DstOps have either scalar type when \p NumElts = 1 or are
5307/// vectors with \p NumElts elements. When Ty.getNumElements() is not multiple
5308/// of \p NumElts last DstOp (leftover) has fewer then \p NumElts elements.
5309static void makeDstOps(SmallVectorImpl<DstOp> &DstOps, LLT Ty,
5310 unsigned NumElts) {
5311 LLT LeftoverTy;
5312 assert(Ty.isVector() && "Expected vector type");
5313 LLT NarrowTy = Ty.changeElementCount(EC: ElementCount::getFixed(MinVal: NumElts));
5314 int NumParts, NumLeftover;
5315 std::tie(args&: NumParts, args&: NumLeftover) =
5316 getNarrowTypeBreakDown(OrigTy: Ty, NarrowTy, LeftoverTy);
5317
5318 assert(NumParts > 0 && "Error in getNarrowTypeBreakDown");
5319 for (int i = 0; i < NumParts; ++i) {
5320 DstOps.push_back(Elt: NarrowTy);
5321 }
5322
5323 if (LeftoverTy.isValid()) {
5324 assert(NumLeftover == 1 && "expected exactly one leftover");
5325 DstOps.push_back(Elt: LeftoverTy);
5326 }
5327}
5328
5329/// Operand \p Op is used on \p N sub-instructions. Fill \p Ops with \p N SrcOps
5330/// made from \p Op depending on operand type.
5331static void broadcastSrcOp(SmallVectorImpl<SrcOp> &Ops, unsigned N,
5332 MachineOperand &Op) {
5333 for (unsigned i = 0; i < N; ++i) {
5334 if (Op.isReg())
5335 Ops.push_back(Elt: Op.getReg());
5336 else if (Op.isImm())
5337 Ops.push_back(Elt: Op.getImm());
5338 else if (Op.isPredicate())
5339 Ops.push_back(Elt: static_cast<CmpInst::Predicate>(Op.getPredicate()));
5340 else
5341 llvm_unreachable("Unsupported type");
5342 }
5343}
5344
5345// Handle splitting vector operations which need to have the same number of
5346// elements in each type index, but each type index may have a different element
5347// type.
5348//
5349// e.g. <4 x s64> = G_SHL <4 x s64>, <4 x s32> ->
5350// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
5351// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
5352//
5353// Also handles some irregular breakdown cases, e.g.
5354// e.g. <3 x s64> = G_SHL <3 x s64>, <3 x s32> ->
5355// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
5356// s64 = G_SHL s64, s32
5357LegalizerHelper::LegalizeResult
5358LegalizerHelper::fewerElementsVectorMultiEltType(
5359 GenericMachineInstr &MI, unsigned NumElts,
5360 std::initializer_list<unsigned> NonVecOpIndices) {
5361 assert(hasSameNumEltsOnAllVectorOperands(MI, MRI, NonVecOpIndices) &&
5362 "Non-compatible opcode or not specified non-vector operands");
5363 unsigned OrigNumElts = MRI.getType(Reg: MI.getReg(Idx: 0)).getNumElements();
5364
5365 unsigned NumInputs = MI.getNumOperands() - MI.getNumDefs();
5366 unsigned NumDefs = MI.getNumDefs();
5367
5368 // Create DstOps (sub-vectors with NumElts elts + Leftover) for each output.
5369 // Build instructions with DstOps to use instruction found by CSE directly.
5370 // CSE copies found instruction into given vreg when building with vreg dest.
5371 SmallVector<SmallVector<DstOp, 8>, 2> OutputOpsPieces(NumDefs);
5372 // Output registers will be taken from created instructions.
5373 SmallVector<SmallVector<Register, 8>, 2> OutputRegs(NumDefs);
5374 for (unsigned i = 0; i < NumDefs; ++i) {
5375 makeDstOps(DstOps&: OutputOpsPieces[i], Ty: MRI.getType(Reg: MI.getReg(Idx: i)), NumElts);
5376 }
5377
5378 // Split vector input operands into sub-vectors with NumElts elts + Leftover.
5379 // Operands listed in NonVecOpIndices will be used as is without splitting;
5380 // examples: compare predicate in icmp and fcmp (op 1), vector select with i1
5381 // scalar condition (op 1), immediate in sext_inreg (op 2).
5382 SmallVector<SmallVector<SrcOp, 8>, 3> InputOpsPieces(NumInputs);
5383 for (unsigned UseIdx = NumDefs, UseNo = 0; UseIdx < MI.getNumOperands();
5384 ++UseIdx, ++UseNo) {
5385 if (is_contained(Set: NonVecOpIndices, Element: UseIdx)) {
5386 broadcastSrcOp(Ops&: InputOpsPieces[UseNo], N: OutputOpsPieces[0].size(),
5387 Op&: MI.getOperand(i: UseIdx));
5388 } else {
5389 SmallVector<Register, 8> SplitPieces;
5390 extractVectorParts(Reg: MI.getReg(Idx: UseIdx), NumElts, VRegs&: SplitPieces, MIRBuilder,
5391 MRI);
5392 llvm::append_range(C&: InputOpsPieces[UseNo], R&: SplitPieces);
5393 }
5394 }
5395
5396 unsigned NumLeftovers = OrigNumElts % NumElts ? 1 : 0;
5397
5398 // Take i-th piece of each input operand split and build sub-vector/scalar
5399 // instruction. Set i-th DstOp(s) from OutputOpsPieces as destination(s).
5400 for (unsigned i = 0; i < OrigNumElts / NumElts + NumLeftovers; ++i) {
5401 SmallVector<DstOp, 2> Defs;
5402 for (unsigned DstNo = 0; DstNo < NumDefs; ++DstNo)
5403 Defs.push_back(Elt: OutputOpsPieces[DstNo][i]);
5404
5405 SmallVector<SrcOp, 3> Uses;
5406 for (unsigned InputNo = 0; InputNo < NumInputs; ++InputNo)
5407 Uses.push_back(Elt: InputOpsPieces[InputNo][i]);
5408
5409 auto I = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: Defs, SrcOps: Uses, Flags: MI.getFlags());
5410 for (unsigned DstNo = 0; DstNo < NumDefs; ++DstNo)
5411 OutputRegs[DstNo].push_back(Elt: I.getReg(Idx: DstNo));
5412 }
5413
5414 // Merge small outputs into MI's output for each def operand.
5415 if (NumLeftovers) {
5416 for (unsigned i = 0; i < NumDefs; ++i)
5417 mergeMixedSubvectors(DstReg: MI.getReg(Idx: i), PartRegs: OutputRegs[i]);
5418 } else {
5419 for (unsigned i = 0; i < NumDefs; ++i)
5420 MIRBuilder.buildMergeLikeInstr(Res: MI.getReg(Idx: i), Ops: OutputRegs[i]);
5421 }
5422
5423 MI.eraseFromParent();
5424 return Legalized;
5425}
5426
5427LegalizerHelper::LegalizeResult
5428LegalizerHelper::fewerElementsVectorPhi(GenericMachineInstr &MI,
5429 unsigned NumElts) {
5430 unsigned OrigNumElts = MRI.getType(Reg: MI.getReg(Idx: 0)).getNumElements();
5431
5432 unsigned NumInputs = MI.getNumOperands() - MI.getNumDefs();
5433 unsigned NumDefs = MI.getNumDefs();
5434
5435 SmallVector<DstOp, 8> OutputOpsPieces;
5436 SmallVector<Register, 8> OutputRegs;
5437 makeDstOps(DstOps&: OutputOpsPieces, Ty: MRI.getType(Reg: MI.getReg(Idx: 0)), NumElts);
5438
5439 // Instructions that perform register split will be inserted in basic block
5440 // where register is defined (basic block is in the next operand).
5441 SmallVector<SmallVector<Register, 8>, 3> InputOpsPieces(NumInputs / 2);
5442 for (unsigned UseIdx = NumDefs, UseNo = 0; UseIdx < MI.getNumOperands();
5443 UseIdx += 2, ++UseNo) {
5444 MachineBasicBlock &OpMBB = *MI.getOperand(i: UseIdx + 1).getMBB();
5445 MIRBuilder.setInsertPt(MBB&: OpMBB, II: OpMBB.getFirstTerminatorForward());
5446 extractVectorParts(Reg: MI.getReg(Idx: UseIdx), NumElts, VRegs&: InputOpsPieces[UseNo],
5447 MIRBuilder, MRI);
5448 }
5449
5450 // Build PHIs with fewer elements.
5451 unsigned NumLeftovers = OrigNumElts % NumElts ? 1 : 0;
5452 MIRBuilder.setInsertPt(MBB&: *MI.getParent(), II: MI);
5453 for (unsigned i = 0; i < OrigNumElts / NumElts + NumLeftovers; ++i) {
5454 auto Phi = MIRBuilder.buildInstr(Opcode: TargetOpcode::G_PHI);
5455 Phi.addDef(
5456 RegNo: MRI.createGenericVirtualRegister(Ty: OutputOpsPieces[i].getLLTTy(MRI)));
5457 OutputRegs.push_back(Elt: Phi.getReg(Idx: 0));
5458
5459 for (unsigned j = 0; j < NumInputs / 2; ++j) {
5460 Phi.addUse(RegNo: InputOpsPieces[j][i]);
5461 Phi.add(MO: MI.getOperand(i: 1 + j * 2 + 1));
5462 }
5463 }
5464
5465 // Set the insert point after the existing PHIs
5466 MachineBasicBlock &MBB = *MI.getParent();
5467 MIRBuilder.setInsertPt(MBB, II: MBB.getFirstNonPHI());
5468
5469 // Merge small outputs into MI's def.
5470 if (NumLeftovers) {
5471 mergeMixedSubvectors(DstReg: MI.getReg(Idx: 0), PartRegs: OutputRegs);
5472 } else {
5473 MIRBuilder.buildMergeLikeInstr(Res: MI.getReg(Idx: 0), Ops: OutputRegs);
5474 }
5475
5476 MI.eraseFromParent();
5477 return Legalized;
5478}
5479
5480LegalizerHelper::LegalizeResult
5481LegalizerHelper::fewerElementsVectorUnmergeValues(MachineInstr &MI,
5482 unsigned TypeIdx,
5483 LLT NarrowTy) {
5484 const int NumDst = MI.getNumOperands() - 1;
5485 const Register SrcReg = MI.getOperand(i: NumDst).getReg();
5486 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5487 LLT SrcTy = MRI.getType(Reg: SrcReg);
5488
5489 if (TypeIdx != 1 || NarrowTy == DstTy)
5490 return UnableToLegalize;
5491
5492 // Requires compatible types. Otherwise SrcReg should have been defined by
5493 // merge-like instruction that would get artifact combined. Most likely
5494 // instruction that defines SrcReg has to perform more/fewer elements
5495 // legalization compatible with NarrowTy.
5496 assert(SrcTy.isVector() && NarrowTy.isVector() && "Expected vector types");
5497 assert((SrcTy.getScalarType() == NarrowTy.getScalarType()) && "bad type");
5498
5499 if ((SrcTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0) ||
5500 (NarrowTy.getSizeInBits() % DstTy.getSizeInBits() != 0))
5501 return UnableToLegalize;
5502
5503 // This is most likely DstTy (smaller then register size) packed in SrcTy
5504 // (larger then register size) and since unmerge was not combined it will be
5505 // lowered to bit sequence extracts from register. Unpack SrcTy to NarrowTy
5506 // (register size) pieces first. Then unpack each of NarrowTy pieces to DstTy.
5507
5508 // %1:_(DstTy), %2, %3, %4 = G_UNMERGE_VALUES %0:_(SrcTy)
5509 //
5510 // %5:_(NarrowTy), %6 = G_UNMERGE_VALUES %0:_(SrcTy) - reg sequence
5511 // %1:_(DstTy), %2 = G_UNMERGE_VALUES %5:_(NarrowTy) - sequence of bits in reg
5512 // %3:_(DstTy), %4 = G_UNMERGE_VALUES %6:_(NarrowTy)
5513 auto Unmerge = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: SrcReg);
5514 const int NumUnmerge = Unmerge->getNumOperands() - 1;
5515 const int PartsPerUnmerge = NumDst / NumUnmerge;
5516
5517 for (int I = 0; I != NumUnmerge; ++I) {
5518 auto MIB = MIRBuilder.buildInstr(Opcode: TargetOpcode::G_UNMERGE_VALUES);
5519
5520 for (int J = 0; J != PartsPerUnmerge; ++J)
5521 MIB.addDef(RegNo: MI.getOperand(i: I * PartsPerUnmerge + J).getReg());
5522 MIB.addUse(RegNo: Unmerge.getReg(Idx: I));
5523 }
5524
5525 MI.eraseFromParent();
5526 return Legalized;
5527}
5528
5529LegalizerHelper::LegalizeResult
5530LegalizerHelper::fewerElementsVectorMerge(MachineInstr &MI, unsigned TypeIdx,
5531 LLT NarrowTy) {
5532 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
5533 // Requires compatible types. Otherwise user of DstReg did not perform unmerge
5534 // that should have been artifact combined. Most likely instruction that uses
5535 // DstReg has to do more/fewer elements legalization compatible with NarrowTy.
5536 assert(DstTy.isVector() && NarrowTy.isVector() && "Expected vector types");
5537 assert((DstTy.getScalarType() == NarrowTy.getScalarType()) && "bad type");
5538 if (NarrowTy == SrcTy)
5539 return UnableToLegalize;
5540
5541 // This attempts to lower part of LCMTy merge/unmerge sequence. Intended use
5542 // is for old mir tests. Since the changes to more/fewer elements it should no
5543 // longer be possible to generate MIR like this when starting from llvm-ir
5544 // because LCMTy approach was replaced with merge/unmerge to vector elements.
5545 if (TypeIdx == 1) {
5546 assert(SrcTy.isVector() && "Expected vector types");
5547 assert((SrcTy.getScalarType() == NarrowTy.getScalarType()) && "bad type");
5548 if ((DstTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0) ||
5549 (NarrowTy.getNumElements() >= SrcTy.getNumElements()))
5550 return UnableToLegalize;
5551 // %2:_(DstTy) = G_CONCAT_VECTORS %0:_(SrcTy), %1:_(SrcTy)
5552 //
5553 // %3:_(EltTy), %4, %5 = G_UNMERGE_VALUES %0:_(SrcTy)
5554 // %6:_(EltTy), %7, %8 = G_UNMERGE_VALUES %1:_(SrcTy)
5555 // %9:_(NarrowTy) = G_BUILD_VECTOR %3:_(EltTy), %4
5556 // %10:_(NarrowTy) = G_BUILD_VECTOR %5:_(EltTy), %6
5557 // %11:_(NarrowTy) = G_BUILD_VECTOR %7:_(EltTy), %8
5558 // %2:_(DstTy) = G_CONCAT_VECTORS %9:_(NarrowTy), %10, %11
5559
5560 SmallVector<Register, 8> Elts;
5561 LLT EltTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getScalarType();
5562 for (unsigned i = 1; i < MI.getNumOperands(); ++i) {
5563 auto Unmerge = MIRBuilder.buildUnmerge(Res: EltTy, Op: MI.getOperand(i).getReg());
5564 for (unsigned j = 0; j < Unmerge->getNumDefs(); ++j)
5565 Elts.push_back(Elt: Unmerge.getReg(Idx: j));
5566 }
5567
5568 SmallVector<Register, 8> NarrowTyElts;
5569 unsigned NumNarrowTyElts = NarrowTy.getNumElements();
5570 unsigned NumNarrowTyPieces = DstTy.getNumElements() / NumNarrowTyElts;
5571 for (unsigned i = 0, Offset = 0; i < NumNarrowTyPieces;
5572 ++i, Offset += NumNarrowTyElts) {
5573 ArrayRef<Register> Pieces(&Elts[Offset], NumNarrowTyElts);
5574 NarrowTyElts.push_back(
5575 Elt: MIRBuilder.buildMergeLikeInstr(Res: NarrowTy, Ops: Pieces).getReg(Idx: 0));
5576 }
5577
5578 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: NarrowTyElts);
5579 MI.eraseFromParent();
5580 return Legalized;
5581 }
5582
5583 assert(TypeIdx == 0 && "Bad type index");
5584 if ((NarrowTy.getSizeInBits() % SrcTy.getSizeInBits() != 0) ||
5585 (DstTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0))
5586 return UnableToLegalize;
5587
5588 // This is most likely SrcTy (smaller then register size) packed in DstTy
5589 // (larger then register size) and since merge was not combined it will be
5590 // lowered to bit sequence packing into register. Merge SrcTy to NarrowTy
5591 // (register size) pieces first. Then merge each of NarrowTy pieces to DstTy.
5592
5593 // %0:_(DstTy) = G_MERGE_VALUES %1:_(SrcTy), %2, %3, %4
5594 //
5595 // %5:_(NarrowTy) = G_MERGE_VALUES %1:_(SrcTy), %2 - sequence of bits in reg
5596 // %6:_(NarrowTy) = G_MERGE_VALUES %3:_(SrcTy), %4
5597 // %0:_(DstTy) = G_MERGE_VALUES %5:_(NarrowTy), %6 - reg sequence
5598 SmallVector<Register, 8> NarrowTyElts;
5599 unsigned NumParts = DstTy.getNumElements() / NarrowTy.getNumElements();
5600 unsigned NumSrcElts = SrcTy.isVector() ? SrcTy.getNumElements() : 1;
5601 unsigned NumElts = NarrowTy.getNumElements() / NumSrcElts;
5602 for (unsigned i = 0; i < NumParts; ++i) {
5603 SmallVector<Register, 8> Sources;
5604 for (unsigned j = 0; j < NumElts; ++j)
5605 Sources.push_back(Elt: MI.getOperand(i: 1 + i * NumElts + j).getReg());
5606 NarrowTyElts.push_back(
5607 Elt: MIRBuilder.buildMergeLikeInstr(Res: NarrowTy, Ops: Sources).getReg(Idx: 0));
5608 }
5609
5610 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: NarrowTyElts);
5611 MI.eraseFromParent();
5612 return Legalized;
5613}
5614
5615LegalizerHelper::LegalizeResult
5616LegalizerHelper::fewerElementsVectorExtractInsertVectorElt(MachineInstr &MI,
5617 unsigned TypeIdx,
5618 LLT NarrowVecTy) {
5619 auto [DstReg, SrcVec] = MI.getFirst2Regs();
5620 Register InsertVal;
5621 bool IsInsert = MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT;
5622
5623 assert((IsInsert ? TypeIdx == 0 : TypeIdx == 1) && "not a vector type index");
5624 if (IsInsert)
5625 InsertVal = MI.getOperand(i: 2).getReg();
5626
5627 Register Idx = MI.getOperand(i: MI.getNumOperands() - 1).getReg();
5628 LLT VecTy = MRI.getType(Reg: SrcVec);
5629
5630 // If the index is a constant, we can really break this down as you would
5631 // expect, and index into the target size pieces.
5632 auto MaybeCst = getIConstantVRegValWithLookThrough(VReg: Idx, MRI);
5633 if (MaybeCst) {
5634 uint64_t IdxVal = MaybeCst->Value.getZExtValue();
5635 // Avoid out of bounds indexing the pieces.
5636 if (IdxVal >= VecTy.getNumElements()) {
5637 MIRBuilder.buildUndef(Res: DstReg);
5638 MI.eraseFromParent();
5639 return Legalized;
5640 }
5641
5642 if (!NarrowVecTy.isVector()) {
5643 SmallVector<Register, 8> SplitPieces;
5644 extractParts(Reg: MI.getOperand(i: 1).getReg(), Ty: NarrowVecTy,
5645 NumParts: VecTy.getNumElements(), VRegs&: SplitPieces, MIRBuilder, MRI);
5646 if (IsInsert) {
5647 SplitPieces[IdxVal] = InsertVal;
5648 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0).getReg(), Ops: SplitPieces);
5649 } else {
5650 MIRBuilder.buildCopy(Res: MI.getOperand(i: 0).getReg(), Op: SplitPieces[IdxVal]);
5651 }
5652 } else {
5653 SmallVector<Register, 8> VecParts;
5654 LLT GCDTy = extractGCDType(Parts&: VecParts, DstTy: VecTy, NarrowTy: NarrowVecTy, SrcReg: SrcVec);
5655
5656 // Build a sequence of NarrowTy pieces in VecParts for this operand.
5657 LLT LCMTy = buildLCMMergePieces(DstTy: VecTy, NarrowTy: NarrowVecTy, GCDTy, VRegs&: VecParts,
5658 PadStrategy: TargetOpcode::G_ANYEXT);
5659
5660 unsigned NewNumElts = NarrowVecTy.getNumElements();
5661
5662 LLT IdxTy = MRI.getType(Reg: Idx);
5663 int64_t PartIdx = IdxVal / NewNumElts;
5664 auto NewIdx =
5665 MIRBuilder.buildConstant(Res: IdxTy, Val: IdxVal - NewNumElts * PartIdx);
5666
5667 if (IsInsert) {
5668 LLT PartTy = MRI.getType(Reg: VecParts[PartIdx]);
5669
5670 // Use the adjusted index to insert into one of the subvectors.
5671 auto InsertPart = MIRBuilder.buildInsertVectorElement(
5672 Res: PartTy, Val: VecParts[PartIdx], Elt: InsertVal, Idx: NewIdx);
5673 VecParts[PartIdx] = InsertPart.getReg(Idx: 0);
5674
5675 // Recombine the inserted subvector with the others to reform the result
5676 // vector.
5677 buildWidenedRemergeToDst(DstReg, LCMTy, RemergeRegs: VecParts);
5678 } else {
5679 MIRBuilder.buildExtractVectorElement(Res: DstReg, Val: VecParts[PartIdx], Idx: NewIdx);
5680 }
5681 }
5682
5683 MI.eraseFromParent();
5684 return Legalized;
5685 }
5686
5687 // With a variable index, we can't perform the operation in a smaller type, so
5688 // we're forced to expand this.
5689 //
5690 // TODO: We could emit a chain of compare/select to figure out which piece to
5691 // index.
5692 return lowerExtractInsertVectorElt(MI);
5693}
5694
5695LegalizerHelper::LegalizeResult
5696LegalizerHelper::reduceLoadStoreWidth(GLoadStore &LdStMI, unsigned TypeIdx,
5697 LLT NarrowTy) {
5698 // FIXME: Don't know how to handle secondary types yet.
5699 if (TypeIdx != 0)
5700 return UnableToLegalize;
5701
5702 if (!NarrowTy.isByteSized()) {
5703 LLVM_DEBUG(dbgs() << "Can't narrow load/store to non-byte-sized type\n");
5704 return UnableToLegalize;
5705 }
5706
5707 // This implementation doesn't work for atomics. Give up instead of doing
5708 // something invalid.
5709 if (LdStMI.isAtomic())
5710 return UnableToLegalize;
5711
5712 bool IsLoad = isa<GLoad>(Val: LdStMI);
5713 Register ValReg = LdStMI.getReg(Idx: 0);
5714 Register AddrReg = LdStMI.getPointerReg();
5715 LLT ValTy = MRI.getType(Reg: ValReg);
5716
5717 // FIXME: Do we need a distinct NarrowMemory legalize action?
5718 if (ValTy.getSizeInBits() != 8 * LdStMI.getMemSize().getValue()) {
5719 LLVM_DEBUG(dbgs() << "Can't narrow extload/truncstore\n");
5720 return UnableToLegalize;
5721 }
5722
5723 int NumParts = -1;
5724 int NumLeftover = -1;
5725 LLT LeftoverTy;
5726 SmallVector<Register, 8> NarrowRegs, NarrowLeftoverRegs;
5727 if (IsLoad) {
5728 std::tie(args&: NumParts, args&: NumLeftover) = getNarrowTypeBreakDown(OrigTy: ValTy, NarrowTy, LeftoverTy);
5729 } else {
5730 if (extractParts(Reg: ValReg, RegTy: ValTy, MainTy: NarrowTy, LeftoverTy, VRegs&: NarrowRegs,
5731 LeftoverVRegs&: NarrowLeftoverRegs, MIRBuilder, MRI)) {
5732 NumParts = NarrowRegs.size();
5733 NumLeftover = NarrowLeftoverRegs.size();
5734 }
5735 }
5736
5737 if (NumParts == -1)
5738 return UnableToLegalize;
5739
5740 LLT PtrTy = MRI.getType(Reg: AddrReg);
5741 const LLT OffsetTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
5742
5743 unsigned TotalSize = ValTy.getSizeInBits();
5744
5745 // Split the load/store into PartTy sized pieces starting at Offset. If this
5746 // is a load, return the new registers in ValRegs. For a store, each elements
5747 // of ValRegs should be PartTy. Returns the next offset that needs to be
5748 // handled.
5749 bool isBigEndian = MIRBuilder.getDataLayout().isBigEndian();
5750 auto MMO = LdStMI.getMMO();
5751 auto splitTypePieces = [=](LLT PartTy, SmallVectorImpl<Register> &ValRegs,
5752 unsigned NumParts, unsigned Offset) -> unsigned {
5753 MachineFunction &MF = MIRBuilder.getMF();
5754 unsigned PartSize = PartTy.getSizeInBits();
5755 for (unsigned Idx = 0, E = NumParts; Idx != E && Offset < TotalSize;
5756 ++Idx) {
5757 unsigned ByteOffset = Offset / 8;
5758 Register NewAddrReg;
5759
5760 MIRBuilder.materializeObjectPtrOffset(Res&: NewAddrReg, Op0: AddrReg, ValueTy: OffsetTy,
5761 Value: ByteOffset);
5762
5763 MachineMemOperand *NewMMO =
5764 MF.getMachineMemOperand(MMO: &MMO, Offset: ByteOffset, Ty: PartTy);
5765
5766 if (IsLoad) {
5767 Register Dst = MRI.createGenericVirtualRegister(Ty: PartTy);
5768 ValRegs.push_back(Elt: Dst);
5769 MIRBuilder.buildLoad(Res: Dst, Addr: NewAddrReg, MMO&: *NewMMO);
5770 } else {
5771 MIRBuilder.buildStore(Val: ValRegs[Idx], Addr: NewAddrReg, MMO&: *NewMMO);
5772 }
5773 Offset = isBigEndian ? Offset - PartSize : Offset + PartSize;
5774 }
5775
5776 return Offset;
5777 };
5778
5779 unsigned Offset = isBigEndian ? TotalSize - NarrowTy.getSizeInBits() : 0;
5780 unsigned HandledOffset =
5781 splitTypePieces(NarrowTy, NarrowRegs, NumParts, Offset);
5782
5783 // Handle the rest of the register if this isn't an even type breakdown.
5784 if (LeftoverTy.isValid())
5785 splitTypePieces(LeftoverTy, NarrowLeftoverRegs, NumLeftover, HandledOffset);
5786
5787 if (IsLoad) {
5788 insertParts(DstReg: ValReg, ResultTy: ValTy, PartTy: NarrowTy, PartRegs: NarrowRegs,
5789 LeftoverTy, LeftoverRegs: NarrowLeftoverRegs);
5790 }
5791
5792 LdStMI.eraseFromParent();
5793 return Legalized;
5794}
5795
5796LegalizerHelper::LegalizeResult
5797LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
5798 LLT NarrowTy) {
5799 using namespace TargetOpcode;
5800 GenericMachineInstr &GMI = cast<GenericMachineInstr>(Val&: MI);
5801 unsigned NumElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
5802
5803 switch (MI.getOpcode()) {
5804 case G_IMPLICIT_DEF:
5805 case G_TRUNC:
5806 case G_AND:
5807 case G_OR:
5808 case G_XOR:
5809 case G_ADD:
5810 case G_SUB:
5811 case G_MUL:
5812 case G_PTR_ADD:
5813 case G_SMULH:
5814 case G_UMULH:
5815 case G_FADD:
5816 case G_FMUL:
5817 case G_FSUB:
5818 case G_FNEG:
5819 case G_FABS:
5820 case G_FCANONICALIZE:
5821 case G_FDIV:
5822 case G_FREM:
5823 case G_FMA:
5824 case G_FMAD:
5825 case G_FPOW:
5826 case G_FEXP:
5827 case G_FEXP2:
5828 case G_FEXP10:
5829 case G_FLOG:
5830 case G_FLOG2:
5831 case G_FLOG10:
5832 case G_FLDEXP:
5833 case G_FNEARBYINT:
5834 case G_FCEIL:
5835 case G_FFLOOR:
5836 case G_FRINT:
5837 case G_INTRINSIC_LRINT:
5838 case G_INTRINSIC_LLRINT:
5839 case G_INTRINSIC_ROUND:
5840 case G_INTRINSIC_ROUNDEVEN:
5841 case G_LROUND:
5842 case G_LLROUND:
5843 case G_INTRINSIC_TRUNC:
5844 case G_FMODF:
5845 case G_FCOS:
5846 case G_FSIN:
5847 case G_FTAN:
5848 case G_FACOS:
5849 case G_FASIN:
5850 case G_FATAN:
5851 case G_FATAN2:
5852 case G_FCOSH:
5853 case G_FSINH:
5854 case G_FTANH:
5855 case G_FSQRT:
5856 case G_BSWAP:
5857 case G_BITREVERSE:
5858 case G_SDIV:
5859 case G_UDIV:
5860 case G_SREM:
5861 case G_UREM:
5862 case G_SDIVREM:
5863 case G_UDIVREM:
5864 case G_SMIN:
5865 case G_SMAX:
5866 case G_UMIN:
5867 case G_UMAX:
5868 case G_ABS:
5869 case G_FMINNUM:
5870 case G_FMAXNUM:
5871 case G_FMINNUM_IEEE:
5872 case G_FMAXNUM_IEEE:
5873 case G_FMINIMUM:
5874 case G_FMAXIMUM:
5875 case G_FMINIMUMNUM:
5876 case G_FMAXIMUMNUM:
5877 case G_FSHL:
5878 case G_FSHR:
5879 case G_ROTL:
5880 case G_ROTR:
5881 case G_FREEZE:
5882 case G_SADDSAT:
5883 case G_SSUBSAT:
5884 case G_UADDSAT:
5885 case G_USUBSAT:
5886 case G_UMULO:
5887 case G_SMULO:
5888 case G_SHL:
5889 case G_LSHR:
5890 case G_ASHR:
5891 case G_SSHLSAT:
5892 case G_USHLSAT:
5893 case G_CTLZ:
5894 case G_CTLZ_ZERO_POISON:
5895 case G_CTTZ:
5896 case G_CTTZ_ZERO_POISON:
5897 case G_CTPOP:
5898 case G_CTLS:
5899 case G_FCOPYSIGN:
5900 case G_ZEXT:
5901 case G_SEXT:
5902 case G_ANYEXT:
5903 case G_FPEXT:
5904 case G_FPTRUNC:
5905 case G_SITOFP:
5906 case G_UITOFP:
5907 case G_FPTOSI:
5908 case G_FPTOUI:
5909 case G_FPTOSI_SAT:
5910 case G_FPTOUI_SAT:
5911 case G_INTTOPTR:
5912 case G_PTRTOINT:
5913 case G_ADDRSPACE_CAST:
5914 case G_UADDO:
5915 case G_USUBO:
5916 case G_UADDE:
5917 case G_USUBE:
5918 case G_SADDO:
5919 case G_SSUBO:
5920 case G_SADDE:
5921 case G_SSUBE:
5922 case G_STRICT_FADD:
5923 case G_STRICT_FSUB:
5924 case G_STRICT_FMUL:
5925 case G_STRICT_FMA:
5926 case G_STRICT_FLDEXP:
5927 case G_FFREXP:
5928 case G_TRUNC_SSAT_S:
5929 case G_TRUNC_SSAT_U:
5930 case G_TRUNC_USAT_U:
5931 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts);
5932 case G_ICMP:
5933 case G_FCMP:
5934 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {1 /*cpm predicate*/});
5935 case G_IS_FPCLASS:
5936 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {2, 3 /*mask,fpsem*/});
5937 case G_SELECT:
5938 if (MRI.getType(Reg: MI.getOperand(i: 1).getReg()).isVector())
5939 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts);
5940 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {1 /*scalar cond*/});
5941 case G_PHI:
5942 return fewerElementsVectorPhi(MI&: GMI, NumElts);
5943 case G_UNMERGE_VALUES:
5944 return fewerElementsVectorUnmergeValues(MI, TypeIdx, NarrowTy);
5945 case G_BUILD_VECTOR:
5946 assert(TypeIdx == 0 && "not a vector type index");
5947 return fewerElementsVectorMerge(MI, TypeIdx, NarrowTy);
5948 case G_CONCAT_VECTORS:
5949 if (TypeIdx != 1) // TODO: This probably does work as expected already.
5950 return UnableToLegalize;
5951 return fewerElementsVectorMerge(MI, TypeIdx, NarrowTy);
5952 case G_EXTRACT_SUBVECTOR: {
5953 Register DstReg = MI.getOperand(i: 0).getReg();
5954 LLT DstTy = MRI.getType(Reg: DstReg);
5955 Register SrcReg = MI.getOperand(i: 1).getReg();
5956 uint64_t InsertionPointImm = MI.getOperand(i: 2).getImm();
5957
5958 // If Dst > NarrowTy bits, then cannot legalize
5959 if (DstTy.getSizeInBits() > NarrowTy.getSizeInBits())
5960 return UnableToLegalize;
5961
5962 // If DstTy's size is not a multiple of NarrowTy's, then cannot legalize
5963 if (!DstTy.getElementCount().isKnownMultipleOf(RHS: NarrowTy.getElementCount()))
5964 return UnableToLegalize;
5965
5966 auto Unmerge = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: SrcReg);
5967 uint64_t RequiredSubvectorIndex =
5968 InsertionPointImm / NarrowTy.getNumElements();
5969 // If Dst and Narrow are both same size, convert to a copy
5970 if (DstTy.getNumElements() == NarrowTy.getNumElements())
5971 MIRBuilder.buildCopy(Res: DstReg, Op: Unmerge.getReg(Idx: RequiredSubvectorIndex));
5972 else
5973 MIRBuilder.buildExtractSubvector(
5974 Res: DstReg, Src: Unmerge.getReg(Idx: RequiredSubvectorIndex),
5975 Index: InsertionPointImm % NarrowTy.getNumElements());
5976
5977 MI.eraseFromParent();
5978 return Legalized;
5979 }
5980 case G_EXTRACT_VECTOR_ELT:
5981 case G_INSERT_VECTOR_ELT:
5982 return fewerElementsVectorExtractInsertVectorElt(MI, TypeIdx, NarrowVecTy: NarrowTy);
5983 case G_LOAD:
5984 case G_STORE:
5985 return reduceLoadStoreWidth(LdStMI&: cast<GLoadStore>(Val&: MI), TypeIdx, NarrowTy);
5986 case G_SEXT_INREG:
5987 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {2 /*imm*/});
5988 GISEL_VECREDUCE_CASES_NONSEQ
5989 return fewerElementsVectorReductions(MI, TypeIdx, NarrowTy);
5990 case TargetOpcode::G_VECREDUCE_SEQ_FADD:
5991 case TargetOpcode::G_VECREDUCE_SEQ_FMUL:
5992 return fewerElementsVectorSeqReductions(MI, TypeIdx, NarrowTy);
5993 case G_SHUFFLE_VECTOR:
5994 return fewerElementsVectorShuffle(MI, TypeIdx, NarrowTy);
5995 case G_FPOWI:
5996 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {2 /*pow*/});
5997 case G_BITCAST:
5998 return fewerElementsBitcast(MI, TypeIdx, NarrowTy);
5999 case G_INTRINSIC_FPTRUNC_ROUND:
6000 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {2});
6001 default:
6002 return UnableToLegalize;
6003 }
6004}
6005
6006LegalizerHelper::LegalizeResult
6007LegalizerHelper::fewerElementsBitcast(MachineInstr &MI, unsigned int TypeIdx,
6008 LLT NarrowTy) {
6009 assert(MI.getOpcode() == TargetOpcode::G_BITCAST &&
6010 "Not a bitcast operation");
6011
6012 if (TypeIdx != 0)
6013 return UnableToLegalize;
6014
6015 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
6016
6017 unsigned NewElemCount =
6018 NarrowTy.getSizeInBits() / SrcTy.getScalarSizeInBits();
6019 SmallVector<Register> SrcVRegs, BitcastVRegs;
6020 if (NewElemCount == 1) {
6021 LLT SrcNarrowTy = SrcTy.getElementType();
6022
6023 auto Unmerge = MIRBuilder.buildUnmerge(Res: SrcNarrowTy, Op: SrcReg);
6024 getUnmergeResults(Regs&: SrcVRegs, MI: *Unmerge);
6025 } else {
6026 LLT SrcNarrowTy =
6027 SrcTy.changeVectorElementCount(EC: ElementCount::getFixed(MinVal: NewElemCount));
6028
6029 // Split the Src and Dst Reg into smaller registers
6030 if (extractGCDType(Parts&: SrcVRegs, DstTy, NarrowTy: SrcNarrowTy, SrcReg) != SrcNarrowTy)
6031 return UnableToLegalize;
6032 }
6033
6034 // Build new smaller bitcast instructions
6035 // Not supporting Leftover types for now but will have to
6036 for (Register Reg : SrcVRegs)
6037 BitcastVRegs.push_back(Elt: MIRBuilder.buildBitcast(Dst: NarrowTy, Src: Reg).getReg(Idx: 0));
6038
6039 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: BitcastVRegs);
6040 MI.eraseFromParent();
6041 return Legalized;
6042}
6043
6044LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorShuffle(
6045 MachineInstr &MI, unsigned int TypeIdx, LLT NarrowTy) {
6046 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
6047 if (TypeIdx != 0)
6048 return UnableToLegalize;
6049
6050 auto [DstReg, DstTy, Src1Reg, Src1Ty, Src2Reg, Src2Ty] =
6051 MI.getFirst3RegLLTs();
6052 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
6053 // The shuffle should be canonicalized by now.
6054 if (DstTy != Src1Ty)
6055 return UnableToLegalize;
6056 if (DstTy != Src2Ty)
6057 return UnableToLegalize;
6058
6059 if (!isPowerOf2_32(Value: DstTy.getNumElements()))
6060 return UnableToLegalize;
6061
6062 // We only support splitting a shuffle into 2, so adjust NarrowTy accordingly.
6063 // Further legalization attempts will be needed to do split further.
6064 NarrowTy =
6065 DstTy.changeElementCount(EC: DstTy.getElementCount().divideCoefficientBy(RHS: 2));
6066 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
6067
6068 SmallVector<Register> SplitSrc1Regs, SplitSrc2Regs;
6069 extractParts(Reg: Src1Reg, Ty: NarrowTy, NumParts: 2, VRegs&: SplitSrc1Regs, MIRBuilder, MRI);
6070 extractParts(Reg: Src2Reg, Ty: NarrowTy, NumParts: 2, VRegs&: SplitSrc2Regs, MIRBuilder, MRI);
6071 Register Inputs[4] = {SplitSrc1Regs[0], SplitSrc1Regs[1], SplitSrc2Regs[0],
6072 SplitSrc2Regs[1]};
6073
6074 Register Hi, Lo;
6075
6076 // If Lo or Hi uses elements from at most two of the four input vectors, then
6077 // express it as a vector shuffle of those two inputs. Otherwise extract the
6078 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
6079 SmallVector<int, 16> Ops;
6080 for (unsigned High = 0; High < 2; ++High) {
6081 Register &Output = High ? Hi : Lo;
6082
6083 // Build a shuffle mask for the output, discovering on the fly which
6084 // input vectors to use as shuffle operands (recorded in InputUsed).
6085 // If building a suitable shuffle vector proves too hard, then bail
6086 // out with useBuildVector set.
6087 unsigned InputUsed[2] = {-1U, -1U}; // Not yet discovered.
6088 unsigned FirstMaskIdx = High * NewElts;
6089 bool UseBuildVector = false;
6090 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
6091 // The mask element. This indexes into the input.
6092 int Idx = Mask[FirstMaskIdx + MaskOffset];
6093
6094 // The input vector this mask element indexes into.
6095 unsigned Input = (unsigned)Idx / NewElts;
6096
6097 if (Input >= std::size(Inputs)) {
6098 // The mask element does not index into any input vector.
6099 Ops.push_back(Elt: -1);
6100 continue;
6101 }
6102
6103 // Turn the index into an offset from the start of the input vector.
6104 Idx -= Input * NewElts;
6105
6106 // Find or create a shuffle vector operand to hold this input.
6107 unsigned OpNo;
6108 for (OpNo = 0; OpNo < std::size(InputUsed); ++OpNo) {
6109 if (InputUsed[OpNo] == Input) {
6110 // This input vector is already an operand.
6111 break;
6112 } else if (InputUsed[OpNo] == -1U) {
6113 // Create a new operand for this input vector.
6114 InputUsed[OpNo] = Input;
6115 break;
6116 }
6117 }
6118
6119 if (OpNo >= std::size(InputUsed)) {
6120 // More than two input vectors used! Give up on trying to create a
6121 // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
6122 UseBuildVector = true;
6123 break;
6124 }
6125
6126 // Add the mask index for the new shuffle vector.
6127 Ops.push_back(Elt: Idx + OpNo * NewElts);
6128 }
6129
6130 if (UseBuildVector) {
6131 LLT EltTy = NarrowTy.getElementType();
6132 SmallVector<Register, 16> SVOps;
6133
6134 // Extract the input elements by hand.
6135 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
6136 // The mask element. This indexes into the input.
6137 int Idx = Mask[FirstMaskIdx + MaskOffset];
6138
6139 // The input vector this mask element indexes into.
6140 unsigned Input = (unsigned)Idx / NewElts;
6141
6142 if (Input >= std::size(Inputs)) {
6143 // The mask element is "undef" or indexes off the end of the input.
6144 SVOps.push_back(Elt: MIRBuilder.buildUndef(Res: EltTy).getReg(Idx: 0));
6145 continue;
6146 }
6147
6148 // Turn the index into an offset from the start of the input vector.
6149 Idx -= Input * NewElts;
6150
6151 // Extract the vector element by hand.
6152 SVOps.push_back(Elt: MIRBuilder
6153 .buildExtractVectorElement(
6154 Res: EltTy, Val: Inputs[Input],
6155 Idx: MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: 32), Val: Idx))
6156 .getReg(Idx: 0));
6157 }
6158
6159 // Construct the Lo/Hi output using a G_BUILD_VECTOR.
6160 Output = MIRBuilder.buildBuildVector(Res: NarrowTy, Ops: SVOps).getReg(Idx: 0);
6161 } else if (InputUsed[0] == -1U) {
6162 // No input vectors were used! The result is undefined.
6163 Output = MIRBuilder.buildUndef(Res: NarrowTy).getReg(Idx: 0);
6164 } else if (NewElts == 1) {
6165 Output = MIRBuilder.buildCopy(Res: NarrowTy, Op: Inputs[InputUsed[0]]).getReg(Idx: 0);
6166 } else {
6167 Register Op0 = Inputs[InputUsed[0]];
6168 // If only one input was used, use an undefined vector for the other.
6169 Register Op1 = InputUsed[1] == -1U
6170 ? MIRBuilder.buildUndef(Res: NarrowTy).getReg(Idx: 0)
6171 : Inputs[InputUsed[1]];
6172 // At least one input vector was used. Create a new shuffle vector.
6173 Output = MIRBuilder.buildShuffleVector(Res: NarrowTy, Src1: Op0, Src2: Op1, Mask: Ops).getReg(Idx: 0);
6174 }
6175
6176 Ops.clear();
6177 }
6178
6179 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: {Lo, Hi});
6180 MI.eraseFromParent();
6181 return Legalized;
6182}
6183
6184LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorReductions(
6185 MachineInstr &MI, unsigned int TypeIdx, LLT NarrowTy) {
6186 auto &RdxMI = cast<GVecReduce>(Val&: MI);
6187
6188 if (TypeIdx != 1)
6189 return UnableToLegalize;
6190
6191 // The semantics of the normal non-sequential reductions allow us to freely
6192 // re-associate the operation.
6193 auto [DstReg, DstTy, SrcReg, SrcTy] = RdxMI.getFirst2RegLLTs();
6194
6195 if (NarrowTy.isVector() &&
6196 (SrcTy.getNumElements() % NarrowTy.getNumElements() != 0))
6197 return UnableToLegalize;
6198
6199 unsigned ScalarOpc = RdxMI.getScalarOpcForReduction();
6200 SmallVector<Register> SplitSrcs;
6201 // If NarrowTy is a scalar then we're being asked to scalarize.
6202 const unsigned NumParts =
6203 NarrowTy.isVector() ? SrcTy.getNumElements() / NarrowTy.getNumElements()
6204 : SrcTy.getNumElements();
6205
6206 extractParts(Reg: SrcReg, Ty: NarrowTy, NumParts, VRegs&: SplitSrcs, MIRBuilder, MRI);
6207 if (NarrowTy.isScalar()) {
6208 if (DstTy != NarrowTy)
6209 return UnableToLegalize; // FIXME: handle implicit extensions.
6210
6211 if (isPowerOf2_32(Value: NumParts)) {
6212 // Generate a tree of scalar operations to reduce the critical path.
6213 SmallVector<Register> PartialResults;
6214 unsigned NumPartsLeft = NumParts;
6215 while (NumPartsLeft > 1) {
6216 for (unsigned Idx = 0; Idx < NumPartsLeft - 1; Idx += 2) {
6217 PartialResults.emplace_back(
6218 Args: MIRBuilder
6219 .buildInstr(Opc: ScalarOpc, DstOps: {NarrowTy},
6220 SrcOps: {SplitSrcs[Idx], SplitSrcs[Idx + 1]},
6221 Flags: MI.getFlags())
6222 .getReg(Idx: 0));
6223 }
6224 SplitSrcs = PartialResults;
6225 PartialResults.clear();
6226 NumPartsLeft = SplitSrcs.size();
6227 }
6228 assert(SplitSrcs.size() == 1);
6229 MIRBuilder.buildCopy(Res: DstReg, Op: SplitSrcs[0]);
6230 MI.eraseFromParent();
6231 return Legalized;
6232 }
6233 // If we can't generate a tree, then just do sequential operations.
6234 Register Acc = SplitSrcs[0];
6235 for (unsigned Idx = 1; Idx < NumParts; ++Idx)
6236 Acc = MIRBuilder
6237 .buildInstr(Opc: ScalarOpc, DstOps: {NarrowTy}, SrcOps: {Acc, SplitSrcs[Idx]},
6238 Flags: MI.getFlags())
6239 .getReg(Idx: 0);
6240 MIRBuilder.buildCopy(Res: DstReg, Op: Acc);
6241 MI.eraseFromParent();
6242 return Legalized;
6243 }
6244 SmallVector<Register> PartialReductions;
6245 for (unsigned Part = 0; Part < NumParts; ++Part) {
6246 PartialReductions.push_back(Elt: MIRBuilder
6247 .buildInstr(Opc: RdxMI.getOpcode(), DstOps: {DstTy},
6248 SrcOps: {SplitSrcs[Part]},
6249 Flags: MI.getFlags())
6250 .getReg(Idx: 0));
6251 }
6252
6253 // If the types involved are powers of 2, we can generate intermediate vector
6254 // ops, before generating a final reduction operation.
6255 if (isPowerOf2_32(Value: SrcTy.getNumElements()) &&
6256 isPowerOf2_32(Value: NarrowTy.getNumElements())) {
6257 return tryNarrowPow2Reduction(MI, SrcReg, SrcTy, NarrowTy, ScalarOpc);
6258 }
6259
6260 Register Acc = PartialReductions[0];
6261 for (unsigned Part = 1; Part < NumParts; ++Part) {
6262 if (Part == NumParts - 1) {
6263 MIRBuilder.buildInstr(Opc: ScalarOpc, DstOps: {DstReg}, SrcOps: {Acc, PartialReductions[Part]},
6264 Flags: MI.getFlags());
6265 } else {
6266 Acc = MIRBuilder
6267 .buildInstr(Opc: ScalarOpc, DstOps: {DstTy}, SrcOps: {Acc, PartialReductions[Part]},
6268 Flags: MI.getFlags())
6269 .getReg(Idx: 0);
6270 }
6271 }
6272 MI.eraseFromParent();
6273 return Legalized;
6274}
6275
6276LegalizerHelper::LegalizeResult
6277LegalizerHelper::fewerElementsVectorSeqReductions(MachineInstr &MI,
6278 unsigned int TypeIdx,
6279 LLT NarrowTy) {
6280 auto [DstReg, DstTy, ScalarReg, ScalarTy, SrcReg, SrcTy] =
6281 MI.getFirst3RegLLTs();
6282 if (!NarrowTy.isScalar() || TypeIdx != 2 || DstTy != ScalarTy ||
6283 DstTy != NarrowTy)
6284 return UnableToLegalize;
6285
6286 assert((MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FADD ||
6287 MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FMUL) &&
6288 "Unexpected vecreduce opcode");
6289 unsigned ScalarOpc = MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FADD
6290 ? TargetOpcode::G_FADD
6291 : TargetOpcode::G_FMUL;
6292
6293 SmallVector<Register> SplitSrcs;
6294 unsigned NumParts = SrcTy.getNumElements();
6295 extractParts(Reg: SrcReg, Ty: NarrowTy, NumParts, VRegs&: SplitSrcs, MIRBuilder, MRI);
6296 Register Acc = ScalarReg;
6297 for (unsigned i = 0; i < NumParts; i++)
6298 Acc = MIRBuilder
6299 .buildInstr(Opc: ScalarOpc, DstOps: {NarrowTy}, SrcOps: {Acc, SplitSrcs[i]},
6300 Flags: MI.getFlags())
6301 .getReg(Idx: 0);
6302
6303 MIRBuilder.buildCopy(Res: DstReg, Op: Acc);
6304 MI.eraseFromParent();
6305 return Legalized;
6306}
6307
6308LegalizerHelper::LegalizeResult
6309LegalizerHelper::tryNarrowPow2Reduction(MachineInstr &MI, Register SrcReg,
6310 LLT SrcTy, LLT NarrowTy,
6311 unsigned ScalarOpc) {
6312 SmallVector<Register> SplitSrcs;
6313 // Split the sources into NarrowTy size pieces.
6314 extractParts(Reg: SrcReg, Ty: NarrowTy,
6315 NumParts: SrcTy.getNumElements() / NarrowTy.getNumElements(), VRegs&: SplitSrcs,
6316 MIRBuilder, MRI);
6317 // We're going to do a tree reduction using vector operations until we have
6318 // one NarrowTy size value left.
6319 while (SplitSrcs.size() > 1) {
6320 SmallVector<Register> PartialRdxs;
6321 for (unsigned Idx = 0; Idx < SplitSrcs.size()-1; Idx += 2) {
6322 Register LHS = SplitSrcs[Idx];
6323 Register RHS = SplitSrcs[Idx + 1];
6324 // Create the intermediate vector op.
6325 Register Res =
6326 MIRBuilder
6327 .buildInstr(Opc: ScalarOpc, DstOps: {NarrowTy}, SrcOps: {LHS, RHS}, Flags: MI.getFlags())
6328 .getReg(Idx: 0);
6329 PartialRdxs.push_back(Elt: Res);
6330 }
6331 SplitSrcs = std::move(PartialRdxs);
6332 }
6333 // Finally generate the requested NarrowTy based reduction.
6334 Observer.changingInstr(MI);
6335 MI.getOperand(i: 1).setReg(SplitSrcs[0]);
6336 Observer.changedInstr(MI);
6337 return Legalized;
6338}
6339
6340LegalizerHelper::LegalizeResult
6341LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt,
6342 const LLT HalfTy, const LLT AmtTy) {
6343
6344 Register InL = MRI.createGenericVirtualRegister(Ty: HalfTy);
6345 Register InH = MRI.createGenericVirtualRegister(Ty: HalfTy);
6346 MIRBuilder.buildUnmerge(Res: {InL, InH}, Op: MI.getOperand(i: 1));
6347
6348 if (Amt.isZero()) {
6349 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0), Ops: {InL, InH});
6350 MI.eraseFromParent();
6351 return Legalized;
6352 }
6353
6354 LLT NVT = HalfTy;
6355 unsigned NVTBits = HalfTy.getSizeInBits();
6356 unsigned VTBits = 2 * NVTBits;
6357
6358 SrcOp Lo(Register(0)), Hi(Register(0));
6359 if (MI.getOpcode() == TargetOpcode::G_SHL) {
6360 if (Amt.ugt(RHS: VTBits)) {
6361 Lo = Hi = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6362 } else if (Amt.ugt(RHS: NVTBits)) {
6363 Lo = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6364 Hi = MIRBuilder.buildShl(Dst: NVT, Src0: InL,
6365 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt - NVTBits));
6366 } else if (Amt == NVTBits) {
6367 Lo = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6368 Hi = InL;
6369 } else {
6370 Lo = MIRBuilder.buildShl(Dst: NVT, Src0: InL, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt));
6371 auto OrLHS =
6372 MIRBuilder.buildShl(Dst: NVT, Src0: InH, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt));
6373 auto OrRHS = MIRBuilder.buildLShr(
6374 Dst: NVT, Src0: InL, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: -Amt + NVTBits));
6375 Hi = MIRBuilder.buildOr(Dst: NVT, Src0: OrLHS, Src1: OrRHS);
6376 }
6377 } else if (MI.getOpcode() == TargetOpcode::G_LSHR) {
6378 if (Amt.ugt(RHS: VTBits)) {
6379 Lo = Hi = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6380 } else if (Amt.ugt(RHS: NVTBits)) {
6381 Lo = MIRBuilder.buildLShr(Dst: NVT, Src0: InH,
6382 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt - NVTBits));
6383 Hi = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6384 } else if (Amt == NVTBits) {
6385 Lo = InH;
6386 Hi = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6387 } else {
6388 auto ShiftAmtConst = MIRBuilder.buildConstant(Res: AmtTy, Val: Amt);
6389
6390 auto OrLHS = MIRBuilder.buildLShr(Dst: NVT, Src0: InL, Src1: ShiftAmtConst);
6391 auto OrRHS = MIRBuilder.buildShl(
6392 Dst: NVT, Src0: InH, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: -Amt + NVTBits));
6393
6394 Lo = MIRBuilder.buildOr(Dst: NVT, Src0: OrLHS, Src1: OrRHS);
6395 Hi = MIRBuilder.buildLShr(Dst: NVT, Src0: InH, Src1: ShiftAmtConst);
6396 }
6397 } else {
6398 if (Amt.ugt(RHS: VTBits)) {
6399 Hi = Lo = MIRBuilder.buildAShr(
6400 Dst: NVT, Src0: InH, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: NVTBits - 1));
6401 } else if (Amt.ugt(RHS: NVTBits)) {
6402 Lo = MIRBuilder.buildAShr(Dst: NVT, Src0: InH,
6403 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt - NVTBits));
6404 Hi = MIRBuilder.buildAShr(Dst: NVT, Src0: InH,
6405 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: NVTBits - 1));
6406 } else if (Amt == NVTBits) {
6407 Lo = InH;
6408 Hi = MIRBuilder.buildAShr(Dst: NVT, Src0: InH,
6409 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: NVTBits - 1));
6410 } else {
6411 auto ShiftAmtConst = MIRBuilder.buildConstant(Res: AmtTy, Val: Amt);
6412
6413 auto OrLHS = MIRBuilder.buildLShr(Dst: NVT, Src0: InL, Src1: ShiftAmtConst);
6414 auto OrRHS = MIRBuilder.buildShl(
6415 Dst: NVT, Src0: InH, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: -Amt + NVTBits));
6416
6417 Lo = MIRBuilder.buildOr(Dst: NVT, Src0: OrLHS, Src1: OrRHS);
6418 Hi = MIRBuilder.buildAShr(Dst: NVT, Src0: InH, Src1: ShiftAmtConst);
6419 }
6420 }
6421
6422 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0), Ops: {Lo, Hi});
6423 MI.eraseFromParent();
6424
6425 return Legalized;
6426}
6427
6428LegalizerHelper::LegalizeResult
6429LegalizerHelper::narrowScalarShift(MachineInstr &MI, unsigned TypeIdx,
6430 LLT RequestedTy) {
6431 if (TypeIdx == 1) {
6432 Observer.changingInstr(MI);
6433 narrowScalarSrc(MI, NarrowTy: RequestedTy, OpIdx: 2);
6434 Observer.changedInstr(MI);
6435 return Legalized;
6436 }
6437
6438 Register DstReg = MI.getOperand(i: 0).getReg();
6439 LLT DstTy = MRI.getType(Reg: DstReg);
6440 if (DstTy.isVector())
6441 return UnableToLegalize;
6442
6443 Register Amt = MI.getOperand(i: 2).getReg();
6444 LLT ShiftAmtTy = MRI.getType(Reg: Amt);
6445 const unsigned DstEltSize = DstTy.getScalarSizeInBits();
6446 if (DstEltSize % 2 != 0)
6447 return UnableToLegalize;
6448
6449 // Check if we should use multi-way splitting instead of recursive binary
6450 // splitting.
6451 //
6452 // Multi-way splitting directly decomposes wide shifts (e.g., 128-bit ->
6453 // 4×32-bit) in a single legalization step, avoiding the recursive overhead
6454 // and dependency chains created by usual binary splitting approach
6455 // (128->64->32).
6456 //
6457 // The >= 8 parts threshold ensures we only use this optimization when binary
6458 // splitting would require multiple recursive passes, avoiding overhead for
6459 // simple 2-way splits where binary approach is sufficient.
6460 if (RequestedTy.isValid() && RequestedTy.isScalar() &&
6461 DstEltSize % RequestedTy.getSizeInBits() == 0) {
6462 const unsigned NumParts = DstEltSize / RequestedTy.getSizeInBits();
6463 // Use multiway if we have 8 or more parts (i.e., would need 3+ recursive
6464 // steps).
6465 if (NumParts >= 8)
6466 return narrowScalarShiftMultiway(MI, TargetTy: RequestedTy);
6467 }
6468
6469 // Fall back to binary splitting:
6470 // Ignore the input type. We can only go to exactly half the size of the
6471 // input. If that isn't small enough, the resulting pieces will be further
6472 // legalized.
6473 const unsigned NewBitSize = DstEltSize / 2;
6474 const LLT HalfTy = DstTy.getScalarType().changeElementSize(NewEltSize: NewBitSize);
6475 const LLT CondTy = LLT::integer(SizeInBits: 1);
6476
6477 if (auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: Amt, MRI)) {
6478 return narrowScalarShiftByConstant(MI, Amt: VRegAndVal->Value, HalfTy,
6479 AmtTy: ShiftAmtTy);
6480 }
6481
6482 // TODO: Expand with known bits.
6483
6484 // Handle the fully general expansion by an unknown amount.
6485 auto NewBits = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: NewBitSize);
6486
6487 Register InL = MRI.createGenericVirtualRegister(Ty: HalfTy);
6488 Register InH = MRI.createGenericVirtualRegister(Ty: HalfTy);
6489 MIRBuilder.buildUnmerge(Res: {InL, InH}, Op: MI.getOperand(i: 1));
6490
6491 auto AmtExcess = MIRBuilder.buildSub(Dst: ShiftAmtTy, Src0: Amt, Src1: NewBits);
6492 auto AmtLack = MIRBuilder.buildSub(Dst: ShiftAmtTy, Src0: NewBits, Src1: Amt);
6493
6494 auto Zero = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: 0);
6495 auto IsShort = MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_ULT, Res: CondTy, Op0: Amt, Op1: NewBits);
6496 auto IsZero = MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: CondTy, Op0: Amt, Op1: Zero);
6497
6498 Register ResultRegs[2];
6499 switch (MI.getOpcode()) {
6500 case TargetOpcode::G_SHL: {
6501 // Short: ShAmt < NewBitSize
6502 auto LoS = MIRBuilder.buildShl(Dst: HalfTy, Src0: InL, Src1: Amt);
6503
6504 auto LoOr = MIRBuilder.buildLShr(Dst: HalfTy, Src0: InL, Src1: AmtLack);
6505 auto HiOr = MIRBuilder.buildShl(Dst: HalfTy, Src0: InH, Src1: Amt);
6506 auto HiS = MIRBuilder.buildOr(Dst: HalfTy, Src0: LoOr, Src1: HiOr);
6507
6508 // Long: ShAmt >= NewBitSize
6509 auto LoL = MIRBuilder.buildConstant(Res: HalfTy, Val: 0); // Lo part is zero.
6510 auto HiL = MIRBuilder.buildShl(Dst: HalfTy, Src0: InL, Src1: AmtExcess); // Hi from Lo part.
6511
6512 auto Lo = MIRBuilder.buildSelect(Res: HalfTy, Tst: IsShort, Op0: LoS, Op1: LoL);
6513 auto Hi = MIRBuilder.buildSelect(
6514 Res: HalfTy, Tst: IsZero, Op0: InH, Op1: MIRBuilder.buildSelect(Res: HalfTy, Tst: IsShort, Op0: HiS, Op1: HiL));
6515
6516 ResultRegs[0] = Lo.getReg(Idx: 0);
6517 ResultRegs[1] = Hi.getReg(Idx: 0);
6518 break;
6519 }
6520 case TargetOpcode::G_LSHR:
6521 case TargetOpcode::G_ASHR: {
6522 // Short: ShAmt < NewBitSize
6523 auto HiS = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {HalfTy}, SrcOps: {InH, Amt});
6524
6525 auto LoOr = MIRBuilder.buildLShr(Dst: HalfTy, Src0: InL, Src1: Amt);
6526 auto HiOr = MIRBuilder.buildShl(Dst: HalfTy, Src0: InH, Src1: AmtLack);
6527 auto LoS = MIRBuilder.buildOr(Dst: HalfTy, Src0: LoOr, Src1: HiOr);
6528
6529 // Long: ShAmt >= NewBitSize
6530 MachineInstrBuilder HiL;
6531 if (MI.getOpcode() == TargetOpcode::G_LSHR) {
6532 HiL = MIRBuilder.buildConstant(Res: HalfTy, Val: 0); // Hi part is zero.
6533 } else {
6534 auto ShiftAmt = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: NewBitSize - 1);
6535 HiL = MIRBuilder.buildAShr(Dst: HalfTy, Src0: InH, Src1: ShiftAmt); // Sign of Hi part.
6536 }
6537 auto LoL = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {HalfTy},
6538 SrcOps: {InH, AmtExcess}); // Lo from Hi part.
6539
6540 auto Lo = MIRBuilder.buildSelect(
6541 Res: HalfTy, Tst: IsZero, Op0: InL, Op1: MIRBuilder.buildSelect(Res: HalfTy, Tst: IsShort, Op0: LoS, Op1: LoL));
6542
6543 auto Hi = MIRBuilder.buildSelect(Res: HalfTy, Tst: IsShort, Op0: HiS, Op1: HiL);
6544
6545 ResultRegs[0] = Lo.getReg(Idx: 0);
6546 ResultRegs[1] = Hi.getReg(Idx: 0);
6547 break;
6548 }
6549 default:
6550 llvm_unreachable("not a shift");
6551 }
6552
6553 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: ResultRegs);
6554 MI.eraseFromParent();
6555 return Legalized;
6556}
6557
6558Register LegalizerHelper::buildConstantShiftPart(unsigned Opcode,
6559 unsigned PartIdx,
6560 unsigned NumParts,
6561 ArrayRef<Register> SrcParts,
6562 const ShiftParams &Params,
6563 LLT TargetTy, LLT ShiftAmtTy) {
6564 auto WordShiftConst = getIConstantVRegVal(VReg: Params.WordShift, MRI);
6565 auto BitShiftConst = getIConstantVRegVal(VReg: Params.BitShift, MRI);
6566 assert(WordShiftConst && BitShiftConst && "Expected constants");
6567
6568 const unsigned ShiftWords = WordShiftConst->getZExtValue();
6569 const unsigned ShiftBits = BitShiftConst->getZExtValue();
6570 const bool NeedsInterWordShift = ShiftBits != 0;
6571
6572 switch (Opcode) {
6573 case TargetOpcode::G_SHL: {
6574 // Data moves from lower indices to higher indices
6575 // If this part would come from a source beyond our range, it's zero
6576 if (PartIdx < ShiftWords)
6577 return Params.Zero;
6578
6579 unsigned SrcIdx = PartIdx - ShiftWords;
6580 if (!NeedsInterWordShift)
6581 return SrcParts[SrcIdx];
6582
6583 // Combine shifted main part with carry from previous part
6584 auto Hi = MIRBuilder.buildShl(Dst: TargetTy, Src0: SrcParts[SrcIdx], Src1: Params.BitShift);
6585 if (SrcIdx > 0) {
6586 auto Lo = MIRBuilder.buildLShr(Dst: TargetTy, Src0: SrcParts[SrcIdx - 1],
6587 Src1: Params.InvBitShift);
6588 return MIRBuilder.buildOr(Dst: TargetTy, Src0: Hi, Src1: Lo).getReg(Idx: 0);
6589 }
6590 return Hi.getReg(Idx: 0);
6591 }
6592
6593 case TargetOpcode::G_LSHR: {
6594 unsigned SrcIdx = PartIdx + ShiftWords;
6595 if (SrcIdx >= NumParts)
6596 return Params.Zero;
6597 if (!NeedsInterWordShift)
6598 return SrcParts[SrcIdx];
6599
6600 // Combine shifted main part with carry from next part
6601 auto Lo = MIRBuilder.buildLShr(Dst: TargetTy, Src0: SrcParts[SrcIdx], Src1: Params.BitShift);
6602 if (SrcIdx + 1 < NumParts) {
6603 auto Hi = MIRBuilder.buildShl(Dst: TargetTy, Src0: SrcParts[SrcIdx + 1],
6604 Src1: Params.InvBitShift);
6605 return MIRBuilder.buildOr(Dst: TargetTy, Src0: Lo, Src1: Hi).getReg(Idx: 0);
6606 }
6607 return Lo.getReg(Idx: 0);
6608 }
6609
6610 case TargetOpcode::G_ASHR: {
6611 // Like LSHR but preserves sign bit
6612 unsigned SrcIdx = PartIdx + ShiftWords;
6613 if (SrcIdx >= NumParts)
6614 return Params.SignBit;
6615 if (!NeedsInterWordShift)
6616 return SrcParts[SrcIdx];
6617
6618 // Only the original MSB part uses arithmetic shift to preserve sign. All
6619 // other parts use logical shift since they're just moving data bits.
6620 auto Lo =
6621 (SrcIdx == NumParts - 1)
6622 ? MIRBuilder.buildAShr(Dst: TargetTy, Src0: SrcParts[SrcIdx], Src1: Params.BitShift)
6623 : MIRBuilder.buildLShr(Dst: TargetTy, Src0: SrcParts[SrcIdx], Src1: Params.BitShift);
6624 Register HiSrc =
6625 (SrcIdx + 1 < NumParts) ? SrcParts[SrcIdx + 1] : Params.SignBit;
6626 auto Hi = MIRBuilder.buildShl(Dst: TargetTy, Src0: HiSrc, Src1: Params.InvBitShift);
6627 return MIRBuilder.buildOr(Dst: TargetTy, Src0: Lo, Src1: Hi).getReg(Idx: 0);
6628 }
6629
6630 default:
6631 llvm_unreachable("not a shift");
6632 }
6633}
6634
6635Register LegalizerHelper::buildVariableShiftPart(unsigned Opcode,
6636 Register MainOperand,
6637 Register ShiftAmt,
6638 LLT TargetTy,
6639 Register CarryOperand) {
6640 // This helper generates a single output part for variable shifts by combining
6641 // the main operand (shifted by BitShift) with carry bits from an adjacent
6642 // part.
6643
6644 // For G_ASHR, individual parts don't have their own sign bit, only the
6645 // complete value does. So we use LSHR for the main operand shift in ASHR
6646 // context.
6647 unsigned MainOpcode = (Opcode == TargetOpcode::G_ASHR)
6648 ? static_cast<unsigned>(TargetOpcode::G_LSHR)
6649 : Opcode;
6650
6651 // Perform the primary shift on the main operand
6652 Register MainShifted =
6653 MIRBuilder.buildInstr(Opc: MainOpcode, DstOps: {TargetTy}, SrcOps: {MainOperand, ShiftAmt})
6654 .getReg(Idx: 0);
6655
6656 // No carry operand available
6657 if (!CarryOperand.isValid())
6658 return MainShifted;
6659
6660 // If BitShift is 0 (word-aligned shift), no inter-word bit movement occurs,
6661 // so carry bits aren't needed.
6662 LLT ShiftAmtTy = MRI.getType(Reg: ShiftAmt);
6663 auto ZeroConst = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: 0);
6664 LLT BoolTy = LLT::integer(SizeInBits: 1);
6665 auto IsZeroBitShift =
6666 MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: BoolTy, Op0: ShiftAmt, Op1: ZeroConst);
6667
6668 // Extract bits from the adjacent part that will "carry over" into this part.
6669 // The carry direction is opposite to the main shift direction, so we can
6670 // align the two shifted values before combining them with OR.
6671
6672 // Determine the carry shift opcode (opposite direction)
6673 unsigned CarryOpcode = (Opcode == TargetOpcode::G_SHL) ? TargetOpcode::G_LSHR
6674 : TargetOpcode::G_SHL;
6675
6676 // Calculate inverse shift amount: BitWidth - ShiftAmt
6677 auto TargetBitsConst =
6678 MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetTy.getScalarSizeInBits());
6679 auto InvShiftAmt = MIRBuilder.buildSub(Dst: ShiftAmtTy, Src0: TargetBitsConst, Src1: ShiftAmt);
6680
6681 // Shift the carry operand
6682 Register CarryBits =
6683 MIRBuilder
6684 .buildInstr(Opc: CarryOpcode, DstOps: {TargetTy}, SrcOps: {CarryOperand, InvShiftAmt})
6685 .getReg(Idx: 0);
6686
6687 // If BitShift is 0, don't include carry bits (InvShiftAmt would equal
6688 // TargetBits which would be poison for the individual carry shift operation).
6689 auto ZeroReg = MIRBuilder.buildConstant(Res: TargetTy, Val: 0);
6690 Register SafeCarryBits =
6691 MIRBuilder.buildSelect(Res: TargetTy, Tst: IsZeroBitShift, Op0: ZeroReg, Op1: CarryBits)
6692 .getReg(Idx: 0);
6693
6694 // Combine the main shifted part with the carry bits
6695 return MIRBuilder.buildOr(Dst: TargetTy, Src0: MainShifted, Src1: SafeCarryBits).getReg(Idx: 0);
6696}
6697
6698LegalizerHelper::LegalizeResult
6699LegalizerHelper::narrowScalarShiftByConstantMultiway(MachineInstr &MI,
6700 const APInt &Amt,
6701 LLT TargetTy,
6702 LLT ShiftAmtTy) {
6703 // Any wide shift can be decomposed into WordShift + BitShift components.
6704 // When shift amount is known constant, directly compute the decomposition
6705 // values and generate constant registers.
6706 Register DstReg = MI.getOperand(i: 0).getReg();
6707 Register SrcReg = MI.getOperand(i: 1).getReg();
6708 LLT DstTy = MRI.getType(Reg: DstReg);
6709
6710 const unsigned DstBits = DstTy.getScalarSizeInBits();
6711 const unsigned TargetBits = TargetTy.getScalarSizeInBits();
6712 const unsigned NumParts = DstBits / TargetBits;
6713
6714 assert(DstBits % TargetBits == 0 && "Target type must evenly divide source");
6715
6716 // When the shift amount is known at compile time, we just calculate which
6717 // source parts contribute to each output part.
6718
6719 SmallVector<Register, 8> SrcParts;
6720 extractParts(Reg: SrcReg, Ty: TargetTy, NumParts, VRegs&: SrcParts, MIRBuilder, MRI);
6721
6722 if (Amt.isZero()) {
6723 // No shift needed, just copy
6724 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: SrcParts);
6725 MI.eraseFromParent();
6726 return Legalized;
6727 }
6728
6729 ShiftParams Params;
6730 const unsigned ShiftWords = Amt.getZExtValue() / TargetBits;
6731 const unsigned ShiftBits = Amt.getZExtValue() % TargetBits;
6732
6733 // Generate constants and values needed by all shift types
6734 Params.WordShift = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: ShiftWords).getReg(Idx: 0);
6735 Params.BitShift = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: ShiftBits).getReg(Idx: 0);
6736 Params.InvBitShift =
6737 MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBits - ShiftBits).getReg(Idx: 0);
6738 Params.Zero = MIRBuilder.buildConstant(Res: TargetTy, Val: 0).getReg(Idx: 0);
6739
6740 // For ASHR, we need the sign-extended value to fill shifted-out positions
6741 if (MI.getOpcode() == TargetOpcode::G_ASHR)
6742 Params.SignBit =
6743 MIRBuilder
6744 .buildAShr(Dst: TargetTy, Src0: SrcParts[SrcParts.size() - 1],
6745 Src1: MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBits - 1))
6746 .getReg(Idx: 0);
6747
6748 SmallVector<Register, 8> DstParts(NumParts);
6749 for (unsigned I = 0; I < NumParts; ++I)
6750 DstParts[I] = buildConstantShiftPart(Opcode: MI.getOpcode(), PartIdx: I, NumParts, SrcParts,
6751 Params, TargetTy, ShiftAmtTy);
6752
6753 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstParts);
6754 MI.eraseFromParent();
6755 return Legalized;
6756}
6757
6758LegalizerHelper::LegalizeResult
6759LegalizerHelper::narrowScalarShiftMultiway(MachineInstr &MI, LLT TargetTy) {
6760 Register DstReg = MI.getOperand(i: 0).getReg();
6761 Register SrcReg = MI.getOperand(i: 1).getReg();
6762 Register AmtReg = MI.getOperand(i: 2).getReg();
6763 LLT DstTy = MRI.getType(Reg: DstReg);
6764 LLT ShiftAmtTy = MRI.getType(Reg: AmtReg);
6765
6766 const unsigned DstBits = DstTy.getScalarSizeInBits();
6767 const unsigned TargetBits = TargetTy.getScalarSizeInBits();
6768 const unsigned NumParts = DstBits / TargetBits;
6769
6770 assert(DstBits % TargetBits == 0 && "Target type must evenly divide source");
6771 assert(isPowerOf2_32(TargetBits) && "Target bit width must be power of 2");
6772
6773 // If the shift amount is known at compile time, we can use direct indexing
6774 // instead of generating select chains in the general case.
6775 if (auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: AmtReg, MRI))
6776 return narrowScalarShiftByConstantMultiway(MI, Amt: VRegAndVal->Value, TargetTy,
6777 ShiftAmtTy);
6778
6779 // For runtime-variable shift amounts, we must generate a more complex
6780 // sequence that handles all possible shift values using select chains.
6781
6782 // Split the input into target-sized pieces
6783 SmallVector<Register, 8> SrcParts;
6784 extractParts(Reg: SrcReg, Ty: TargetTy, NumParts, VRegs&: SrcParts, MIRBuilder, MRI);
6785
6786 // Shifting by zero should be a no-op.
6787 auto ZeroAmtConst = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: 0);
6788 LLT BoolTy = LLT::integer(SizeInBits: 1);
6789 auto IsZeroShift =
6790 MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: BoolTy, Op0: AmtReg, Op1: ZeroAmtConst);
6791
6792 // Any wide shift can be decomposed into two components:
6793 // 1. WordShift: number of complete target-sized words to shift
6794 // 2. BitShift: number of bits to shift within each word
6795 //
6796 // Example: 128-bit >> 50 with 32-bit target:
6797 // WordShift = 50 / 32 = 1 (shift right by 1 complete word)
6798 // BitShift = 50 % 32 = 18 (shift each word right by 18 bits)
6799 unsigned TargetBitsLog2 = Log2_32(Value: TargetBits);
6800 auto TargetBitsLog2Const =
6801 MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBitsLog2);
6802 auto TargetBitsMask = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBits - 1);
6803
6804 Register WordShift =
6805 MIRBuilder.buildLShr(Dst: ShiftAmtTy, Src0: AmtReg, Src1: TargetBitsLog2Const).getReg(Idx: 0);
6806 Register BitShift =
6807 MIRBuilder.buildAnd(Dst: ShiftAmtTy, Src0: AmtReg, Src1: TargetBitsMask).getReg(Idx: 0);
6808
6809 // Fill values:
6810 // - SHL/LSHR: fill with zeros
6811 // - ASHR: fill with sign-extended MSB
6812 Register ZeroReg = MIRBuilder.buildConstant(Res: TargetTy, Val: 0).getReg(Idx: 0);
6813
6814 Register FillValue;
6815 if (MI.getOpcode() == TargetOpcode::G_ASHR) {
6816 auto TargetBitsMinusOneConst =
6817 MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBits - 1);
6818 FillValue = MIRBuilder
6819 .buildAShr(Dst: TargetTy, Src0: SrcParts[NumParts - 1],
6820 Src1: TargetBitsMinusOneConst)
6821 .getReg(Idx: 0);
6822 } else {
6823 FillValue = ZeroReg;
6824 }
6825
6826 SmallVector<Register, 8> DstParts(NumParts);
6827
6828 // For each output part, generate a select chain that chooses the correct
6829 // result based on the runtime WordShift value. This handles all possible
6830 // word shift amounts by pre-calculating what each would produce.
6831 for (unsigned I = 0; I < NumParts; ++I) {
6832 // Initialize with appropriate default value for this shift type
6833 Register InBoundsResult = FillValue;
6834
6835 // clang-format off
6836 // Build a branchless select chain by pre-computing results for all possible
6837 // WordShift values (0 to NumParts-1). Each iteration nests a new select:
6838 //
6839 // K=0: select(WordShift==0, result0, FillValue)
6840 // K=1: select(WordShift==1, result1, select(WordShift==0, result0, FillValue))
6841 // K=2: select(WordShift==2, result2, select(WordShift==1, result1, select(...)))
6842 // clang-format on
6843 for (unsigned K = 0; K < NumParts; ++K) {
6844 auto WordShiftKConst = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: K);
6845 auto IsWordShiftK = MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: BoolTy,
6846 Op0: WordShift, Op1: WordShiftKConst);
6847
6848 // Calculate source indices for this word shift
6849 //
6850 // For 4-part 128-bit value with K=1 word shift:
6851 // SHL: [3][2][1][0] << K => [2][1][0][Z]
6852 // -> (MainIdx = I-K, CarryIdx = I-K-1)
6853 // LSHR: [3][2][1][0] >> K => [Z][3][2][1]
6854 // -> (MainIdx = I+K, CarryIdx = I+K+1)
6855 int MainSrcIdx;
6856 int CarrySrcIdx; // Index for the word that provides the carried-in bits.
6857
6858 switch (MI.getOpcode()) {
6859 case TargetOpcode::G_SHL:
6860 MainSrcIdx = (int)I - (int)K;
6861 CarrySrcIdx = MainSrcIdx - 1;
6862 break;
6863 case TargetOpcode::G_LSHR:
6864 case TargetOpcode::G_ASHR:
6865 MainSrcIdx = (int)I + (int)K;
6866 CarrySrcIdx = MainSrcIdx + 1;
6867 break;
6868 default:
6869 llvm_unreachable("Not a shift");
6870 }
6871
6872 // Check bounds and build the result for this word shift
6873 Register ResultForK;
6874 if (MainSrcIdx >= 0 && MainSrcIdx < (int)NumParts) {
6875 Register MainOp = SrcParts[MainSrcIdx];
6876 Register CarryOp;
6877
6878 // Determine carry operand with bounds checking
6879 if (CarrySrcIdx >= 0 && CarrySrcIdx < (int)NumParts)
6880 CarryOp = SrcParts[CarrySrcIdx];
6881 else if (MI.getOpcode() == TargetOpcode::G_ASHR &&
6882 CarrySrcIdx >= (int)NumParts)
6883 CarryOp = FillValue; // Use sign extension
6884
6885 ResultForK = buildVariableShiftPart(Opcode: MI.getOpcode(), MainOperand: MainOp, ShiftAmt: BitShift,
6886 TargetTy, CarryOperand: CarryOp);
6887 } else {
6888 // Out of bounds - use fill value for this k
6889 ResultForK = FillValue;
6890 }
6891
6892 // Select this result if WordShift equals k
6893 InBoundsResult =
6894 MIRBuilder
6895 .buildSelect(Res: TargetTy, Tst: IsWordShiftK, Op0: ResultForK, Op1: InBoundsResult)
6896 .getReg(Idx: 0);
6897 }
6898
6899 // Handle zero-shift special case: if shift is 0, use original input
6900 DstParts[I] =
6901 MIRBuilder
6902 .buildSelect(Res: TargetTy, Tst: IsZeroShift, Op0: SrcParts[I], Op1: InBoundsResult)
6903 .getReg(Idx: 0);
6904 }
6905
6906 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstParts);
6907 MI.eraseFromParent();
6908 return Legalized;
6909}
6910
6911LegalizerHelper::LegalizeResult
6912LegalizerHelper::moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
6913 LLT MoreTy) {
6914 assert(TypeIdx == 0 && "Expecting only Idx 0");
6915
6916 Observer.changingInstr(MI);
6917 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
6918 MachineBasicBlock &OpMBB = *MI.getOperand(i: I + 1).getMBB();
6919 MIRBuilder.setInsertPt(MBB&: OpMBB, II: OpMBB.getFirstTerminator());
6920 moreElementsVectorSrc(MI, MoreTy, OpIdx: I);
6921 }
6922
6923 MachineBasicBlock &MBB = *MI.getParent();
6924 MIRBuilder.setInsertPt(MBB, II: --MBB.getFirstNonPHI());
6925 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
6926 Observer.changedInstr(MI);
6927 return Legalized;
6928}
6929
6930MachineInstrBuilder LegalizerHelper::getNeutralElementForVecReduce(
6931 unsigned Opcode, MachineIRBuilder &MIRBuilder, LLT Ty) {
6932 assert(Ty.isScalar() && "Expected scalar type to make neutral element for");
6933
6934 switch (Opcode) {
6935 default:
6936 llvm_unreachable(
6937 "getNeutralElementForVecReduce called with invalid opcode!");
6938 case TargetOpcode::G_VECREDUCE_ADD:
6939 case TargetOpcode::G_VECREDUCE_OR:
6940 case TargetOpcode::G_VECREDUCE_XOR:
6941 case TargetOpcode::G_VECREDUCE_UMAX:
6942 return MIRBuilder.buildConstant(Res: Ty, Val: 0);
6943 case TargetOpcode::G_VECREDUCE_MUL:
6944 return MIRBuilder.buildConstant(Res: Ty, Val: 1);
6945 case TargetOpcode::G_VECREDUCE_AND:
6946 case TargetOpcode::G_VECREDUCE_UMIN:
6947 return MIRBuilder.buildConstant(
6948 Res: Ty, Val: APInt::getAllOnes(numBits: Ty.getScalarSizeInBits()));
6949 case TargetOpcode::G_VECREDUCE_SMAX:
6950 return MIRBuilder.buildConstant(
6951 Res: Ty, Val: APInt::getSignedMinValue(numBits: Ty.getSizeInBits()));
6952 case TargetOpcode::G_VECREDUCE_SMIN:
6953 return MIRBuilder.buildConstant(
6954 Res: Ty, Val: APInt::getSignedMaxValue(numBits: Ty.getSizeInBits()));
6955 case TargetOpcode::G_VECREDUCE_FADD:
6956 return MIRBuilder.buildFConstant(Res: Ty, Val: -0.0);
6957 case TargetOpcode::G_VECREDUCE_FMUL:
6958 return MIRBuilder.buildFConstant(Res: Ty, Val: 1.0);
6959 case TargetOpcode::G_VECREDUCE_FMINIMUM:
6960 case TargetOpcode::G_VECREDUCE_FMAXIMUM:
6961 assert(false && "getNeutralElementForVecReduce unimplemented for "
6962 "G_VECREDUCE_FMINIMUM and G_VECREDUCE_FMAXIMUM!");
6963 }
6964 llvm_unreachable("switch expected to return!");
6965}
6966
6967LegalizerHelper::LegalizeResult
6968LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
6969 LLT MoreTy) {
6970 unsigned Opc = MI.getOpcode();
6971 switch (Opc) {
6972 case TargetOpcode::G_IMPLICIT_DEF:
6973 case TargetOpcode::G_LOAD: {
6974 if (TypeIdx != 0)
6975 return UnableToLegalize;
6976 Observer.changingInstr(MI);
6977 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
6978 Observer.changedInstr(MI);
6979 return Legalized;
6980 }
6981 case TargetOpcode::G_STORE:
6982 if (TypeIdx != 0)
6983 return UnableToLegalize;
6984 Observer.changingInstr(MI);
6985 moreElementsVectorSrc(MI, MoreTy, OpIdx: 0);
6986 Observer.changedInstr(MI);
6987 return Legalized;
6988 case TargetOpcode::G_AND:
6989 case TargetOpcode::G_OR:
6990 case TargetOpcode::G_XOR:
6991 case TargetOpcode::G_ADD:
6992 case TargetOpcode::G_SUB:
6993 case TargetOpcode::G_MUL:
6994 case TargetOpcode::G_FADD:
6995 case TargetOpcode::G_FSUB:
6996 case TargetOpcode::G_FMUL:
6997 case TargetOpcode::G_FDIV:
6998 case TargetOpcode::G_FCOPYSIGN:
6999 case TargetOpcode::G_UADDSAT:
7000 case TargetOpcode::G_USUBSAT:
7001 case TargetOpcode::G_SADDSAT:
7002 case TargetOpcode::G_SSUBSAT:
7003 case TargetOpcode::G_SMIN:
7004 case TargetOpcode::G_SMAX:
7005 case TargetOpcode::G_UMIN:
7006 case TargetOpcode::G_UMAX:
7007 case TargetOpcode::G_FMINNUM:
7008 case TargetOpcode::G_FMAXNUM:
7009 case TargetOpcode::G_FMINNUM_IEEE:
7010 case TargetOpcode::G_FMAXNUM_IEEE:
7011 case TargetOpcode::G_FMINIMUM:
7012 case TargetOpcode::G_FMAXIMUM:
7013 case TargetOpcode::G_FMINIMUMNUM:
7014 case TargetOpcode::G_FMAXIMUMNUM:
7015 case TargetOpcode::G_STRICT_FADD:
7016 case TargetOpcode::G_STRICT_FSUB:
7017 case TargetOpcode::G_STRICT_FMUL: {
7018 Observer.changingInstr(MI);
7019 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
7020 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
7021 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7022 Observer.changedInstr(MI);
7023 return Legalized;
7024 }
7025 case TargetOpcode::G_SHL:
7026 case TargetOpcode::G_ASHR:
7027 case TargetOpcode::G_LSHR: {
7028 Observer.changingInstr(MI);
7029 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
7030 // The shift operand may have a different scalar type from the source and
7031 // destination operands.
7032 LLT ShiftMoreTy = MoreTy.changeElementType(
7033 NewEltTy: MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getElementType());
7034 moreElementsVectorSrc(MI, MoreTy: ShiftMoreTy, OpIdx: 2);
7035 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7036 Observer.changedInstr(MI);
7037 return Legalized;
7038 }
7039 case TargetOpcode::G_FMA:
7040 case TargetOpcode::G_STRICT_FMA:
7041 case TargetOpcode::G_FSHR:
7042 case TargetOpcode::G_FSHL: {
7043 Observer.changingInstr(MI);
7044 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
7045 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
7046 moreElementsVectorSrc(MI, MoreTy, OpIdx: 3);
7047 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7048 Observer.changedInstr(MI);
7049 return Legalized;
7050 }
7051 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
7052 case TargetOpcode::G_EXTRACT:
7053 if (TypeIdx != 1)
7054 return UnableToLegalize;
7055 Observer.changingInstr(MI);
7056 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
7057 Observer.changedInstr(MI);
7058 return Legalized;
7059 case TargetOpcode::G_INSERT:
7060 case TargetOpcode::G_INSERT_VECTOR_ELT:
7061 case TargetOpcode::G_FREEZE:
7062 case TargetOpcode::G_FNEG:
7063 case TargetOpcode::G_FABS:
7064 case TargetOpcode::G_FSQRT:
7065 case TargetOpcode::G_FCEIL:
7066 case TargetOpcode::G_FFLOOR:
7067 case TargetOpcode::G_FNEARBYINT:
7068 case TargetOpcode::G_FRINT:
7069 case TargetOpcode::G_INTRINSIC_ROUND:
7070 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
7071 case TargetOpcode::G_INTRINSIC_TRUNC:
7072 case TargetOpcode::G_BITREVERSE:
7073 case TargetOpcode::G_BSWAP:
7074 case TargetOpcode::G_FCANONICALIZE:
7075 case TargetOpcode::G_SEXT_INREG:
7076 case TargetOpcode::G_ABS:
7077 case TargetOpcode::G_CTLZ:
7078 case TargetOpcode::G_CTPOP:
7079 if (TypeIdx != 0)
7080 return UnableToLegalize;
7081 Observer.changingInstr(MI);
7082 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
7083 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7084 Observer.changedInstr(MI);
7085 return Legalized;
7086 case TargetOpcode::G_SELECT: {
7087 auto [DstReg, DstTy, CondReg, CondTy] = MI.getFirst2RegLLTs();
7088 if (TypeIdx == 1) {
7089 if (!CondTy.isScalar() ||
7090 DstTy.getElementCount() != MoreTy.getElementCount())
7091 return UnableToLegalize;
7092
7093 // This is turning a scalar select of vectors into a vector
7094 // select. Broadcast the select condition.
7095 auto ShufSplat = MIRBuilder.buildShuffleSplat(Res: MoreTy, Src: CondReg);
7096 Observer.changingInstr(MI);
7097 MI.getOperand(i: 1).setReg(ShufSplat.getReg(Idx: 0));
7098 Observer.changedInstr(MI);
7099 return Legalized;
7100 }
7101
7102 if (CondTy.isVector())
7103 return UnableToLegalize;
7104
7105 Observer.changingInstr(MI);
7106 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
7107 moreElementsVectorSrc(MI, MoreTy, OpIdx: 3);
7108 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7109 Observer.changedInstr(MI);
7110 return Legalized;
7111 }
7112 case TargetOpcode::G_UNMERGE_VALUES:
7113 return UnableToLegalize;
7114 case TargetOpcode::G_PHI:
7115 return moreElementsVectorPhi(MI, TypeIdx, MoreTy);
7116 case TargetOpcode::G_SHUFFLE_VECTOR:
7117 return moreElementsVectorShuffle(MI, TypeIdx, MoreTy);
7118 case TargetOpcode::G_BUILD_VECTOR: {
7119 SmallVector<SrcOp, 8> Elts;
7120 for (auto Op : MI.uses()) {
7121 Elts.push_back(Elt: Op.getReg());
7122 }
7123
7124 for (unsigned i = Elts.size(); i < MoreTy.getNumElements(); ++i) {
7125 Elts.push_back(Elt: MIRBuilder.buildUndef(Res: MoreTy.getScalarType()));
7126 }
7127
7128 MIRBuilder.buildDeleteTrailingVectorElements(
7129 Res: MI.getOperand(i: 0).getReg(), Op0: MIRBuilder.buildInstr(Opc, DstOps: {MoreTy}, SrcOps: Elts));
7130 MI.eraseFromParent();
7131 return Legalized;
7132 }
7133 case TargetOpcode::G_SEXT:
7134 case TargetOpcode::G_ZEXT:
7135 case TargetOpcode::G_ANYEXT:
7136 case TargetOpcode::G_TRUNC:
7137 case TargetOpcode::G_FPTRUNC:
7138 case TargetOpcode::G_FPEXT:
7139 case TargetOpcode::G_FPTOSI:
7140 case TargetOpcode::G_FPTOUI:
7141 case TargetOpcode::G_FPTOSI_SAT:
7142 case TargetOpcode::G_FPTOUI_SAT:
7143 case TargetOpcode::G_SITOFP:
7144 case TargetOpcode::G_UITOFP:
7145 case TargetOpcode::G_TRUNC_SSAT_S:
7146 case TargetOpcode::G_TRUNC_SSAT_U:
7147 case TargetOpcode::G_TRUNC_USAT_U: {
7148 Observer.changingInstr(MI);
7149 LLT SrcExtTy;
7150 LLT DstExtTy;
7151 if (TypeIdx == 0) {
7152 DstExtTy = MoreTy;
7153 SrcExtTy = MoreTy.changeElementType(
7154 NewEltTy: MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getElementType());
7155 } else {
7156 DstExtTy = MoreTy.changeElementType(
7157 NewEltTy: MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getElementType());
7158 SrcExtTy = MoreTy;
7159 }
7160 moreElementsVectorSrc(MI, MoreTy: SrcExtTy, OpIdx: 1);
7161 moreElementsVectorDst(MI, WideTy: DstExtTy, OpIdx: 0);
7162 Observer.changedInstr(MI);
7163 return Legalized;
7164 }
7165 case TargetOpcode::G_ICMP:
7166 case TargetOpcode::G_FCMP: {
7167 if (TypeIdx != 1)
7168 return UnableToLegalize;
7169
7170 Observer.changingInstr(MI);
7171 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
7172 moreElementsVectorSrc(MI, MoreTy, OpIdx: 3);
7173 LLT CondTy = MoreTy.changeVectorElementType(
7174 NewEltTy: MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getElementType());
7175 moreElementsVectorDst(MI, WideTy: CondTy, OpIdx: 0);
7176 Observer.changedInstr(MI);
7177 return Legalized;
7178 }
7179 case TargetOpcode::G_BITCAST: {
7180 if (TypeIdx != 0)
7181 return UnableToLegalize;
7182
7183 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
7184 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7185
7186 unsigned coefficient = SrcTy.getNumElements() * MoreTy.getNumElements();
7187 if (coefficient % DstTy.getNumElements() != 0)
7188 return UnableToLegalize;
7189
7190 coefficient = coefficient / DstTy.getNumElements();
7191
7192 LLT NewTy = SrcTy.changeElementCount(
7193 EC: ElementCount::get(MinVal: coefficient, Scalable: MoreTy.isScalable()));
7194 Observer.changingInstr(MI);
7195 moreElementsVectorSrc(MI, MoreTy: NewTy, OpIdx: 1);
7196 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7197 Observer.changedInstr(MI);
7198 return Legalized;
7199 }
7200 case TargetOpcode::G_VECREDUCE_FADD:
7201 case TargetOpcode::G_VECREDUCE_FMUL:
7202 case TargetOpcode::G_VECREDUCE_ADD:
7203 case TargetOpcode::G_VECREDUCE_MUL:
7204 case TargetOpcode::G_VECREDUCE_AND:
7205 case TargetOpcode::G_VECREDUCE_OR:
7206 case TargetOpcode::G_VECREDUCE_XOR:
7207 case TargetOpcode::G_VECREDUCE_SMAX:
7208 case TargetOpcode::G_VECREDUCE_SMIN:
7209 case TargetOpcode::G_VECREDUCE_UMAX:
7210 case TargetOpcode::G_VECREDUCE_UMIN: {
7211 LLT OrigTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
7212 MachineOperand &MO = MI.getOperand(i: 1);
7213 auto NewVec = MIRBuilder.buildPadVectorWithUndefElements(Res: MoreTy, Op0: MO);
7214 auto NeutralElement = getNeutralElementForVecReduce(
7215 Opcode: MI.getOpcode(), MIRBuilder, Ty: MoreTy.getElementType());
7216
7217 LLT IdxTy(TLI.getVectorIdxLLT(DL: MIRBuilder.getDataLayout()));
7218 for (size_t i = OrigTy.getNumElements(), e = MoreTy.getNumElements();
7219 i != e; i++) {
7220 auto Idx = MIRBuilder.buildConstant(Res: IdxTy, Val: i);
7221 NewVec = MIRBuilder.buildInsertVectorElement(Res: MoreTy, Val: NewVec,
7222 Elt: NeutralElement, Idx);
7223 }
7224
7225 Observer.changingInstr(MI);
7226 MO.setReg(NewVec.getReg(Idx: 0));
7227 Observer.changedInstr(MI);
7228 return Legalized;
7229 }
7230
7231 default:
7232 return UnableToLegalize;
7233 }
7234}
7235
7236LegalizerHelper::LegalizeResult
7237LegalizerHelper::equalizeVectorShuffleLengths(MachineInstr &MI) {
7238 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7239 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
7240 unsigned MaskNumElts = Mask.size();
7241 unsigned SrcNumElts = SrcTy.getNumElements();
7242 LLT DestEltTy = DstTy.getElementType();
7243
7244 if (MaskNumElts == SrcNumElts)
7245 return Legalized;
7246
7247 if (MaskNumElts < SrcNumElts) {
7248 // Extend mask to match new destination vector size with
7249 // undef values.
7250 SmallVector<int, 16> NewMask(SrcNumElts, -1);
7251 llvm::copy(Range&: Mask, Out: NewMask.begin());
7252
7253 moreElementsVectorDst(MI, WideTy: SrcTy, OpIdx: 0);
7254 MIRBuilder.setInstrAndDebugLoc(MI);
7255 MIRBuilder.buildShuffleVector(Res: MI.getOperand(i: 0).getReg(),
7256 Src1: MI.getOperand(i: 1).getReg(),
7257 Src2: MI.getOperand(i: 2).getReg(), Mask: NewMask);
7258 MI.eraseFromParent();
7259
7260 return Legalized;
7261 }
7262
7263 unsigned PaddedMaskNumElts = alignTo(Value: MaskNumElts, Align: SrcNumElts);
7264 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
7265 LLT PaddedTy =
7266 DstTy.changeVectorElementCount(EC: ElementCount::getFixed(MinVal: PaddedMaskNumElts));
7267
7268 // Create new source vectors by concatenating the initial
7269 // source vectors with undefined vectors of the same size.
7270 auto Undef = MIRBuilder.buildUndef(Res: SrcTy);
7271 SmallVector<Register, 8> MOps1(NumConcat, Undef.getReg(Idx: 0));
7272 SmallVector<Register, 8> MOps2(NumConcat, Undef.getReg(Idx: 0));
7273 MOps1[0] = MI.getOperand(i: 1).getReg();
7274 MOps2[0] = MI.getOperand(i: 2).getReg();
7275
7276 auto Src1 = MIRBuilder.buildConcatVectors(Res: PaddedTy, Ops: MOps1);
7277 auto Src2 = MIRBuilder.buildConcatVectors(Res: PaddedTy, Ops: MOps2);
7278
7279 // Readjust mask for new input vector length.
7280 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
7281 for (unsigned I = 0; I != MaskNumElts; ++I) {
7282 int Idx = Mask[I];
7283 if (Idx >= static_cast<int>(SrcNumElts))
7284 Idx += PaddedMaskNumElts - SrcNumElts;
7285 MappedOps[I] = Idx;
7286 }
7287
7288 // If we got more elements than required, extract subvector.
7289 if (MaskNumElts != PaddedMaskNumElts) {
7290 auto Shuffle =
7291 MIRBuilder.buildShuffleVector(Res: PaddedTy, Src1, Src2, Mask: MappedOps);
7292
7293 SmallVector<Register, 16> Elts(MaskNumElts);
7294 for (unsigned I = 0; I < MaskNumElts; ++I) {
7295 Elts[I] =
7296 MIRBuilder.buildExtractVectorElementConstant(Res: DestEltTy, Val: Shuffle, Idx: I)
7297 .getReg(Idx: 0);
7298 }
7299 MIRBuilder.buildBuildVector(Res: DstReg, Ops: Elts);
7300 } else {
7301 MIRBuilder.buildShuffleVector(Res: DstReg, Src1, Src2, Mask: MappedOps);
7302 }
7303
7304 MI.eraseFromParent();
7305 return LegalizerHelper::LegalizeResult::Legalized;
7306}
7307
7308LegalizerHelper::LegalizeResult
7309LegalizerHelper::moreElementsVectorShuffle(MachineInstr &MI,
7310 unsigned int TypeIdx, LLT MoreTy) {
7311 auto [DstTy, Src1Ty, Src2Ty] = MI.getFirst3LLTs();
7312 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
7313 unsigned NumElts = DstTy.getNumElements();
7314 unsigned WidenNumElts = MoreTy.getNumElements();
7315
7316 if (DstTy.isVector() && Src1Ty.isVector() &&
7317 DstTy.getNumElements() != Src1Ty.getNumElements()) {
7318 return equalizeVectorShuffleLengths(MI);
7319 }
7320
7321 if (TypeIdx != 0)
7322 return UnableToLegalize;
7323
7324 // Expect a canonicalized shuffle.
7325 if (DstTy != Src1Ty || DstTy != Src2Ty)
7326 return UnableToLegalize;
7327
7328 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
7329 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
7330
7331 // Adjust mask based on new input vector length.
7332 SmallVector<int, 16> NewMask(WidenNumElts, -1);
7333 for (unsigned I = 0; I != NumElts; ++I) {
7334 int Idx = Mask[I];
7335 if (Idx < static_cast<int>(NumElts))
7336 NewMask[I] = Idx;
7337 else
7338 NewMask[I] = Idx - NumElts + WidenNumElts;
7339 }
7340 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7341 MIRBuilder.setInstrAndDebugLoc(MI);
7342 MIRBuilder.buildShuffleVector(Res: MI.getOperand(i: 0).getReg(),
7343 Src1: MI.getOperand(i: 1).getReg(),
7344 Src2: MI.getOperand(i: 2).getReg(), Mask: NewMask);
7345 MI.eraseFromParent();
7346 return Legalized;
7347}
7348
7349void LegalizerHelper::multiplyRegisters(SmallVectorImpl<Register> &DstRegs,
7350 ArrayRef<Register> Src1Regs,
7351 ArrayRef<Register> Src2Regs,
7352 LLT NarrowTy) {
7353 MachineIRBuilder &B = MIRBuilder;
7354 unsigned SrcParts = Src1Regs.size();
7355 unsigned DstParts = DstRegs.size();
7356
7357 unsigned DstIdx = 0; // Low bits of the result.
7358 Register FactorSum =
7359 B.buildMul(Dst: NarrowTy, Src0: Src1Regs[DstIdx], Src1: Src2Regs[DstIdx]).getReg(Idx: 0);
7360 DstRegs[DstIdx] = FactorSum;
7361
7362 Register CarrySumPrevDstIdx;
7363 SmallVector<Register, 4> Factors;
7364
7365 for (DstIdx = 1; DstIdx < DstParts; DstIdx++) {
7366 // Collect high parts of muls from previous DstIdx.
7367 for (unsigned i = DstIdx < SrcParts ? 0 : DstIdx - SrcParts;
7368 i <= std::min(a: DstIdx - 1, b: SrcParts - 1); ++i) {
7369 MachineInstrBuilder Umulh =
7370 B.buildUMulH(Dst: NarrowTy, Src0: Src1Regs[DstIdx - 1 - i], Src1: Src2Regs[i]);
7371 Factors.push_back(Elt: Umulh.getReg(Idx: 0));
7372 }
7373 // Collect low parts of muls for DstIdx. Visit the diagonal starting with
7374 // the low Src1 part, so multiply-add selectors can use it as the first
7375 // accumulated cross product.
7376 unsigned LowStart = DstIdx + 1 < SrcParts ? 0 : DstIdx - SrcParts + 1;
7377 unsigned LowEnd = std::min(a: DstIdx, b: SrcParts - 1);
7378 for (unsigned RevI = LowEnd + 1; RevI != LowStart; --RevI) {
7379 unsigned i = RevI - 1;
7380 MachineInstrBuilder Mul =
7381 B.buildMul(Dst: NarrowTy, Src0: Src1Regs[DstIdx - i], Src1: Src2Regs[i]);
7382 Factors.push_back(Elt: Mul.getReg(Idx: 0));
7383 }
7384 // Add CarrySum from additions calculated for previous DstIdx.
7385 if (DstIdx != 1) {
7386 Factors.push_back(Elt: CarrySumPrevDstIdx);
7387 }
7388
7389 Register CarrySum;
7390 // Add all factors and accumulate all carries into CarrySum.
7391 if (DstIdx != DstParts - 1) {
7392 MachineInstrBuilder Uaddo =
7393 B.buildUAddo(Res: NarrowTy, CarryOut: LLT::integer(SizeInBits: 1), Op0: Factors[0], Op1: Factors[1]);
7394 FactorSum = Uaddo.getReg(Idx: 0);
7395 CarrySum = B.buildZExt(Res: NarrowTy, Op: Uaddo.getReg(Idx: 1)).getReg(Idx: 0);
7396 for (unsigned i = 2; i < Factors.size(); ++i) {
7397 MachineInstrBuilder Uaddo =
7398 B.buildUAddo(Res: NarrowTy, CarryOut: LLT::integer(SizeInBits: 1), Op0: FactorSum, Op1: Factors[i]);
7399 FactorSum = Uaddo.getReg(Idx: 0);
7400 MachineInstrBuilder Carry = B.buildZExt(Res: NarrowTy, Op: Uaddo.getReg(Idx: 1));
7401 CarrySum = B.buildAdd(Dst: NarrowTy, Src0: CarrySum, Src1: Carry).getReg(Idx: 0);
7402 }
7403 } else {
7404 // Since value for the next index is not calculated, neither is CarrySum.
7405 FactorSum = B.buildAdd(Dst: NarrowTy, Src0: Factors[0], Src1: Factors[1]).getReg(Idx: 0);
7406 for (unsigned i = 2; i < Factors.size(); ++i)
7407 FactorSum = B.buildAdd(Dst: NarrowTy, Src0: FactorSum, Src1: Factors[i]).getReg(Idx: 0);
7408 }
7409
7410 CarrySumPrevDstIdx = CarrySum;
7411 DstRegs[DstIdx] = FactorSum;
7412 Factors.clear();
7413 }
7414}
7415
7416LegalizerHelper::LegalizeResult
7417LegalizerHelper::narrowScalarAddSub(MachineInstr &MI, unsigned TypeIdx,
7418 LLT NarrowTy) {
7419 if (TypeIdx != 0)
7420 return UnableToLegalize;
7421
7422 Register DstReg = MI.getOperand(i: 0).getReg();
7423 LLT DstType = MRI.getType(Reg: DstReg);
7424 // FIXME: add support for vector types
7425 if (DstType.isVector())
7426 return UnableToLegalize;
7427
7428 unsigned Opcode = MI.getOpcode();
7429 unsigned OpO, OpE, OpF;
7430 switch (Opcode) {
7431 case TargetOpcode::G_SADDO:
7432 case TargetOpcode::G_SADDE:
7433 case TargetOpcode::G_UADDO:
7434 case TargetOpcode::G_UADDE:
7435 case TargetOpcode::G_ADD:
7436 OpO = TargetOpcode::G_UADDO;
7437 OpE = TargetOpcode::G_UADDE;
7438 OpF = TargetOpcode::G_UADDE;
7439 if (Opcode == TargetOpcode::G_SADDO || Opcode == TargetOpcode::G_SADDE)
7440 OpF = TargetOpcode::G_SADDE;
7441 break;
7442 case TargetOpcode::G_SSUBO:
7443 case TargetOpcode::G_SSUBE:
7444 case TargetOpcode::G_USUBO:
7445 case TargetOpcode::G_USUBE:
7446 case TargetOpcode::G_SUB:
7447 OpO = TargetOpcode::G_USUBO;
7448 OpE = TargetOpcode::G_USUBE;
7449 OpF = TargetOpcode::G_USUBE;
7450 if (Opcode == TargetOpcode::G_SSUBO || Opcode == TargetOpcode::G_SSUBE)
7451 OpF = TargetOpcode::G_SSUBE;
7452 break;
7453 default:
7454 llvm_unreachable("Unexpected add/sub opcode!");
7455 }
7456
7457 // 1 for a plain add/sub, 2 if this is an operation with a carry-out.
7458 unsigned NumDefs = MI.getNumExplicitDefs();
7459 Register Src1 = MI.getOperand(i: NumDefs).getReg();
7460 Register Src2 = MI.getOperand(i: NumDefs + 1).getReg();
7461 Register CarryDst, CarryIn;
7462 if (NumDefs == 2)
7463 CarryDst = MI.getOperand(i: 1).getReg();
7464 if (MI.getNumOperands() == NumDefs + 3)
7465 CarryIn = MI.getOperand(i: NumDefs + 2).getReg();
7466
7467 LLT RegTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7468 LLT LeftoverTy, DummyTy;
7469 SmallVector<Register, 2> Src1Regs, Src2Regs, Src1Left, Src2Left, DstRegs;
7470 extractParts(Reg: Src1, RegTy, MainTy: NarrowTy, LeftoverTy, VRegs&: Src1Regs, LeftoverVRegs&: Src1Left,
7471 MIRBuilder, MRI);
7472 extractParts(Reg: Src2, RegTy, MainTy: NarrowTy, LeftoverTy&: DummyTy, VRegs&: Src2Regs, LeftoverVRegs&: Src2Left, MIRBuilder,
7473 MRI);
7474
7475 int NarrowParts = Src1Regs.size();
7476 Src1Regs.append(RHS: Src1Left);
7477 Src2Regs.append(RHS: Src2Left);
7478 DstRegs.reserve(N: Src1Regs.size());
7479
7480 for (int i = 0, e = Src1Regs.size(); i != e; ++i) {
7481 Register DstReg =
7482 MRI.createGenericVirtualRegister(Ty: MRI.getType(Reg: Src1Regs[i]));
7483 Register CarryOut;
7484 // Forward the final carry-out to the destination register
7485 if (i == e - 1 && CarryDst)
7486 CarryOut = CarryDst;
7487 else
7488 CarryOut = MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: 1));
7489
7490 if (!CarryIn) {
7491 MIRBuilder.buildInstr(Opc: OpO, DstOps: {DstReg, CarryOut},
7492 SrcOps: {Src1Regs[i], Src2Regs[i]});
7493 } else if (i == e - 1) {
7494 MIRBuilder.buildInstr(Opc: OpF, DstOps: {DstReg, CarryOut},
7495 SrcOps: {Src1Regs[i], Src2Regs[i], CarryIn});
7496 } else {
7497 MIRBuilder.buildInstr(Opc: OpE, DstOps: {DstReg, CarryOut},
7498 SrcOps: {Src1Regs[i], Src2Regs[i], CarryIn});
7499 }
7500
7501 DstRegs.push_back(Elt: DstReg);
7502 CarryIn = CarryOut;
7503 }
7504 insertParts(DstReg: MI.getOperand(i: 0).getReg(), ResultTy: RegTy, PartTy: NarrowTy,
7505 PartRegs: ArrayRef(DstRegs).take_front(N: NarrowParts), LeftoverTy,
7506 LeftoverRegs: ArrayRef(DstRegs).drop_front(N: NarrowParts));
7507
7508 MI.eraseFromParent();
7509 return Legalized;
7510}
7511
7512LegalizerHelper::LegalizeResult
7513LegalizerHelper::narrowScalarMul(MachineInstr &MI, LLT NarrowTy) {
7514 auto [DstReg, Src1, Src2] = MI.getFirst3Regs();
7515
7516 LLT Ty = MRI.getType(Reg: DstReg);
7517 if (Ty.isVector())
7518 return UnableToLegalize;
7519
7520 unsigned Size = Ty.getSizeInBits();
7521 unsigned NarrowSize = NarrowTy.getSizeInBits();
7522 if (Size % NarrowSize != 0)
7523 return UnableToLegalize;
7524
7525 unsigned NumParts = Size / NarrowSize;
7526 bool IsMulHigh = MI.getOpcode() == TargetOpcode::G_UMULH;
7527 unsigned DstTmpParts = NumParts * (IsMulHigh ? 2 : 1);
7528
7529 SmallVector<Register, 2> Src1Parts, Src2Parts;
7530 SmallVector<Register, 2> DstTmpRegs(DstTmpParts);
7531 extractParts(Reg: Src1, Ty: NarrowTy, NumParts, VRegs&: Src1Parts, MIRBuilder, MRI);
7532 extractParts(Reg: Src2, Ty: NarrowTy, NumParts, VRegs&: Src2Parts, MIRBuilder, MRI);
7533 multiplyRegisters(DstRegs&: DstTmpRegs, Src1Regs: Src1Parts, Src2Regs: Src2Parts, NarrowTy);
7534
7535 // Take only high half of registers if this is high mul.
7536 ArrayRef<Register> DstRegs(&DstTmpRegs[DstTmpParts - NumParts], NumParts);
7537 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
7538 MI.eraseFromParent();
7539 return Legalized;
7540}
7541
7542LegalizerHelper::LegalizeResult
7543LegalizerHelper::narrowScalarFPTOI(MachineInstr &MI, unsigned TypeIdx,
7544 LLT NarrowTy) {
7545 if (TypeIdx != 0)
7546 return UnableToLegalize;
7547
7548 bool IsSigned = MI.getOpcode() == TargetOpcode::G_FPTOSI;
7549
7550 Register Src = MI.getOperand(i: 1).getReg();
7551 LLT SrcTy = MRI.getType(Reg: Src);
7552
7553 // If all finite floats fit into the narrowed integer type, we can just swap
7554 // out the result type. Only IEEE half qualifies: bfloat is also 16 bits wide
7555 // but has float's exponent range. LLT::float16() is equivalent to
7556 // LLT::scalar(16) on targets without extended LLTs.
7557 if (SrcTy.getScalarType() != LLT::float16() ||
7558 NarrowTy.getScalarSizeInBits() < (IsSigned ? 17u : 16u))
7559 return UnableToLegalize;
7560
7561 Observer.changingInstr(MI);
7562 narrowScalarDst(MI, NarrowTy, OpIdx: 0,
7563 ExtOpcode: IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT);
7564 Observer.changedInstr(MI);
7565 return Legalized;
7566}
7567
7568LegalizerHelper::LegalizeResult
7569LegalizerHelper::narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx,
7570 LLT NarrowTy) {
7571 if (TypeIdx != 1)
7572 return UnableToLegalize;
7573
7574 uint64_t NarrowSize = NarrowTy.getSizeInBits();
7575
7576 int64_t SizeOp1 = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
7577 // FIXME: add support for when SizeOp1 isn't an exact multiple of
7578 // NarrowSize.
7579 if (SizeOp1 % NarrowSize != 0)
7580 return UnableToLegalize;
7581 int NumParts = SizeOp1 / NarrowSize;
7582
7583 SmallVector<Register, 2> SrcRegs, DstRegs;
7584 extractParts(Reg: MI.getOperand(i: 1).getReg(), Ty: NarrowTy, NumParts, VRegs&: SrcRegs,
7585 MIRBuilder, MRI);
7586
7587 Register OpReg = MI.getOperand(i: 0).getReg();
7588 uint64_t OpStart = MI.getOperand(i: 2).getImm();
7589 uint64_t OpSize = MRI.getType(Reg: OpReg).getSizeInBits();
7590 for (int i = 0; i < NumParts; ++i) {
7591 unsigned SrcStart = i * NarrowSize;
7592
7593 if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
7594 // No part of the extract uses this subregister, ignore it.
7595 continue;
7596 } else if (SrcStart == OpStart && NarrowTy == MRI.getType(Reg: OpReg)) {
7597 // The entire subregister is extracted, forward the value.
7598 DstRegs.push_back(Elt: SrcRegs[i]);
7599 continue;
7600 }
7601
7602 // OpSegStart is where this destination segment would start in OpReg if it
7603 // extended infinitely in both directions.
7604 int64_t ExtractOffset;
7605 uint64_t SegSize;
7606 if (OpStart < SrcStart) {
7607 ExtractOffset = 0;
7608 SegSize = std::min(a: NarrowSize, b: OpStart + OpSize - SrcStart);
7609 } else {
7610 ExtractOffset = OpStart - SrcStart;
7611 SegSize = std::min(a: SrcStart + NarrowSize - OpStart, b: OpSize);
7612 }
7613
7614 Register SegReg = SrcRegs[i];
7615 if (ExtractOffset != 0 || SegSize != NarrowSize) {
7616 // A genuine extract is needed.
7617 SegReg = MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: SegSize));
7618 MIRBuilder.buildExtract(Res: SegReg, Src: SrcRegs[i], Index: ExtractOffset);
7619 }
7620
7621 DstRegs.push_back(Elt: SegReg);
7622 }
7623
7624 Register DstReg = MI.getOperand(i: 0).getReg();
7625 if (MRI.getType(Reg: DstReg).isVector())
7626 MIRBuilder.buildBuildVector(Res: DstReg, Ops: DstRegs);
7627 else if (DstRegs.size() > 1)
7628 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
7629 else
7630 MIRBuilder.buildCopy(Res: DstReg, Op: DstRegs[0]);
7631 MI.eraseFromParent();
7632 return Legalized;
7633}
7634
7635LegalizerHelper::LegalizeResult
7636LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx,
7637 LLT NarrowTy) {
7638 // FIXME: Don't know how to handle secondary types yet.
7639 if (TypeIdx != 0)
7640 return UnableToLegalize;
7641
7642 SmallVector<Register, 2> SrcRegs, LeftoverRegs, DstRegs;
7643 LLT RegTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7644 LLT LeftoverTy;
7645 extractParts(Reg: MI.getOperand(i: 1).getReg(), RegTy, MainTy: NarrowTy, LeftoverTy, VRegs&: SrcRegs,
7646 LeftoverVRegs&: LeftoverRegs, MIRBuilder, MRI);
7647
7648 SrcRegs.append(RHS: LeftoverRegs);
7649
7650 uint64_t NarrowSize = NarrowTy.getSizeInBits();
7651 Register OpReg = MI.getOperand(i: 2).getReg();
7652 uint64_t OpStart = MI.getOperand(i: 3).getImm();
7653 uint64_t OpSize = MRI.getType(Reg: OpReg).getSizeInBits();
7654 for (int I = 0, E = SrcRegs.size(); I != E; ++I) {
7655 unsigned DstStart = I * NarrowSize;
7656
7657 if (DstStart == OpStart && NarrowTy == MRI.getType(Reg: OpReg)) {
7658 // The entire subregister is defined by this insert, forward the new
7659 // value.
7660 DstRegs.push_back(Elt: OpReg);
7661 continue;
7662 }
7663
7664 Register SrcReg = SrcRegs[I];
7665 if (MRI.getType(Reg: SrcRegs[I]) == LeftoverTy) {
7666 // The leftover reg is smaller than NarrowTy, so we need to extend it.
7667 SrcReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
7668 MIRBuilder.buildAnyExt(Res: SrcReg, Op: SrcRegs[I]);
7669 }
7670
7671 if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
7672 // No part of the insert affects this subregister, forward the original.
7673 DstRegs.push_back(Elt: SrcReg);
7674 continue;
7675 }
7676
7677 // OpSegStart is where this destination segment would start in OpReg if it
7678 // extended infinitely in both directions.
7679 int64_t ExtractOffset, InsertOffset;
7680 uint64_t SegSize;
7681 if (OpStart < DstStart) {
7682 InsertOffset = 0;
7683 ExtractOffset = DstStart - OpStart;
7684 SegSize = std::min(a: NarrowSize, b: OpStart + OpSize - DstStart);
7685 } else {
7686 InsertOffset = OpStart - DstStart;
7687 ExtractOffset = 0;
7688 SegSize = std::min(a: NarrowSize - InsertOffset, b: OpSize);
7689 }
7690
7691 Register SegReg = OpReg;
7692 if (ExtractOffset != 0 || SegSize != OpSize) {
7693 // A genuine extract is needed.
7694 SegReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: SegSize));
7695 MIRBuilder.buildExtract(Res: SegReg, Src: OpReg, Index: ExtractOffset);
7696 }
7697
7698 Register DstReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
7699 MIRBuilder.buildInsert(Res: DstReg, Src: SrcReg, Op: SegReg, Index: InsertOffset);
7700 DstRegs.push_back(Elt: DstReg);
7701 }
7702
7703 uint64_t WideSize = DstRegs.size() * NarrowSize;
7704 Register DstReg = MI.getOperand(i: 0).getReg();
7705 if (WideSize > RegTy.getSizeInBits()) {
7706 Register MergeReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: WideSize));
7707 MIRBuilder.buildMergeLikeInstr(Res: MergeReg, Ops: DstRegs);
7708 MIRBuilder.buildTrunc(Res: DstReg, Op: MergeReg);
7709 } else
7710 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
7711
7712 MI.eraseFromParent();
7713 return Legalized;
7714}
7715
7716LegalizerHelper::LegalizeResult
7717LegalizerHelper::narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx,
7718 LLT NarrowTy) {
7719 Register DstReg = MI.getOperand(i: 0).getReg();
7720 LLT DstTy = MRI.getType(Reg: DstReg);
7721
7722 assert(MI.getNumOperands() == 3 && TypeIdx == 0);
7723
7724 SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
7725 SmallVector<Register, 4> Src0Regs, Src0LeftoverRegs;
7726 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
7727 LLT LeftoverTy;
7728 if (!extractParts(Reg: MI.getOperand(i: 1).getReg(), RegTy: DstTy, MainTy: NarrowTy, LeftoverTy,
7729 VRegs&: Src0Regs, LeftoverVRegs&: Src0LeftoverRegs, MIRBuilder, MRI))
7730 return UnableToLegalize;
7731
7732 LLT Unused;
7733 if (!extractParts(Reg: MI.getOperand(i: 2).getReg(), RegTy: DstTy, MainTy: NarrowTy, LeftoverTy&: Unused,
7734 VRegs&: Src1Regs, LeftoverVRegs&: Src1LeftoverRegs, MIRBuilder, MRI))
7735 llvm_unreachable("inconsistent extractParts result");
7736
7737 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
7738 auto Inst = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {NarrowTy},
7739 SrcOps: {Src0Regs[I], Src1Regs[I]});
7740 DstRegs.push_back(Elt: Inst.getReg(Idx: 0));
7741 }
7742
7743 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
7744 auto Inst = MIRBuilder.buildInstr(
7745 Opc: MI.getOpcode(),
7746 DstOps: {LeftoverTy}, SrcOps: {Src0LeftoverRegs[I], Src1LeftoverRegs[I]});
7747 DstLeftoverRegs.push_back(Elt: Inst.getReg(Idx: 0));
7748 }
7749
7750 insertParts(DstReg, ResultTy: DstTy, PartTy: NarrowTy, PartRegs: DstRegs,
7751 LeftoverTy, LeftoverRegs: DstLeftoverRegs);
7752
7753 MI.eraseFromParent();
7754 return Legalized;
7755}
7756
7757LegalizerHelper::LegalizeResult
7758LegalizerHelper::narrowScalarExt(MachineInstr &MI, unsigned TypeIdx,
7759 LLT NarrowTy) {
7760 if (TypeIdx != 0)
7761 return UnableToLegalize;
7762
7763 auto [DstReg, SrcReg] = MI.getFirst2Regs();
7764
7765 LLT DstTy = MRI.getType(Reg: DstReg);
7766 if (DstTy.isVector())
7767 return UnableToLegalize;
7768
7769 SmallVector<Register, 8> Parts;
7770 LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg);
7771 LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, VRegs&: Parts, PadStrategy: MI.getOpcode());
7772 buildWidenedRemergeToDst(DstReg, LCMTy, RemergeRegs: Parts);
7773
7774 MI.eraseFromParent();
7775 return Legalized;
7776}
7777
7778LegalizerHelper::LegalizeResult
7779LegalizerHelper::narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx,
7780 LLT NarrowTy) {
7781 if (TypeIdx != 0)
7782 return UnableToLegalize;
7783
7784 Register CondReg = MI.getOperand(i: 1).getReg();
7785 LLT CondTy = MRI.getType(Reg: CondReg);
7786 if (CondTy.isVector()) // TODO: Handle vselect
7787 return UnableToLegalize;
7788
7789 Register DstReg = MI.getOperand(i: 0).getReg();
7790 LLT DstTy = MRI.getType(Reg: DstReg);
7791
7792 SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
7793 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
7794 SmallVector<Register, 4> Src2Regs, Src2LeftoverRegs;
7795 LLT LeftoverTy;
7796 if (!extractParts(Reg: MI.getOperand(i: 2).getReg(), RegTy: DstTy, MainTy: NarrowTy, LeftoverTy,
7797 VRegs&: Src1Regs, LeftoverVRegs&: Src1LeftoverRegs, MIRBuilder, MRI))
7798 return UnableToLegalize;
7799
7800 LLT Unused;
7801 if (!extractParts(Reg: MI.getOperand(i: 3).getReg(), RegTy: DstTy, MainTy: NarrowTy, LeftoverTy&: Unused,
7802 VRegs&: Src2Regs, LeftoverVRegs&: Src2LeftoverRegs, MIRBuilder, MRI))
7803 llvm_unreachable("inconsistent extractParts result");
7804
7805 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
7806 auto Select = MIRBuilder.buildSelect(Res: NarrowTy,
7807 Tst: CondReg, Op0: Src1Regs[I], Op1: Src2Regs[I]);
7808 DstRegs.push_back(Elt: Select.getReg(Idx: 0));
7809 }
7810
7811 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
7812 auto Select = MIRBuilder.buildSelect(
7813 Res: LeftoverTy, Tst: CondReg, Op0: Src1LeftoverRegs[I], Op1: Src2LeftoverRegs[I]);
7814 DstLeftoverRegs.push_back(Elt: Select.getReg(Idx: 0));
7815 }
7816
7817 insertParts(DstReg, ResultTy: DstTy, PartTy: NarrowTy, PartRegs: DstRegs,
7818 LeftoverTy, LeftoverRegs: DstLeftoverRegs);
7819
7820 MI.eraseFromParent();
7821 return Legalized;
7822}
7823
7824LegalizerHelper::LegalizeResult
7825LegalizerHelper::narrowScalarCTLZ(MachineInstr &MI, unsigned TypeIdx,
7826 LLT NarrowTy) {
7827 if (TypeIdx != 1)
7828 return UnableToLegalize;
7829
7830 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7831 unsigned NarrowSize = NarrowTy.getSizeInBits();
7832
7833 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
7834 const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_POISON;
7835
7836 MachineIRBuilder &B = MIRBuilder;
7837 auto UnmergeSrc = B.buildUnmerge(Res: NarrowTy, Op: SrcReg);
7838 // ctlz(Hi:Lo) -> Hi == 0 ? (NarrowSize + ctlz(Lo)) : ctlz(Hi)
7839 auto C_0 = B.buildConstant(Res: NarrowTy, Val: 0);
7840 auto HiIsZero = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: LLT::integer(SizeInBits: 1),
7841 Op0: UnmergeSrc.getReg(Idx: 1), Op1: C_0);
7842 auto LoCTLZ = IsUndef ? B.buildCTLZ_ZERO_POISON(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 0))
7843 : B.buildCTLZ(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 0));
7844 auto C_NarrowSize = B.buildConstant(Res: DstTy, Val: NarrowSize);
7845 auto HiIsZeroCTLZ = B.buildAdd(Dst: DstTy, Src0: LoCTLZ, Src1: C_NarrowSize);
7846 auto HiCTLZ = B.buildCTLZ_ZERO_POISON(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 1));
7847 B.buildSelect(Res: DstReg, Tst: HiIsZero, Op0: HiIsZeroCTLZ, Op1: HiCTLZ);
7848
7849 MI.eraseFromParent();
7850 return Legalized;
7851 }
7852
7853 return UnableToLegalize;
7854}
7855
7856LegalizerHelper::LegalizeResult
7857LegalizerHelper::narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx,
7858 LLT NarrowTy) {
7859 if (TypeIdx != 1)
7860 return UnableToLegalize;
7861
7862 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7863 unsigned NarrowSize = NarrowTy.getSizeInBits();
7864
7865 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
7866 const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTTZ_ZERO_POISON;
7867
7868 MachineIRBuilder &B = MIRBuilder;
7869 auto UnmergeSrc = B.buildUnmerge(Res: NarrowTy, Op: SrcReg);
7870 // cttz(Hi:Lo) -> Lo == 0 ? (cttz(Hi) + NarrowSize) : cttz(Lo)
7871 auto C_0 = B.buildConstant(Res: NarrowTy, Val: 0);
7872 auto LoIsZero = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: LLT::integer(SizeInBits: 1),
7873 Op0: UnmergeSrc.getReg(Idx: 0), Op1: C_0);
7874 auto HiCTTZ = IsUndef ? B.buildCTTZ_ZERO_POISON(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 1))
7875 : B.buildCTTZ(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 1));
7876 auto C_NarrowSize = B.buildConstant(Res: DstTy, Val: NarrowSize);
7877 auto LoIsZeroCTTZ = B.buildAdd(Dst: DstTy, Src0: HiCTTZ, Src1: C_NarrowSize);
7878 auto LoCTTZ = B.buildCTTZ_ZERO_POISON(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 0));
7879 B.buildSelect(Res: DstReg, Tst: LoIsZero, Op0: LoIsZeroCTTZ, Op1: LoCTTZ);
7880
7881 MI.eraseFromParent();
7882 return Legalized;
7883 }
7884
7885 return UnableToLegalize;
7886}
7887
7888LegalizerHelper::LegalizeResult
7889LegalizerHelper::narrowScalarCTLS(MachineInstr &MI, unsigned TypeIdx,
7890 LLT NarrowTy) {
7891 if (TypeIdx != 1)
7892 return UnableToLegalize;
7893
7894 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7895 unsigned NarrowSize = NarrowTy.getSizeInBits();
7896
7897 if (!SrcTy.isScalar() || SrcTy.getSizeInBits() != 2 * NarrowSize)
7898 return UnableToLegalize;
7899
7900 MachineIRBuilder &B = MIRBuilder;
7901
7902 auto UnmergeSrc = B.buildUnmerge(Res: NarrowTy, Op: SrcReg);
7903 Register Lo = UnmergeSrc.getReg(Idx: 0);
7904 Register Hi = UnmergeSrc.getReg(Idx: 1);
7905
7906 auto ShAmt = B.buildConstant(Res: NarrowTy, Val: NarrowSize - 1);
7907 auto Sign = B.buildAShr(Dst: NarrowTy, Src0: Hi, Src1: ShAmt);
7908
7909 auto HiIsSign = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: LLT::scalar(SizeInBits: 1), Op0: Hi, Op1: Sign);
7910
7911 // Invert Lo if Hi is negative. Then count the leading zeros. If there are no
7912 // leading zeros, then the MSB of Lo is different than the MSB of Hi.
7913 // Otherwise the leading zeros represent additional sign bits of the original
7914 // value.
7915 auto LoInv = B.buildXor(Dst: DstTy, Src0: Lo, Src1: Sign);
7916 auto LoCTLZ = B.buildCTLZ(Dst: DstTy, Src0: LoInv);
7917
7918 // Add NarrowSize-1 to LoCTLZ. This is the full CTLS if Hi is all sign bits.
7919 auto C_NarrowSizeM1 = B.buildConstant(Res: DstTy, Val: NarrowSize - 1);
7920 auto HiIsSignCTLS = B.buildAdd(Dst: DstTy, Src0: LoCTLZ, Src1: C_NarrowSizeM1);
7921
7922 auto HiCTLS = B.buildCTLS(Dst: DstTy, Src0: Hi);
7923
7924 B.buildSelect(Res: DstReg, Tst: HiIsSign, Op0: HiIsSignCTLS, Op1: HiCTLS);
7925
7926 MI.eraseFromParent();
7927 return Legalized;
7928}
7929
7930LegalizerHelper::LegalizeResult
7931LegalizerHelper::narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx,
7932 LLT NarrowTy) {
7933 if (TypeIdx != 1)
7934 return UnableToLegalize;
7935
7936 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7937 unsigned NarrowSize = NarrowTy.getSizeInBits();
7938
7939 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
7940 auto UnmergeSrc = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: MI.getOperand(i: 1));
7941
7942 auto LoCTPOP = MIRBuilder.buildCTPOP(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 0));
7943 auto HiCTPOP = MIRBuilder.buildCTPOP(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 1));
7944 MIRBuilder.buildAdd(Dst: DstReg, Src0: HiCTPOP, Src1: LoCTPOP);
7945
7946 MI.eraseFromParent();
7947 return Legalized;
7948 }
7949
7950 return UnableToLegalize;
7951}
7952
7953LegalizerHelper::LegalizeResult
7954LegalizerHelper::narrowScalarFLDEXP(MachineInstr &MI, unsigned TypeIdx,
7955 LLT NarrowTy) {
7956 if (TypeIdx != 1)
7957 return UnableToLegalize;
7958
7959 MachineIRBuilder &B = MIRBuilder;
7960 Register ExpReg = MI.getOperand(i: 2).getReg();
7961 LLT ExpTy = MRI.getType(Reg: ExpReg);
7962
7963 unsigned ClampSize = NarrowTy.getScalarSizeInBits();
7964
7965 // Clamp the exponent to the range of the target type.
7966 auto MinExp = B.buildConstant(Res: ExpTy, Val: minIntN(N: ClampSize));
7967 auto ClampMin = B.buildSMax(Dst: ExpTy, Src0: ExpReg, Src1: MinExp);
7968 auto MaxExp = B.buildConstant(Res: ExpTy, Val: maxIntN(N: ClampSize));
7969 auto Clamp = B.buildSMin(Dst: ExpTy, Src0: ClampMin, Src1: MaxExp);
7970
7971 auto Trunc = B.buildTrunc(Res: NarrowTy, Op: Clamp);
7972 Observer.changingInstr(MI);
7973 MI.getOperand(i: 2).setReg(Trunc.getReg(Idx: 0));
7974 Observer.changedInstr(MI);
7975 return Legalized;
7976}
7977
7978LegalizerHelper::LegalizeResult
7979LegalizerHelper::lowerBitCount(MachineInstr &MI) {
7980 unsigned Opc = MI.getOpcode();
7981 const auto &TII = MIRBuilder.getTII();
7982 auto isSupported = [this](const LegalityQuery &Q) {
7983 auto QAction = LI.getAction(Query: Q).Action;
7984 return QAction == Legal || QAction == Libcall || QAction == Custom;
7985 };
7986 switch (Opc) {
7987 default:
7988 return UnableToLegalize;
7989 case TargetOpcode::G_CTLZ_ZERO_POISON: {
7990 // This trivially expands to CTLZ.
7991 Observer.changingInstr(MI);
7992 MI.setDesc(TII.get(Opcode: TargetOpcode::G_CTLZ));
7993 Observer.changedInstr(MI);
7994 return Legalized;
7995 }
7996 case TargetOpcode::G_CTLZ: {
7997 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7998 unsigned Len = SrcTy.getScalarSizeInBits();
7999
8000 if (isSupported({TargetOpcode::G_CTLZ_ZERO_POISON, {DstTy, SrcTy}})) {
8001 // If CTLZ_ZERO_POISON is supported, emit that and a select for zero.
8002 auto CtlzZU = MIRBuilder.buildCTLZ_ZERO_POISON(Dst: DstTy, Src0: SrcReg);
8003 auto ZeroSrc = MIRBuilder.buildConstant(Res: SrcTy, Val: 0);
8004 auto ICmp = MIRBuilder.buildICmp(
8005 Pred: CmpInst::ICMP_EQ, Res: SrcTy.changeElementSize(NewEltSize: 1), Op0: SrcReg, Op1: ZeroSrc);
8006 auto LenConst = MIRBuilder.buildConstant(Res: DstTy, Val: Len);
8007 MIRBuilder.buildSelect(Res: DstReg, Tst: ICmp, Op0: LenConst, Op1: CtlzZU);
8008 MI.eraseFromParent();
8009 return Legalized;
8010 }
8011 // for now, we do this:
8012 // NewLen = NextPowerOf2(Len);
8013 // x = x | (x >> 1);
8014 // x = x | (x >> 2);
8015 // ...
8016 // x = x | (x >>16);
8017 // x = x | (x >>32); // for 64-bit input
8018 // Upto NewLen/2
8019 // return Len - popcount(x);
8020 //
8021 // Ref: "Hacker's Delight" by Henry Warren
8022 Register Op = SrcReg;
8023 unsigned NewLen = PowerOf2Ceil(A: Len);
8024 for (unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) {
8025 auto MIBShiftAmt = MIRBuilder.buildConstant(Res: SrcTy, Val: 1ULL << i);
8026 auto MIBOp = MIRBuilder.buildOr(
8027 Dst: SrcTy, Src0: Op, Src1: MIRBuilder.buildLShr(Dst: SrcTy, Src0: Op, Src1: MIBShiftAmt));
8028 Op = MIBOp.getReg(Idx: 0);
8029 }
8030 auto MIBPop = MIRBuilder.buildCTPOP(Dst: DstTy, Src0: Op);
8031 MIRBuilder.buildSub(Dst: MI.getOperand(i: 0), Src0: MIRBuilder.buildConstant(Res: DstTy, Val: Len),
8032 Src1: MIBPop);
8033 MI.eraseFromParent();
8034 return Legalized;
8035 }
8036 case TargetOpcode::G_CTTZ_ZERO_POISON: {
8037 // This trivially expands to CTTZ.
8038 Observer.changingInstr(MI);
8039 MI.setDesc(TII.get(Opcode: TargetOpcode::G_CTTZ));
8040 Observer.changedInstr(MI);
8041 return Legalized;
8042 }
8043 case TargetOpcode::G_CTTZ: {
8044 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
8045
8046 unsigned Len = SrcTy.getScalarSizeInBits();
8047 if (isSupported({TargetOpcode::G_CTTZ_ZERO_POISON, {DstTy, SrcTy}})) {
8048 // If CTTZ_ZERO_POISON is legal or custom, emit that and a select with
8049 // zero.
8050 auto CttzZU = MIRBuilder.buildCTTZ_ZERO_POISON(Dst: DstTy, Src0: SrcReg);
8051 auto Zero = MIRBuilder.buildConstant(Res: SrcTy, Val: 0);
8052 auto ICmp = MIRBuilder.buildICmp(
8053 Pred: CmpInst::ICMP_EQ, Res: DstTy.changeElementSize(NewEltSize: 1), Op0: SrcReg, Op1: Zero);
8054 auto LenConst = MIRBuilder.buildConstant(Res: DstTy, Val: Len);
8055 MIRBuilder.buildSelect(Res: DstReg, Tst: ICmp, Op0: LenConst, Op1: CttzZU);
8056 MI.eraseFromParent();
8057 return Legalized;
8058 }
8059 // for now, we use: { return popcount(~x & (x - 1)); }
8060 // unless the target has ctlz but not ctpop, in which case we use:
8061 // { return 32 - nlz(~x & (x-1)); }
8062 // Ref: "Hacker's Delight" by Henry Warren
8063 auto MIBCstNeg1 = MIRBuilder.buildConstant(Res: SrcTy, Val: -1);
8064 auto MIBNot = MIRBuilder.buildXor(Dst: SrcTy, Src0: SrcReg, Src1: MIBCstNeg1);
8065 auto MIBTmp = MIRBuilder.buildAnd(
8066 Dst: SrcTy, Src0: MIBNot, Src1: MIRBuilder.buildAdd(Dst: SrcTy, Src0: SrcReg, Src1: MIBCstNeg1));
8067 if (!isSupported({TargetOpcode::G_CTPOP, {SrcTy, SrcTy}}) &&
8068 isSupported({TargetOpcode::G_CTLZ, {SrcTy, SrcTy}})) {
8069 auto MIBCstLen = MIRBuilder.buildConstant(Res: SrcTy, Val: Len);
8070 MIRBuilder.buildSub(Dst: MI.getOperand(i: 0), Src0: MIBCstLen,
8071 Src1: MIRBuilder.buildCTLZ(Dst: SrcTy, Src0: MIBTmp));
8072 MI.eraseFromParent();
8073 return Legalized;
8074 }
8075 Observer.changingInstr(MI);
8076 MI.setDesc(TII.get(Opcode: TargetOpcode::G_CTPOP));
8077 MI.getOperand(i: 1).setReg(MIBTmp.getReg(Idx: 0));
8078 Observer.changedInstr(MI);
8079 return Legalized;
8080 }
8081 case TargetOpcode::G_CTPOP: {
8082 Register SrcReg = MI.getOperand(i: 1).getReg();
8083 LLT Ty = MRI.getType(Reg: SrcReg);
8084 unsigned Size = Ty.getScalarSizeInBits();
8085 MachineIRBuilder &B = MIRBuilder;
8086
8087 // Bail out on irregular type lengths.
8088 if (Size > 128 || Size % 8 != 0)
8089 return UnableToLegalize;
8090
8091 // Count set bits in blocks of 2 bits. Default approach would be
8092 // B2Count = { val & 0x55555555 } + { (val >> 1) & 0x55555555 }
8093 // We use following formula instead:
8094 // B2Count = val - { (val >> 1) & 0x55555555 }
8095 // since it gives same result in blocks of 2 with one instruction less.
8096 auto C_1 = B.buildConstant(Res: Ty, Val: 1);
8097 auto B2Set1LoTo1Hi = B.buildLShr(Dst: Ty, Src0: SrcReg, Src1: C_1);
8098 APInt B2Mask1HiTo0 = APInt::getSplat(NewLen: Size, V: APInt(8, 0x55));
8099 auto C_B2Mask1HiTo0 = B.buildConstant(Res: Ty, Val: B2Mask1HiTo0);
8100 auto B2Count1Hi = B.buildAnd(Dst: Ty, Src0: B2Set1LoTo1Hi, Src1: C_B2Mask1HiTo0);
8101 auto B2Count = B.buildSub(Dst: Ty, Src0: SrcReg, Src1: B2Count1Hi);
8102
8103 // In order to get count in blocks of 4 add values from adjacent block of 2.
8104 // B4Count = { B2Count & 0x33333333 } + { (B2Count >> 2) & 0x33333333 }
8105 auto C_2 = B.buildConstant(Res: Ty, Val: 2);
8106 auto B4Set2LoTo2Hi = B.buildLShr(Dst: Ty, Src0: B2Count, Src1: C_2);
8107 APInt B4Mask2HiTo0 = APInt::getSplat(NewLen: Size, V: APInt(8, 0x33));
8108 auto C_B4Mask2HiTo0 = B.buildConstant(Res: Ty, Val: B4Mask2HiTo0);
8109 auto B4HiB2Count = B.buildAnd(Dst: Ty, Src0: B4Set2LoTo2Hi, Src1: C_B4Mask2HiTo0);
8110 auto B4LoB2Count = B.buildAnd(Dst: Ty, Src0: B2Count, Src1: C_B4Mask2HiTo0);
8111 auto B4Count = B.buildAdd(Dst: Ty, Src0: B4HiB2Count, Src1: B4LoB2Count);
8112
8113 // For count in blocks of 8 bits we don't have to mask high 4 bits before
8114 // addition since count value sits in range {0,...,8} and 4 bits are enough
8115 // to hold such binary values. After addition high 4 bits still hold count
8116 // of set bits in high 4 bit block, set them to zero and get 8 bit result.
8117 // B8Count = { B4Count + (B4Count >> 4) } & 0x0F0F0F0F
8118 auto C_4 = B.buildConstant(Res: Ty, Val: 4);
8119 auto B8HiB4Count = B.buildLShr(Dst: Ty, Src0: B4Count, Src1: C_4);
8120 auto B8CountDirty4Hi = B.buildAdd(Dst: Ty, Src0: B8HiB4Count, Src1: B4Count);
8121 APInt B8Mask4HiTo0 = APInt::getSplat(NewLen: Size, V: APInt(8, 0x0F));
8122 auto C_B8Mask4HiTo0 = B.buildConstant(Res: Ty, Val: B8Mask4HiTo0);
8123 auto B8Count = B.buildAnd(Dst: Ty, Src0: B8CountDirty4Hi, Src1: C_B8Mask4HiTo0);
8124
8125 assert(Size <= 128 && "Scalar size is too large for CTPOP lower algorithm");
8126
8127 // Avoid the multiply when shift-add is cheaper.
8128 if (Size == 16 && !Ty.isVector()) {
8129 // v = (v + (v >> 8)) & 0xFF;
8130 auto C_8 = B.buildConstant(Res: Ty, Val: 8);
8131 auto HighSum = B.buildLShr(Dst: Ty, Src0: B8Count, Src1: C_8);
8132 auto Res = B.buildAdd(Dst: Ty, Src0: B8Count, Src1: HighSum);
8133 B.buildAnd(Dst: MI.getOperand(i: 0).getReg(), Src0: Res, Src1: B.buildConstant(Res: Ty, Val: 0xFF));
8134 MI.eraseFromParent();
8135 return Legalized;
8136 }
8137
8138 // 8 bits can hold CTPOP result of 128 bit int or smaller. Mul with this
8139 // bitmask will set 8 msb in ResTmp to sum of all B8Counts in 8 bit blocks.
8140 auto MulMask = B.buildConstant(Res: Ty, Val: APInt::getSplat(NewLen: Size, V: APInt(8, 0x01)));
8141
8142 // Shift count result from 8 high bits to low bits.
8143 auto C_SizeM8 = B.buildConstant(Res: Ty, Val: Size - 8);
8144
8145 auto IsMulSupported = [this](const LLT Ty) {
8146 auto Action = LI.getAction(Query: {TargetOpcode::G_MUL, {Ty}}).Action;
8147 return Action == Legal || Action == WidenScalar || Action == Custom;
8148 };
8149 if (IsMulSupported(Ty)) {
8150 auto ResTmp = B.buildMul(Dst: Ty, Src0: B8Count, Src1: MulMask);
8151 B.buildLShr(Dst: MI.getOperand(i: 0).getReg(), Src0: ResTmp, Src1: C_SizeM8);
8152 } else {
8153 auto ResTmp = B8Count;
8154 for (unsigned Shift = 8; Shift < Size; Shift *= 2) {
8155 auto ShiftC = B.buildConstant(Res: Ty, Val: Shift);
8156 auto Shl = B.buildShl(Dst: Ty, Src0: ResTmp, Src1: ShiftC);
8157 ResTmp = B.buildAdd(Dst: Ty, Src0: ResTmp, Src1: Shl);
8158 }
8159 B.buildLShr(Dst: MI.getOperand(i: 0).getReg(), Src0: ResTmp, Src1: C_SizeM8);
8160 }
8161 MI.eraseFromParent();
8162 return Legalized;
8163 }
8164 case TargetOpcode::G_CTLS: {
8165 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
8166
8167 // ctls(x) -> ctlz(x ^ (x >> (N - 1))) - 1
8168 auto SignIdxC =
8169 MIRBuilder.buildConstant(Res: SrcTy, Val: SrcTy.getScalarSizeInBits() - 1);
8170 auto OneC = MIRBuilder.buildConstant(Res: DstTy, Val: 1);
8171
8172 auto Shr = MIRBuilder.buildAShr(Dst: SrcTy, Src0: SrcReg, Src1: SignIdxC);
8173
8174 auto Xor = MIRBuilder.buildXor(Dst: SrcTy, Src0: SrcReg, Src1: Shr);
8175 auto Ctlz = MIRBuilder.buildCTLZ(Dst: DstTy, Src0: Xor);
8176
8177 MIRBuilder.buildSub(Dst: DstReg, Src0: Ctlz, Src1: OneC);
8178 MI.eraseFromParent();
8179 return Legalized;
8180 }
8181 }
8182}
8183
8184// Check that (every element of) Reg is undef or not an exact multiple of BW.
8185static bool isNonZeroModBitWidthOrUndef(const MachineRegisterInfo &MRI,
8186 Register Reg, unsigned BW) {
8187 return matchUnaryPredicate(
8188 MRI, Reg,
8189 Match: [=](const Constant *C) {
8190 // Null constant here means an undef.
8191 const ConstantInt *CI = dyn_cast_or_null<ConstantInt>(Val: C);
8192 return !CI || CI->getValue().urem(RHS: BW) != 0;
8193 },
8194 /*AllowUndefs*/ true);
8195}
8196
8197LegalizerHelper::LegalizeResult
8198LegalizerHelper::lowerFunnelShiftWithInverse(MachineInstr &MI) {
8199 auto [Dst, X, Y, Z] = MI.getFirst4Regs();
8200 LLT Ty = MRI.getType(Reg: Dst);
8201 LLT ShTy = MRI.getType(Reg: Z);
8202
8203 unsigned BW = Ty.getScalarSizeInBits();
8204
8205 if (!isPowerOf2_32(Value: BW))
8206 return UnableToLegalize;
8207
8208 const bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL;
8209 unsigned RevOpcode = IsFSHL ? TargetOpcode::G_FSHR : TargetOpcode::G_FSHL;
8210
8211 if (isNonZeroModBitWidthOrUndef(MRI, Reg: Z, BW)) {
8212 // fshl X, Y, Z -> fshr X, Y, -Z
8213 // fshr X, Y, Z -> fshl X, Y, -Z
8214 auto Zero = MIRBuilder.buildConstant(Res: ShTy, Val: 0);
8215 Z = MIRBuilder.buildSub(Dst: Ty, Src0: Zero, Src1: Z).getReg(Idx: 0);
8216 } else {
8217 // fshl X, Y, Z -> fshr (srl X, 1), (fshr X, Y, 1), ~Z
8218 // fshr X, Y, Z -> fshl (fshl X, Y, 1), (shl Y, 1), ~Z
8219 auto One = MIRBuilder.buildConstant(Res: ShTy, Val: 1);
8220 if (IsFSHL) {
8221 Y = MIRBuilder.buildInstr(Opc: RevOpcode, DstOps: {Ty}, SrcOps: {X, Y, One}).getReg(Idx: 0);
8222 X = MIRBuilder.buildLShr(Dst: Ty, Src0: X, Src1: One).getReg(Idx: 0);
8223 } else {
8224 X = MIRBuilder.buildInstr(Opc: RevOpcode, DstOps: {Ty}, SrcOps: {X, Y, One}).getReg(Idx: 0);
8225 Y = MIRBuilder.buildShl(Dst: Ty, Src0: Y, Src1: One).getReg(Idx: 0);
8226 }
8227
8228 Z = MIRBuilder.buildNot(Dst: ShTy, Src0: Z).getReg(Idx: 0);
8229 }
8230
8231 MIRBuilder.buildInstr(Opc: RevOpcode, DstOps: {Dst}, SrcOps: {X, Y, Z});
8232 MI.eraseFromParent();
8233 return Legalized;
8234}
8235
8236LegalizerHelper::LegalizeResult
8237LegalizerHelper::lowerFunnelShiftAsShifts(MachineInstr &MI) {
8238 auto [Dst, X, Y, Z] = MI.getFirst4Regs();
8239 LLT Ty = MRI.getType(Reg: Dst);
8240 LLT ShTy = MRI.getType(Reg: Z);
8241
8242 const unsigned BW = Ty.getScalarSizeInBits();
8243 const bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL;
8244
8245 Register ShX, ShY;
8246 Register ShAmt, InvShAmt;
8247
8248 // FIXME: Emit optimized urem by constant instead of letting it expand later.
8249 if (isNonZeroModBitWidthOrUndef(MRI, Reg: Z, BW)) {
8250 // fshl: X << C | Y >> (BW - C)
8251 // fshr: X << (BW - C) | Y >> C
8252 // where C = Z % BW is not zero
8253 auto BitWidthC = MIRBuilder.buildConstant(Res: ShTy, Val: BW);
8254 ShAmt = MIRBuilder.buildURem(Dst: ShTy, Src0: Z, Src1: BitWidthC).getReg(Idx: 0);
8255 InvShAmt = MIRBuilder.buildSub(Dst: ShTy, Src0: BitWidthC, Src1: ShAmt).getReg(Idx: 0);
8256 ShX = MIRBuilder.buildShl(Dst: Ty, Src0: X, Src1: IsFSHL ? ShAmt : InvShAmt).getReg(Idx: 0);
8257 ShY = MIRBuilder.buildLShr(Dst: Ty, Src0: Y, Src1: IsFSHL ? InvShAmt : ShAmt).getReg(Idx: 0);
8258 } else {
8259 // fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))
8260 // fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)
8261 auto Mask = MIRBuilder.buildConstant(Res: ShTy, Val: BW - 1);
8262 if (isPowerOf2_32(Value: BW)) {
8263 // Z % BW -> Z & (BW - 1)
8264 ShAmt = MIRBuilder.buildAnd(Dst: ShTy, Src0: Z, Src1: Mask).getReg(Idx: 0);
8265 // (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
8266 auto NotZ = MIRBuilder.buildNot(Dst: ShTy, Src0: Z);
8267 InvShAmt = MIRBuilder.buildAnd(Dst: ShTy, Src0: NotZ, Src1: Mask).getReg(Idx: 0);
8268 } else {
8269 auto BitWidthC = MIRBuilder.buildConstant(Res: ShTy, Val: BW);
8270 ShAmt = MIRBuilder.buildURem(Dst: ShTy, Src0: Z, Src1: BitWidthC).getReg(Idx: 0);
8271 InvShAmt = MIRBuilder.buildSub(Dst: ShTy, Src0: Mask, Src1: ShAmt).getReg(Idx: 0);
8272 }
8273
8274 auto One = MIRBuilder.buildConstant(Res: ShTy, Val: 1);
8275 if (IsFSHL) {
8276 ShX = MIRBuilder.buildShl(Dst: Ty, Src0: X, Src1: ShAmt).getReg(Idx: 0);
8277 auto ShY1 = MIRBuilder.buildLShr(Dst: Ty, Src0: Y, Src1: One);
8278 ShY = MIRBuilder.buildLShr(Dst: Ty, Src0: ShY1, Src1: InvShAmt).getReg(Idx: 0);
8279 } else {
8280 auto ShX1 = MIRBuilder.buildShl(Dst: Ty, Src0: X, Src1: One);
8281 ShX = MIRBuilder.buildShl(Dst: Ty, Src0: ShX1, Src1: InvShAmt).getReg(Idx: 0);
8282 ShY = MIRBuilder.buildLShr(Dst: Ty, Src0: Y, Src1: ShAmt).getReg(Idx: 0);
8283 }
8284 }
8285
8286 MIRBuilder.buildOr(Dst, Src0: ShX, Src1: ShY, Flags: MachineInstr::Disjoint);
8287 MI.eraseFromParent();
8288 return Legalized;
8289}
8290
8291LegalizerHelper::LegalizeResult
8292LegalizerHelper::lowerFunnelShift(MachineInstr &MI) {
8293 // These operations approximately do the following (while avoiding undefined
8294 // shifts by BW):
8295 // G_FSHL: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
8296 // G_FSHR: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
8297 Register Dst = MI.getOperand(i: 0).getReg();
8298 LLT Ty = MRI.getType(Reg: Dst);
8299 LLT ShTy = MRI.getType(Reg: MI.getOperand(i: 3).getReg());
8300
8301 bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL;
8302 unsigned RevOpcode = IsFSHL ? TargetOpcode::G_FSHR : TargetOpcode::G_FSHL;
8303
8304 // TODO: Use smarter heuristic that accounts for vector legalization.
8305 if (LI.getAction(Query: {RevOpcode, {Ty, ShTy}}).Action == Lower)
8306 return lowerFunnelShiftAsShifts(MI);
8307
8308 // This only works for powers of 2, fallback to shifts if it fails.
8309 LegalizerHelper::LegalizeResult Result = lowerFunnelShiftWithInverse(MI);
8310 if (Result == UnableToLegalize)
8311 return lowerFunnelShiftAsShifts(MI);
8312 return Result;
8313}
8314
8315LegalizerHelper::LegalizeResult LegalizerHelper::lowerEXT(MachineInstr &MI) {
8316 auto [Dst, Src] = MI.getFirst2Regs();
8317 LLT DstTy = MRI.getType(Reg: Dst);
8318 LLT SrcTy = MRI.getType(Reg: Src);
8319
8320 uint32_t DstTySize = DstTy.getSizeInBits();
8321 uint32_t DstTyScalarSize = DstTy.getScalarSizeInBits();
8322 uint32_t SrcTyScalarSize = SrcTy.getScalarSizeInBits();
8323
8324 if (!isPowerOf2_32(Value: DstTySize) || !isPowerOf2_32(Value: DstTyScalarSize) ||
8325 !isPowerOf2_32(Value: SrcTyScalarSize))
8326 return UnableToLegalize;
8327
8328 // The step between extend is too large, split it by creating an intermediate
8329 // extend instruction
8330 if (SrcTyScalarSize * 2 < DstTyScalarSize) {
8331 LLT MidTy = SrcTy.changeElementSize(NewEltSize: SrcTyScalarSize * 2);
8332 // If the destination type is illegal, split it into multiple statements
8333 // zext x -> zext(merge(zext(unmerge), zext(unmerge)))
8334 auto NewExt = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {MidTy}, SrcOps: {Src});
8335 // Unmerge the vector
8336 LLT EltTy = MidTy.changeElementCount(
8337 EC: MidTy.getElementCount().divideCoefficientBy(RHS: 2));
8338 auto UnmergeSrc = MIRBuilder.buildUnmerge(Res: EltTy, Op: NewExt);
8339
8340 // ZExt the vectors
8341 LLT ZExtResTy = DstTy.changeElementCount(
8342 EC: DstTy.getElementCount().divideCoefficientBy(RHS: 2));
8343 auto ZExtRes1 = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {ZExtResTy},
8344 SrcOps: {UnmergeSrc.getReg(Idx: 0)});
8345 auto ZExtRes2 = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {ZExtResTy},
8346 SrcOps: {UnmergeSrc.getReg(Idx: 1)});
8347
8348 // Merge the ending vectors
8349 MIRBuilder.buildMergeLikeInstr(Res: Dst, Ops: {ZExtRes1, ZExtRes2});
8350
8351 MI.eraseFromParent();
8352 return Legalized;
8353 }
8354 return UnableToLegalize;
8355}
8356
8357LegalizerHelper::LegalizeResult LegalizerHelper::lowerTRUNC(MachineInstr &MI) {
8358 // MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
8359 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
8360 // Similar to how operand splitting is done in SelectiondDAG, we can handle
8361 // %res(v8s8) = G_TRUNC %in(v8s32) by generating:
8362 // %inlo(<4x s32>), %inhi(<4 x s32>) = G_UNMERGE %in(<8 x s32>)
8363 // %lo16(<4 x s16>) = G_TRUNC %inlo
8364 // %hi16(<4 x s16>) = G_TRUNC %inhi
8365 // %in16(<8 x s16>) = G_CONCAT_VECTORS %lo16, %hi16
8366 // %res(<8 x s8>) = G_TRUNC %in16
8367
8368 assert(MI.getOpcode() == TargetOpcode::G_TRUNC);
8369
8370 Register DstReg = MI.getOperand(i: 0).getReg();
8371 Register SrcReg = MI.getOperand(i: 1).getReg();
8372 LLT DstTy = MRI.getType(Reg: DstReg);
8373 LLT SrcTy = MRI.getType(Reg: SrcReg);
8374
8375 if (DstTy.isVector() && isPowerOf2_32(Value: DstTy.getNumElements()) &&
8376 isPowerOf2_32(Value: DstTy.getScalarSizeInBits()) &&
8377 isPowerOf2_32(Value: SrcTy.getNumElements()) &&
8378 isPowerOf2_32(Value: SrcTy.getScalarSizeInBits())) {
8379 // Split input type.
8380 LLT SplitSrcTy = SrcTy.changeElementCount(
8381 EC: SrcTy.getElementCount().divideCoefficientBy(RHS: 2));
8382
8383 // First, split the source into two smaller vectors.
8384 SmallVector<Register, 2> SplitSrcs;
8385 extractParts(Reg: SrcReg, Ty: SplitSrcTy, NumParts: 2, VRegs&: SplitSrcs, MIRBuilder, MRI);
8386
8387 // Truncate the splits into intermediate narrower elements.
8388 LLT InterTy;
8389 if (DstTy.getScalarSizeInBits() * 2 < SrcTy.getScalarSizeInBits())
8390 InterTy = SplitSrcTy.changeElementSize(NewEltSize: DstTy.getScalarSizeInBits() * 2);
8391 else
8392 InterTy = SplitSrcTy.changeElementSize(NewEltSize: DstTy.getScalarSizeInBits());
8393 for (Register &Src : SplitSrcs)
8394 Src = MIRBuilder.buildTrunc(Res: InterTy, Op: Src).getReg(Idx: 0);
8395
8396 // Combine the new truncates into one vector
8397 auto Merge = MIRBuilder.buildMergeLikeInstr(
8398 Res: DstTy.changeElementSize(NewEltSize: InterTy.getScalarSizeInBits()), Ops: SplitSrcs);
8399
8400 // Truncate the new vector to the final result type
8401 if (DstTy.getScalarSizeInBits() * 2 < SrcTy.getScalarSizeInBits())
8402 MIRBuilder.buildTrunc(Res: MI.getOperand(i: 0).getReg(), Op: Merge.getReg(Idx: 0));
8403 else
8404 MIRBuilder.buildCopy(Res: MI.getOperand(i: 0).getReg(), Op: Merge.getReg(Idx: 0));
8405
8406 MI.eraseFromParent();
8407
8408 return Legalized;
8409 }
8410 return UnableToLegalize;
8411}
8412
8413LegalizerHelper::LegalizeResult
8414LegalizerHelper::lowerRotateWithReverseRotate(MachineInstr &MI) {
8415 auto [Dst, DstTy, Src, SrcTy, Amt, AmtTy] = MI.getFirst3RegLLTs();
8416 auto Zero = MIRBuilder.buildConstant(Res: AmtTy, Val: 0);
8417 bool IsLeft = MI.getOpcode() == TargetOpcode::G_ROTL;
8418 unsigned RevRot = IsLeft ? TargetOpcode::G_ROTR : TargetOpcode::G_ROTL;
8419 auto Neg = MIRBuilder.buildSub(Dst: AmtTy, Src0: Zero, Src1: Amt);
8420 MIRBuilder.buildInstr(Opc: RevRot, DstOps: {Dst}, SrcOps: {Src, Neg});
8421 MI.eraseFromParent();
8422 return Legalized;
8423}
8424
8425LegalizerHelper::LegalizeResult LegalizerHelper::lowerRotate(MachineInstr &MI) {
8426 auto [Dst, DstTy, Src, SrcTy, Amt, AmtTy] = MI.getFirst3RegLLTs();
8427
8428 unsigned EltSizeInBits = DstTy.getScalarSizeInBits();
8429 bool IsLeft = MI.getOpcode() == TargetOpcode::G_ROTL;
8430
8431 MIRBuilder.setInstrAndDebugLoc(MI);
8432
8433 // If a rotate in the other direction is supported, use it.
8434 unsigned RevRot = IsLeft ? TargetOpcode::G_ROTR : TargetOpcode::G_ROTL;
8435 if (LI.isLegalOrCustom(Query: {RevRot, {DstTy, SrcTy}}) &&
8436 isPowerOf2_32(Value: EltSizeInBits))
8437 return lowerRotateWithReverseRotate(MI);
8438
8439 // If a funnel shift is supported, use it.
8440 unsigned FShOpc = IsLeft ? TargetOpcode::G_FSHL : TargetOpcode::G_FSHR;
8441 unsigned RevFsh = !IsLeft ? TargetOpcode::G_FSHL : TargetOpcode::G_FSHR;
8442 bool IsFShLegal = false;
8443 if ((IsFShLegal = LI.isLegalOrCustom(Query: {FShOpc, {DstTy, AmtTy}})) ||
8444 LI.isLegalOrCustom(Query: {RevFsh, {DstTy, AmtTy}})) {
8445 auto buildFunnelShift = [&](unsigned Opc, Register R1, Register R2,
8446 Register R3) {
8447 MIRBuilder.buildInstr(Opc, DstOps: {R1}, SrcOps: {R2, R2, R3});
8448 MI.eraseFromParent();
8449 return Legalized;
8450 };
8451 // If a funnel shift in the other direction is supported, use it.
8452 if (IsFShLegal) {
8453 return buildFunnelShift(FShOpc, Dst, Src, Amt);
8454 } else if (isPowerOf2_32(Value: EltSizeInBits)) {
8455 Amt = MIRBuilder.buildNeg(Dst: DstTy, Src0: Amt).getReg(Idx: 0);
8456 return buildFunnelShift(RevFsh, Dst, Src, Amt);
8457 }
8458 }
8459
8460 auto Zero = MIRBuilder.buildConstant(Res: AmtTy, Val: 0);
8461 unsigned ShOpc = IsLeft ? TargetOpcode::G_SHL : TargetOpcode::G_LSHR;
8462 unsigned RevShiftOpc = IsLeft ? TargetOpcode::G_LSHR : TargetOpcode::G_SHL;
8463 auto BitWidthMinusOneC = MIRBuilder.buildConstant(Res: AmtTy, Val: EltSizeInBits - 1);
8464 Register ShVal;
8465 Register RevShiftVal;
8466 if (isPowerOf2_32(Value: EltSizeInBits)) {
8467 // (rotl x, c) -> x << (c & (w - 1)) | x >> (-c & (w - 1))
8468 // (rotr x, c) -> x >> (c & (w - 1)) | x << (-c & (w - 1))
8469 auto NegAmt = MIRBuilder.buildSub(Dst: AmtTy, Src0: Zero, Src1: Amt);
8470 auto ShAmt = MIRBuilder.buildAnd(Dst: AmtTy, Src0: Amt, Src1: BitWidthMinusOneC);
8471 ShVal = MIRBuilder.buildInstr(Opc: ShOpc, DstOps: {DstTy}, SrcOps: {Src, ShAmt}).getReg(Idx: 0);
8472 auto RevAmt = MIRBuilder.buildAnd(Dst: AmtTy, Src0: NegAmt, Src1: BitWidthMinusOneC);
8473 RevShiftVal =
8474 MIRBuilder.buildInstr(Opc: RevShiftOpc, DstOps: {DstTy}, SrcOps: {Src, RevAmt}).getReg(Idx: 0);
8475 } else {
8476 // (rotl x, c) -> x << (c % w) | x >> 1 >> (w - 1 - (c % w))
8477 // (rotr x, c) -> x >> (c % w) | x << 1 << (w - 1 - (c % w))
8478 auto BitWidthC = MIRBuilder.buildConstant(Res: AmtTy, Val: EltSizeInBits);
8479 auto ShAmt = MIRBuilder.buildURem(Dst: AmtTy, Src0: Amt, Src1: BitWidthC);
8480 ShVal = MIRBuilder.buildInstr(Opc: ShOpc, DstOps: {DstTy}, SrcOps: {Src, ShAmt}).getReg(Idx: 0);
8481 auto RevAmt = MIRBuilder.buildSub(Dst: AmtTy, Src0: BitWidthMinusOneC, Src1: ShAmt);
8482 auto One = MIRBuilder.buildConstant(Res: AmtTy, Val: 1);
8483 auto Inner = MIRBuilder.buildInstr(Opc: RevShiftOpc, DstOps: {DstTy}, SrcOps: {Src, One});
8484 RevShiftVal =
8485 MIRBuilder.buildInstr(Opc: RevShiftOpc, DstOps: {DstTy}, SrcOps: {Inner, RevAmt}).getReg(Idx: 0);
8486 }
8487 MIRBuilder.buildOr(Dst, Src0: ShVal, Src1: RevShiftVal, Flags: MachineInstr::Disjoint);
8488 MI.eraseFromParent();
8489 return Legalized;
8490}
8491
8492// Expand s32 = G_UITOFP s64 to an IEEE float representation using bit
8493// operations and G_SITOFP
8494LegalizerHelper::LegalizeResult
8495LegalizerHelper::lowerU64ToF32WithSITOFP(MachineInstr &MI) {
8496 auto [Dst, Src] = MI.getFirst2Regs();
8497 const LLT S64 = LLT::scalar(SizeInBits: 64);
8498 const LLT S32 = LLT::scalar(SizeInBits: 32);
8499 const LLT S1 = LLT::scalar(SizeInBits: 1);
8500
8501 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S32);
8502
8503 // For i64 < INT_MAX we simply reuse SITOFP.
8504 // Otherwise, divide i64 by 2, round result by ORing with the lowest bit
8505 // saved before division, convert to float by SITOFP, multiply the result
8506 // by 2.
8507 auto One = MIRBuilder.buildConstant(Res: S64, Val: 1);
8508 auto Zero = MIRBuilder.buildConstant(Res: S64, Val: 0);
8509 // Result if Src < INT_MAX
8510 auto SmallResult = MIRBuilder.buildSITOFP(Dst: S32, Src0: Src);
8511 // Result if Src >= INT_MAX
8512 auto Halved = MIRBuilder.buildLShr(Dst: S64, Src0: Src, Src1: One);
8513 auto LowerBit = MIRBuilder.buildAnd(Dst: S64, Src0: Src, Src1: One);
8514 auto RoundedHalved = MIRBuilder.buildOr(Dst: S64, Src0: Halved, Src1: LowerBit);
8515 auto HalvedFP = MIRBuilder.buildSITOFP(Dst: S32, Src0: RoundedHalved);
8516 auto LargeResult = MIRBuilder.buildFAdd(Dst: S32, Src0: HalvedFP, Src1: HalvedFP);
8517 // Check if the original value is larger than INT_MAX by comparing with
8518 // zero to pick one of the two conversions.
8519 auto IsLarge =
8520 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_SLT, Res: S1, Op0: Src, Op1: Zero);
8521 MIRBuilder.buildSelect(Res: Dst, Tst: IsLarge, Op0: LargeResult, Op1: SmallResult);
8522
8523 MI.eraseFromParent();
8524 return Legalized;
8525}
8526
8527// Expand s64 = G_UITOFP s64 using bit and float arithmetic operations to an
8528// IEEE double representation.
8529LegalizerHelper::LegalizeResult
8530LegalizerHelper::lowerU64ToF64BitFloatOps(MachineInstr &MI) {
8531 auto [Dst, Src] = MI.getFirst2Regs();
8532 const LLT S64 = LLT::scalar(SizeInBits: 64);
8533 const LLT S32 = LLT::scalar(SizeInBits: 32);
8534
8535 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S64);
8536
8537 // We create double value from 32 bit parts with 32 exponent difference.
8538 // Note that + and - are float operations that adjust the implicit leading
8539 // one, the bases 2^52 and 2^84 are for illustrative purposes.
8540 //
8541 // X = 2^52 * 1.0...LowBits
8542 // Y = 2^84 * 1.0...HighBits
8543 // Scratch = 2^84 * 1.0...HighBits - 2^84 * 1.0 - 2^52 * 1.0
8544 // = - 2^52 * 1.0...HighBits
8545 // Result = - 2^52 * 1.0...HighBits + 2^52 * 1.0...LowBits
8546 auto TwoP52 = MIRBuilder.buildConstant(Res: S64, UINT64_C(0x4330000000000000));
8547 auto TwoP84 = MIRBuilder.buildConstant(Res: S64, UINT64_C(0x4530000000000000));
8548 auto TwoP52P84 = llvm::bit_cast<double>(UINT64_C(0x4530000000100000));
8549 auto TwoP52P84FP = MIRBuilder.buildFConstant(Res: S64, Val: TwoP52P84);
8550 auto HalfWidth = MIRBuilder.buildConstant(Res: S64, Val: 32);
8551
8552 auto LowBits = MIRBuilder.buildTrunc(Res: S32, Op: Src);
8553 LowBits = MIRBuilder.buildZExt(Res: S64, Op: LowBits);
8554 auto LowBitsFP = MIRBuilder.buildOr(Dst: S64, Src0: TwoP52, Src1: LowBits);
8555 auto HighBits = MIRBuilder.buildLShr(Dst: S64, Src0: Src, Src1: HalfWidth);
8556 auto HighBitsFP = MIRBuilder.buildOr(Dst: S64, Src0: TwoP84, Src1: HighBits);
8557 auto Scratch = MIRBuilder.buildFSub(Dst: S64, Src0: HighBitsFP, Src1: TwoP52P84FP);
8558 MIRBuilder.buildFAdd(Dst, Src0: Scratch, Src1: LowBitsFP);
8559
8560 MI.eraseFromParent();
8561 return Legalized;
8562}
8563
8564/// i64->fp16 itofp can be lowered to i64->f64,f64->f32,f32->f16. We cannot
8565/// convert fpround f64->f16 without double-rounding, so we manually perform the
8566/// lowering here where we know it is valid.
8567static LegalizerHelper::LegalizeResult
8568loweri64tof16ITOFP(MachineInstr &MI, Register Dst, LLT DstTy, Register Src,
8569 LLT SrcTy, MachineIRBuilder &MIRBuilder) {
8570 auto DstFpTy =
8571 SrcTy.changeElementType(NewEltTy: LLT::floatIEEE(SizeInBits: SrcTy.getScalarSizeInBits()));
8572 auto M1 = MI.getOpcode() == TargetOpcode::G_UITOFP
8573 ? MIRBuilder.buildUITOFP(Dst: DstFpTy, Src0: Src)
8574 : MIRBuilder.buildSITOFP(Dst: DstFpTy, Src0: Src);
8575 LLT F32Ty = DstFpTy.changeElementSize(NewEltSize: 32);
8576 auto M2 = MIRBuilder.buildFPTrunc(Res: F32Ty, Op: M1);
8577 MIRBuilder.buildFPTrunc(Res: Dst, Op: M2);
8578 MI.eraseFromParent();
8579 return LegalizerHelper::Legalized;
8580}
8581
8582LegalizerHelper::LegalizeResult LegalizerHelper::lowerUITOFP(MachineInstr &MI) {
8583 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8584
8585 if (SrcTy == LLT::scalar(SizeInBits: 1)) {
8586 auto True = MIRBuilder.buildFConstant(Res: DstTy, Val: 1.0);
8587 auto False = MIRBuilder.buildFConstant(Res: DstTy, Val: 0.0);
8588 MIRBuilder.buildSelect(Res: Dst, Tst: Src, Op0: True, Op1: False);
8589 MI.eraseFromParent();
8590 return Legalized;
8591 }
8592
8593 if (DstTy.getScalarSizeInBits() == 16 && SrcTy.getScalarSizeInBits() == 64)
8594 return loweri64tof16ITOFP(MI, Dst, DstTy, Src, SrcTy, MIRBuilder);
8595
8596 if (SrcTy != LLT::scalar(SizeInBits: 64))
8597 return UnableToLegalize;
8598
8599 if (DstTy == LLT::scalar(SizeInBits: 32))
8600 // TODO: SelectionDAG has several alternative expansions to port which may
8601 // be more reasonable depending on the available instructions. We also need
8602 // a more advanced mechanism to choose an optimal version depending on
8603 // target features such as sitofp or CTLZ availability.
8604 return lowerU64ToF32WithSITOFP(MI);
8605
8606 if (DstTy == LLT::scalar(SizeInBits: 64))
8607 return lowerU64ToF64BitFloatOps(MI);
8608
8609 return UnableToLegalize;
8610}
8611
8612LegalizerHelper::LegalizeResult LegalizerHelper::lowerSITOFP(MachineInstr &MI) {
8613 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8614
8615 const LLT I64 = LLT::integer(SizeInBits: 64);
8616 const LLT I32 = LLT::integer(SizeInBits: 32);
8617 const LLT I1 = LLT::integer(SizeInBits: 1);
8618
8619 if (SrcTy == I1) {
8620 auto True = MIRBuilder.buildFConstant(Res: DstTy, Val: -1.0);
8621 auto False = MIRBuilder.buildFConstant(Res: DstTy, Val: 0.0);
8622 MIRBuilder.buildSelect(Res: Dst, Tst: Src, Op0: True, Op1: False);
8623 MI.eraseFromParent();
8624 return Legalized;
8625 }
8626
8627 if (DstTy.getScalarSizeInBits() == 16 && SrcTy.getScalarSizeInBits() == 64)
8628 return loweri64tof16ITOFP(MI, Dst, DstTy, Src, SrcTy, MIRBuilder);
8629
8630 if (SrcTy != I64)
8631 return UnableToLegalize;
8632
8633 if (DstTy.getScalarSizeInBits() == 32) {
8634 // signed cl2f(long l) {
8635 // long s = l >> 63;
8636 // float r = cul2f((l + s) ^ s);
8637 // return s ? -r : r;
8638 // }
8639 Register L = Src;
8640 auto SignBit = MIRBuilder.buildConstant(Res: I64, Val: 63);
8641 auto S = MIRBuilder.buildAShr(Dst: I64, Src0: L, Src1: SignBit);
8642
8643 auto LPlusS = MIRBuilder.buildAdd(Dst: I64, Src0: L, Src1: S);
8644 auto Xor = MIRBuilder.buildXor(Dst: I64, Src0: LPlusS, Src1: S);
8645 auto R = MIRBuilder.buildUITOFP(Dst: I32, Src0: Xor);
8646
8647 auto RNeg = MIRBuilder.buildFNeg(Dst: I32, Src0: R);
8648 auto SignNotZero = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: I1, Op0: S,
8649 Op1: MIRBuilder.buildConstant(Res: I64, Val: 0));
8650 MIRBuilder.buildSelect(Res: Dst, Tst: SignNotZero, Op0: RNeg, Op1: R);
8651 MI.eraseFromParent();
8652 return Legalized;
8653 }
8654
8655 return UnableToLegalize;
8656}
8657
8658LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOUI(MachineInstr &MI) {
8659 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8660 const LLT S64 = LLT::scalar(SizeInBits: 64);
8661 const LLT S32 = LLT::scalar(SizeInBits: 32);
8662
8663 if (SrcTy != S64 && SrcTy != S32)
8664 return UnableToLegalize;
8665 if (DstTy != S32 && DstTy != S64)
8666 return UnableToLegalize;
8667
8668 // FPTOSI gives same result as FPTOUI for positive signed integers.
8669 // FPTOUI needs to deal with fp values that convert to unsigned integers
8670 // greater or equal to 2^31 for float or 2^63 for double. For brevity 2^Exp.
8671
8672 APInt TwoPExpInt = APInt::getSignMask(BitWidth: DstTy.getSizeInBits());
8673 APFloat TwoPExpFP(SrcTy.getSizeInBits() == 32 ? APFloat::IEEEsingle()
8674 : APFloat::IEEEdouble(),
8675 APInt::getZero(numBits: SrcTy.getSizeInBits()));
8676 TwoPExpFP.convertFromAPInt(Input: TwoPExpInt, IsSigned: false, RM: APFloat::rmNearestTiesToEven);
8677
8678 MachineInstrBuilder FPTOSI = MIRBuilder.buildFPTOSI(Dst: DstTy, Src0: Src);
8679
8680 MachineInstrBuilder Threshold = MIRBuilder.buildFConstant(Res: SrcTy, Val: TwoPExpFP);
8681 // For fp Value greater or equal to Threshold(2^Exp), we use FPTOSI on
8682 // (Value - 2^Exp) and add 2^Exp by setting highest bit in result to 1.
8683 MachineInstrBuilder FSub = MIRBuilder.buildFSub(Dst: SrcTy, Src0: Src, Src1: Threshold);
8684 MachineInstrBuilder ResLowBits = MIRBuilder.buildFPTOSI(Dst: DstTy, Src0: FSub);
8685 MachineInstrBuilder ResHighBit = MIRBuilder.buildConstant(Res: DstTy, Val: TwoPExpInt);
8686 MachineInstrBuilder Res = MIRBuilder.buildXor(Dst: DstTy, Src0: ResLowBits, Src1: ResHighBit);
8687
8688 const LLT S1 = LLT::scalar(SizeInBits: 1);
8689
8690 MachineInstrBuilder FCMP =
8691 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_ULT, Res: S1, Op0: Src, Op1: Threshold);
8692 MIRBuilder.buildSelect(Res: Dst, Tst: FCMP, Op0: FPTOSI, Op1: Res);
8693
8694 MI.eraseFromParent();
8695 return Legalized;
8696}
8697
8698LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOSI(MachineInstr &MI) {
8699 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8700 const LLT S64 = LLT::scalar(SizeInBits: 64);
8701 const LLT S32 = LLT::scalar(SizeInBits: 32);
8702
8703 // FIXME: Only f32 to i64 conversions are supported.
8704 if (SrcTy.getScalarType() != S32 || DstTy.getScalarType() != S64)
8705 return UnableToLegalize;
8706
8707 // Expand f32 -> i64 conversion
8708 // This algorithm comes from compiler-rt's implementation of fixsfdi:
8709 // https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/fixsfdi.c
8710
8711 unsigned SrcEltBits = SrcTy.getScalarSizeInBits();
8712
8713 auto ExponentMask = MIRBuilder.buildConstant(Res: SrcTy, Val: 0x7F800000);
8714 auto ExponentLoBit = MIRBuilder.buildConstant(Res: SrcTy, Val: 23);
8715
8716 auto AndExpMask = MIRBuilder.buildAnd(Dst: SrcTy, Src0: Src, Src1: ExponentMask);
8717 auto ExponentBits = MIRBuilder.buildLShr(Dst: SrcTy, Src0: AndExpMask, Src1: ExponentLoBit);
8718
8719 auto SignMask = MIRBuilder.buildConstant(Res: SrcTy,
8720 Val: APInt::getSignMask(BitWidth: SrcEltBits));
8721 auto AndSignMask = MIRBuilder.buildAnd(Dst: SrcTy, Src0: Src, Src1: SignMask);
8722 auto SignLowBit = MIRBuilder.buildConstant(Res: SrcTy, Val: SrcEltBits - 1);
8723 auto Sign = MIRBuilder.buildAShr(Dst: SrcTy, Src0: AndSignMask, Src1: SignLowBit);
8724 Sign = MIRBuilder.buildSExt(Res: DstTy, Op: Sign);
8725
8726 auto MantissaMask = MIRBuilder.buildConstant(Res: SrcTy, Val: 0x007FFFFF);
8727 auto AndMantissaMask = MIRBuilder.buildAnd(Dst: SrcTy, Src0: Src, Src1: MantissaMask);
8728 auto K = MIRBuilder.buildConstant(Res: SrcTy, Val: 0x00800000);
8729
8730 auto R = MIRBuilder.buildOr(Dst: SrcTy, Src0: AndMantissaMask, Src1: K);
8731 R = MIRBuilder.buildZExt(Res: DstTy, Op: R);
8732
8733 auto Bias = MIRBuilder.buildConstant(Res: SrcTy, Val: 127);
8734 auto Exponent = MIRBuilder.buildSub(Dst: SrcTy, Src0: ExponentBits, Src1: Bias);
8735 auto SubExponent = MIRBuilder.buildSub(Dst: SrcTy, Src0: Exponent, Src1: ExponentLoBit);
8736 auto ExponentSub = MIRBuilder.buildSub(Dst: SrcTy, Src0: ExponentLoBit, Src1: Exponent);
8737
8738 auto Shl = MIRBuilder.buildShl(Dst: DstTy, Src0: R, Src1: SubExponent);
8739 auto Srl = MIRBuilder.buildLShr(Dst: DstTy, Src0: R, Src1: ExponentSub);
8740
8741 const LLT S1 = LLT::scalar(SizeInBits: 1);
8742 auto CmpGt = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SGT,
8743 Res: S1, Op0: Exponent, Op1: ExponentLoBit);
8744
8745 R = MIRBuilder.buildSelect(Res: DstTy, Tst: CmpGt, Op0: Shl, Op1: Srl);
8746
8747 auto XorSign = MIRBuilder.buildXor(Dst: DstTy, Src0: R, Src1: Sign);
8748 auto Ret = MIRBuilder.buildSub(Dst: DstTy, Src0: XorSign, Src1: Sign);
8749
8750 auto ZeroSrcTy = MIRBuilder.buildConstant(Res: SrcTy, Val: 0);
8751
8752 auto ExponentLt0 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT,
8753 Res: S1, Op0: Exponent, Op1: ZeroSrcTy);
8754
8755 auto ZeroDstTy = MIRBuilder.buildConstant(Res: DstTy, Val: 0);
8756 MIRBuilder.buildSelect(Res: Dst, Tst: ExponentLt0, Op0: ZeroDstTy, Op1: Ret);
8757
8758 MI.eraseFromParent();
8759 return Legalized;
8760}
8761
8762LegalizerHelper::LegalizeResult
8763LegalizerHelper::lowerFPTOINT_SAT(MachineInstr &MI) {
8764 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8765
8766 bool IsSigned = MI.getOpcode() == TargetOpcode::G_FPTOSI_SAT;
8767 unsigned SatWidth = DstTy.getScalarSizeInBits();
8768
8769 // Determine minimum and maximum integer values and their corresponding
8770 // floating-point values.
8771 APInt MinInt, MaxInt;
8772 if (IsSigned) {
8773 MinInt = APInt::getSignedMinValue(numBits: SatWidth);
8774 MaxInt = APInt::getSignedMaxValue(numBits: SatWidth);
8775 } else {
8776 MinInt = APInt::getMinValue(numBits: SatWidth);
8777 MaxInt = APInt::getMaxValue(numBits: SatWidth);
8778 }
8779
8780 const fltSemantics &Semantics = getFltSemanticForLLT(Ty: SrcTy.getScalarType());
8781 APFloat MinFloat(Semantics);
8782 APFloat MaxFloat(Semantics);
8783
8784 APFloat::opStatus MinStatus =
8785 MinFloat.convertFromAPInt(Input: MinInt, IsSigned, RM: APFloat::rmTowardZero);
8786 APFloat::opStatus MaxStatus =
8787 MaxFloat.convertFromAPInt(Input: MaxInt, IsSigned, RM: APFloat::rmTowardZero);
8788 bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact) &&
8789 !(MaxStatus & APFloat::opStatus::opInexact);
8790
8791 // If the integer bounds are exactly representable as floats, emit a
8792 // min+max+fptoi sequence. Otherwise we have to use a sequence of comparisons
8793 // and selects.
8794 if (AreExactFloatBounds) {
8795 // Clamp Src by MinFloat from below. If Src is NaN the result is MinFloat.
8796 auto MaxC = MIRBuilder.buildFConstant(Res: SrcTy, Val: MinFloat);
8797 auto MaxP =
8798 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OGT, Res: LLT::integer(SizeInBits: 1), Op0: Src, Op1: MaxC);
8799 auto Max = MIRBuilder.buildSelect(Res: SrcTy, Tst: MaxP, Op0: Src, Op1: MaxC);
8800 // Clamp by MaxFloat from above. NaN cannot occur.
8801 auto MinC = MIRBuilder.buildFConstant(Res: SrcTy, Val: MaxFloat);
8802 auto MinP = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OLT, Res: LLT::integer(SizeInBits: 1), Op0: Max,
8803 Op1: MinC, Flags: MachineInstr::FmNoNans);
8804 auto Min =
8805 MIRBuilder.buildSelect(Res: SrcTy, Tst: MinP, Op0: Max, Op1: MinC, Flags: MachineInstr::FmNoNans);
8806 // Convert clamped value to integer. In the unsigned case we're done,
8807 // because we mapped NaN to MinFloat, which will cast to zero.
8808 if (!IsSigned) {
8809 MIRBuilder.buildFPTOUI(Dst, Src0: Min);
8810 MI.eraseFromParent();
8811 return Legalized;
8812 }
8813
8814 // Otherwise, select 0 if Src is NaN.
8815 auto FpToInt = MIRBuilder.buildFPTOSI(Dst: DstTy, Src0: Min);
8816 auto IsZero =
8817 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_UNO, Res: LLT::integer(SizeInBits: 1), Op0: Src, Op1: Src);
8818 MIRBuilder.buildSelect(Res: Dst, Tst: IsZero, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: 0),
8819 Op1: FpToInt);
8820 MI.eraseFromParent();
8821 return Legalized;
8822 }
8823
8824 // Result of direct conversion. The assumption here is that the operation is
8825 // non-trapping and it's fine to apply it to an out-of-range value if we
8826 // select it away later.
8827 auto FpToInt = IsSigned ? MIRBuilder.buildFPTOSI(Dst: DstTy, Src0: Src)
8828 : MIRBuilder.buildFPTOUI(Dst: DstTy, Src0: Src);
8829
8830 // If Src ULT MinFloat, select MinInt. In particular, this also selects
8831 // MinInt if Src is NaN.
8832 auto ULT = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_ULT, Res: LLT::integer(SizeInBits: 1), Op0: Src,
8833 Op1: MIRBuilder.buildFConstant(Res: SrcTy, Val: MinFloat));
8834 auto Max = MIRBuilder.buildSelect(
8835 Res: DstTy, Tst: ULT, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: MinInt), Op1: FpToInt);
8836 // If Src OGT MaxFloat, select MaxInt.
8837 auto OGT = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OGT, Res: LLT::integer(SizeInBits: 1), Op0: Src,
8838 Op1: MIRBuilder.buildFConstant(Res: SrcTy, Val: MaxFloat));
8839
8840 // In the unsigned case we are done, because we mapped NaN to MinInt, which
8841 // is already zero.
8842 if (!IsSigned) {
8843 MIRBuilder.buildSelect(Res: Dst, Tst: OGT, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: MaxInt),
8844 Op1: Max);
8845 MI.eraseFromParent();
8846 return Legalized;
8847 }
8848
8849 // Otherwise, select 0 if Src is NaN.
8850 auto Min = MIRBuilder.buildSelect(
8851 Res: DstTy, Tst: OGT, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: MaxInt), Op1: Max);
8852 auto IsZero =
8853 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_UNO, Res: LLT::integer(SizeInBits: 1), Op0: Src, Op1: Src);
8854 MIRBuilder.buildSelect(Res: Dst, Tst: IsZero, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: 0), Op1: Min);
8855 MI.eraseFromParent();
8856 return Legalized;
8857}
8858
8859// Floating-point conversions using truncating and extending loads and stores.
8860LegalizerHelper::LegalizeResult
8861LegalizerHelper::lowerFPExtAndTruncMem(MachineInstr &MI) {
8862 assert((MI.getOpcode() == TargetOpcode::G_FPEXT ||
8863 MI.getOpcode() == TargetOpcode::G_FPTRUNC) &&
8864 "Only G_FPEXT and G_FPTRUNC are expected");
8865
8866 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
8867 MachinePointerInfo PtrInfo;
8868 unsigned StoreOpc;
8869 unsigned LoadOpc;
8870 LLT StackTy;
8871 if (MI.getOpcode() == TargetOpcode::G_FPEXT) {
8872 StackTy = SrcTy;
8873 StoreOpc = TargetOpcode::G_STORE;
8874 LoadOpc = TargetOpcode::G_FPEXTLOAD;
8875 } else {
8876 StackTy = DstTy;
8877 StoreOpc = TargetOpcode::G_FPTRUNCSTORE;
8878 LoadOpc = TargetOpcode::G_LOAD;
8879 }
8880
8881 Align StackTyAlign = getStackTemporaryAlignment(Ty: StackTy);
8882 auto StackTemp =
8883 createStackTemporary(Bytes: StackTy.getSizeInBytes(), Alignment: StackTyAlign, PtrInfo);
8884
8885 MachineFunction &MF = MIRBuilder.getMF();
8886 auto *StoreMMO = MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
8887 MemTy: StackTy, BaseAlignment: StackTyAlign);
8888 MIRBuilder.buildStoreInstr(Opcode: StoreOpc, Val: SrcReg, Addr: StackTemp, MMO&: *StoreMMO);
8889
8890 auto *LoadMMO = MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOLoad,
8891 MemTy: StackTy, BaseAlignment: StackTyAlign);
8892 MIRBuilder.buildLoadInstr(Opcode: LoadOpc, Res: DstReg, Addr: StackTemp, MMO&: *LoadMMO);
8893
8894 MI.eraseFromParent();
8895 return Legalized;
8896}
8897
8898// Expand a bf16 -> f32/f64 fpext with a shift and bitcast. This is based on the
8899// SDAG ISD::BF16_TO_FP lowering.
8900LegalizerHelper::LegalizeResult
8901LegalizerHelper::lowerFPEXT_BF16(MachineInstr &MI) {
8902 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
8903 assert(SrcTy.getScalarType().isBFloat16() &&
8904 "expected a bf16 source for bf16 fpext lowering");
8905
8906 LLT I16Ty = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 16));
8907 LLT I32Ty = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 32));
8908 LLT F32Ty = SrcTy.changeElementType(NewEltTy: LLT::float32());
8909
8910 auto SrcI =
8911 MIRBuilder.buildAnyExt(Res: I32Ty, Op: MIRBuilder.buildBitcast(Dst: I16Ty, Src: SrcReg));
8912 auto Shl =
8913 MIRBuilder.buildShl(Dst: I32Ty, Src0: SrcI, Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 16));
8914
8915 if (DstTy.getScalarType().isFloat32())
8916 MIRBuilder.buildBitcast(Dst: DstReg, Src: Shl);
8917 else
8918 MIRBuilder.buildFPExt(Res: DstReg, Op: MIRBuilder.buildBitcast(Dst: F32Ty, Src: Shl));
8919
8920 MI.eraseFromParent();
8921 return Legalized;
8922}
8923
8924LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPEXT(MachineInstr &MI) {
8925 auto [DstTy, SrcTy] = MI.getFirst2LLTs();
8926 if (SrcTy.getScalarType().isBFloat16() &&
8927 (DstTy.getScalarType().isFloat32() || DstTy.getScalarType().isFloat64()))
8928 return lowerFPEXT_BF16(MI);
8929
8930 return lowerFPExtAndTruncMem(MI);
8931}
8932
8933// f64 -> f16 conversion using round-to-nearest-even rounding mode.
8934LegalizerHelper::LegalizeResult
8935LegalizerHelper::lowerFPTRUNC_F64_TO_F16(MachineInstr &MI) {
8936 const LLT S1 = LLT::scalar(SizeInBits: 1);
8937 const LLT I32 = LLT::integer(SizeInBits: 32);
8938
8939 auto [Dst, Src] = MI.getFirst2Regs();
8940 assert(MRI.getType(Dst).getScalarType() == LLT::float16() &&
8941 MRI.getType(Src).getScalarType() == LLT::float64());
8942
8943 if (MRI.getType(Reg: Src).isVector()) // TODO: Handle vectors directly.
8944 return UnableToLegalize;
8945
8946 if (MI.getFlag(Flag: MachineInstr::FmAfn)) {
8947 unsigned Flags = MI.getFlags();
8948 auto Src32 = MIRBuilder.buildFPTrunc(Res: LLT::float32(), Op: Src, Flags);
8949 MIRBuilder.buildFPTrunc(Res: Dst, Op: Src32, Flags);
8950 MI.eraseFromParent();
8951 return Legalized;
8952 }
8953
8954 const unsigned ExpMask = 0x7ff;
8955 const unsigned ExpBiasf64 = 1023;
8956 const unsigned ExpBiasf16 = 15;
8957
8958 auto Unmerge = MIRBuilder.buildUnmerge(Res: I32, Op: Src);
8959 Register U = Unmerge.getReg(Idx: 0);
8960 Register UH = Unmerge.getReg(Idx: 1);
8961
8962 auto E = MIRBuilder.buildLShr(Dst: I32, Src0: UH, Src1: MIRBuilder.buildConstant(Res: I32, Val: 20));
8963 E = MIRBuilder.buildAnd(Dst: I32, Src0: E, Src1: MIRBuilder.buildConstant(Res: I32, Val: ExpMask));
8964
8965 // Subtract the fp64 exponent bias (1023) to get the real exponent and
8966 // add the f16 bias (15) to get the biased exponent for the f16 format.
8967 E = MIRBuilder.buildAdd(
8968 Dst: I32, Src0: E, Src1: MIRBuilder.buildConstant(Res: I32, Val: -ExpBiasf64 + ExpBiasf16));
8969
8970 auto M = MIRBuilder.buildLShr(Dst: I32, Src0: UH, Src1: MIRBuilder.buildConstant(Res: I32, Val: 8));
8971 M = MIRBuilder.buildAnd(Dst: I32, Src0: M, Src1: MIRBuilder.buildConstant(Res: I32, Val: 0xffe));
8972
8973 auto MaskedSig =
8974 MIRBuilder.buildAnd(Dst: I32, Src0: UH, Src1: MIRBuilder.buildConstant(Res: I32, Val: 0x1ff));
8975 MaskedSig = MIRBuilder.buildOr(Dst: I32, Src0: MaskedSig, Src1: U);
8976
8977 auto Zero = MIRBuilder.buildConstant(Res: I32, Val: 0);
8978 auto SigCmpNE0 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: S1, Op0: MaskedSig, Op1: Zero);
8979 auto Lo40Set = MIRBuilder.buildZExt(Res: I32, Op: SigCmpNE0);
8980 M = MIRBuilder.buildOr(Dst: I32, Src0: M, Src1: Lo40Set);
8981
8982 // (M != 0 ? 0x0200 : 0) | 0x7c00;
8983 auto Bits0x200 = MIRBuilder.buildConstant(Res: I32, Val: 0x0200);
8984 auto CmpM_NE0 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: S1, Op0: M, Op1: Zero);
8985 auto SelectCC = MIRBuilder.buildSelect(Res: I32, Tst: CmpM_NE0, Op0: Bits0x200, Op1: Zero);
8986
8987 auto Bits0x7c00 = MIRBuilder.buildConstant(Res: I32, Val: 0x7c00);
8988 auto I = MIRBuilder.buildOr(Dst: I32, Src0: SelectCC, Src1: Bits0x7c00);
8989
8990 // N = M | (E << 12);
8991 auto EShl12 = MIRBuilder.buildShl(Dst: I32, Src0: E, Src1: MIRBuilder.buildConstant(Res: I32, Val: 12));
8992 auto N = MIRBuilder.buildOr(Dst: I32, Src0: M, Src1: EShl12);
8993
8994 // B = clamp(1-E, 0, 13);
8995 auto One = MIRBuilder.buildConstant(Res: I32, Val: 1);
8996 auto OneSubExp = MIRBuilder.buildSub(Dst: I32, Src0: One, Src1: E);
8997 auto B = MIRBuilder.buildSMax(Dst: I32, Src0: OneSubExp, Src1: Zero);
8998 B = MIRBuilder.buildSMin(Dst: I32, Src0: B, Src1: MIRBuilder.buildConstant(Res: I32, Val: 13));
8999
9000 auto SigSetHigh =
9001 MIRBuilder.buildOr(Dst: I32, Src0: M, Src1: MIRBuilder.buildConstant(Res: I32, Val: 0x1000));
9002
9003 auto D = MIRBuilder.buildLShr(Dst: I32, Src0: SigSetHigh, Src1: B);
9004 auto D0 = MIRBuilder.buildShl(Dst: I32, Src0: D, Src1: B);
9005
9006 auto D0_NE_SigSetHigh = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: S1,
9007 Op0: D0, Op1: SigSetHigh);
9008 auto D1 = MIRBuilder.buildZExt(Res: I32, Op: D0_NE_SigSetHigh);
9009 D = MIRBuilder.buildOr(Dst: I32, Src0: D, Src1: D1);
9010
9011 auto CmpELtOne = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: S1, Op0: E, Op1: One);
9012 auto V = MIRBuilder.buildSelect(Res: I32, Tst: CmpELtOne, Op0: D, Op1: N);
9013
9014 auto VLow3 = MIRBuilder.buildAnd(Dst: I32, Src0: V, Src1: MIRBuilder.buildConstant(Res: I32, Val: 7));
9015 V = MIRBuilder.buildLShr(Dst: I32, Src0: V, Src1: MIRBuilder.buildConstant(Res: I32, Val: 2));
9016
9017 auto VLow3Eq3 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: S1, Op0: VLow3,
9018 Op1: MIRBuilder.buildConstant(Res: I32, Val: 3));
9019 auto V0 = MIRBuilder.buildZExt(Res: I32, Op: VLow3Eq3);
9020
9021 auto VLow3Gt5 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SGT, Res: S1, Op0: VLow3,
9022 Op1: MIRBuilder.buildConstant(Res: I32, Val: 5));
9023 auto V1 = MIRBuilder.buildZExt(Res: I32, Op: VLow3Gt5);
9024
9025 V1 = MIRBuilder.buildOr(Dst: I32, Src0: V0, Src1: V1);
9026 V = MIRBuilder.buildAdd(Dst: I32, Src0: V, Src1: V1);
9027
9028 auto CmpEGt30 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SGT, Res: S1, Op0: E,
9029 Op1: MIRBuilder.buildConstant(Res: I32, Val: 30));
9030 V = MIRBuilder.buildSelect(Res: I32, Tst: CmpEGt30,
9031 Op0: MIRBuilder.buildConstant(Res: I32, Val: 0x7c00), Op1: V);
9032
9033 auto CmpEGt1039 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: S1, Op0: E,
9034 Op1: MIRBuilder.buildConstant(Res: I32, Val: 1039));
9035 V = MIRBuilder.buildSelect(Res: I32, Tst: CmpEGt1039, Op0: I, Op1: V);
9036
9037 // Extract the sign bit.
9038 auto Sign = MIRBuilder.buildLShr(Dst: I32, Src0: UH, Src1: MIRBuilder.buildConstant(Res: I32, Val: 16));
9039 Sign = MIRBuilder.buildAnd(Dst: I32, Src0: Sign, Src1: MIRBuilder.buildConstant(Res: I32, Val: 0x8000));
9040
9041 // Insert the sign bit
9042 V = MIRBuilder.buildOr(Dst: I32, Src0: Sign, Src1: V);
9043
9044 MIRBuilder.buildTrunc(Res: Dst, Op: V);
9045 MI.eraseFromParent();
9046 return Legalized;
9047}
9048
9049// f32 -> bf16 conversion using round-to-nearest-even rounding mode.
9050LegalizerHelper::LegalizeResult
9051LegalizerHelper::lowerFPTRUNC_F32_TO_BF16(MachineInstr &MI) {
9052 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
9053 assert(DstTy.getScalarType() == LLT::bfloat16() &&
9054 SrcTy.getScalarType() == LLT::float32());
9055
9056 LLT I1Ty = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9057 LLT I16Ty = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 16));
9058 LLT I32Ty = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 32));
9059
9060 auto SrcI = MIRBuilder.buildBitcast(Dst: I32Ty, Src: SrcReg);
9061
9062 // Conversions should set NaN's quiet bit. This also prevents NaNs from
9063 // turning into infinities.
9064 auto NaN = MIRBuilder.buildOr(Dst: I32Ty, Src0: SrcI,
9065 Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 0x400000));
9066
9067 // Factor in the contribution of the low 16 bits.
9068 auto Lsb =
9069 MIRBuilder.buildLShr(Dst: I32Ty, Src0: SrcI, Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 16));
9070 Lsb = MIRBuilder.buildAnd(Dst: I32Ty, Src0: Lsb, Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 1));
9071 auto RoundingBias =
9072 MIRBuilder.buildAdd(Dst: I32Ty, Src0: Lsb, Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 0x7fff));
9073 auto Add = MIRBuilder.buildAdd(Dst: I32Ty, Src0: SrcI, Src1: RoundingBias);
9074
9075 // Don't round if we had a NaN, we don't want to turn 0x7fffffff into
9076 // 0x80000000.
9077 if (!MI.getFlag(Flag: MachineInstr::FmNoNans)) {
9078 auto IsNaN = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_UNO, Res: I1Ty, Op0: SrcReg,
9079 Op1: MIRBuilder.buildFConstant(Res: SrcTy, Val: 0));
9080 Add = MIRBuilder.buildSelect(Res: I32Ty, Tst: IsNaN, Op0: NaN, Op1: Add);
9081 }
9082
9083 // Now that we have rounded, shift the bits into position.
9084 auto Srl =
9085 MIRBuilder.buildLShr(Dst: I32Ty, Src0: Add, Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 16));
9086 auto Trunc = MIRBuilder.buildTrunc(Res: I16Ty, Op: Srl);
9087 MIRBuilder.buildBitcast(Dst: DstReg, Src: Trunc);
9088 MI.eraseFromParent();
9089 return Legalized;
9090}
9091
9092// Round a wide fp value to ResultTy's element size, forcing inexact
9093// results to the odd value so a subsequent narrowing round is correct. This
9094// avoids double-rounding when narrowing e.g. f64 -> f32 -> bf16. See Boldo &
9095// Melquiond, "When double rounding is odd" (2005).
9096Register LegalizerHelper::lowerRoundInexactToOdd(LLT ResultTy, Register Op) {
9097 LLT OperandTy = MRI.getType(Reg: Op);
9098 if (OperandTy.getScalarType() == ResultTy.getScalarType())
9099 return Op;
9100
9101 LLT ResultIntTy =
9102 ResultTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: ResultTy.getScalarSizeInBits()));
9103 LLT ResultCCTy = ResultTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9104 LLT OperandCCTy = OperandTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9105
9106 auto Narrow = MIRBuilder.buildFPTrunc(Res: ResultTy, Op);
9107 auto NarrowAsWide = MIRBuilder.buildFPExt(Res: OperandTy, Op: Narrow);
9108
9109 auto NarrowBits = MIRBuilder.buildBitcast(Dst: ResultIntTy, Src: Narrow);
9110 auto One = MIRBuilder.buildConstant(Res: ResultIntTy, Val: 1);
9111 auto NegativeOne = MIRBuilder.buildConstant(Res: ResultIntTy, Val: -1);
9112 auto Zero = MIRBuilder.buildConstant(Res: ResultIntTy, Val: 0);
9113 auto And = MIRBuilder.buildAnd(Dst: ResultIntTy, Src0: NarrowBits, Src1: One);
9114 // The result is already odd so we don't need to do anything.
9115 auto AlreadyOdd =
9116 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: ResultCCTy, Op0: And, Op1: Zero);
9117
9118 // We keep results which are exact, odd or NaN.
9119 auto KeepNarrow =
9120 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_UEQ, Res: OperandCCTy, Op0: Op, Op1: NarrowAsWide);
9121 KeepNarrow = MIRBuilder.buildOr(Dst: OperandCCTy, Src0: KeepNarrow, Src1: AlreadyOdd);
9122 // We morally performed a round-down if AbsNarrow is smaller than AbsWide.
9123 auto AbsWide = MIRBuilder.buildFAbs(Dst: OperandTy, Src0: Op);
9124 auto AbsNarrowAsWide = MIRBuilder.buildFAbs(Dst: OperandTy, Src0: NarrowAsWide);
9125 auto NarrowIsRd = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OGT, Res: OperandCCTy,
9126 Op0: AbsWide, Op1: AbsNarrowAsWide);
9127 // If narrow is the rounded-down value, pick the rounded-up value as it will
9128 // be odd; otherwise adjust down.
9129 auto Adjust =
9130 MIRBuilder.buildSelect(Res: ResultIntTy, Tst: NarrowIsRd, Op0: One, Op1: NegativeOne);
9131 auto Adjusted = MIRBuilder.buildAdd(Dst: ResultIntTy, Src0: NarrowBits, Src1: Adjust);
9132 auto Res =
9133 MIRBuilder.buildSelect(Res: ResultIntTy, Tst: KeepNarrow, Op0: NarrowBits, Op1: Adjusted);
9134 return MIRBuilder.buildBitcast(Dst: ResultTy, Src: Res).getReg(Idx: 0);
9135}
9136
9137// f64 -> bf16 conversion, correcting for double rounding.
9138LegalizerHelper::LegalizeResult
9139LegalizerHelper::lowerFPTRUNC_F64_TO_BF16(MachineInstr &MI) {
9140 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
9141 assert(DstTy.getScalarType() == LLT::bfloat16() &&
9142 SrcTy.getScalarType() == LLT::float64());
9143
9144 LLT F32Ty = SrcTy.changeElementType(NewEltTy: LLT::float32());
9145 Register OddF32 = lowerRoundInexactToOdd(ResultTy: F32Ty, Op: SrcReg);
9146 MIRBuilder.buildFPTrunc(Res: DstReg, Op: OddF32, Flags: MI.getFlags());
9147 MI.eraseFromParent();
9148 return Legalized;
9149}
9150
9151LegalizerHelper::LegalizeResult
9152LegalizerHelper::lowerFPTRUNC(MachineInstr &MI) {
9153 auto [DstTy, SrcTy] = MI.getFirst2LLTs();
9154 if (DstTy.getScalarType().isFloat16() && SrcTy.getScalarType().isFloat64())
9155 return lowerFPTRUNC_F64_TO_F16(MI);
9156
9157 if (DstTy.getScalarType().isBFloat16() && SrcTy.getScalarType().isFloat32())
9158 return lowerFPTRUNC_F32_TO_BF16(MI);
9159
9160 if (DstTy.getScalarType().isBFloat16() && SrcTy.getScalarType().isFloat64())
9161 return lowerFPTRUNC_F64_TO_BF16(MI);
9162
9163 return lowerFPExtAndTruncMem(MI);
9164}
9165
9166LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPOWI(MachineInstr &MI) {
9167 auto [Dst, Src0, Src1] = MI.getFirst3Regs();
9168 LLT Ty = MRI.getType(Reg: Dst);
9169
9170 auto CvtSrc1 = MIRBuilder.buildSITOFP(Dst: Ty, Src0: Src1);
9171 MIRBuilder.buildFPow(Dst, Src0, Src1: CvtSrc1, Flags: MI.getFlags());
9172 MI.eraseFromParent();
9173 return Legalized;
9174}
9175
9176LegalizerHelper::LegalizeResult LegalizerHelper::lowerFMODF(MachineInstr &MI) {
9177 auto [DstFrac, DstInt, Src] = MI.getFirst3Regs();
9178 LLT Ty = MRI.getType(Reg: Src);
9179 auto Flags = MI.getFlags();
9180 const LLT CondTy = Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9181
9182 auto IntPart = MIRBuilder.buildIntrinsicTrunc(Dst: Ty, Src0: Src, Flags);
9183 auto FracPart = MIRBuilder.buildFSub(Dst: Ty, Src0: Src, Src1: IntPart, Flags);
9184
9185 Register FracToUse;
9186 if (MI.getFlag(Flag: MachineInstr::FmNoInfs)) {
9187 FracToUse = FracPart.getReg(Idx: 0);
9188 } else {
9189 auto Abs = MIRBuilder.buildFAbs(Dst: Ty, Src0: Src, Flags);
9190 const fltSemantics &Semantics = getFltSemanticForLLT(Ty: Ty.getScalarType());
9191 auto Inf = MIRBuilder.buildFConstant(Res: Ty, Val: APFloat::getInf(Sem: Semantics));
9192 auto IsInf = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OEQ, Res: CondTy, Op0: Abs, Op1: Inf);
9193 auto Zero = MIRBuilder.buildFConstant(Res: Ty, Val: 0.0);
9194 auto Select = MIRBuilder.buildSelect(Res: Ty, Tst: IsInf, Op0: Zero, Op1: FracPart);
9195 FracToUse = Select.getReg(Idx: 0);
9196 }
9197
9198 MIRBuilder.buildFCopysign(Dst: DstFrac, Src0: FracToUse, Src1: Src, Flags);
9199 MIRBuilder.buildCopy(Res: DstInt, Op: IntPart.getReg(Idx: 0));
9200
9201 MI.eraseFromParent();
9202 return Legalized;
9203}
9204
9205static CmpInst::Predicate minMaxToCompare(unsigned Opc) {
9206 switch (Opc) {
9207 case TargetOpcode::G_SMIN:
9208 return CmpInst::ICMP_SLT;
9209 case TargetOpcode::G_SMAX:
9210 return CmpInst::ICMP_SGT;
9211 case TargetOpcode::G_UMIN:
9212 return CmpInst::ICMP_ULT;
9213 case TargetOpcode::G_UMAX:
9214 return CmpInst::ICMP_UGT;
9215 default:
9216 llvm_unreachable("not in integer min/max");
9217 }
9218}
9219
9220LegalizerHelper::LegalizeResult LegalizerHelper::lowerMinMax(MachineInstr &MI) {
9221 auto [Dst, Src0, Src1] = MI.getFirst3Regs();
9222
9223 const CmpInst::Predicate Pred = minMaxToCompare(Opc: MI.getOpcode());
9224 LLT CmpType = MRI.getType(Reg: Dst).changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9225
9226 auto Cmp = MIRBuilder.buildICmp(Pred, Res: CmpType, Op0: Src0, Op1: Src1);
9227 MIRBuilder.buildSelect(Res: Dst, Tst: Cmp, Op0: Src0, Op1: Src1);
9228
9229 MI.eraseFromParent();
9230 return Legalized;
9231}
9232
9233LegalizerHelper::LegalizeResult
9234LegalizerHelper::lowerThreewayCompare(MachineInstr &MI) {
9235 GSUCmp *Cmp = cast<GSUCmp>(Val: &MI);
9236
9237 Register Dst = Cmp->getReg(Idx: 0);
9238 LLT DstTy = MRI.getType(Reg: Dst);
9239 LLT SrcTy = MRI.getType(Reg: Cmp->getReg(Idx: 1));
9240 LLT CmpTy = DstTy.changeElementSize(NewEltSize: 1);
9241
9242 CmpInst::Predicate LTPredicate = Cmp->isSigned()
9243 ? CmpInst::Predicate::ICMP_SLT
9244 : CmpInst::Predicate::ICMP_ULT;
9245 CmpInst::Predicate GTPredicate = Cmp->isSigned()
9246 ? CmpInst::Predicate::ICMP_SGT
9247 : CmpInst::Predicate::ICMP_UGT;
9248
9249 auto Zero = MIRBuilder.buildConstant(Res: DstTy, Val: 0);
9250 auto IsGT = MIRBuilder.buildICmp(Pred: GTPredicate, Res: CmpTy, Op0: Cmp->getLHSReg(),
9251 Op1: Cmp->getRHSReg());
9252 auto IsLT = MIRBuilder.buildICmp(Pred: LTPredicate, Res: CmpTy, Op0: Cmp->getLHSReg(),
9253 Op1: Cmp->getRHSReg());
9254
9255 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
9256 auto BC = TLI.getBooleanContents(isVec: DstTy.isVector(), /*isFP=*/isFloat: false);
9257 if (TLI.preferSelectsOverBooleanArithmetic(
9258 VT: getApproximateEVTForLLT(Ty: SrcTy, Ctx)) ||
9259 BC == TargetLowering::UndefinedBooleanContent) {
9260 auto One = MIRBuilder.buildConstant(Res: DstTy, Val: 1);
9261 auto SelectZeroOrOne = MIRBuilder.buildSelect(Res: DstTy, Tst: IsGT, Op0: One, Op1: Zero);
9262
9263 auto MinusOne = MIRBuilder.buildConstant(Res: DstTy, Val: -1);
9264 MIRBuilder.buildSelect(Res: Dst, Tst: IsLT, Op0: MinusOne, Op1: SelectZeroOrOne);
9265 } else {
9266 if (BC == TargetLowering::ZeroOrNegativeOneBooleanContent)
9267 std::swap(a&: IsGT, b&: IsLT);
9268 // Extend boolean results to DstTy, which is at least i2, before subtracting
9269 // them.
9270 unsigned BoolExtOp =
9271 MIRBuilder.getBoolExtOp(IsVec: DstTy.isVector(), /*isFP=*/IsFP: false);
9272 IsGT = MIRBuilder.buildInstr(Opc: BoolExtOp, DstOps: {DstTy}, SrcOps: {IsGT});
9273 IsLT = MIRBuilder.buildInstr(Opc: BoolExtOp, DstOps: {DstTy}, SrcOps: {IsLT});
9274 MIRBuilder.buildSub(Dst, Src0: IsGT, Src1: IsLT);
9275 }
9276
9277 MI.eraseFromParent();
9278 return Legalized;
9279}
9280
9281LegalizerHelper::LegalizeResult
9282LegalizerHelper::lowerFCopySign(MachineInstr &MI) {
9283 auto [Dst, DstTy, Src0, Src0Ty, Src1, Src1Ty] = MI.getFirst3RegLLTs();
9284 const int Src0Size = Src0Ty.getScalarSizeInBits();
9285 const int Src1Size = Src1Ty.getScalarSizeInBits();
9286
9287 LLT DstIntTy =
9288 DstTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: DstTy.getScalarSizeInBits()));
9289 LLT Src0IntTy = Src0Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: Src0Size));
9290 LLT Src1IntTy = Src1Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: Src1Size));
9291
9292 Register Src0Int = Src0;
9293 Register Src1Int = Src1;
9294
9295 if (!(Src0Ty.getScalarType().isAnyScalar() ||
9296 Src0Ty.getScalarType().isInteger()))
9297 Src0Int = MIRBuilder.buildBitcast(Dst: Src0IntTy, Src: Src0).getReg(Idx: 0);
9298
9299 if (!(Src1Ty.getScalarType().isAnyScalar() ||
9300 Src1Ty.getScalarType().isInteger()))
9301 Src1Int = MIRBuilder.buildBitcast(Dst: Src1IntTy, Src: Src1).getReg(Idx: 0);
9302
9303 auto SignBitMask =
9304 MIRBuilder.buildConstant(Res: Src0IntTy, Val: APInt::getSignMask(BitWidth: Src0Size));
9305
9306 auto NotSignBitMask = MIRBuilder.buildConstant(
9307 Res: Src0IntTy, Val: APInt::getLowBitsSet(numBits: Src0Size, loBitsSet: Src0Size - 1));
9308
9309 Register And0 =
9310 MIRBuilder.buildAnd(Dst: Src0IntTy, Src0: Src0Int, Src1: NotSignBitMask).getReg(Idx: 0);
9311 Register And1;
9312 if (Src0Ty == Src1Ty) {
9313 And1 = MIRBuilder.buildAnd(Dst: Src1IntTy, Src0: Src1Int, Src1: SignBitMask).getReg(Idx: 0);
9314 } else if (Src0Size > Src1Size) {
9315 auto ShiftAmt = MIRBuilder.buildConstant(Res: Src0IntTy, Val: Src0Size - Src1Size);
9316 auto Zext = MIRBuilder.buildZExt(Res: Src0IntTy, Op: Src1Int);
9317 auto Shift = MIRBuilder.buildShl(Dst: Src0IntTy, Src0: Zext, Src1: ShiftAmt);
9318 And1 = MIRBuilder.buildAnd(Dst: Src0IntTy, Src0: Shift, Src1: SignBitMask).getReg(Idx: 0);
9319 } else {
9320 auto ShiftAmt = MIRBuilder.buildConstant(Res: Src1IntTy, Val: Src1Size - Src0Size);
9321 auto Shift = MIRBuilder.buildLShr(Dst: Src1IntTy, Src0: Src1Int, Src1: ShiftAmt);
9322 auto Trunc = MIRBuilder.buildTrunc(Res: Src0IntTy, Op: Shift);
9323 And1 = MIRBuilder.buildAnd(Dst: Src0IntTy, Src0: Trunc, Src1: SignBitMask).getReg(Idx: 0);
9324 }
9325
9326 // Be careful about setting nsz/nnan/ninf on every instruction, since the
9327 // constants are a nan and -0.0, but the final result should preserve
9328 // everything.
9329 unsigned Flags = MI.getFlags();
9330
9331 // We masked the sign bit and the not-sign bit, so these are disjoint.
9332 Flags |= MachineInstr::Disjoint;
9333
9334 if (DstTy == DstIntTy)
9335 MIRBuilder.buildOr(Dst, Src0: And0, Src1: And1, Flags).getReg(Idx: 0);
9336 else {
9337 Register NewDst = MIRBuilder.buildOr(Dst: DstIntTy, Src0: And0, Src1: And1, Flags).getReg(Idx: 0);
9338 MIRBuilder.buildBitcast(Dst, Src: NewDst);
9339 }
9340
9341 MI.eraseFromParent();
9342 return Legalized;
9343}
9344
9345LegalizerHelper::LegalizeResult
9346LegalizerHelper::lowerFMinNumMaxNum(MachineInstr &MI) {
9347 // FIXME: fminnum/fmaxnum and fminimumnum/fmaximumnum should not have
9348 // identical handling. fminimumnum/fmaximumnum also need a path that do not
9349 // depend on fminnum/fmaxnum.
9350
9351 unsigned NewOp;
9352 switch (MI.getOpcode()) {
9353 case TargetOpcode::G_FMINNUM:
9354 NewOp = TargetOpcode::G_FMINNUM_IEEE;
9355 break;
9356 case TargetOpcode::G_FMINIMUMNUM:
9357 NewOp = TargetOpcode::G_FMINNUM;
9358 break;
9359 case TargetOpcode::G_FMAXNUM:
9360 NewOp = TargetOpcode::G_FMAXNUM_IEEE;
9361 break;
9362 case TargetOpcode::G_FMAXIMUMNUM:
9363 NewOp = TargetOpcode::G_FMAXNUM;
9364 break;
9365 default:
9366 llvm_unreachable("unexpected min/max opcode");
9367 }
9368
9369 auto [Dst, Src0, Src1] = MI.getFirst3Regs();
9370 LLT Ty = MRI.getType(Reg: Dst);
9371
9372 if (!MI.getFlag(Flag: MachineInstr::FmNoNans)) {
9373 // Insert canonicalizes if it's possible we need to quiet to get correct
9374 // sNaN behavior.
9375
9376 // Note this must be done here, and not as an optimization combine in the
9377 // absence of a dedicate quiet-snan instruction as we're using an
9378 // omni-purpose G_FCANONICALIZE.
9379 if (!VT->isKnownNeverSNaN(Val: Src0))
9380 Src0 = MIRBuilder.buildFCanonicalize(Dst: Ty, Src0, Flags: MI.getFlags()).getReg(Idx: 0);
9381
9382 if (!VT->isKnownNeverSNaN(Val: Src1))
9383 Src1 = MIRBuilder.buildFCanonicalize(Dst: Ty, Src0: Src1, Flags: MI.getFlags()).getReg(Idx: 0);
9384 }
9385
9386 // If there are no nans, it's safe to simply replace this with the non-IEEE
9387 // version.
9388 MIRBuilder.buildInstr(Opc: NewOp, DstOps: {Dst}, SrcOps: {Src0, Src1}, Flags: MI.getFlags());
9389 MI.eraseFromParent();
9390 return Legalized;
9391}
9392
9393LegalizerHelper::LegalizeResult
9394LegalizerHelper::lowerFMinimumMaximum(MachineInstr &MI) {
9395 unsigned Opc = MI.getOpcode();
9396 auto [Dst, Src0, Src1] = MI.getFirst3Regs();
9397 LLT Ty = MRI.getType(Reg: Dst);
9398 const LLT CmpTy = Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9399
9400 bool IsMax = (Opc == TargetOpcode::G_FMAXIMUM);
9401 unsigned OpcIeee =
9402 IsMax ? TargetOpcode::G_FMAXNUM_IEEE : TargetOpcode::G_FMINNUM_IEEE;
9403 unsigned OpcNonIeee =
9404 IsMax ? TargetOpcode::G_FMAXNUM : TargetOpcode::G_FMINNUM;
9405 bool MinMaxMustRespectOrderedZero = false;
9406 Register Res;
9407
9408 // IEEE variants don't need canonicalization
9409 if (LI.isLegalOrCustom(Query: {OpcIeee, Ty})) {
9410 Res = MIRBuilder.buildInstr(Opc: OpcIeee, DstOps: {Ty}, SrcOps: {Src0, Src1}).getReg(Idx: 0);
9411 MinMaxMustRespectOrderedZero = true;
9412 } else if (LI.isLegalOrCustom(Query: {OpcNonIeee, Ty})) {
9413 Res = MIRBuilder.buildInstr(Opc: OpcNonIeee, DstOps: {Ty}, SrcOps: {Src0, Src1}).getReg(Idx: 0);
9414 } else {
9415 auto Compare = MIRBuilder.buildFCmp(
9416 Pred: IsMax ? CmpInst::FCMP_OGT : CmpInst::FCMP_OLT, Res: CmpTy, Op0: Src0, Op1: Src1);
9417 Res = MIRBuilder.buildSelect(Res: Ty, Tst: Compare, Op0: Src0, Op1: Src1).getReg(Idx: 0);
9418 }
9419
9420 // Propagate any NaN of both operands
9421 if (!MI.getFlag(Flag: MachineInstr::FmNoNans) &&
9422 (!VT->isKnownNeverNaN(Val: Src0) || !VT->isKnownNeverNaN(Val: Src1))) {
9423 auto IsOrdered = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_ORD, Res: CmpTy, Op0: Src0, Op1: Src1);
9424
9425 LLT ElementTy = Ty.isScalar() ? Ty : Ty.getElementType();
9426 APFloat NaNValue = APFloat::getNaN(Sem: getFltSemanticForLLT(Ty: ElementTy));
9427 Register NaN = MIRBuilder.buildFConstant(Res: ElementTy, Val: NaNValue).getReg(Idx: 0);
9428 if (Ty.isVector())
9429 NaN = MIRBuilder.buildSplatBuildVector(Res: Ty, Src: NaN).getReg(Idx: 0);
9430
9431 Res = MIRBuilder.buildSelect(Res: Ty, Tst: IsOrdered, Op0: Res, Op1: NaN).getReg(Idx: 0);
9432 }
9433
9434 // fminimum/fmaximum requires -0.0 less than +0.0
9435 if (!MinMaxMustRespectOrderedZero && !MI.getFlag(Flag: MachineInstr::FmNsz)) {
9436 GISelValueTracking VT(MIRBuilder.getMF());
9437 KnownFPClass Src0Info = VT.computeKnownFPClass(R: Src0, InterestedClasses: fcZero);
9438 KnownFPClass Src1Info = VT.computeKnownFPClass(R: Src1, InterestedClasses: fcZero);
9439
9440 if (!Src0Info.isKnownNeverZero() && !Src1Info.isKnownNeverZero()) {
9441 const unsigned Flags = MI.getFlags();
9442 Register Zero = MIRBuilder.buildFConstant(Res: Ty, Val: 0.0).getReg(Idx: 0);
9443 auto IsZero = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OEQ, Res: CmpTy, Op0: Res, Op1: Zero);
9444
9445 unsigned TestClass = IsMax ? fcPosZero : fcNegZero;
9446
9447 auto LHSTestZero = MIRBuilder.buildIsFPClass(Res: CmpTy, Src: Src0, Mask: TestClass);
9448 auto LHSSelect =
9449 MIRBuilder.buildSelect(Res: Ty, Tst: LHSTestZero, Op0: Src0, Op1: Res, Flags);
9450
9451 auto RHSTestZero = MIRBuilder.buildIsFPClass(Res: CmpTy, Src: Src1, Mask: TestClass);
9452 auto RHSSelect =
9453 MIRBuilder.buildSelect(Res: Ty, Tst: RHSTestZero, Op0: Src1, Op1: LHSSelect, Flags);
9454
9455 Res = MIRBuilder.buildSelect(Res: Ty, Tst: IsZero, Op0: RHSSelect, Op1: Res, Flags).getReg(Idx: 0);
9456 }
9457 }
9458
9459 MIRBuilder.buildCopy(Res: Dst, Op: Res);
9460 MI.eraseFromParent();
9461 return Legalized;
9462}
9463
9464LegalizerHelper::LegalizeResult LegalizerHelper::lowerFMad(MachineInstr &MI) {
9465 // Expand G_FMAD a, b, c -> G_FADD (G_FMUL a, b), c
9466 Register DstReg = MI.getOperand(i: 0).getReg();
9467 LLT Ty = MRI.getType(Reg: DstReg);
9468 unsigned Flags = MI.getFlags();
9469
9470 auto Mul = MIRBuilder.buildFMul(Dst: Ty, Src0: MI.getOperand(i: 1), Src1: MI.getOperand(i: 2),
9471 Flags);
9472 MIRBuilder.buildFAdd(Dst: DstReg, Src0: Mul, Src1: MI.getOperand(i: 3), Flags);
9473 MI.eraseFromParent();
9474 return Legalized;
9475}
9476
9477LegalizerHelper::LegalizeResult
9478LegalizerHelper::lowerIntrinsicRound(MachineInstr &MI) {
9479 auto [DstReg, X] = MI.getFirst2Regs();
9480 const unsigned Flags = MI.getFlags();
9481 const LLT Ty = MRI.getType(Reg: DstReg);
9482 const LLT CondTy = Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9483
9484 // round(x) =>
9485 // t = trunc(x);
9486 // d = fabs(x - t);
9487 // o = copysign(d >= 0.5 ? 1.0 : 0.0, x);
9488 // return t + o;
9489
9490 auto T = MIRBuilder.buildIntrinsicTrunc(Dst: Ty, Src0: X, Flags);
9491
9492 auto Diff = MIRBuilder.buildFSub(Dst: Ty, Src0: X, Src1: T, Flags);
9493 auto AbsDiff = MIRBuilder.buildFAbs(Dst: Ty, Src0: Diff, Flags);
9494
9495 auto Half = MIRBuilder.buildFConstant(Res: Ty, Val: 0.5);
9496 auto Cmp =
9497 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OGE, Res: CondTy, Op0: AbsDiff, Op1: Half, Flags);
9498
9499 // Could emit G_UITOFP instead
9500 auto One = MIRBuilder.buildFConstant(Res: Ty, Val: 1.0);
9501 auto Zero = MIRBuilder.buildFConstant(Res: Ty, Val: 0.0);
9502 auto BoolFP = MIRBuilder.buildSelect(Res: Ty, Tst: Cmp, Op0: One, Op1: Zero);
9503 auto SignedOffset = MIRBuilder.buildFCopysign(Dst: Ty, Src0: BoolFP, Src1: X);
9504
9505 MIRBuilder.buildFAdd(Dst: DstReg, Src0: T, Src1: SignedOffset, Flags);
9506
9507 MI.eraseFromParent();
9508 return Legalized;
9509}
9510
9511LegalizerHelper::LegalizeResult LegalizerHelper::lowerFFloor(MachineInstr &MI) {
9512 auto [DstReg, SrcReg] = MI.getFirst2Regs();
9513 unsigned Flags = MI.getFlags();
9514 LLT Ty = MRI.getType(Reg: DstReg);
9515 const LLT CondTy = Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9516
9517 // result = trunc(src);
9518 // if (src < 0.0 && src != result)
9519 // result += -1.0.
9520
9521 auto Trunc = MIRBuilder.buildIntrinsicTrunc(Dst: Ty, Src0: SrcReg, Flags);
9522 auto Zero = MIRBuilder.buildFConstant(Res: Ty, Val: 0.0);
9523
9524 auto Lt0 = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OLT, Res: CondTy,
9525 Op0: SrcReg, Op1: Zero, Flags);
9526 auto NeTrunc = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_ONE, Res: CondTy,
9527 Op0: SrcReg, Op1: Trunc, Flags);
9528 auto And = MIRBuilder.buildAnd(Dst: CondTy, Src0: Lt0, Src1: NeTrunc);
9529 auto AddVal = MIRBuilder.buildSITOFP(Dst: Ty, Src0: And);
9530
9531 MIRBuilder.buildFAdd(Dst: DstReg, Src0: Trunc, Src1: AddVal, Flags);
9532 MI.eraseFromParent();
9533 return Legalized;
9534}
9535
9536LegalizerHelper::LegalizeResult
9537LegalizerHelper::lowerMergeValues(MachineInstr &MI) {
9538 const unsigned NumOps = MI.getNumOperands();
9539 auto [DstReg, DstTy, Src0Reg, Src0Ty] = MI.getFirst2RegLLTs();
9540 unsigned PartSize = Src0Ty.getSizeInBits();
9541
9542 LLT WideTy = LLT::integer(SizeInBits: DstTy.getSizeInBits());
9543 Register ResultReg = MIRBuilder.buildZExt(Res: WideTy, Op: Src0Reg).getReg(Idx: 0);
9544
9545 for (unsigned I = 2; I != NumOps; ++I) {
9546 const unsigned Offset = (I - 1) * PartSize;
9547
9548 Register SrcReg = MI.getOperand(i: I).getReg();
9549 auto ZextInput = MIRBuilder.buildZExt(Res: WideTy, Op: SrcReg);
9550
9551 Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg :
9552 MRI.createGenericVirtualRegister(Ty: WideTy);
9553
9554 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: Offset);
9555 auto Shl = MIRBuilder.buildShl(Dst: WideTy, Src0: ZextInput, Src1: ShiftAmt);
9556 MIRBuilder.buildOr(Dst: NextResult, Src0: ResultReg, Src1: Shl);
9557 ResultReg = NextResult;
9558 }
9559
9560 if (DstTy.isPointer()) {
9561 if (MIRBuilder.getDataLayout().isNonIntegralAddressSpace(
9562 AddrSpace: DstTy.getAddressSpace())) {
9563 LLVM_DEBUG(dbgs() << "Not casting nonintegral address space\n");
9564 return UnableToLegalize;
9565 }
9566
9567 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: ResultReg);
9568 } else if (WideTy != DstTy) {
9569 MIRBuilder.buildBitcast(Dst: DstReg, Src: ResultReg);
9570 }
9571
9572 MI.eraseFromParent();
9573 return Legalized;
9574}
9575
9576LegalizerHelper::LegalizeResult
9577LegalizerHelper::lowerUnmergeValues(MachineInstr &MI) {
9578 const unsigned NumDst = MI.getNumOperands() - 1;
9579 Register SrcReg = MI.getOperand(i: NumDst).getReg();
9580 Register Dst0Reg = MI.getOperand(i: 0).getReg();
9581 LLT DstTy = MRI.getType(Reg: Dst0Reg);
9582 if (DstTy.isPointer())
9583 return UnableToLegalize; // TODO
9584
9585 SrcReg = coerceToInteger(Val: SrcReg);
9586 if (!SrcReg)
9587 return UnableToLegalize;
9588
9589 // Expand scalarizing unmerge as bitcast to integer and shift.
9590 LLT IntTy = MRI.getType(Reg: SrcReg);
9591
9592 MIRBuilder.buildTrunc(Res: Dst0Reg, Op: SrcReg);
9593
9594 const unsigned DstSize = DstTy.getSizeInBits();
9595 unsigned Offset = DstSize;
9596 for (unsigned I = 1; I != NumDst; ++I, Offset += DstSize) {
9597 auto ShiftAmt = MIRBuilder.buildConstant(Res: IntTy, Val: Offset);
9598 auto Shift = MIRBuilder.buildLShr(Dst: IntTy, Src0: SrcReg, Src1: ShiftAmt);
9599 MIRBuilder.buildTrunc(Res: MI.getOperand(i: I), Op: Shift);
9600 }
9601
9602 MI.eraseFromParent();
9603 return Legalized;
9604}
9605
9606/// Lower a vector extract or insert by writing the vector to a stack temporary
9607/// and reloading the element or vector.
9608///
9609/// %dst = G_EXTRACT_VECTOR_ELT %vec, %idx
9610/// =>
9611/// %stack_temp = G_FRAME_INDEX
9612/// G_STORE %vec, %stack_temp
9613/// %idx = clamp(%idx, %vec.getNumElements())
9614/// %element_ptr = G_PTR_ADD %stack_temp, %idx
9615/// %dst = G_LOAD %element_ptr
9616LegalizerHelper::LegalizeResult
9617LegalizerHelper::lowerExtractInsertVectorElt(MachineInstr &MI) {
9618 Register DstReg = MI.getOperand(i: 0).getReg();
9619 Register SrcVec = MI.getOperand(i: 1).getReg();
9620 Register InsertVal;
9621 if (MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)
9622 InsertVal = MI.getOperand(i: 2).getReg();
9623
9624 Register Idx = MI.getOperand(i: MI.getNumOperands() - 1).getReg();
9625
9626 LLT VecTy = MRI.getType(Reg: SrcVec);
9627 LLT EltTy = VecTy.getElementType();
9628 unsigned NumElts = VecTy.getNumElements();
9629
9630 int64_t IdxVal;
9631 if (mi_match(R: Idx, MRI, P: m_ICst(Cst&: IdxVal)) && IdxVal <= NumElts) {
9632 SmallVector<Register, 8> SrcRegs;
9633 extractParts(Reg: SrcVec, Ty: EltTy, NumParts: NumElts, VRegs&: SrcRegs, MIRBuilder, MRI);
9634
9635 if (InsertVal) {
9636 SrcRegs[IdxVal] = MI.getOperand(i: 2).getReg();
9637 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: SrcRegs);
9638 } else {
9639 MIRBuilder.buildCopy(Res: DstReg, Op: SrcRegs[IdxVal]);
9640 }
9641
9642 MI.eraseFromParent();
9643 return Legalized;
9644 }
9645
9646 if (!EltTy.isByteSized()) { // Not implemented.
9647 LLVM_DEBUG(dbgs() << "Can't handle non-byte element vectors yet\n");
9648 return UnableToLegalize;
9649 }
9650
9651 unsigned EltBytes = EltTy.getSizeInBytes();
9652 Align VecAlign = getStackTemporaryAlignment(Ty: VecTy);
9653 Align EltAlign;
9654
9655 MachinePointerInfo PtrInfo;
9656 auto StackTemp = createStackTemporary(
9657 Bytes: TypeSize::getFixed(ExactSize: VecTy.getSizeInBytes()), Alignment: VecAlign, PtrInfo);
9658 MIRBuilder.buildStore(Val: SrcVec, Addr: StackTemp, PtrInfo, Alignment: VecAlign);
9659
9660 // Get the pointer to the element, and be sure not to hit undefined behavior
9661 // if the index is out of bounds.
9662 Register EltPtr = getVectorElementPointer(VecPtr: StackTemp.getReg(Idx: 0), VecTy, Index: Idx);
9663
9664 if (mi_match(R: Idx, MRI, P: m_ICst(Cst&: IdxVal))) {
9665 int64_t Offset = IdxVal * EltBytes;
9666 PtrInfo = PtrInfo.getWithOffset(O: Offset);
9667 EltAlign = commonAlignment(A: VecAlign, Offset);
9668 } else {
9669 // We lose information with a variable offset.
9670 EltAlign = getStackTemporaryAlignment(Ty: EltTy);
9671 PtrInfo = MachinePointerInfo(MRI.getType(Reg: EltPtr).getAddressSpace());
9672 }
9673
9674 if (InsertVal) {
9675 // Write the inserted element
9676 MIRBuilder.buildStore(Val: InsertVal, Addr: EltPtr, PtrInfo, Alignment: EltAlign);
9677
9678 // Reload the whole vector.
9679 MIRBuilder.buildLoad(Res: DstReg, Addr: StackTemp, PtrInfo, Alignment: VecAlign);
9680 } else {
9681 MIRBuilder.buildLoad(Res: DstReg, Addr: EltPtr, PtrInfo, Alignment: EltAlign);
9682 }
9683
9684 MI.eraseFromParent();
9685 return Legalized;
9686}
9687
9688LegalizerHelper::LegalizeResult
9689LegalizerHelper::lowerShuffleVector(MachineInstr &MI) {
9690 auto [DstReg, DstTy, Src0Reg, Src0Ty, Src1Reg, Src1Ty] =
9691 MI.getFirst3RegLLTs();
9692 LLT IdxTy = LLT::scalar(SizeInBits: 32);
9693
9694 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
9695 Register Undef;
9696 SmallVector<Register, 32> BuildVec;
9697 LLT EltTy = DstTy.getScalarType();
9698
9699 DenseMap<unsigned, Register> CachedExtract;
9700
9701 for (int Idx : Mask) {
9702 if (Idx < 0) {
9703 if (!Undef.isValid())
9704 Undef = MIRBuilder.buildUndef(Res: EltTy).getReg(Idx: 0);
9705 BuildVec.push_back(Elt: Undef);
9706 continue;
9707 }
9708
9709 assert(!Src0Ty.isScalar() && "Unexpected scalar G_SHUFFLE_VECTOR");
9710
9711 int NumElts = Src0Ty.getNumElements();
9712 Register SrcVec = Idx < NumElts ? Src0Reg : Src1Reg;
9713 int ExtractIdx = Idx < NumElts ? Idx : Idx - NumElts;
9714 auto [It, Inserted] = CachedExtract.try_emplace(Key: Idx);
9715 if (Inserted) {
9716 auto IdxK = MIRBuilder.buildConstant(Res: IdxTy, Val: ExtractIdx);
9717 It->second =
9718 MIRBuilder.buildExtractVectorElement(Res: EltTy, Val: SrcVec, Idx: IdxK).getReg(Idx: 0);
9719 }
9720 BuildVec.push_back(Elt: It->second);
9721 }
9722
9723 assert(DstTy.isVector() && "Unexpected scalar G_SHUFFLE_VECTOR");
9724 MIRBuilder.buildBuildVector(Res: DstReg, Ops: BuildVec);
9725 MI.eraseFromParent();
9726 return Legalized;
9727}
9728
9729LegalizerHelper::LegalizeResult
9730LegalizerHelper::lowerVECTOR_COMPRESS(llvm::MachineInstr &MI) {
9731 auto [Dst, DstTy, Vec, VecTy, Mask, MaskTy, Passthru, PassthruTy] =
9732 MI.getFirst4RegLLTs();
9733
9734 if (VecTy.isScalableVector())
9735 report_fatal_error(reason: "Cannot expand masked_compress for scalable vectors.");
9736
9737 Align VecAlign = getStackTemporaryAlignment(Ty: VecTy);
9738 MachinePointerInfo PtrInfo;
9739 Register StackPtr =
9740 createStackTemporary(Bytes: TypeSize::getFixed(ExactSize: VecTy.getSizeInBytes()), Alignment: VecAlign,
9741 PtrInfo)
9742 .getReg(Idx: 0);
9743 MachinePointerInfo ValPtrInfo =
9744 MachinePointerInfo::getUnknownStack(MF&: *MI.getMF());
9745
9746 LLT IdxTy = LLT::integer(SizeInBits: 32);
9747 LLT ValTy = VecTy.getElementType();
9748 Align ValAlign = getStackTemporaryAlignment(Ty: ValTy);
9749
9750 auto OutPos = MIRBuilder.buildConstant(Res: IdxTy, Val: 0);
9751
9752 bool HasPassthru = !mi_match(R: Passthru, MRI, P: m_GImplicitDef());
9753
9754 if (HasPassthru)
9755 MIRBuilder.buildStore(Val: Passthru, Addr: StackPtr, PtrInfo, Alignment: VecAlign);
9756
9757 Register LastWriteVal;
9758 std::optional<APInt> PassthruSplatVal =
9759 isConstantOrConstantSplatVector(Def: Passthru, MRI);
9760
9761 if (PassthruSplatVal.has_value()) {
9762 LastWriteVal =
9763 MIRBuilder.buildConstant(Res: ValTy, Val: PassthruSplatVal.value()).getReg(Idx: 0);
9764 } else if (HasPassthru) {
9765 auto Popcount = MIRBuilder.buildZExt(Res: MaskTy.changeElementSize(NewEltSize: 32), Op: Mask);
9766 Popcount = MIRBuilder.buildInstr(Opc: TargetOpcode::G_VECREDUCE_ADD,
9767 DstOps: {LLT::integer(SizeInBits: 32)}, SrcOps: {Popcount});
9768
9769 Register LastElmtPtr =
9770 getVectorElementPointer(VecPtr: StackPtr, VecTy, Index: Popcount.getReg(Idx: 0));
9771 LastWriteVal =
9772 MIRBuilder.buildLoad(Res: ValTy, Addr: LastElmtPtr, PtrInfo: ValPtrInfo, Alignment: ValAlign)
9773 .getReg(Idx: 0);
9774 }
9775
9776 unsigned NumElmts = VecTy.getNumElements();
9777 for (unsigned I = 0; I < NumElmts; ++I) {
9778 auto Idx = MIRBuilder.buildConstant(Res: IdxTy, Val: I);
9779 auto Val = MIRBuilder.buildExtractVectorElement(Res: ValTy, Val: Vec, Idx);
9780 Register ElmtPtr =
9781 getVectorElementPointer(VecPtr: StackPtr, VecTy, Index: OutPos.getReg(Idx: 0));
9782 MIRBuilder.buildStore(Val, Addr: ElmtPtr, PtrInfo: ValPtrInfo, Alignment: ValAlign);
9783
9784 LLT MaskITy = MaskTy.getElementType();
9785 auto MaskI = MIRBuilder.buildExtractVectorElement(Res: MaskITy, Val: Mask, Idx);
9786 if (MaskITy.getSizeInBits() > 1)
9787 MaskI = MIRBuilder.buildTrunc(Res: LLT::integer(SizeInBits: 1), Op: MaskI);
9788
9789 MaskI = MIRBuilder.buildZExt(Res: IdxTy, Op: MaskI);
9790 OutPos = MIRBuilder.buildAdd(Dst: IdxTy, Src0: OutPos, Src1: MaskI);
9791
9792 if (HasPassthru && I == NumElmts - 1) {
9793 auto EndOfVector =
9794 MIRBuilder.buildConstant(Res: IdxTy, Val: VecTy.getNumElements() - 1);
9795 auto AllLanesSelected = MIRBuilder.buildICmp(
9796 Pred: CmpInst::ICMP_UGT, Res: LLT::integer(SizeInBits: 1), Op0: OutPos, Op1: EndOfVector);
9797 OutPos = MIRBuilder.buildInstr(Opc: TargetOpcode::G_UMIN, DstOps: {IdxTy},
9798 SrcOps: {OutPos, EndOfVector});
9799 ElmtPtr = getVectorElementPointer(VecPtr: StackPtr, VecTy, Index: OutPos.getReg(Idx: 0));
9800
9801 LastWriteVal =
9802 MIRBuilder.buildSelect(Res: ValTy, Tst: AllLanesSelected, Op0: Val, Op1: LastWriteVal)
9803 .getReg(Idx: 0);
9804 MIRBuilder.buildStore(Val: LastWriteVal, Addr: ElmtPtr, PtrInfo: ValPtrInfo, Alignment: ValAlign);
9805 }
9806 }
9807
9808 // TODO: Use StackPtr's FrameIndex alignment.
9809 MIRBuilder.buildLoad(Res: Dst, Addr: StackPtr, PtrInfo, Alignment: VecAlign);
9810
9811 MI.eraseFromParent();
9812 return Legalized;
9813}
9814
9815Register LegalizerHelper::getDynStackAllocTargetPtr(Register SPReg,
9816 Register AllocSize,
9817 Align Alignment,
9818 LLT PtrTy) {
9819 LLT IntPtrTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
9820
9821 auto SPTmp = MIRBuilder.buildCopy(Res: PtrTy, Op: SPReg);
9822 SPTmp = MIRBuilder.buildCast(Dst: IntPtrTy, Src: SPTmp);
9823
9824 // Subtract the final alloc from the SP. We use G_PTRTOINT here so we don't
9825 // have to generate an extra instruction to negate the alloc and then use
9826 // G_PTR_ADD to add the negative offset.
9827 auto Alloc = MIRBuilder.buildSub(Dst: IntPtrTy, Src0: SPTmp, Src1: AllocSize);
9828 if (Alignment > Align(1)) {
9829 APInt AlignMask(IntPtrTy.getSizeInBits(), Alignment.value(), true);
9830 AlignMask.negate();
9831 auto AlignCst = MIRBuilder.buildConstant(Res: IntPtrTy, Val: AlignMask);
9832 Alloc = MIRBuilder.buildAnd(Dst: IntPtrTy, Src0: Alloc, Src1: AlignCst);
9833 }
9834
9835 return MIRBuilder.buildCast(Dst: PtrTy, Src: Alloc).getReg(Idx: 0);
9836}
9837
9838LegalizerHelper::LegalizeResult
9839LegalizerHelper::lowerDynStackAlloc(MachineInstr &MI) {
9840 const auto &MF = *MI.getMF();
9841 const auto &TFI = *MF.getSubtarget().getFrameLowering();
9842 if (TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp)
9843 return UnableToLegalize;
9844
9845 Register Dst = MI.getOperand(i: 0).getReg();
9846 Register AllocSize = MI.getOperand(i: 1).getReg();
9847 Align Alignment = assumeAligned(Value: MI.getOperand(i: 2).getImm());
9848
9849 LLT PtrTy = MRI.getType(Reg: Dst);
9850 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
9851 Register SPTmp =
9852 getDynStackAllocTargetPtr(SPReg, AllocSize, Alignment, PtrTy);
9853
9854 MIRBuilder.buildCopy(Res: SPReg, Op: SPTmp);
9855 MIRBuilder.buildCopy(Res: Dst, Op: SPTmp);
9856
9857 MI.eraseFromParent();
9858 return Legalized;
9859}
9860
9861LegalizerHelper::LegalizeResult
9862LegalizerHelper::lowerStackSave(MachineInstr &MI) {
9863 Register StackPtr = TLI.getStackPointerRegisterToSaveRestore();
9864 if (!StackPtr)
9865 return UnableToLegalize;
9866
9867 MIRBuilder.buildCopy(Res: MI.getOperand(i: 0), Op: StackPtr);
9868 MI.eraseFromParent();
9869 return Legalized;
9870}
9871
9872LegalizerHelper::LegalizeResult
9873LegalizerHelper::lowerStackRestore(MachineInstr &MI) {
9874 Register StackPtr = TLI.getStackPointerRegisterToSaveRestore();
9875 if (!StackPtr)
9876 return UnableToLegalize;
9877
9878 MIRBuilder.buildCopy(Res: StackPtr, Op: MI.getOperand(i: 0));
9879 MI.eraseFromParent();
9880 return Legalized;
9881}
9882
9883LegalizerHelper::LegalizeResult
9884LegalizerHelper::lowerExtract(MachineInstr &MI) {
9885 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
9886 unsigned Offset = MI.getOperand(i: 2).getImm();
9887
9888 // Extract sub-vector or one element
9889 if (SrcTy.isVector()) {
9890 unsigned SrcEltSize = SrcTy.getElementType().getSizeInBits();
9891 unsigned DstSize = DstTy.getSizeInBits();
9892
9893 if ((Offset % SrcEltSize == 0) && (DstSize % SrcEltSize == 0) &&
9894 (Offset + DstSize <= SrcTy.getSizeInBits())) {
9895 // Unmerge and allow access to each Src element for the artifact combiner.
9896 auto Unmerge = MIRBuilder.buildUnmerge(Res: SrcTy.getElementType(), Op: SrcReg);
9897
9898 // Take element(s) we need to extract and copy it (merge them).
9899 SmallVector<Register, 8> SubVectorElts;
9900 for (unsigned Idx = Offset / SrcEltSize;
9901 Idx < (Offset + DstSize) / SrcEltSize; ++Idx) {
9902 SubVectorElts.push_back(Elt: Unmerge.getReg(Idx));
9903 }
9904 if (SubVectorElts.size() == 1)
9905 MIRBuilder.buildCopy(Res: DstReg, Op: SubVectorElts[0]);
9906 else
9907 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: SubVectorElts);
9908
9909 MI.eraseFromParent();
9910 return Legalized;
9911 }
9912 }
9913
9914 const DataLayout &DL = MIRBuilder.getDataLayout();
9915 if ((SrcTy.isPointer() &&
9916 DL.isNonIntegralAddressSpace(AddrSpace: SrcTy.getAddressSpace())) ||
9917 (DstTy.isPointer() &&
9918 DL.isNonIntegralAddressSpace(AddrSpace: DstTy.getAddressSpace()))) {
9919 LLVM_DEBUG(dbgs() << "Not casting non-integral address space integer\n");
9920 return UnableToLegalize;
9921 }
9922
9923 if ((DstTy.isScalar() || DstTy.isPointer()) &&
9924 (SrcTy.isScalar() || SrcTy.isPointer() ||
9925 (SrcTy.isVector() && DstTy == SrcTy.getElementType()))) {
9926 LLT SrcIntTy = SrcTy;
9927 if (!SrcTy.isScalar()) {
9928 SrcIntTy = LLT::integer(SizeInBits: SrcTy.getSizeInBits());
9929 SrcReg = MIRBuilder.buildCast(Dst: SrcIntTy, Src: SrcReg).getReg(Idx: 0);
9930 }
9931
9932 Register ResultReg = DstReg;
9933 if (DstTy.isPointer())
9934 ResultReg =
9935 MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: DstTy.getSizeInBits()));
9936
9937 if (Offset == 0)
9938 MIRBuilder.buildTrunc(Res: ResultReg, Op: SrcReg);
9939 else {
9940 auto ShiftAmt = MIRBuilder.buildConstant(Res: SrcIntTy, Val: Offset);
9941 auto Shr = MIRBuilder.buildLShr(Dst: SrcIntTy, Src0: SrcReg, Src1: ShiftAmt);
9942 MIRBuilder.buildTrunc(Res: ResultReg, Op: Shr);
9943 }
9944
9945 if (DstTy.isPointer())
9946 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: ResultReg);
9947
9948 MI.eraseFromParent();
9949 return Legalized;
9950 }
9951
9952 return UnableToLegalize;
9953}
9954
9955LegalizerHelper::LegalizeResult LegalizerHelper::lowerInsert(MachineInstr &MI) {
9956 auto [Dst, Src, InsertSrc] = MI.getFirst3Regs();
9957 uint64_t Offset = MI.getOperand(i: 3).getImm();
9958
9959 LLT DstTy = MRI.getType(Reg: Src);
9960 LLT InsertTy = MRI.getType(Reg: InsertSrc);
9961
9962 const DataLayout &DL = MIRBuilder.getDataLayout();
9963 bool IsNonIntegralInsert =
9964 InsertTy.isPointerOrPointerVector() &&
9965 DL.isNonIntegralAddressSpace(AddrSpace: InsertTy.getAddressSpace());
9966 bool IsNonIntegralDst = DstTy.isPointerOrPointerVector() &&
9967 DL.isNonIntegralAddressSpace(AddrSpace: DstTy.getAddressSpace());
9968
9969 // Insert sub-vector or one element
9970 if (DstTy.isVector()) {
9971 LLT EltTy = DstTy.getElementType();
9972
9973 if ((IsNonIntegralInsert || IsNonIntegralDst) && InsertTy != EltTy) {
9974 LLVM_DEBUG(dbgs() << "Not casting non-integral address space integer\n");
9975 return UnableToLegalize;
9976 }
9977
9978 unsigned EltSize = EltTy.getSizeInBits();
9979 unsigned InsertSize = InsertTy.getSizeInBits();
9980
9981 if ((Offset % EltSize == 0) && (InsertSize % EltSize == 0) &&
9982 (Offset + InsertSize <= DstTy.getSizeInBits())) {
9983 auto UnmergeSrc = MIRBuilder.buildUnmerge(Res: EltTy, Op: Src);
9984 SmallVector<Register, 8> DstElts;
9985 unsigned Idx = 0;
9986 // Elements from Src before insert start Offset
9987 for (; Idx < Offset / EltSize; ++Idx) {
9988 DstElts.push_back(Elt: UnmergeSrc.getReg(Idx));
9989 }
9990
9991 // Replace elements in Src with elements from InsertSrc
9992 if (InsertTy.getSizeInBits() > EltSize) {
9993 auto UnmergeInsertSrc = MIRBuilder.buildUnmerge(Res: EltTy, Op: InsertSrc);
9994 for (unsigned i = 0; Idx < (Offset + InsertSize) / EltSize;
9995 ++Idx, ++i) {
9996 DstElts.push_back(Elt: UnmergeInsertSrc.getReg(Idx: i));
9997 }
9998 } else {
9999 if (InsertTy.isPointer() && !EltTy.isPointer())
10000 InsertSrc = MIRBuilder.buildPtrToInt(Dst: EltTy, Src: InsertSrc).getReg(Idx: 0);
10001 else if (!InsertTy.isPointer() && EltTy.isPointer())
10002 InsertSrc = MIRBuilder.buildIntToPtr(Dst: EltTy, Src: InsertSrc).getReg(Idx: 0);
10003 DstElts.push_back(Elt: InsertSrc);
10004 ++Idx;
10005 }
10006
10007 // Remaining elements from Src after insert
10008 for (; Idx < DstTy.getNumElements(); ++Idx) {
10009 DstElts.push_back(Elt: UnmergeSrc.getReg(Idx));
10010 }
10011
10012 MIRBuilder.buildMergeLikeInstr(Res: Dst, Ops: DstElts);
10013 MI.eraseFromParent();
10014 return Legalized;
10015 }
10016 }
10017
10018 if (InsertTy.isVector() ||
10019 (DstTy.isVector() && DstTy.getElementType() != InsertTy))
10020 return UnableToLegalize;
10021
10022 if (IsNonIntegralDst || IsNonIntegralInsert) {
10023 LLVM_DEBUG(dbgs() << "Not casting non-integral address space integer\n");
10024 return UnableToLegalize;
10025 }
10026
10027 LLT IntDstTy = DstTy;
10028
10029 if (!DstTy.isScalar()) {
10030 IntDstTy = LLT::scalar(SizeInBits: DstTy.getSizeInBits());
10031 Src = MIRBuilder.buildCast(Dst: IntDstTy, Src).getReg(Idx: 0);
10032 }
10033
10034 if (!InsertTy.isScalar()) {
10035 const LLT IntInsertTy = LLT::scalar(SizeInBits: InsertTy.getSizeInBits());
10036 InsertSrc = MIRBuilder.buildPtrToInt(Dst: IntInsertTy, Src: InsertSrc).getReg(Idx: 0);
10037 }
10038
10039 Register ExtInsSrc = MIRBuilder.buildZExt(Res: IntDstTy, Op: InsertSrc).getReg(Idx: 0);
10040 if (Offset != 0) {
10041 auto ShiftAmt = MIRBuilder.buildConstant(Res: IntDstTy, Val: Offset);
10042 ExtInsSrc = MIRBuilder.buildShl(Dst: IntDstTy, Src0: ExtInsSrc, Src1: ShiftAmt).getReg(Idx: 0);
10043 }
10044
10045 APInt MaskVal = APInt::getBitsSetWithWrap(
10046 numBits: DstTy.getSizeInBits(), loBit: Offset + InsertTy.getSizeInBits(), hiBit: Offset);
10047
10048 auto Mask = MIRBuilder.buildConstant(Res: IntDstTy, Val: MaskVal);
10049 auto MaskedSrc = MIRBuilder.buildAnd(Dst: IntDstTy, Src0: Src, Src1: Mask);
10050 auto Or = MIRBuilder.buildOr(Dst: IntDstTy, Src0: MaskedSrc, Src1: ExtInsSrc);
10051
10052 MIRBuilder.buildCast(Dst, Src: Or);
10053 MI.eraseFromParent();
10054 return Legalized;
10055}
10056
10057LegalizerHelper::LegalizeResult
10058LegalizerHelper::lowerSADDO_SSUBO(MachineInstr &MI) {
10059 auto [Dst0, Dst0Ty, Dst1, Dst1Ty, LHS, LHSTy, RHS, RHSTy] =
10060 MI.getFirst4RegLLTs();
10061 const bool IsAdd = MI.getOpcode() == TargetOpcode::G_SADDO;
10062
10063 LLT Ty = Dst0Ty;
10064 LLT BoolTy = Dst1Ty;
10065
10066 Register NewDst0 = MRI.cloneVirtualRegister(VReg: Dst0);
10067
10068 if (IsAdd)
10069 MIRBuilder.buildAdd(Dst: NewDst0, Src0: LHS, Src1: RHS);
10070 else
10071 MIRBuilder.buildSub(Dst: NewDst0, Src0: LHS, Src1: RHS);
10072
10073 // TODO: If SADDSAT/SSUBSAT is legal, compare results to detect overflow.
10074
10075 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
10076
10077 if (IsAdd) {
10078 // For an addition, the result should be less than one of the operands (LHS)
10079 // if and only if the other operand (RHS) is negative, otherwise there will
10080 // be overflow.
10081 auto ResultLowerThanLHS =
10082 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: NewDst0, Op1: LHS);
10083 auto RHSNegative =
10084 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: RHS, Op1: Zero);
10085 MIRBuilder.buildXor(Dst: Dst1, Src0: RHSNegative, Src1: ResultLowerThanLHS);
10086 } else {
10087 // For subtraction, overflow occurs when the signed comparison of operands
10088 // doesn't match the sign of the result.
10089 auto LHSLessThanRHS =
10090 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: LHS, Op1: RHS);
10091 auto ResultNegative =
10092 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: NewDst0, Op1: Zero);
10093 MIRBuilder.buildXor(Dst: Dst1, Src0: LHSLessThanRHS, Src1: ResultNegative);
10094 }
10095
10096 MIRBuilder.buildCopy(Res: Dst0, Op: NewDst0);
10097 MI.eraseFromParent();
10098
10099 return Legalized;
10100}
10101
10102LegalizerHelper::LegalizeResult LegalizerHelper::lowerSADDE(MachineInstr &MI) {
10103 auto [Res, OvOut, LHS, RHS, CarryIn] = MI.getFirst5Regs();
10104 const LLT Ty = MRI.getType(Reg: Res);
10105
10106 // sum = LHS + RHS + zext(CarryIn)
10107 auto Tmp = MIRBuilder.buildAdd(Dst: Ty, Src0: LHS, Src1: RHS);
10108 auto CarryZ = MIRBuilder.buildZExt(Res: Ty, Op: CarryIn);
10109 auto Sum = MIRBuilder.buildAdd(Dst: Ty, Src0: Tmp, Src1: CarryZ);
10110 MIRBuilder.buildCopy(Res, Op: Sum);
10111
10112 // OvOut = icmp slt ((sum ^ lhs) & (sum ^ rhs)), 0
10113 auto AX = MIRBuilder.buildXor(Dst: Ty, Src0: Sum, Src1: LHS);
10114 auto BX = MIRBuilder.buildXor(Dst: Ty, Src0: Sum, Src1: RHS);
10115 auto T = MIRBuilder.buildAnd(Dst: Ty, Src0: AX, Src1: BX);
10116
10117 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
10118 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: OvOut, Op0: T, Op1: Zero);
10119
10120 MI.eraseFromParent();
10121 return Legalized;
10122}
10123
10124LegalizerHelper::LegalizeResult LegalizerHelper::lowerSSUBE(MachineInstr &MI) {
10125 auto [Res, OvOut, LHS, RHS, CarryIn] = MI.getFirst5Regs();
10126 const LLT Ty = MRI.getType(Reg: Res);
10127
10128 // Diff = LHS - (RHS + zext(CarryIn))
10129 auto CarryZ = MIRBuilder.buildZExt(Res: Ty, Op: CarryIn);
10130 auto RHSPlusCI = MIRBuilder.buildAdd(Dst: Ty, Src0: RHS, Src1: CarryZ);
10131 auto Diff = MIRBuilder.buildSub(Dst: Ty, Src0: LHS, Src1: RHSPlusCI);
10132 MIRBuilder.buildCopy(Res, Op: Diff);
10133
10134 // ov = msb((LHS ^ RHS) & (LHS ^ Diff))
10135 auto X1 = MIRBuilder.buildXor(Dst: Ty, Src0: LHS, Src1: RHS);
10136 auto X2 = MIRBuilder.buildXor(Dst: Ty, Src0: LHS, Src1: Diff);
10137 auto T = MIRBuilder.buildAnd(Dst: Ty, Src0: X1, Src1: X2);
10138 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
10139 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: OvOut, Op0: T, Op1: Zero);
10140
10141 MI.eraseFromParent();
10142 return Legalized;
10143}
10144
10145LegalizerHelper::LegalizeResult
10146LegalizerHelper::lowerAddSubSatToMinMax(MachineInstr &MI) {
10147 auto [Res, LHS, RHS] = MI.getFirst3Regs();
10148 LLT Ty = MRI.getType(Reg: Res);
10149 bool IsSigned;
10150 bool IsAdd;
10151 unsigned BaseOp;
10152 switch (MI.getOpcode()) {
10153 default:
10154 llvm_unreachable("unexpected addsat/subsat opcode");
10155 case TargetOpcode::G_UADDSAT:
10156 IsSigned = false;
10157 IsAdd = true;
10158 BaseOp = TargetOpcode::G_ADD;
10159 break;
10160 case TargetOpcode::G_SADDSAT:
10161 IsSigned = true;
10162 IsAdd = true;
10163 BaseOp = TargetOpcode::G_ADD;
10164 break;
10165 case TargetOpcode::G_USUBSAT:
10166 IsSigned = false;
10167 IsAdd = false;
10168 BaseOp = TargetOpcode::G_SUB;
10169 break;
10170 case TargetOpcode::G_SSUBSAT:
10171 IsSigned = true;
10172 IsAdd = false;
10173 BaseOp = TargetOpcode::G_SUB;
10174 break;
10175 }
10176
10177 if (IsSigned) {
10178 // sadd.sat(a, b) ->
10179 // hi = 0x7fffffff - smax(a, 0)
10180 // lo = 0x80000000 - smin(a, 0)
10181 // a + smin(smax(lo, b), hi)
10182 // ssub.sat(a, b) ->
10183 // lo = smax(a, -1) - 0x7fffffff
10184 // hi = smin(a, -1) - 0x80000000
10185 // a - smin(smax(lo, b), hi)
10186 // TODO: AMDGPU can use a "median of 3" instruction here:
10187 // a +/- med3(lo, b, hi)
10188 uint64_t NumBits = Ty.getScalarSizeInBits();
10189 auto MaxVal =
10190 MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMaxValue(numBits: NumBits));
10191 auto MinVal =
10192 MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMinValue(numBits: NumBits));
10193 MachineInstrBuilder Hi, Lo;
10194 if (IsAdd) {
10195 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
10196 Hi = MIRBuilder.buildSub(Dst: Ty, Src0: MaxVal, Src1: MIRBuilder.buildSMax(Dst: Ty, Src0: LHS, Src1: Zero));
10197 Lo = MIRBuilder.buildSub(Dst: Ty, Src0: MinVal, Src1: MIRBuilder.buildSMin(Dst: Ty, Src0: LHS, Src1: Zero));
10198 } else {
10199 auto NegOne = MIRBuilder.buildConstant(Res: Ty, Val: -1);
10200 Lo = MIRBuilder.buildSub(Dst: Ty, Src0: MIRBuilder.buildSMax(Dst: Ty, Src0: LHS, Src1: NegOne),
10201 Src1: MaxVal);
10202 Hi = MIRBuilder.buildSub(Dst: Ty, Src0: MIRBuilder.buildSMin(Dst: Ty, Src0: LHS, Src1: NegOne),
10203 Src1: MinVal);
10204 }
10205 auto RHSClamped =
10206 MIRBuilder.buildSMin(Dst: Ty, Src0: MIRBuilder.buildSMax(Dst: Ty, Src0: Lo, Src1: RHS), Src1: Hi);
10207 MIRBuilder.buildInstr(Opc: BaseOp, DstOps: {Res}, SrcOps: {LHS, RHSClamped});
10208 } else {
10209 // uadd.sat(a, b) -> a + umin(~a, b)
10210 // usub.sat(a, b) -> a - umin(a, b)
10211 Register Not = IsAdd ? MIRBuilder.buildNot(Dst: Ty, Src0: LHS).getReg(Idx: 0) : LHS;
10212 auto Min = MIRBuilder.buildUMin(Dst: Ty, Src0: Not, Src1: RHS);
10213 MIRBuilder.buildInstr(Opc: BaseOp, DstOps: {Res}, SrcOps: {LHS, Min});
10214 }
10215
10216 MI.eraseFromParent();
10217 return Legalized;
10218}
10219
10220LegalizerHelper::LegalizeResult
10221LegalizerHelper::lowerAddSubSatToAddoSubo(MachineInstr &MI) {
10222 auto [Res, LHS, RHS] = MI.getFirst3Regs();
10223 LLT Ty = MRI.getType(Reg: Res);
10224 LLT BoolTy = Ty.changeElementSize(NewEltSize: 1);
10225 bool IsSigned;
10226 bool IsAdd;
10227 unsigned OverflowOp;
10228 switch (MI.getOpcode()) {
10229 default:
10230 llvm_unreachable("unexpected addsat/subsat opcode");
10231 case TargetOpcode::G_UADDSAT:
10232 IsSigned = false;
10233 IsAdd = true;
10234 OverflowOp = TargetOpcode::G_UADDO;
10235 break;
10236 case TargetOpcode::G_SADDSAT:
10237 IsSigned = true;
10238 IsAdd = true;
10239 OverflowOp = TargetOpcode::G_SADDO;
10240 break;
10241 case TargetOpcode::G_USUBSAT:
10242 IsSigned = false;
10243 IsAdd = false;
10244 OverflowOp = TargetOpcode::G_USUBO;
10245 break;
10246 case TargetOpcode::G_SSUBSAT:
10247 IsSigned = true;
10248 IsAdd = false;
10249 OverflowOp = TargetOpcode::G_SSUBO;
10250 break;
10251 }
10252
10253 auto OverflowRes =
10254 MIRBuilder.buildInstr(Opc: OverflowOp, DstOps: {Ty, BoolTy}, SrcOps: {LHS, RHS});
10255 Register Tmp = OverflowRes.getReg(Idx: 0);
10256 Register Ov = OverflowRes.getReg(Idx: 1);
10257 MachineInstrBuilder Clamp;
10258 if (IsSigned) {
10259 // sadd.sat(a, b) ->
10260 // {tmp, ov} = saddo(a, b)
10261 // ov ? (tmp >>s 31) + 0x80000000 : r
10262 // ssub.sat(a, b) ->
10263 // {tmp, ov} = ssubo(a, b)
10264 // ov ? (tmp >>s 31) + 0x80000000 : r
10265 uint64_t NumBits = Ty.getScalarSizeInBits();
10266 auto ShiftAmount = MIRBuilder.buildConstant(Res: Ty, Val: NumBits - 1);
10267 auto Sign = MIRBuilder.buildAShr(Dst: Ty, Src0: Tmp, Src1: ShiftAmount);
10268 auto MinVal =
10269 MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMinValue(numBits: NumBits));
10270 Clamp = MIRBuilder.buildAdd(Dst: Ty, Src0: Sign, Src1: MinVal);
10271 } else {
10272 // uadd.sat(a, b) ->
10273 // {tmp, ov} = uaddo(a, b)
10274 // ov ? 0xffffffff : tmp
10275 // usub.sat(a, b) ->
10276 // {tmp, ov} = usubo(a, b)
10277 // ov ? 0 : tmp
10278 Clamp = MIRBuilder.buildConstant(Res: Ty, Val: IsAdd ? -1 : 0);
10279 }
10280 MIRBuilder.buildSelect(Res, Tst: Ov, Op0: Clamp, Op1: Tmp);
10281
10282 MI.eraseFromParent();
10283 return Legalized;
10284}
10285
10286LegalizerHelper::LegalizeResult
10287LegalizerHelper::lowerShlSat(MachineInstr &MI) {
10288 assert((MI.getOpcode() == TargetOpcode::G_SSHLSAT ||
10289 MI.getOpcode() == TargetOpcode::G_USHLSAT) &&
10290 "Expected shlsat opcode!");
10291 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SSHLSAT;
10292 auto [Res, LHS, RHS] = MI.getFirst3Regs();
10293 LLT Ty = MRI.getType(Reg: Res);
10294 LLT BoolTy = Ty.changeElementSize(NewEltSize: 1);
10295
10296 unsigned BW = Ty.getScalarSizeInBits();
10297 auto Result = MIRBuilder.buildShl(Dst: Ty, Src0: LHS, Src1: RHS);
10298 auto Orig = IsSigned ? MIRBuilder.buildAShr(Dst: Ty, Src0: Result, Src1: RHS)
10299 : MIRBuilder.buildLShr(Dst: Ty, Src0: Result, Src1: RHS);
10300
10301 MachineInstrBuilder SatVal;
10302 if (IsSigned) {
10303 auto SatMin = MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMinValue(numBits: BW));
10304 auto SatMax = MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMaxValue(numBits: BW));
10305 auto Cmp = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: LHS,
10306 Op1: MIRBuilder.buildConstant(Res: Ty, Val: 0));
10307 SatVal = MIRBuilder.buildSelect(Res: Ty, Tst: Cmp, Op0: SatMin, Op1: SatMax);
10308 } else {
10309 SatVal = MIRBuilder.buildConstant(Res: Ty, Val: APInt::getMaxValue(numBits: BW));
10310 }
10311 auto Ov = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: BoolTy, Op0: LHS, Op1: Orig);
10312 MIRBuilder.buildSelect(Res, Tst: Ov, Op0: SatVal, Op1: Result);
10313
10314 MI.eraseFromParent();
10315 return Legalized;
10316}
10317
10318LegalizerHelper::LegalizeResult
10319LegalizerHelper::lowerTruncSat(MachineInstr &MI) {
10320 unsigned Opc = MI.getOpcode();
10321 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
10322 unsigned DstSize = DstTy.getScalarSizeInBits();
10323 unsigned SrcSize = SrcTy.getScalarSizeInBits();
10324
10325 if (Opc == TargetOpcode::G_TRUNC_SSAT_S) {
10326 auto Max = MIRBuilder.buildConstant(
10327 Res: SrcTy, Val: APInt::getSignedMaxValue(numBits: DstSize).sext(width: SrcSize));
10328 Src = MIRBuilder.buildSMin(Dst: SrcTy, Src0: Src, Src1: Max).getReg(Idx: 0);
10329 auto Min = MIRBuilder.buildConstant(
10330 Res: SrcTy, Val: APInt::getSignedMinValue(numBits: DstSize).sext(width: SrcSize));
10331 Src = MIRBuilder.buildSMax(Dst: SrcTy, Src0: Src, Src1: Min).getReg(Idx: 0);
10332 } else if (Opc == TargetOpcode::G_TRUNC_USAT_U) {
10333 auto Max = MIRBuilder.buildConstant(
10334 Res: SrcTy, Val: APInt::getAllOnes(numBits: DstSize).zext(width: SrcSize));
10335 Src = MIRBuilder.buildUMin(Dst: SrcTy, Src0: Src, Src1: Max).getReg(Idx: 0);
10336 } else if (Opc == TargetOpcode::G_TRUNC_SSAT_U) {
10337 auto Max = MIRBuilder.buildConstant(
10338 Res: SrcTy, Val: APInt::getAllOnes(numBits: DstSize).zext(width: SrcSize));
10339 Src = MIRBuilder.buildSMin(Dst: SrcTy, Src0: Src, Src1: Max).getReg(Idx: 0);
10340 auto Min = MIRBuilder.buildConstant(Res: SrcTy, Val: APInt::getZero(numBits: SrcSize));
10341 Src = MIRBuilder.buildSMax(Dst: SrcTy, Src0: Src, Src1: Min).getReg(Idx: 0);
10342 } else {
10343 llvm_unreachable("Expected truncsat opcode!");
10344 }
10345
10346 MIRBuilder.buildTrunc(Res: Dst, Op: Src);
10347 MI.eraseFromParent();
10348 return Legalized;
10349}
10350
10351LegalizerHelper::LegalizeResult LegalizerHelper::lowerBswap(MachineInstr &MI) {
10352 auto [Dst, Src] = MI.getFirst2Regs();
10353 const LLT Ty = MRI.getType(Reg: Src);
10354 unsigned SizeInBytes = (Ty.getScalarSizeInBits() + 7) / 8;
10355 unsigned BaseShiftAmt = (SizeInBytes - 1) * 8;
10356
10357 // Swap most and least significant byte, set remaining bytes in Res to zero.
10358 auto ShiftAmt = MIRBuilder.buildConstant(Res: Ty, Val: BaseShiftAmt);
10359 auto LSByteShiftedLeft = MIRBuilder.buildShl(Dst: Ty, Src0: Src, Src1: ShiftAmt);
10360 auto MSByteShiftedRight = MIRBuilder.buildLShr(Dst: Ty, Src0: Src, Src1: ShiftAmt);
10361 auto Res = MIRBuilder.buildOr(Dst: Ty, Src0: MSByteShiftedRight, Src1: LSByteShiftedLeft);
10362
10363 // Set i-th high/low byte in Res to i-th low/high byte from Src.
10364 for (unsigned i = 1; i < SizeInBytes / 2; ++i) {
10365 // AND with Mask leaves byte i unchanged and sets remaining bytes to 0.
10366 APInt APMask = APInt::getBitsSet(numBits: SizeInBytes * 8, loBit: i * 8, hiBit: i * 8 + 8);
10367 auto Mask = MIRBuilder.buildConstant(Res: Ty, Val: APMask);
10368 auto ShiftAmt = MIRBuilder.buildConstant(Res: Ty, Val: BaseShiftAmt - 16 * i);
10369 // Low byte shifted left to place of high byte: (Src & Mask) << ShiftAmt.
10370 auto LoByte = MIRBuilder.buildAnd(Dst: Ty, Src0: Src, Src1: Mask);
10371 auto LoShiftedLeft = MIRBuilder.buildShl(Dst: Ty, Src0: LoByte, Src1: ShiftAmt);
10372 Res = MIRBuilder.buildOr(Dst: Ty, Src0: Res, Src1: LoShiftedLeft);
10373 // High byte shifted right to place of low byte: (Src >> ShiftAmt) & Mask.
10374 auto SrcShiftedRight = MIRBuilder.buildLShr(Dst: Ty, Src0: Src, Src1: ShiftAmt);
10375 auto HiShiftedRight = MIRBuilder.buildAnd(Dst: Ty, Src0: SrcShiftedRight, Src1: Mask);
10376 Res = MIRBuilder.buildOr(Dst: Ty, Src0: Res, Src1: HiShiftedRight);
10377 }
10378 Res.getInstr()->getOperand(i: 0).setReg(Dst);
10379
10380 MI.eraseFromParent();
10381 return Legalized;
10382}
10383
10384//{ (Src & Mask) >> N } | { (Src << N) & Mask }
10385static MachineInstrBuilder SwapN(unsigned N, DstOp Dst, MachineIRBuilder &B,
10386 MachineInstrBuilder Src, const APInt &Mask) {
10387 const LLT Ty = Dst.getLLTTy(MRI: *B.getMRI());
10388 MachineInstrBuilder C_N = B.buildConstant(Res: Ty, Val: N);
10389 MachineInstrBuilder MaskLoNTo0 = B.buildConstant(Res: Ty, Val: Mask);
10390 auto LHS = B.buildLShr(Dst: Ty, Src0: B.buildAnd(Dst: Ty, Src0: Src, Src1: MaskLoNTo0), Src1: C_N);
10391 auto RHS = B.buildAnd(Dst: Ty, Src0: B.buildShl(Dst: Ty, Src0: Src, Src1: C_N), Src1: MaskLoNTo0);
10392 return B.buildOr(Dst, Src0: LHS, Src1: RHS);
10393}
10394
10395LegalizerHelper::LegalizeResult
10396LegalizerHelper::lowerBitreverse(MachineInstr &MI) {
10397 auto [Dst, Src] = MI.getFirst2Regs();
10398 const LLT SrcTy = MRI.getType(Reg: Src);
10399 unsigned Size = SrcTy.getScalarSizeInBits();
10400 unsigned VSize = SrcTy.getSizeInBits();
10401
10402 if (Size >= 8) {
10403 if (SrcTy.isVector() && (VSize % 8 == 0) &&
10404 (LI.isLegal(Query: {TargetOpcode::G_BITREVERSE,
10405 {LLT::fixed_vector(NumElements: VSize / 8, ScalarTy: LLT::integer(SizeInBits: 8)),
10406 LLT::fixed_vector(NumElements: VSize / 8, ScalarTy: LLT::integer(SizeInBits: 8))}}))) {
10407 // If bitreverse is legal for i8 vector of the same size, then cast
10408 // to i8 vector type.
10409 // e.g. v4s32 -> v16s8
10410 LLT VTy = LLT::fixed_vector(NumElements: VSize / 8, ScalarTy: LLT::integer(SizeInBits: 8));
10411 auto BSWAP = MIRBuilder.buildBSwap(Dst: SrcTy, Src0: Src);
10412 auto Cast = MIRBuilder.buildBitcast(Dst: VTy, Src: BSWAP);
10413 auto RBIT = MIRBuilder.buildBitReverse(Dst: VTy, Src: Cast);
10414 MIRBuilder.buildBitcast(Dst, Src: RBIT);
10415 } else {
10416 MachineInstrBuilder BSWAP =
10417 MIRBuilder.buildInstr(Opc: TargetOpcode::G_BSWAP, DstOps: {SrcTy}, SrcOps: {Src});
10418
10419 // swap high and low 4 bits in 8 bit blocks 7654|3210 -> 3210|7654
10420 // [(val & 0xF0F0F0F0) >> 4] | [(val & 0x0F0F0F0F) << 4]
10421 // -> [(val & 0xF0F0F0F0) >> 4] | [(val << 4) & 0xF0F0F0F0]
10422 MachineInstrBuilder Swap4 = SwapN(N: 4, Dst: SrcTy, B&: MIRBuilder, Src: BSWAP,
10423 Mask: APInt::getSplat(NewLen: Size, V: APInt(8, 0xF0)));
10424
10425 // swap high and low 2 bits in 4 bit blocks 32|10 76|54 -> 10|32 54|76
10426 // [(val & 0xCCCCCCCC) >> 2] & [(val & 0x33333333) << 2]
10427 // -> [(val & 0xCCCCCCCC) >> 2] & [(val << 2) & 0xCCCCCCCC]
10428 MachineInstrBuilder Swap2 = SwapN(N: 2, Dst: SrcTy, B&: MIRBuilder, Src: Swap4,
10429 Mask: APInt::getSplat(NewLen: Size, V: APInt(8, 0xCC)));
10430
10431 // swap high and low 1 bit in 2 bit blocks 1|0 3|2 5|4 7|6 -> 0|1 2|3 4|5
10432 // 6|7
10433 // [(val & 0xAAAAAAAA) >> 1] & [(val & 0x55555555) << 1]
10434 // -> [(val & 0xAAAAAAAA) >> 1] & [(val << 1) & 0xAAAAAAAA]
10435 SwapN(N: 1, Dst, B&: MIRBuilder, Src: Swap2, Mask: APInt::getSplat(NewLen: Size, V: APInt(8, 0xAA)));
10436 }
10437 } else {
10438 // Expand bitreverse for types smaller than 8 bits.
10439 MachineInstrBuilder Tmp;
10440 for (unsigned I = 0, J = Size - 1; I < Size; ++I, --J) {
10441 MachineInstrBuilder Tmp2;
10442 if (I < J) {
10443 auto ShAmt = MIRBuilder.buildConstant(Res: SrcTy, Val: J - I);
10444 Tmp2 = MIRBuilder.buildShl(Dst: SrcTy, Src0: Src, Src1: ShAmt);
10445 } else {
10446 auto ShAmt = MIRBuilder.buildConstant(Res: SrcTy, Val: I - J);
10447 Tmp2 = MIRBuilder.buildLShr(Dst: SrcTy, Src0: Src, Src1: ShAmt);
10448 }
10449
10450 auto Mask = MIRBuilder.buildConstant(Res: SrcTy, Val: 1ULL << J);
10451 Tmp2 = MIRBuilder.buildAnd(Dst: SrcTy, Src0: Tmp2, Src1: Mask);
10452 if (I == 0)
10453 Tmp = Tmp2;
10454 else
10455 Tmp = MIRBuilder.buildOr(Dst: SrcTy, Src0: Tmp, Src1: Tmp2);
10456 }
10457 MIRBuilder.buildCopy(Res: Dst, Op: Tmp);
10458 }
10459
10460 MI.eraseFromParent();
10461 return Legalized;
10462}
10463
10464LegalizerHelper::LegalizeResult
10465LegalizerHelper::lowerReadWriteRegister(MachineInstr &MI) {
10466 MachineFunction &MF = MIRBuilder.getMF();
10467
10468 bool IsRead = MI.getOpcode() == TargetOpcode::G_READ_REGISTER;
10469 int NameOpIdx = IsRead ? 1 : 0;
10470 int ValRegIndex = IsRead ? 0 : 1;
10471
10472 Register ValReg = MI.getOperand(i: ValRegIndex).getReg();
10473 const LLT Ty = MRI.getType(Reg: ValReg);
10474 const MDString *RegStr = cast<MDString>(
10475 Val: cast<MDNode>(Val: MI.getOperand(i: NameOpIdx).getMetadata())->getOperand(I: 0));
10476
10477 Register PhysReg = TLI.getRegisterByName(RegName: RegStr->getString().data(), Ty, MF);
10478 if (!PhysReg) {
10479 const Function &Fn = MF.getFunction();
10480 Fn.getContext().diagnose(DI: DiagnosticInfoGenericWithLoc(
10481 "invalid register \"" + Twine(RegStr->getString().data()) + "\" for " +
10482 (IsRead ? "llvm.read_register" : "llvm.write_register"),
10483 Fn, MI.getDebugLoc()));
10484 if (IsRead)
10485 MIRBuilder.buildUndef(Res: ValReg);
10486
10487 MI.eraseFromParent();
10488 return Legalized;
10489 }
10490
10491 if (IsRead)
10492 MIRBuilder.buildCopy(Res: ValReg, Op: PhysReg);
10493 else
10494 MIRBuilder.buildCopy(Res: PhysReg, Op: ValReg);
10495
10496 MI.eraseFromParent();
10497 return Legalized;
10498}
10499
10500LegalizerHelper::LegalizeResult
10501LegalizerHelper::lowerSMULH_UMULH(MachineInstr &MI) {
10502 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SMULH;
10503 unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
10504 Register Result = MI.getOperand(i: 0).getReg();
10505 LLT OrigTy = MRI.getType(Reg: Result);
10506 auto SizeInBits = OrigTy.getScalarSizeInBits();
10507 LLT WideTy = OrigTy.changeElementSize(NewEltSize: SizeInBits * 2);
10508
10509 auto LHS = MIRBuilder.buildInstr(Opc: ExtOp, DstOps: {WideTy}, SrcOps: {MI.getOperand(i: 1)});
10510 auto RHS = MIRBuilder.buildInstr(Opc: ExtOp, DstOps: {WideTy}, SrcOps: {MI.getOperand(i: 2)});
10511 auto Mul = MIRBuilder.buildMul(Dst: WideTy, Src0: LHS, Src1: RHS);
10512 unsigned ShiftOp = IsSigned ? TargetOpcode::G_ASHR : TargetOpcode::G_LSHR;
10513
10514 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: SizeInBits);
10515 auto Shifted = MIRBuilder.buildInstr(Opc: ShiftOp, DstOps: {WideTy}, SrcOps: {Mul, ShiftAmt});
10516 MIRBuilder.buildTrunc(Res: Result, Op: Shifted);
10517
10518 MI.eraseFromParent();
10519 return Legalized;
10520}
10521
10522LegalizerHelper::LegalizeResult
10523LegalizerHelper::lowerISFPCLASS(MachineInstr &MI) {
10524 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
10525 FPClassTest Mask = static_cast<FPClassTest>(MI.getOperand(i: 2).getImm());
10526
10527 if (Mask == fcNone) {
10528 MIRBuilder.buildConstant(Res: DstReg, Val: 0);
10529 MI.eraseFromParent();
10530 return Legalized;
10531 }
10532 if (Mask == fcAllFlags) {
10533 MIRBuilder.buildConstant(Res: DstReg, Val: 1);
10534 MI.eraseFromParent();
10535 return Legalized;
10536 }
10537
10538 // TODO: Try inverting the test with getInvertedFPClassTest like the DAG
10539 // version
10540
10541 unsigned BitSize = SrcTy.getScalarSizeInBits();
10542 const fltSemantics &Semantics = getFltSemanticForLLT(Ty: SrcTy.getScalarType());
10543
10544 LLT IntTy = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: BitSize));
10545 auto AsInt = SrcTy == IntTy ? MIRBuilder.buildCopy(Res: IntTy, Op: SrcReg)
10546 : MIRBuilder.buildBitcast(Dst: IntTy, Src: SrcReg);
10547
10548 // Various masks.
10549 APInt SignBit = APInt::getSignMask(BitWidth: BitSize);
10550 APInt ValueMask = APInt::getSignedMaxValue(numBits: BitSize); // All bits but sign.
10551 APInt Inf = APFloat::getInf(Sem: Semantics).bitcastToAPInt(); // Exp and int bit.
10552 APInt ExpMask = Inf;
10553 APInt AllOneMantissa = APFloat::getLargest(Sem: Semantics).bitcastToAPInt() & ~Inf;
10554 APInt QNaNBitMask =
10555 APInt::getOneBitSet(numBits: BitSize, BitNo: AllOneMantissa.getActiveBits() - 1);
10556 APInt InversionMask = APInt::getAllOnes(numBits: DstTy.getScalarSizeInBits());
10557
10558 auto SignBitC = MIRBuilder.buildConstant(Res: IntTy, Val: SignBit);
10559 auto ValueMaskC = MIRBuilder.buildConstant(Res: IntTy, Val: ValueMask);
10560 auto InfC = MIRBuilder.buildConstant(Res: IntTy, Val: Inf);
10561 auto ExpMaskC = MIRBuilder.buildConstant(Res: IntTy, Val: ExpMask);
10562 auto ZeroC = MIRBuilder.buildConstant(Res: IntTy, Val: 0);
10563
10564 auto Abs = MIRBuilder.buildAnd(Dst: IntTy, Src0: AsInt, Src1: ValueMaskC);
10565 auto Sign =
10566 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_NE, Res: DstTy, Op0: AsInt, Op1: Abs);
10567
10568 auto Res = MIRBuilder.buildConstant(Res: DstTy, Val: 0);
10569 // Clang doesn't support capture of structured bindings:
10570 LLT DstTyCopy = DstTy;
10571 const auto appendToRes = [&](MachineInstrBuilder ToAppend) {
10572 Res = MIRBuilder.buildOr(Dst: DstTyCopy, Src0: Res, Src1: ToAppend);
10573 };
10574
10575 // Tests that involve more than one class should be processed first.
10576 if ((Mask & fcFinite) == fcFinite) {
10577 // finite(V) ==> abs(V) u< exp_mask
10578 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: Abs,
10579 Op1: ExpMaskC));
10580 Mask &= ~fcFinite;
10581 } else if ((Mask & fcFinite) == fcPosFinite) {
10582 // finite(V) && V > 0 ==> V u< exp_mask
10583 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: AsInt,
10584 Op1: ExpMaskC));
10585 Mask &= ~fcPosFinite;
10586 } else if ((Mask & fcFinite) == fcNegFinite) {
10587 // finite(V) && V < 0 ==> abs(V) u< exp_mask && signbit == 1
10588 auto Cmp = MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: Abs,
10589 Op1: ExpMaskC);
10590 auto And = MIRBuilder.buildAnd(Dst: DstTy, Src0: Cmp, Src1: Sign);
10591 appendToRes(And);
10592 Mask &= ~fcNegFinite;
10593 }
10594
10595 if (FPClassTest PartialCheck = Mask & (fcZero | fcSubnormal)) {
10596 // fcZero | fcSubnormal => test all exponent bits are 0
10597 // TODO: Handle sign bit specific cases
10598 // TODO: Handle inverted case
10599 if (PartialCheck == (fcZero | fcSubnormal)) {
10600 auto ExpBits = MIRBuilder.buildAnd(Dst: IntTy, Src0: AsInt, Src1: ExpMaskC);
10601 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10602 Op0: ExpBits, Op1: ZeroC));
10603 Mask &= ~PartialCheck;
10604 }
10605 }
10606
10607 // Check for individual classes.
10608 if (FPClassTest PartialCheck = Mask & fcZero) {
10609 if (PartialCheck == fcPosZero)
10610 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10611 Op0: AsInt, Op1: ZeroC));
10612 else if (PartialCheck == fcZero)
10613 appendToRes(
10614 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy, Op0: Abs, Op1: ZeroC));
10615 else // fcNegZero
10616 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10617 Op0: AsInt, Op1: SignBitC));
10618 }
10619
10620 if (FPClassTest PartialCheck = Mask & fcSubnormal) {
10621 // issubnormal(V) ==> unsigned(abs(V) - 1) u< (all mantissa bits set)
10622 // issubnormal(V) && V>0 ==> unsigned(V - 1) u< (all mantissa bits set)
10623 auto V = (PartialCheck == fcPosSubnormal) ? AsInt : Abs;
10624 auto OneC = MIRBuilder.buildConstant(Res: IntTy, Val: 1);
10625 auto VMinusOne = MIRBuilder.buildSub(Dst: IntTy, Src0: V, Src1: OneC);
10626 auto SubnormalRes =
10627 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: VMinusOne,
10628 Op1: MIRBuilder.buildConstant(Res: IntTy, Val: AllOneMantissa));
10629 if (PartialCheck == fcNegSubnormal)
10630 SubnormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: SubnormalRes, Src1: Sign);
10631 appendToRes(SubnormalRes);
10632 }
10633
10634 if (FPClassTest PartialCheck = Mask & fcInf) {
10635 if (PartialCheck == fcPosInf)
10636 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10637 Op0: AsInt, Op1: InfC));
10638 else if (PartialCheck == fcInf)
10639 appendToRes(
10640 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy, Op0: Abs, Op1: InfC));
10641 else { // fcNegInf
10642 APInt NegInf = APFloat::getInf(Sem: Semantics, Negative: true).bitcastToAPInt();
10643 auto NegInfC = MIRBuilder.buildConstant(Res: IntTy, Val: NegInf);
10644 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10645 Op0: AsInt, Op1: NegInfC));
10646 }
10647 }
10648
10649 if (FPClassTest PartialCheck = Mask & fcNan) {
10650 auto InfWithQnanBitC = MIRBuilder.buildConstant(Res: IntTy, Val: Inf | QNaNBitMask);
10651 if (PartialCheck == fcNan) {
10652 // isnan(V) ==> abs(V) u> int(inf)
10653 appendToRes(
10654 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: DstTy, Op0: Abs, Op1: InfC));
10655 } else if (PartialCheck == fcQNan) {
10656 // isquiet(V) ==> abs(V) u>= (unsigned(Inf) | quiet_bit)
10657 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGE, Res: DstTy, Op0: Abs,
10658 Op1: InfWithQnanBitC));
10659 } else { // fcSNan
10660 // issignaling(V) ==> abs(V) u> unsigned(Inf) &&
10661 // abs(V) u< (unsigned(Inf) | quiet_bit)
10662 auto IsNan =
10663 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: DstTy, Op0: Abs, Op1: InfC);
10664 auto IsNotQnan = MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy,
10665 Op0: Abs, Op1: InfWithQnanBitC);
10666 appendToRes(MIRBuilder.buildAnd(Dst: DstTy, Src0: IsNan, Src1: IsNotQnan));
10667 }
10668 }
10669
10670 if (FPClassTest PartialCheck = Mask & fcNormal) {
10671 // isnormal(V) ==> (0 u< exp u< max_exp) ==> (unsigned(exp-1) u<
10672 // (max_exp-1))
10673 APInt ExpLSB = ExpMask & ~(ExpMask.shl(shiftAmt: 1));
10674 auto ExpMinusOne = MIRBuilder.buildSub(
10675 Dst: IntTy, Src0: Abs, Src1: MIRBuilder.buildConstant(Res: IntTy, Val: ExpLSB));
10676 APInt MaxExpMinusOne = ExpMask - ExpLSB;
10677 auto NormalRes =
10678 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: ExpMinusOne,
10679 Op1: MIRBuilder.buildConstant(Res: IntTy, Val: MaxExpMinusOne));
10680 if (PartialCheck == fcNegNormal)
10681 NormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: NormalRes, Src1: Sign);
10682 else if (PartialCheck == fcPosNormal) {
10683 auto PosSign = MIRBuilder.buildXor(
10684 Dst: DstTy, Src0: Sign, Src1: MIRBuilder.buildConstant(Res: DstTy, Val: InversionMask));
10685 NormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: NormalRes, Src1: PosSign);
10686 }
10687 appendToRes(NormalRes);
10688 }
10689
10690 MIRBuilder.buildCopy(Res: DstReg, Op: Res);
10691 MI.eraseFromParent();
10692 return Legalized;
10693}
10694
10695LegalizerHelper::LegalizeResult LegalizerHelper::lowerSelect(MachineInstr &MI) {
10696 // Implement G_SELECT in terms of XOR, AND, OR.
10697 auto [DstReg, DstTy, MaskReg, MaskTy, Op1Reg, Op1Ty, Op2Reg, Op2Ty] =
10698 MI.getFirst4RegLLTs();
10699
10700 LLT Op1TyInt =
10701 Op1Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: Op1Ty.getScalarSizeInBits()));
10702
10703 bool IsEltPtr = DstTy.isPointerOrPointerVector();
10704 if (IsEltPtr) {
10705 LLT ScalarPtrTy = LLT::integer(SizeInBits: DstTy.getScalarSizeInBits());
10706 LLT NewTy = DstTy.changeElementType(NewEltTy: ScalarPtrTy);
10707 Op1Reg = MIRBuilder.buildPtrToInt(Dst: NewTy, Src: Op1Reg).getReg(Idx: 0);
10708 Op1Ty = MRI.getType(Reg: Op1Reg);
10709 Op2Reg = MIRBuilder.buildPtrToInt(Dst: NewTy, Src: Op2Reg).getReg(Idx: 0);
10710 Op2Ty = MRI.getType(Reg: Op2Reg);
10711 DstTy = NewTy;
10712 }
10713
10714 if (MaskTy.isScalar()) {
10715 // Turn the scalar condition into a vector condition mask if needed.
10716
10717 Register MaskElt = MaskReg;
10718
10719 // The condition was potentially zero extended before, but we want a sign
10720 // extended boolean.
10721 if (MaskTy != LLT::scalar(SizeInBits: 1))
10722 MaskElt = MIRBuilder.buildSExtInReg(Res: MaskTy, Op: MaskElt, ImmOp: 1).getReg(Idx: 0);
10723
10724 // Continue the sign extension (or truncate) to match the data type.
10725 MaskTy = DstTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: DstTy.getScalarSizeInBits()));
10726 MaskElt =
10727 MIRBuilder.buildSExtOrTrunc(Res: MaskTy.getScalarType(), Op: MaskElt).getReg(Idx: 0);
10728
10729 if (DstTy.isVector()) {
10730 // Generate a vector splat idiom.
10731 auto ShufSplat = MIRBuilder.buildShuffleSplat(Res: MaskTy, Src: MaskElt);
10732 MaskReg = ShufSplat.getReg(Idx: 0);
10733 } else {
10734 MaskReg = MaskElt;
10735 }
10736 } else if (!DstTy.isVector()) {
10737 // Cannot handle the case that mask is a vector and dst is a scalar.
10738 return UnableToLegalize;
10739 }
10740
10741 if (MaskTy.getSizeInBits() != DstTy.getSizeInBits()) {
10742 return UnableToLegalize;
10743 }
10744
10745 if (!Op1Ty.getScalarType().isAnyScalar() &&
10746 !Op1Ty.getScalarType().isInteger())
10747 Op1Reg = MIRBuilder.buildBitcast(Dst: Op1TyInt, Src: Op1Reg).getReg(Idx: 0);
10748
10749 if (!Op2Ty.getScalarType().isAnyScalar() &&
10750 !Op2Ty.getScalarType().isInteger()) {
10751 auto Op2TyInt =
10752 Op2Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: Op2Ty.getScalarSizeInBits()));
10753 Op2Reg = MIRBuilder.buildBitcast(Dst: Op2TyInt, Src: Op2Reg).getReg(Idx: 0);
10754 }
10755
10756 auto NotMask = MIRBuilder.buildNot(Dst: MaskTy, Src0: MaskReg);
10757 auto NewOp1 = MIRBuilder.buildAnd(Dst: MaskTy, Src0: Op1Reg, Src1: MaskReg);
10758 auto NewOp2 = MIRBuilder.buildAnd(Dst: MaskTy, Src0: Op2Reg, Src1: NotMask);
10759 if (IsEltPtr) {
10760 auto Or = MIRBuilder.buildOr(Dst: DstTy, Src0: NewOp1, Src1: NewOp2);
10761 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: Or);
10762 } else {
10763 if (DstTy == Op1TyInt)
10764 MIRBuilder.buildOr(Dst: DstReg, Src0: NewOp1, Src1: NewOp2);
10765 else {
10766 auto Or = MIRBuilder.buildOr(Dst: Op1TyInt, Src0: NewOp1, Src1: NewOp2);
10767 MIRBuilder.buildBitcast(Dst: DstReg, Src: Or.getReg(Idx: 0));
10768 }
10769 }
10770 MI.eraseFromParent();
10771 return Legalized;
10772}
10773
10774LegalizerHelper::LegalizeResult LegalizerHelper::lowerDIVREM(MachineInstr &MI) {
10775 // Split DIVREM into individual instructions.
10776 unsigned Opcode = MI.getOpcode();
10777
10778 MIRBuilder.buildInstr(
10779 Opc: Opcode == TargetOpcode::G_SDIVREM ? TargetOpcode::G_SDIV
10780 : TargetOpcode::G_UDIV,
10781 DstOps: {MI.getOperand(i: 0).getReg()}, SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3)});
10782 MIRBuilder.buildInstr(
10783 Opc: Opcode == TargetOpcode::G_SDIVREM ? TargetOpcode::G_SREM
10784 : TargetOpcode::G_UREM,
10785 DstOps: {MI.getOperand(i: 1).getReg()}, SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3)});
10786 MI.eraseFromParent();
10787 return Legalized;
10788}
10789
10790LegalizerHelper::LegalizeResult
10791LegalizerHelper::lowerAbsToAddXor(MachineInstr &MI) {
10792 // Expand %res = G_ABS %a into:
10793 // %v1 = G_ASHR %a, scalar_size-1
10794 // %v2 = G_ADD %a, %v1
10795 // %res = G_XOR %v2, %v1
10796 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
10797 Register OpReg = MI.getOperand(i: 1).getReg();
10798 auto ShiftAmt =
10799 MIRBuilder.buildConstant(Res: DstTy, Val: DstTy.getScalarSizeInBits() - 1);
10800 auto Shift = MIRBuilder.buildAShr(Dst: DstTy, Src0: OpReg, Src1: ShiftAmt);
10801 auto Add = MIRBuilder.buildAdd(Dst: DstTy, Src0: OpReg, Src1: Shift);
10802 MIRBuilder.buildXor(Dst: MI.getOperand(i: 0).getReg(), Src0: Add, Src1: Shift);
10803 MI.eraseFromParent();
10804 return Legalized;
10805}
10806
10807LegalizerHelper::LegalizeResult
10808LegalizerHelper::lowerAbsToMaxNeg(MachineInstr &MI) {
10809 // Expand %res = G_ABS %a into:
10810 // %v1 = G_CONSTANT 0
10811 // %v2 = G_SUB %v1, %a
10812 // %res = G_SMAX %a, %v2
10813 Register SrcReg = MI.getOperand(i: 1).getReg();
10814 LLT Ty = MRI.getType(Reg: SrcReg);
10815 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
10816 auto Sub = MIRBuilder.buildSub(Dst: Ty, Src0: Zero, Src1: SrcReg);
10817 MIRBuilder.buildSMax(Dst: MI.getOperand(i: 0), Src0: SrcReg, Src1: Sub);
10818 MI.eraseFromParent();
10819 return Legalized;
10820}
10821
10822LegalizerHelper::LegalizeResult
10823LegalizerHelper::lowerAbsToCNeg(MachineInstr &MI) {
10824 Register SrcReg = MI.getOperand(i: 1).getReg();
10825 Register DestReg = MI.getOperand(i: 0).getReg();
10826 LLT Ty = MRI.getType(Reg: SrcReg), IType = LLT::integer(SizeInBits: 1);
10827 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0).getReg(Idx: 0);
10828 auto Sub = MIRBuilder.buildSub(Dst: Ty, Src0: Zero, Src1: SrcReg).getReg(Idx: 0);
10829 auto ICmp = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SGT, Res: IType, Op0: SrcReg, Op1: Zero);
10830 MIRBuilder.buildSelect(Res: DestReg, Tst: ICmp, Op0: SrcReg, Op1: Sub);
10831 MI.eraseFromParent();
10832 return Legalized;
10833}
10834
10835LegalizerHelper::LegalizeResult
10836LegalizerHelper::lowerAbsDiffToSelect(MachineInstr &MI) {
10837 assert((MI.getOpcode() == TargetOpcode::G_ABDS ||
10838 MI.getOpcode() == TargetOpcode::G_ABDU) &&
10839 "Expected G_ABDS or G_ABDU instruction");
10840
10841 auto [DstReg, LHS, RHS] = MI.getFirst3Regs();
10842 LLT Ty = MRI.getType(Reg: LHS);
10843
10844 // abds(lhs, rhs) -> select(sgt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
10845 // abdu(lhs, rhs) -> select(ugt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
10846 Register LHSSub = MIRBuilder.buildSub(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10847 Register RHSSub = MIRBuilder.buildSub(Dst: Ty, Src0: RHS, Src1: LHS).getReg(Idx: 0);
10848 CmpInst::Predicate Pred = (MI.getOpcode() == TargetOpcode::G_ABDS)
10849 ? CmpInst::ICMP_SGT
10850 : CmpInst::ICMP_UGT;
10851 auto ICmp = MIRBuilder.buildICmp(Pred, Res: LLT::scalar(SizeInBits: 1), Op0: LHS, Op1: RHS);
10852 MIRBuilder.buildSelect(Res: DstReg, Tst: ICmp, Op0: LHSSub, Op1: RHSSub);
10853
10854 MI.eraseFromParent();
10855 return Legalized;
10856}
10857
10858LegalizerHelper::LegalizeResult
10859LegalizerHelper::lowerAbsDiffToMinMax(MachineInstr &MI) {
10860 assert((MI.getOpcode() == TargetOpcode::G_ABDS ||
10861 MI.getOpcode() == TargetOpcode::G_ABDU) &&
10862 "Expected G_ABDS or G_ABDU instruction");
10863
10864 auto [DstReg, LHS, RHS] = MI.getFirst3Regs();
10865 LLT Ty = MRI.getType(Reg: LHS);
10866
10867 // abds(lhs, rhs) -→ sub(smax(lhs, rhs), smin(lhs, rhs))
10868 // abdu(lhs, rhs) -→ sub(umax(lhs, rhs), umin(lhs, rhs))
10869 Register MaxReg, MinReg;
10870 if (MI.getOpcode() == TargetOpcode::G_ABDS) {
10871 MaxReg = MIRBuilder.buildSMax(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10872 MinReg = MIRBuilder.buildSMin(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10873 } else {
10874 MaxReg = MIRBuilder.buildUMax(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10875 MinReg = MIRBuilder.buildUMin(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10876 }
10877 MIRBuilder.buildSub(Dst: DstReg, Src0: MaxReg, Src1: MinReg);
10878
10879 MI.eraseFromParent();
10880 return Legalized;
10881}
10882
10883LegalizerHelper::LegalizeResult LegalizerHelper::lowerFAbs(MachineInstr &MI) {
10884 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
10885 LLT TyInt =
10886 DstTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: DstTy.getScalarSizeInBits()));
10887 Register CastedSrc = SrcReg;
10888
10889 if (!(SrcTy.getScalarType().isAnyScalar() ||
10890 SrcTy.getScalarType().isInteger())) {
10891 auto SrcTyInt =
10892 SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: SrcTy.getScalarSizeInBits()));
10893 CastedSrc = MIRBuilder.buildBitcast(Dst: SrcTyInt, Src: SrcReg).getReg(Idx: 0);
10894 }
10895
10896 if (MRI.getType(Reg: DstReg) != TyInt) {
10897 // Reset sign bit
10898 Register NewDst =
10899 MIRBuilder
10900 .buildAnd(Dst: TyInt, Src0: CastedSrc,
10901 Src1: MIRBuilder.buildConstant(
10902 Res: TyInt, Val: APInt::getSignedMaxValue(
10903 numBits: DstTy.getScalarSizeInBits())))
10904 .getReg(Idx: 0);
10905
10906 MIRBuilder.buildBitcast(Dst: DstReg, Src: NewDst);
10907 } else
10908 MIRBuilder
10909 .buildAnd(
10910 Dst: DstReg, Src0: CastedSrc,
10911 Src1: MIRBuilder.buildConstant(
10912 Res: TyInt, Val: APInt::getSignedMaxValue(numBits: DstTy.getScalarSizeInBits())))
10913 .getReg(Idx: 0);
10914
10915 MI.eraseFromParent();
10916 return Legalized;
10917}
10918
10919LegalizerHelper::LegalizeResult
10920LegalizerHelper::lowerVectorReduction(MachineInstr &MI) {
10921 Register SrcReg = MI.getOperand(i: 1).getReg();
10922 LLT SrcTy = MRI.getType(Reg: SrcReg);
10923 LLT DstTy = MRI.getType(Reg: SrcReg);
10924
10925 // The source could be a scalar if the IR type was <1 x sN>.
10926 if (SrcTy.isScalar()) {
10927 if (DstTy.getSizeInBits() > SrcTy.getSizeInBits())
10928 return UnableToLegalize; // FIXME: handle extension.
10929 // This can be just a plain copy.
10930 Observer.changingInstr(MI);
10931 MI.setDesc(MIRBuilder.getTII().get(Opcode: TargetOpcode::COPY));
10932 Observer.changedInstr(MI);
10933 return Legalized;
10934 }
10935 return UnableToLegalize;
10936}
10937
10938LegalizerHelper::LegalizeResult LegalizerHelper::lowerVAArg(MachineInstr &MI) {
10939 MachineFunction &MF = *MI.getMF();
10940 const DataLayout &DL = MIRBuilder.getDataLayout();
10941 LLVMContext &Ctx = MF.getFunction().getContext();
10942 Register ListPtr = MI.getOperand(i: 1).getReg();
10943 LLT PtrTy = MRI.getType(Reg: ListPtr);
10944
10945 // LstPtr is a pointer to the head of the list. Get the address
10946 // of the head of the list.
10947 Align PtrAlignment = DL.getABITypeAlign(Ty: getTypeForLLT(Ty: PtrTy, C&: Ctx));
10948 MachineMemOperand *PtrLoadMMO = MF.getMachineMemOperand(
10949 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOLoad, MemTy: PtrTy, BaseAlignment: PtrAlignment);
10950 auto VAList = MIRBuilder.buildLoad(Res: PtrTy, Addr: ListPtr, MMO&: *PtrLoadMMO).getReg(Idx: 0);
10951
10952 const Align A(MI.getOperand(i: 2).getImm());
10953 LLT PtrTyAsScalarTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
10954 if (A > TLI.getMinStackArgumentAlignment()) {
10955 Register AlignAmt =
10956 MIRBuilder.buildConstant(Res: PtrTyAsScalarTy, Val: A.value() - 1).getReg(Idx: 0);
10957 auto AddDst = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: VAList, Op1: AlignAmt);
10958 auto AndDst = MIRBuilder.buildMaskLowPtrBits(Res: PtrTy, Op0: AddDst, NumBits: Log2(A));
10959 VAList = AndDst.getReg(Idx: 0);
10960 }
10961
10962 // Increment the pointer, VAList, to the next vaarg
10963 // The list should be bumped by the size of element in the current head of
10964 // list.
10965 Register Dst = MI.getOperand(i: 0).getReg();
10966 LLT LLTTy = MRI.getType(Reg: Dst);
10967 Type *Ty = getTypeForLLT(Ty: LLTTy, C&: Ctx);
10968 auto IncAmt =
10969 MIRBuilder.buildConstant(Res: PtrTyAsScalarTy, Val: DL.getTypeAllocSize(Ty));
10970 auto Succ = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: VAList, Op1: IncAmt);
10971
10972 // Store the increment VAList to the legalized pointer
10973 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
10974 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOStore, MemTy: PtrTy, BaseAlignment: PtrAlignment);
10975 MIRBuilder.buildStore(Val: Succ, Addr: ListPtr, MMO&: *StoreMMO);
10976 // Load the actual argument out of the pointer VAList
10977 Align EltAlignment = DL.getABITypeAlign(Ty);
10978 MachineMemOperand *EltLoadMMO = MF.getMachineMemOperand(
10979 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOLoad, MemTy: LLTTy, BaseAlignment: EltAlignment);
10980 MIRBuilder.buildLoad(Res: Dst, Addr: VAList, MMO&: *EltLoadMMO);
10981
10982 MI.eraseFromParent();
10983 return Legalized;
10984}
10985
10986LegalizerHelper::LegalizeResult LegalizerHelper::lowerMulfix(MachineInstr &MI) {
10987 unsigned OpCode = MI.getOpcode();
10988 assert((OpCode == TargetOpcode::G_SMULFIX ||
10989 OpCode == TargetOpcode::G_UMULFIX ||
10990 OpCode == TargetOpcode::G_SMULFIXSAT ||
10991 OpCode == TargetOpcode::G_UMULFIXSAT) &&
10992 "Operator must be either G_SMULFIX[SAT] or G_UMULFIX[SAT]!");
10993 auto [Dst, LHS, RHS] = MI.getFirst3Regs();
10994 LLT Ty = MRI.getType(Reg: Dst);
10995 unsigned Scale = MI.getOperand(i: 3).getImm();
10996
10997 bool Saturating = (OpCode == TargetOpcode::G_SMULFIXSAT ||
10998 OpCode == TargetOpcode::G_UMULFIXSAT);
10999 bool IsSigned = (OpCode == TargetOpcode::G_SMULFIX ||
11000 OpCode == TargetOpcode::G_SMULFIXSAT);
11001
11002 if (!Saturating && Scale == 0) {
11003 MIRBuilder.buildMul(Dst, Src0: LHS, Src1: RHS);
11004 MI.eraseFromParent();
11005 return Legalized;
11006 }
11007
11008 // TODO: Port other lowering paths from SelectionDAG.
11009 LLT WideTy = Ty.changeElementSize(NewEltSize: Ty.getScalarSizeInBits() * 2);
11010 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: Scale);
11011 MachineInstrBuilder ExtLHS{}, ExtRHS{}, Shift{};
11012 if (IsSigned) {
11013 ExtLHS = MIRBuilder.buildSExt(Res: WideTy, Op: LHS);
11014 ExtRHS = MIRBuilder.buildSExt(Res: WideTy, Op: RHS);
11015 } else {
11016 ExtLHS = MIRBuilder.buildZExt(Res: WideTy, Op: LHS);
11017 ExtRHS = MIRBuilder.buildZExt(Res: WideTy, Op: RHS);
11018 }
11019
11020 auto Mul = MIRBuilder.buildMul(Dst: WideTy, Src0: ExtLHS, Src1: ExtRHS);
11021 if (IsSigned)
11022 Shift = MIRBuilder.buildAShr(Dst: WideTy, Src0: Mul, Src1: ShiftAmt);
11023 else
11024 Shift = MIRBuilder.buildLShr(Dst: WideTy, Src0: Mul, Src1: ShiftAmt);
11025
11026 if (!Saturating)
11027 MIRBuilder.buildTrunc(Res: Dst, Op: Shift);
11028 else if (IsSigned)
11029 MIRBuilder.buildTruncSSatS(Res: Dst, Op: Shift);
11030 else
11031 MIRBuilder.buildTruncUSatU(Res: Dst, Op: Shift);
11032
11033 MI.eraseFromParent();
11034 return Legalized;
11035}
11036
11037// Get a vectorized representation of the memset value operand, GISel edition.
11038static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB) {
11039 MachineRegisterInfo &MRI = *MIB.getMRI();
11040 unsigned NumBits = Ty.getScalarSizeInBits();
11041 auto ValVRegAndVal = getIConstantVRegValWithLookThrough(VReg: Val, MRI);
11042 if (!Ty.isVector() && ValVRegAndVal) {
11043 APInt Scalar = ValVRegAndVal->Value.trunc(width: 8);
11044 APInt SplatVal = APInt::getSplat(NewLen: NumBits, V: Scalar);
11045 return MIB.buildConstant(Res: Ty, Val: SplatVal).getReg(Idx: 0);
11046 }
11047
11048 // Extend the byte value to the larger type, and then multiply by a magic
11049 // value 0x010101... in order to replicate it across every byte.
11050 // Unless it's zero, in which case just emit a larger G_CONSTANT 0.
11051 if (ValVRegAndVal && ValVRegAndVal->Value == 0) {
11052 return MIB.buildConstant(Res: Ty, Val: 0).getReg(Idx: 0);
11053 }
11054
11055 LLT ExtType = Ty.getScalarType();
11056 auto ZExt = MIB.buildZExtOrTrunc(Res: ExtType, Op: Val);
11057 if (NumBits > 8) {
11058 APInt Magic = APInt::getSplat(NewLen: NumBits, V: APInt(8, 0x01));
11059 auto MagicMI = MIB.buildConstant(Res: ExtType, Val: Magic);
11060 Val = MIB.buildMul(Dst: ExtType, Src0: ZExt, Src1: MagicMI).getReg(Idx: 0);
11061 }
11062
11063 // For vector types create a G_BUILD_VECTOR.
11064 if (Ty.isVector())
11065 Val = MIB.buildSplatBuildVector(Res: Ty, Src: Val).getReg(Idx: 0);
11066
11067 return Val;
11068}
11069
11070LegalizerHelper::LegalizeResult
11071LegalizerHelper::lowerMemset(MachineInstr &MI, Register Dst, Register Val,
11072 uint64_t KnownLen, Align Alignment,
11073 bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
11074 auto &MF = *MI.getParent()->getParent();
11075 const auto &TLI = *MF.getSubtarget().getTargetLowering();
11076 auto &DL = MF.getDataLayout();
11077 LLVMContext &C = MF.getFunction().getContext();
11078
11079 assert(KnownLen != 0 && "Have a zero length memset length!");
11080 assert(!MemOps.empty() && "Expected at least one memory op");
11081
11082 MachineFrameInfo &MFI = MF.getFrameInfo();
11083 MachineInstr *FIDef = getOpcodeDef(Opcode: TargetOpcode::G_FRAME_INDEX, Reg: Dst, MRI);
11084 const auto &DstMMO = **MI.memoperands_begin();
11085
11086 if (DstAlignCanChange) {
11087 // Get an estimate of the type from the LLT.
11088 Type *IRTy = getTypeForLLT(Ty: MemOps[0], C);
11089 Align NewAlign = DL.getABITypeAlign(Ty: IRTy);
11090 if (NewAlign > Alignment) {
11091 Alignment = NewAlign;
11092 unsigned FI = FIDef->getOperand(i: 1).getIndex();
11093 // Give the stack frame object a larger alignment if needed.
11094 if (MFI.getObjectAlign(ObjectIdx: FI) < Alignment)
11095 MFI.setObjectAlignment(ObjectIdx: FI, Alignment);
11096 }
11097 }
11098
11099 MachineIRBuilder MIB(MI);
11100 // Find the largest store and generate the bit pattern for it.
11101 LLT LargestTy = MemOps[0];
11102 for (unsigned i = 1; i < MemOps.size(); i++)
11103 if (MemOps[i].getSizeInBits() > LargestTy.getSizeInBits())
11104 LargestTy = MemOps[i];
11105
11106 // The memset stored value is always defined as an s8, so in order to make it
11107 // work with larger store types we need to repeat the bit pattern across the
11108 // wider type.
11109 Register MemSetValue = getMemsetValue(Val, Ty: LargestTy, MIB);
11110
11111 if (!MemSetValue)
11112 return UnableToLegalize;
11113
11114 // Generate the stores. For each store type in the list, we generate the
11115 // matching store of that type to the destination address.
11116 LLT PtrTy = MRI.getType(Reg: Dst);
11117 unsigned DstOff = 0;
11118 unsigned Size = KnownLen;
11119 for (unsigned I = 0; I < MemOps.size(); I++) {
11120 LLT Ty = MemOps[I];
11121 unsigned TySize = Ty.getSizeInBytes();
11122
11123 if (TySize > Size) {
11124 // Issuing a load / store pair that overlaps with the previous pair.
11125 // Adjust the offset accordingly.
11126 assert(I == MemOps.size() - 1 && I != 0);
11127 DstOff -= TySize - Size;
11128 }
11129
11130 // If this store is smaller than the largest store see whether we can get
11131 // the smaller value for free with a truncate.
11132 Register Value = MemSetValue;
11133 if (Ty.getSizeInBits() < LargestTy.getSizeInBits()) {
11134 MVT VT = getMVTForLLT(Ty);
11135 MVT LargestVT = getMVTForLLT(Ty: LargestTy);
11136 if (!LargestTy.isVector() && !Ty.isVector() &&
11137 TLI.isTruncateFree(FromVT: LargestVT, ToVT: VT))
11138 Value = MIB.buildTrunc(Res: Ty, Op: MemSetValue).getReg(Idx: 0);
11139 else
11140 Value = getMemsetValue(Val, Ty, MIB);
11141 if (!Value)
11142 return UnableToLegalize;
11143 }
11144
11145 auto *StoreMMO = MF.getMachineMemOperand(MMO: &DstMMO, Offset: DstOff, Ty);
11146
11147 Register Ptr = Dst;
11148 if (DstOff != 0) {
11149 auto Offset =
11150 MIB.buildConstant(Res: LLT::integer(SizeInBits: PtrTy.getSizeInBits()), Val: DstOff);
11151 Ptr = MIB.buildObjectPtrOffset(Res: PtrTy, Op0: Dst, Op1: Offset).getReg(Idx: 0);
11152 }
11153
11154 MIB.buildStore(Val: Value, Addr: Ptr, MMO&: *StoreMMO);
11155 DstOff += Ty.getSizeInBytes();
11156 Size -= TySize;
11157 }
11158
11159 MI.eraseFromParent();
11160 return Legalized;
11161}
11162
11163LegalizerHelper::LegalizeResult
11164LegalizerHelper::lowerMemcpy(MachineInstr &MI, Register Dst, Register Src,
11165 uint64_t KnownLen, Align Alignment,
11166 bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
11167 auto &MF = *MI.getParent()->getParent();
11168 auto &DL = MF.getDataLayout();
11169 LLVMContext &C = MF.getFunction().getContext();
11170
11171 assert(KnownLen != 0 && "Have a zero length memcpy length!");
11172 assert(!MemOps.empty() && "Expected at least one memory op");
11173
11174 MachineFrameInfo &MFI = MF.getFrameInfo();
11175 MachineInstr *FIDef = getOpcodeDef(Opcode: TargetOpcode::G_FRAME_INDEX, Reg: Dst, MRI);
11176
11177 // FIXME: infer better src pointer alignment like SelectionDAG does here.
11178 // FIXME: also use the equivalent of isMemSrcFromConstant and alwaysinlining
11179 // if the memcpy is in a tail call position.
11180
11181 const auto &DstMMO = **MI.memoperands_begin();
11182 const auto &SrcMMO = **std::next(x: MI.memoperands_begin());
11183
11184 if (DstAlignCanChange) {
11185 // Get an estimate of the type from the LLT.
11186 Type *IRTy = getTypeForLLT(Ty: MemOps[0], C);
11187 Align NewAlign = DL.getABITypeAlign(Ty: IRTy);
11188
11189 // Don't promote to an alignment that would require dynamic stack
11190 // realignment.
11191 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
11192 if (!TRI->hasStackRealignment(MF))
11193 if (MaybeAlign StackAlign = DL.getStackAlignment())
11194 NewAlign = std::min(a: NewAlign, b: *StackAlign);
11195
11196 if (NewAlign > Alignment) {
11197 Alignment = NewAlign;
11198 unsigned FI = FIDef->getOperand(i: 1).getIndex();
11199 // Give the stack frame object a larger alignment if needed.
11200 if (MFI.getObjectAlign(ObjectIdx: FI) < Alignment)
11201 MFI.setObjectAlignment(ObjectIdx: FI, Alignment);
11202 }
11203 }
11204
11205 LLVM_DEBUG(dbgs() << "Inlining memcpy: " << MI << " into loads & stores\n");
11206
11207 MachineIRBuilder MIB(MI);
11208 // Now we need to emit a pair of load and stores for each of the types we've
11209 // collected. I.e. for each type, generate a load from the source pointer of
11210 // that type width, and then generate a corresponding store to the dest buffer
11211 // of that value loaded. This can result in a sequence of loads and stores
11212 // mixed types, depending on what the target specifies as good types to use.
11213 LLT SrcTy = MRI.getType(Reg: Src);
11214 LLT DstTy = MRI.getType(Reg: Dst);
11215 unsigned SrcIndexSize = DL.getIndexSizeInBits(AS: SrcTy.getAddressSpace());
11216 unsigned DstIndexSize = DL.getIndexSizeInBits(AS: DstTy.getAddressSpace());
11217 unsigned CurrOffset = 0;
11218 unsigned Size = KnownLen;
11219 for (auto CopyTy : MemOps) {
11220 TypeSize TySize = CopyTy.getSizeInBytes();
11221
11222 // Issuing a load / store pair that overlaps with the previous pair. Adjust
11223 // the offset accordingly.
11224 if (TySize > Size) {
11225 unsigned Overlap = TySize - Size;
11226 assert(Overlap < CurrOffset &&
11227 "overlapping memcpy load/store spans the whole region or more");
11228 CurrOffset -= Overlap;
11229 }
11230
11231 // Construct MMOs for the accesses.
11232 auto *LoadMMO = MF.getMachineMemOperand(MMO: &SrcMMO, Offset: CurrOffset, Size: TySize);
11233 auto *StoreMMO = MF.getMachineMemOperand(MMO: &DstMMO, Offset: CurrOffset, Size: TySize);
11234
11235 // Create the load.
11236 Register LoadPtr = Src;
11237 Register Offset;
11238 if (CurrOffset != 0) {
11239 Offset =
11240 MIB.buildConstant(Res: LLT::integer(SizeInBits: SrcIndexSize), Val: CurrOffset).getReg(Idx: 0);
11241 LoadPtr = MIB.buildObjectPtrOffset(Res: SrcTy, Op0: Src, Op1: Offset).getReg(Idx: 0);
11242 }
11243 auto LdVal = MIB.buildLoad(Res: CopyTy, Addr: LoadPtr, MMO&: *LoadMMO);
11244
11245 // Create the store.
11246 Register StorePtr = Dst;
11247 if (CurrOffset != 0) {
11248 if (SrcIndexSize != DstIndexSize) {
11249 Offset =
11250 MIB.buildConstant(Res: LLT::integer(SizeInBits: DstIndexSize), Val: CurrOffset).getReg(Idx: 0);
11251 }
11252 StorePtr = MIB.buildObjectPtrOffset(Res: DstTy, Op0: Dst, Op1: Offset).getReg(Idx: 0);
11253 }
11254 MIB.buildStore(Val: LdVal, Addr: StorePtr, MMO&: *StoreMMO);
11255 CurrOffset += TySize;
11256 Size -= TySize;
11257 }
11258
11259 MI.eraseFromParent();
11260 return Legalized;
11261}
11262
11263LegalizerHelper::LegalizeResult
11264LegalizerHelper::lowerMemmove(MachineInstr &MI, Register Dst, Register Src,
11265 uint64_t KnownLen, Align Alignment,
11266 bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
11267 auto &MF = *MI.getParent()->getParent();
11268 auto &DL = MF.getDataLayout();
11269 LLVMContext &C = MF.getFunction().getContext();
11270
11271 assert(KnownLen != 0 && "Have a zero length memmove length!");
11272 assert(!MemOps.empty() && "Expected at least one memory op");
11273
11274 MachineFrameInfo &MFI = MF.getFrameInfo();
11275 MachineInstr *FIDef = getOpcodeDef(Opcode: TargetOpcode::G_FRAME_INDEX, Reg: Dst, MRI);
11276 const auto &DstMMO = **MI.memoperands_begin();
11277 const auto &SrcMMO = **std::next(x: MI.memoperands_begin());
11278
11279 if (DstAlignCanChange) {
11280 // Get an estimate of the type from the LLT.
11281 Type *IRTy = getTypeForLLT(Ty: MemOps[0], C);
11282 Align NewAlign = DL.getABITypeAlign(Ty: IRTy);
11283
11284 // Don't promote to an alignment that would require dynamic stack
11285 // realignment.
11286 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
11287 if (!TRI->hasStackRealignment(MF))
11288 if (MaybeAlign StackAlign = DL.getStackAlignment())
11289 NewAlign = std::min(a: NewAlign, b: *StackAlign);
11290
11291 if (NewAlign > Alignment) {
11292 Alignment = NewAlign;
11293 unsigned FI = FIDef->getOperand(i: 1).getIndex();
11294 // Give the stack frame object a larger alignment if needed.
11295 if (MFI.getObjectAlign(ObjectIdx: FI) < Alignment)
11296 MFI.setObjectAlignment(ObjectIdx: FI, Alignment);
11297 }
11298 }
11299
11300 LLVM_DEBUG(dbgs() << "Inlining memmove: " << MI << " into loads & stores\n");
11301
11302 MachineIRBuilder MIB(MI);
11303 // Memmove requires that we perform the loads first before issuing the stores.
11304 // Apart from that, this loop is pretty much doing the same thing as the
11305 // memcpy codegen function.
11306 unsigned CurrOffset = 0;
11307 unsigned Size = KnownLen;
11308 SmallVector<Register, 16> LoadVals;
11309 for (auto CopyTy : MemOps) {
11310 TypeSize TySize = CopyTy.getSizeInBytes();
11311
11312 // Issuing a load that overlaps with the previous load. Adjust the offset
11313 // accordingly.
11314 if (TySize > Size) {
11315 unsigned Overlap = TySize - Size;
11316 assert(Overlap < CurrOffset &&
11317 "overlapping memmove load spans the whole region or more");
11318 CurrOffset -= Overlap;
11319 }
11320
11321 // Construct MMO for the load.
11322 auto *LoadMMO = MF.getMachineMemOperand(MMO: &SrcMMO, Offset: CurrOffset, Size: TySize);
11323
11324 // Create the load.
11325 Register LoadPtr = Src;
11326 if (CurrOffset != 0) {
11327 LLT SrcTy = MRI.getType(Reg: Src);
11328 auto Offset =
11329 MIB.buildConstant(Res: LLT::integer(SizeInBits: SrcTy.getSizeInBits()), Val: CurrOffset);
11330 LoadPtr = MIB.buildObjectPtrOffset(Res: SrcTy, Op0: Src, Op1: Offset).getReg(Idx: 0);
11331 }
11332 LoadVals.push_back(Elt: MIB.buildLoad(Res: CopyTy, Addr: LoadPtr, MMO&: *LoadMMO).getReg(Idx: 0));
11333 CurrOffset += TySize;
11334 Size -= TySize;
11335 }
11336
11337 CurrOffset = 0;
11338 Size = KnownLen;
11339 for (unsigned I = 0; I < MemOps.size(); ++I) {
11340 LLT CopyTy = MemOps[I];
11341 TypeSize TySize = CopyTy.getSizeInBytes();
11342
11343 // Issuing a store that overlaps with the previous store. Adjust the offset
11344 // accordingly.
11345 if (TySize > Size) {
11346 unsigned Overlap = TySize - Size;
11347 assert(Overlap < CurrOffset &&
11348 "overlapping memmove store spans the whole region or more");
11349 CurrOffset -= Overlap;
11350 }
11351
11352 // Now store the values loaded.
11353 auto *StoreMMO = MF.getMachineMemOperand(MMO: &DstMMO, Offset: CurrOffset, Size: TySize);
11354
11355 Register StorePtr = Dst;
11356 if (CurrOffset != 0) {
11357 LLT DstTy = MRI.getType(Reg: Dst);
11358 auto Offset =
11359 MIB.buildConstant(Res: LLT::integer(SizeInBits: DstTy.getSizeInBits()), Val: CurrOffset);
11360 StorePtr = MIB.buildObjectPtrOffset(Res: DstTy, Op0: Dst, Op1: Offset).getReg(Idx: 0);
11361 }
11362 MIB.buildStore(Val: LoadVals[I], Addr: StorePtr, MMO&: *StoreMMO);
11363 CurrOffset += TySize;
11364 Size -= TySize;
11365 }
11366 MI.eraseFromParent();
11367 return Legalized;
11368}
11369
11370LegalizerHelper::LegalizeResult LegalizerHelper::lowerMemCpyFamily(
11371 MachineInstr &MI, Register Dst, Register Src, uint64_t KnownLen,
11372 Align Alignment, bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
11373 const unsigned Opc = MI.getOpcode();
11374 assert((Opc == TargetOpcode::G_MEMCPY ||
11375 Opc == TargetOpcode::G_MEMCPY_INLINE ||
11376 Opc == TargetOpcode::G_MEMMOVE || Opc == TargetOpcode::G_MEMSET ||
11377 Opc == TargetOpcode::G_MEMSET_INLINE) &&
11378 "Expected memcpy like instruction");
11379
11380 if (KnownLen == 0) {
11381 MI.eraseFromParent();
11382 return Legalized;
11383 }
11384
11385 if (Opc == TargetOpcode::G_MEMCPY || Opc == TargetOpcode::G_MEMCPY_INLINE) {
11386 return lowerMemcpy(MI, Dst, Src, KnownLen, Alignment, DstAlignCanChange,
11387 MemOps);
11388 }
11389 if (Opc == TargetOpcode::G_MEMMOVE)
11390 return lowerMemmove(MI, Dst, Src, KnownLen, Alignment, DstAlignCanChange,
11391 MemOps);
11392 if (Opc == TargetOpcode::G_MEMSET || Opc == TargetOpcode::G_MEMSET_INLINE)
11393 return lowerMemset(MI, Dst, Val: Src, KnownLen, Alignment, DstAlignCanChange,
11394 MemOps);
11395 return UnableToLegalize;
11396}
11397
11398LegalizerHelper::LegalizeResult
11399LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen) {
11400 Register Dst, Src;
11401 uint64_t KnownLen;
11402 Align Alignment;
11403 bool DstAlignCanChange;
11404 std::vector<LLT> MemOps;
11405 if (!canLowerMemCpyFamily(MI, MRI, MaxLen, Dst, Src, KnownLen, Alignment,
11406 DstAlignCanChange, MemOps))
11407 return UnableToLegalize;
11408 return lowerMemCpyFamily(MI, Dst, Src, KnownLen, Alignment, DstAlignCanChange,
11409 MemOps);
11410}
11411