1//===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file implements the LegalizerHelper class to legalize
10/// individual instructions and the LegalizeMachineIR wrapper pass for the
11/// primary legalization.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
16#include "llvm/CodeGen/GlobalISel/CallLowering.h"
17#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
18#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
19#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
20#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
21#include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h"
22#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
23#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24#include "llvm/CodeGen/GlobalISel/Utils.h"
25#include "llvm/CodeGen/LowLevelTypeUtils.h"
26#include "llvm/CodeGen/MachineConstantPool.h"
27#include "llvm/CodeGen/MachineFrameInfo.h"
28#include "llvm/CodeGen/MachineRegisterInfo.h"
29#include "llvm/CodeGen/RuntimeLibcallUtil.h"
30#include "llvm/CodeGen/TargetFrameLowering.h"
31#include "llvm/CodeGen/TargetInstrInfo.h"
32#include "llvm/CodeGen/TargetLowering.h"
33#include "llvm/CodeGen/TargetOpcodes.h"
34#include "llvm/CodeGen/TargetSubtargetInfo.h"
35#include "llvm/IR/Instructions.h"
36#include "llvm/Support/Debug.h"
37#include "llvm/Support/MathExtras.h"
38#include "llvm/Support/raw_ostream.h"
39#include "llvm/Target/TargetMachine.h"
40#include <cassert>
41#include <numeric>
42#include <optional>
43
44#define DEBUG_TYPE "legalizer"
45
46using namespace llvm;
47using namespace LegalizeActions;
48using namespace MIPatternMatch;
49
50/// Try to break down \p OrigTy into \p NarrowTy sized pieces.
51///
52/// Returns the number of \p NarrowTy elements needed to reconstruct \p OrigTy,
53/// with any leftover piece as type \p LeftoverTy
54///
55/// Returns -1 in the first element of the pair if the breakdown is not
56/// satisfiable.
57static std::pair<int, int>
58getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) {
59 assert(!LeftoverTy.isValid() && "this is an out argument");
60
61 unsigned Size = OrigTy.getSizeInBits();
62 unsigned NarrowSize = NarrowTy.getSizeInBits();
63 unsigned NumParts = Size / NarrowSize;
64 unsigned LeftoverSize = Size - NumParts * NarrowSize;
65 assert(Size > NarrowSize);
66
67 if (LeftoverSize == 0)
68 return {NumParts, 0};
69
70 if (NarrowTy.isVector()) {
71 unsigned EltSize = OrigTy.getScalarSizeInBits();
72 if (LeftoverSize % EltSize != 0)
73 return {-1, -1};
74 LeftoverTy = OrigTy.changeElementCount(
75 EC: ElementCount::getFixed(MinVal: LeftoverSize / EltSize));
76 } else {
77 LeftoverTy = LLT::integer(SizeInBits: LeftoverSize);
78 }
79
80 int NumLeftover = LeftoverSize / LeftoverTy.getSizeInBits();
81 return std::make_pair(x&: NumParts, y&: NumLeftover);
82}
83
84static Type *getFloatTypeForLLT(LLVMContext &Ctx, LLT Ty) {
85
86 if (!Ty.isScalar())
87 return nullptr;
88
89 switch (Ty.getSizeInBits()) {
90 case 16:
91 return Type::getHalfTy(C&: Ctx);
92 case 32:
93 return Type::getFloatTy(C&: Ctx);
94 case 64:
95 return Type::getDoubleTy(C&: Ctx);
96 case 80:
97 return Type::getX86_FP80Ty(C&: Ctx);
98 case 128:
99 return Type::getFP128Ty(C&: Ctx);
100 default:
101 return nullptr;
102 }
103}
104
105LegalizerHelper::LegalizerHelper(MachineFunction &MF,
106 GISelChangeObserver &Observer,
107 MachineIRBuilder &Builder,
108 const LibcallLoweringInfo *Libcalls)
109 : MIRBuilder(Builder), Observer(Observer), MRI(MF.getRegInfo()),
110 LI(*MF.getSubtarget().getLegalizerInfo()),
111 TLI(*MF.getSubtarget().getTargetLowering()), Libcalls(Libcalls) {}
112
113LegalizerHelper::LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI,
114 GISelChangeObserver &Observer,
115 MachineIRBuilder &B,
116 const LibcallLoweringInfo *Libcalls,
117 GISelValueTracking *VT)
118 : MIRBuilder(B), Observer(Observer), MRI(MF.getRegInfo()), LI(LI),
119 TLI(*MF.getSubtarget().getTargetLowering()), Libcalls(Libcalls), VT(VT) {}
120
121LegalizerHelper::LegalizeResult
122LegalizerHelper::legalizeInstrStep(MachineInstr &MI,
123 LostDebugLocObserver &LocObserver) {
124 LLVM_DEBUG(dbgs() << "\nLegalizing: " << MI);
125
126 MIRBuilder.setInstrAndDebugLoc(MI);
127
128 if (isa<GIntrinsic>(Val: MI))
129 return LI.legalizeIntrinsic(Helper&: *this, MI) ? Legalized : UnableToLegalize;
130 auto Step = LI.getAction(MI, MRI);
131 switch (Step.Action) {
132 case Legal:
133 LLVM_DEBUG(dbgs() << ".. Already legal\n");
134 return AlreadyLegal;
135 case Libcall:
136 LLVM_DEBUG(dbgs() << ".. Convert to libcall\n");
137 return libcall(MI, LocObserver);
138 case NarrowScalar:
139 LLVM_DEBUG(dbgs() << ".. Narrow scalar\n");
140 return narrowScalar(MI, TypeIdx: Step.TypeIdx, NarrowTy: Step.NewType);
141 case WidenScalar:
142 LLVM_DEBUG(dbgs() << ".. Widen scalar\n");
143 return widenScalar(MI, TypeIdx: Step.TypeIdx, WideTy: Step.NewType);
144 case Bitcast:
145 LLVM_DEBUG(dbgs() << ".. Bitcast type\n");
146 return bitcast(MI, TypeIdx: Step.TypeIdx, Ty: Step.NewType);
147 case Lower:
148 LLVM_DEBUG(dbgs() << ".. Lower\n");
149 return lower(MI, TypeIdx: Step.TypeIdx, Ty: Step.NewType);
150 case FewerElements:
151 LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n");
152 return fewerElementsVector(MI, TypeIdx: Step.TypeIdx, NarrowTy: Step.NewType);
153 case MoreElements:
154 LLVM_DEBUG(dbgs() << ".. Increase number of elements\n");
155 return moreElementsVector(MI, TypeIdx: Step.TypeIdx, MoreTy: Step.NewType);
156 case Custom:
157 LLVM_DEBUG(dbgs() << ".. Custom legalization\n");
158 return LI.legalizeCustom(Helper&: *this, MI, LocObserver) ? Legalized
159 : UnableToLegalize;
160 default:
161 LLVM_DEBUG(dbgs() << ".. Unable to legalize\n");
162 return UnableToLegalize;
163 }
164}
165
166void LegalizerHelper::insertParts(Register DstReg,
167 LLT ResultTy, LLT PartTy,
168 ArrayRef<Register> PartRegs,
169 LLT LeftoverTy,
170 ArrayRef<Register> LeftoverRegs) {
171 if (!LeftoverTy.isValid()) {
172 assert(LeftoverRegs.empty());
173
174 if (!ResultTy.isVector()) {
175 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: PartRegs);
176 return;
177 }
178
179 if (PartTy.isVector())
180 MIRBuilder.buildConcatVectors(Res: DstReg, Ops: PartRegs);
181 else
182 MIRBuilder.buildBuildVector(Res: DstReg, Ops: PartRegs);
183 return;
184 }
185
186 // Merge sub-vectors with different number of elements and insert into DstReg.
187 if (ResultTy.isVector()) {
188 assert(LeftoverRegs.size() == 1 && "Expected one leftover register");
189 SmallVector<Register, 8> AllRegs(PartRegs);
190 AllRegs.append(in_start: LeftoverRegs.begin(), in_end: LeftoverRegs.end());
191 return mergeMixedSubvectors(DstReg, PartRegs: AllRegs);
192 }
193
194 SmallVector<Register> GCDRegs;
195 LLT GCDTy = getGCDType(OrigTy: getGCDType(OrigTy: ResultTy, TargetTy: LeftoverTy), TargetTy: PartTy);
196 for (auto PartReg : concat<const Register>(Ranges&: PartRegs, Ranges&: LeftoverRegs))
197 extractGCDType(Parts&: GCDRegs, GCDTy, SrcReg: PartReg);
198 LLT ResultLCMTy = buildLCMMergePieces(DstTy: ResultTy, NarrowTy: LeftoverTy, GCDTy, VRegs&: GCDRegs);
199 buildWidenedRemergeToDst(DstReg, LCMTy: ResultLCMTy, RemergeRegs: GCDRegs);
200}
201
202void LegalizerHelper::appendVectorElts(SmallVectorImpl<Register> &Elts,
203 Register Reg) {
204 LLT Ty = MRI.getType(Reg);
205 SmallVector<Register, 8> RegElts;
206 extractParts(Reg, Ty: Ty.getScalarType(), NumParts: Ty.getNumElements(), VRegs&: RegElts,
207 MIRBuilder, MRI);
208 Elts.append(RHS: RegElts);
209}
210
211/// Merge \p PartRegs with different types into \p DstReg.
212void LegalizerHelper::mergeMixedSubvectors(Register DstReg,
213 ArrayRef<Register> PartRegs) {
214 SmallVector<Register, 8> AllElts;
215 for (unsigned i = 0; i < PartRegs.size() - 1; ++i)
216 appendVectorElts(Elts&: AllElts, Reg: PartRegs[i]);
217
218 Register Leftover = PartRegs[PartRegs.size() - 1];
219 if (!MRI.getType(Reg: Leftover).isVector())
220 AllElts.push_back(Elt: Leftover);
221 else
222 appendVectorElts(Elts&: AllElts, Reg: Leftover);
223
224 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: AllElts);
225}
226
227/// Append the result registers of G_UNMERGE_VALUES \p MI to \p Regs.
228static void getUnmergeResults(SmallVectorImpl<Register> &Regs,
229 const MachineInstr &MI) {
230 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
231
232 const int StartIdx = Regs.size();
233 const int NumResults = MI.getNumOperands() - 1;
234 Regs.resize(N: Regs.size() + NumResults);
235 for (int I = 0; I != NumResults; ++I)
236 Regs[StartIdx + I] = MI.getOperand(i: I).getReg();
237}
238
239void LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts,
240 LLT GCDTy, Register SrcReg) {
241 LLT SrcTy = MRI.getType(Reg: SrcReg);
242 if (SrcTy == GCDTy) {
243 // If the source already evenly divides the result type, we don't need to do
244 // anything.
245 Parts.push_back(Elt: SrcReg);
246 } else {
247 // Need to split into common type sized pieces.
248 auto Unmerge = MIRBuilder.buildUnmerge(Res: GCDTy, Op: SrcReg);
249 getUnmergeResults(Regs&: Parts, MI: *Unmerge);
250 }
251}
252
253LLT LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts, LLT DstTy,
254 LLT NarrowTy, Register SrcReg) {
255 LLT SrcTy = MRI.getType(Reg: SrcReg);
256 LLT GCDTy = getGCDType(OrigTy: getGCDType(OrigTy: SrcTy, TargetTy: NarrowTy), TargetTy: DstTy);
257 extractGCDType(Parts, GCDTy, SrcReg);
258 return GCDTy;
259}
260
261LLT LegalizerHelper::buildLCMMergePieces(LLT DstTy, LLT NarrowTy, LLT GCDTy,
262 SmallVectorImpl<Register> &VRegs,
263 unsigned PadStrategy) {
264 LLT LCMTy = getLCMType(OrigTy: DstTy, TargetTy: NarrowTy);
265
266 int NumParts = LCMTy.getSizeInBits() / NarrowTy.getSizeInBits();
267 int NumSubParts = NarrowTy.getSizeInBits() / GCDTy.getSizeInBits();
268 int NumOrigSrc = VRegs.size();
269
270 Register PadReg;
271
272 // Get a value we can use to pad the source value if the sources won't evenly
273 // cover the result type.
274 if (NumOrigSrc < NumParts * NumSubParts) {
275 if (PadStrategy == TargetOpcode::G_ZEXT)
276 PadReg = MIRBuilder.buildConstant(Res: GCDTy, Val: 0).getReg(Idx: 0);
277 else if (PadStrategy == TargetOpcode::G_ANYEXT)
278 PadReg = MIRBuilder.buildUndef(Res: GCDTy).getReg(Idx: 0);
279 else {
280 assert(PadStrategy == TargetOpcode::G_SEXT);
281
282 // Shift the sign bit of the low register through the high register.
283 auto ShiftAmt =
284 MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: GCDTy.getSizeInBits() - 1);
285 PadReg = MIRBuilder.buildAShr(Dst: GCDTy, Src0: VRegs.back(), Src1: ShiftAmt).getReg(Idx: 0);
286 }
287 }
288
289 // Registers for the final merge to be produced.
290 SmallVector<Register, 4> Remerge(NumParts);
291
292 // Registers needed for intermediate merges, which will be merged into a
293 // source for Remerge.
294 SmallVector<Register, 4> SubMerge(NumSubParts);
295
296 // Once we've fully read off the end of the original source bits, we can reuse
297 // the same high bits for remaining padding elements.
298 Register AllPadReg;
299
300 // Build merges to the LCM type to cover the original result type.
301 for (int I = 0; I != NumParts; ++I) {
302 bool AllMergePartsArePadding = true;
303
304 // Build the requested merges to the requested type.
305 for (int J = 0; J != NumSubParts; ++J) {
306 int Idx = I * NumSubParts + J;
307 if (Idx >= NumOrigSrc) {
308 SubMerge[J] = PadReg;
309 continue;
310 }
311
312 SubMerge[J] = VRegs[Idx];
313
314 // There are meaningful bits here we can't reuse later.
315 AllMergePartsArePadding = false;
316 }
317
318 // If we've filled up a complete piece with padding bits, we can directly
319 // emit the natural sized constant if applicable, rather than a merge of
320 // smaller constants.
321 if (AllMergePartsArePadding && !AllPadReg) {
322 if (PadStrategy == TargetOpcode::G_ANYEXT)
323 AllPadReg = MIRBuilder.buildUndef(Res: NarrowTy).getReg(Idx: 0);
324 else if (PadStrategy == TargetOpcode::G_ZEXT)
325 AllPadReg = MIRBuilder.buildConstant(Res: NarrowTy, Val: 0).getReg(Idx: 0);
326
327 // If this is a sign extension, we can't materialize a trivial constant
328 // with the right type and have to produce a merge.
329 }
330
331 if (AllPadReg) {
332 // Avoid creating additional instructions if we're just adding additional
333 // copies of padding bits.
334 Remerge[I] = AllPadReg;
335 continue;
336 }
337
338 if (NumSubParts == 1)
339 Remerge[I] = SubMerge[0];
340 else
341 Remerge[I] = MIRBuilder.buildMergeLikeInstr(Res: NarrowTy, Ops: SubMerge).getReg(Idx: 0);
342
343 // In the sign extend padding case, re-use the first all-signbit merge.
344 if (AllMergePartsArePadding && !AllPadReg)
345 AllPadReg = Remerge[I];
346 }
347
348 VRegs = std::move(Remerge);
349 return LCMTy;
350}
351
352void LegalizerHelper::buildWidenedRemergeToDst(Register DstReg, LLT LCMTy,
353 ArrayRef<Register> RemergeRegs) {
354 LLT DstTy = MRI.getType(Reg: DstReg);
355
356 // Create the merge to the widened source, and extract the relevant bits into
357 // the result.
358
359 if (DstTy == LCMTy) {
360 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: RemergeRegs);
361 return;
362 }
363
364 auto Remerge = MIRBuilder.buildMergeLikeInstr(Res: LCMTy, Ops: RemergeRegs);
365 if (DstTy.isScalar() && LCMTy.isScalar()) {
366 MIRBuilder.buildTrunc(Res: DstReg, Op: Remerge);
367 return;
368 }
369
370 if (LCMTy.isVector()) {
371 unsigned NumDefs = LCMTy.getSizeInBits() / DstTy.getSizeInBits();
372 SmallVector<Register, 8> UnmergeDefs(NumDefs);
373 UnmergeDefs[0] = DstReg;
374 for (unsigned I = 1; I != NumDefs; ++I)
375 UnmergeDefs[I] = MRI.createGenericVirtualRegister(Ty: DstTy);
376
377 MIRBuilder.buildUnmerge(Res: UnmergeDefs,
378 Op: MIRBuilder.buildMergeLikeInstr(Res: LCMTy, Ops: RemergeRegs));
379 return;
380 }
381
382 llvm_unreachable("unhandled case");
383}
384
385static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
386#define RTLIBCASE_INT(LibcallPrefix) \
387 do { \
388 switch (Size) { \
389 case 32: \
390 return RTLIB::LibcallPrefix##32; \
391 case 64: \
392 return RTLIB::LibcallPrefix##64; \
393 case 128: \
394 return RTLIB::LibcallPrefix##128; \
395 default: \
396 llvm_unreachable("unexpected size"); \
397 } \
398 } while (0)
399
400#define RTLIBCASE(LibcallPrefix) \
401 do { \
402 switch (Size) { \
403 case 32: \
404 return RTLIB::LibcallPrefix##32; \
405 case 64: \
406 return RTLIB::LibcallPrefix##64; \
407 case 80: \
408 return RTLIB::LibcallPrefix##80; \
409 case 128: \
410 return RTLIB::LibcallPrefix##128; \
411 default: \
412 llvm_unreachable("unexpected size"); \
413 } \
414 } while (0)
415
416 switch (Opcode) {
417 case TargetOpcode::G_LROUND:
418 RTLIBCASE(LROUND_F);
419 case TargetOpcode::G_LLROUND:
420 RTLIBCASE(LLROUND_F);
421 case TargetOpcode::G_MUL:
422 RTLIBCASE_INT(MUL_I);
423 case TargetOpcode::G_SDIV:
424 RTLIBCASE_INT(SDIV_I);
425 case TargetOpcode::G_UDIV:
426 RTLIBCASE_INT(UDIV_I);
427 case TargetOpcode::G_SREM:
428 RTLIBCASE_INT(SREM_I);
429 case TargetOpcode::G_UREM:
430 RTLIBCASE_INT(UREM_I);
431 case TargetOpcode::G_CTLZ_ZERO_POISON:
432 RTLIBCASE_INT(CTLZ_I);
433 case TargetOpcode::G_FADD:
434 RTLIBCASE(ADD_F);
435 case TargetOpcode::G_FSUB:
436 RTLIBCASE(SUB_F);
437 case TargetOpcode::G_FMUL:
438 RTLIBCASE(MUL_F);
439 case TargetOpcode::G_FDIV:
440 RTLIBCASE(DIV_F);
441 case TargetOpcode::G_FEXP:
442 RTLIBCASE(EXP_F);
443 case TargetOpcode::G_FEXP2:
444 RTLIBCASE(EXP2_F);
445 case TargetOpcode::G_FEXP10:
446 RTLIBCASE(EXP10_F);
447 case TargetOpcode::G_FREM:
448 RTLIBCASE(REM_F);
449 case TargetOpcode::G_FPOW:
450 RTLIBCASE(POW_F);
451 case TargetOpcode::G_FPOWI:
452 RTLIBCASE(POWI_F);
453 case TargetOpcode::G_FMA:
454 RTLIBCASE(FMA_F);
455 case TargetOpcode::G_FSIN:
456 RTLIBCASE(SIN_F);
457 case TargetOpcode::G_FCOS:
458 RTLIBCASE(COS_F);
459 case TargetOpcode::G_FTAN:
460 RTLIBCASE(TAN_F);
461 case TargetOpcode::G_FASIN:
462 RTLIBCASE(ASIN_F);
463 case TargetOpcode::G_FACOS:
464 RTLIBCASE(ACOS_F);
465 case TargetOpcode::G_FATAN:
466 RTLIBCASE(ATAN_F);
467 case TargetOpcode::G_FATAN2:
468 RTLIBCASE(ATAN2_F);
469 case TargetOpcode::G_FSINH:
470 RTLIBCASE(SINH_F);
471 case TargetOpcode::G_FCOSH:
472 RTLIBCASE(COSH_F);
473 case TargetOpcode::G_FTANH:
474 RTLIBCASE(TANH_F);
475 case TargetOpcode::G_FSINCOS:
476 RTLIBCASE(SINCOS_F);
477 case TargetOpcode::G_FMODF:
478 RTLIBCASE(MODF_F);
479 case TargetOpcode::G_FLOG10:
480 RTLIBCASE(LOG10_F);
481 case TargetOpcode::G_FLOG:
482 RTLIBCASE(LOG_F);
483 case TargetOpcode::G_FLOG2:
484 RTLIBCASE(LOG2_F);
485 case TargetOpcode::G_FLDEXP:
486 RTLIBCASE(LDEXP_F);
487 case TargetOpcode::G_FCEIL:
488 RTLIBCASE(CEIL_F);
489 case TargetOpcode::G_FFLOOR:
490 RTLIBCASE(FLOOR_F);
491 case TargetOpcode::G_FMINNUM:
492 RTLIBCASE(FMIN_F);
493 case TargetOpcode::G_FMAXNUM:
494 RTLIBCASE(FMAX_F);
495 case TargetOpcode::G_FMINIMUMNUM:
496 RTLIBCASE(FMINIMUM_NUM_F);
497 case TargetOpcode::G_FMAXIMUMNUM:
498 RTLIBCASE(FMAXIMUM_NUM_F);
499 case TargetOpcode::G_FSQRT:
500 RTLIBCASE(SQRT_F);
501 case TargetOpcode::G_FRINT:
502 RTLIBCASE(RINT_F);
503 case TargetOpcode::G_FNEARBYINT:
504 RTLIBCASE(NEARBYINT_F);
505 case TargetOpcode::G_INTRINSIC_TRUNC:
506 RTLIBCASE(TRUNC_F);
507 case TargetOpcode::G_INTRINSIC_ROUND:
508 RTLIBCASE(ROUND_F);
509 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
510 RTLIBCASE(ROUNDEVEN_F);
511 case TargetOpcode::G_INTRINSIC_LRINT:
512 RTLIBCASE(LRINT_F);
513 case TargetOpcode::G_INTRINSIC_LLRINT:
514 RTLIBCASE(LLRINT_F);
515 }
516 llvm_unreachable("Unknown libcall function");
517#undef RTLIBCASE_INT
518#undef RTLIBCASE
519}
520
521/// True if an instruction is in tail position in its caller. Intended for
522/// legalizing libcalls as tail calls when possible.
523static bool isLibCallInTailPosition(const CallLowering::ArgInfo &Result,
524 MachineInstr &MI,
525 const TargetInstrInfo &TII,
526 MachineRegisterInfo &MRI) {
527 MachineBasicBlock &MBB = *MI.getParent();
528 const Function &F = MBB.getParent()->getFunction();
529
530 // Conservatively require the attributes of the call to match those of
531 // the return. Ignore NoAlias and NonNull because they don't affect the
532 // call sequence.
533 AttributeList CallerAttrs = F.getAttributes();
534 if (AttrBuilder(F.getContext(), CallerAttrs.getRetAttrs())
535 .removeAttribute(Val: Attribute::NoAlias)
536 .removeAttribute(Val: Attribute::NonNull)
537 .hasAttributes())
538 return false;
539
540 // It's not safe to eliminate the sign / zero extension of the return value.
541 if (CallerAttrs.hasRetAttr(Kind: Attribute::ZExt) ||
542 CallerAttrs.hasRetAttr(Kind: Attribute::SExt))
543 return false;
544
545 // Only tail call if the following instruction is a standard return or if we
546 // have a `thisreturn` callee, and a sequence like:
547 //
548 // G_MEMCPY %0, %1, %2
549 // $x0 = COPY %0
550 // RET_ReallyLR implicit $x0
551 auto Next = next_nodbg(It: MI.getIterator(), End: MBB.instr_end());
552 if (Next != MBB.instr_end() && Next->isCopy()) {
553 if (MI.getOpcode() == TargetOpcode::G_BZERO)
554 return false;
555
556 // For MEMCPY/MOMMOVE/MEMSET these will be the first use (the dst), as the
557 // mempy/etc routines return the same parameter. For other it will be the
558 // returned value.
559 Register VReg = MI.getOperand(i: 0).getReg();
560 if (!VReg.isVirtual() || VReg != Next->getOperand(i: 1).getReg())
561 return false;
562
563 Register PReg = Next->getOperand(i: 0).getReg();
564 if (!PReg.isPhysical())
565 return false;
566
567 auto Ret = next_nodbg(It: Next, End: MBB.instr_end());
568 if (Ret == MBB.instr_end() || !Ret->isReturn())
569 return false;
570
571 if (Ret->getNumImplicitOperands() != 1)
572 return false;
573
574 if (!Ret->getOperand(i: 0).isReg() || PReg != Ret->getOperand(i: 0).getReg())
575 return false;
576
577 // Skip over the COPY that we just validated.
578 Next = Ret;
579 }
580
581 if (Next == MBB.instr_end() || TII.isTailCall(Inst: *Next) || !Next->isReturn())
582 return false;
583
584 return true;
585}
586
587LegalizerHelper::LegalizeResult LegalizerHelper::createLibcall(
588 const char *Name, const CallLowering::ArgInfo &Result,
589 ArrayRef<CallLowering::ArgInfo> Args, const CallingConv::ID CC,
590 LostDebugLocObserver &LocObserver, MachineInstr *MI) const {
591 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
592
593 CallLowering::CallLoweringInfo Info;
594 Info.CallConv = CC;
595 Info.Callee = MachineOperand::CreateES(SymName: Name);
596 Info.OrigRet = Result;
597 if (MI)
598 Info.IsTailCall =
599 (Result.Ty->isVoidTy() ||
600 Result.Ty == MIRBuilder.getMF().getFunction().getReturnType()) &&
601 isLibCallInTailPosition(Result, MI&: *MI, TII: MIRBuilder.getTII(),
602 MRI&: *MIRBuilder.getMRI());
603
604 llvm::append_range(C&: Info.OrigArgs, R&: Args);
605 if (!CLI.lowerCall(MIRBuilder, Info))
606 return LegalizerHelper::UnableToLegalize;
607
608 if (MI && Info.LoweredTailCall) {
609 assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?");
610
611 // Check debug locations before removing the return.
612 LocObserver.checkpoint(CheckDebugLocs: true);
613
614 // We must have a return following the call (or debug insts) to get past
615 // isLibCallInTailPosition.
616 do {
617 MachineInstr *Next = MI->getNextNode();
618 assert(Next &&
619 (Next->isCopy() || Next->isReturn() || Next->isDebugInstr()) &&
620 "Expected instr following MI to be return or debug inst?");
621 // We lowered a tail call, so the call is now the return from the block.
622 // Delete the old return.
623 Next->eraseFromParent();
624 } while (MI->getNextNode());
625
626 // We expect to lose the debug location from the return.
627 LocObserver.checkpoint(CheckDebugLocs: false);
628 }
629 return LegalizerHelper::Legalized;
630}
631
632LegalizerHelper::LegalizeResult LegalizerHelper::createLibcall(
633 RTLIB::Libcall Libcall, const CallLowering::ArgInfo &Result,
634 ArrayRef<CallLowering::ArgInfo> Args, LostDebugLocObserver &LocObserver,
635 MachineInstr *MI) const {
636 if (!Libcalls)
637 return LegalizerHelper::UnableToLegalize;
638
639 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: Libcall);
640 if (LibcallImpl == RTLIB::Unsupported)
641 return LegalizerHelper::UnableToLegalize;
642
643 StringRef Name = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: LibcallImpl);
644 const CallingConv::ID CC = Libcalls->getLibcallImplCallingConv(Call: LibcallImpl);
645 return createLibcall(Name: Name.data(), Result, Args, CC, LocObserver, MI);
646}
647
648// Useful for libcalls where all operands have the same type.
649LegalizerHelper::LegalizeResult
650LegalizerHelper::simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder,
651 unsigned Size, Type *OpType,
652 LostDebugLocObserver &LocObserver) const {
653 auto Libcall = getRTLibDesc(Opcode: MI.getOpcode(), Size);
654
655 // FIXME: What does the original arg index mean here?
656 SmallVector<CallLowering::ArgInfo, 3> Args;
657 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands()))
658 Args.push_back(Elt: {MO.getReg(), OpType, 0});
659 return createLibcall(Libcall, Result: {MI.getOperand(i: 0).getReg(), OpType, 0}, Args,
660 LocObserver, MI: &MI);
661}
662
663LegalizerHelper::LegalizeResult LegalizerHelper::emitSincosLibcall(
664 MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size, Type *OpType,
665 LostDebugLocObserver &LocObserver) {
666 MachineFunction &MF = *MI.getMF();
667 MachineRegisterInfo &MRI = MF.getRegInfo();
668
669 Register DstSin = MI.getOperand(i: 0).getReg();
670 Register DstCos = MI.getOperand(i: 1).getReg();
671 Register Src = MI.getOperand(i: 2).getReg();
672 LLT DstTy = MRI.getType(Reg: DstSin);
673
674 int MemSize = DstTy.getSizeInBytes();
675 Align Alignment = getStackTemporaryAlignment(Type: DstTy);
676 const DataLayout &DL = MIRBuilder.getDataLayout();
677 unsigned AddrSpace = DL.getAllocaAddrSpace();
678 MachinePointerInfo PtrInfo;
679
680 Register StackPtrSin =
681 createStackTemporary(Bytes: TypeSize::getFixed(ExactSize: MemSize), Alignment, PtrInfo)
682 .getReg(Idx: 0);
683 Register StackPtrCos =
684 createStackTemporary(Bytes: TypeSize::getFixed(ExactSize: MemSize), Alignment, PtrInfo)
685 .getReg(Idx: 0);
686
687 auto &Ctx = MF.getFunction().getContext();
688 auto LibcallResult = createLibcall(
689 Libcall: getRTLibDesc(Opcode: MI.getOpcode(), Size), Result: {{0}, Type::getVoidTy(C&: Ctx), 0},
690 Args: {{Src, OpType, 0},
691 {StackPtrSin, PointerType::get(C&: Ctx, AddressSpace: AddrSpace), 1},
692 {StackPtrCos, PointerType::get(C&: Ctx, AddressSpace: AddrSpace), 2}},
693 LocObserver, MI: &MI);
694
695 if (LibcallResult != LegalizeResult::Legalized)
696 return LegalizerHelper::UnableToLegalize;
697
698 MachineMemOperand *LoadMMOSin = MF.getMachineMemOperand(
699 PtrInfo, F: MachineMemOperand::MOLoad, Size: MemSize, BaseAlignment: Alignment);
700 MachineMemOperand *LoadMMOCos = MF.getMachineMemOperand(
701 PtrInfo, F: MachineMemOperand::MOLoad, Size: MemSize, BaseAlignment: Alignment);
702
703 MIRBuilder.buildLoad(Res: DstSin, Addr: StackPtrSin, MMO&: *LoadMMOSin);
704 MIRBuilder.buildLoad(Res: DstCos, Addr: StackPtrCos, MMO&: *LoadMMOCos);
705 MI.eraseFromParent();
706
707 return LegalizerHelper::Legalized;
708}
709
710LegalizerHelper::LegalizeResult
711LegalizerHelper::emitModfLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder,
712 unsigned Size, Type *OpType,
713 LostDebugLocObserver &LocObserver) {
714 MachineFunction &MF = MIRBuilder.getMF();
715 MachineRegisterInfo &MRI = MF.getRegInfo();
716
717 Register DstFrac = MI.getOperand(i: 0).getReg();
718 Register DstInt = MI.getOperand(i: 1).getReg();
719 Register Src = MI.getOperand(i: 2).getReg();
720 LLT DstTy = MRI.getType(Reg: DstFrac);
721
722 int MemSize = DstTy.getSizeInBytes();
723 Align Alignment = getStackTemporaryAlignment(Type: DstTy);
724 const DataLayout &DL = MIRBuilder.getDataLayout();
725 unsigned AddrSpace = DL.getAllocaAddrSpace();
726 MachinePointerInfo PtrInfo;
727
728 Register StackPtrInt =
729 createStackTemporary(Bytes: TypeSize::getFixed(ExactSize: MemSize), Alignment, PtrInfo)
730 .getReg(Idx: 0);
731
732 auto &Ctx = MF.getFunction().getContext();
733 auto LibcallResult = createLibcall(
734 Libcall: getRTLibDesc(Opcode: MI.getOpcode(), Size), Result: {DstFrac, OpType, 0},
735 Args: {{Src, OpType, 0}, {StackPtrInt, PointerType::get(C&: Ctx, AddressSpace: AddrSpace), 1}},
736 LocObserver, MI: &MI);
737
738 if (LibcallResult != LegalizeResult::Legalized)
739 return LegalizerHelper::UnableToLegalize;
740
741 MachineMemOperand *LoadMMOInt = MF.getMachineMemOperand(
742 PtrInfo, F: MachineMemOperand::MOLoad, Size: MemSize, BaseAlignment: Alignment);
743
744 MIRBuilder.buildLoad(Res: DstInt, Addr: StackPtrInt, MMO&: *LoadMMOInt);
745 MI.eraseFromParent();
746
747 return LegalizerHelper::Legalized;
748}
749
750static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
751 Type *FromType) {
752 auto ToMVT = MVT::getVT(Ty: ToType);
753 auto FromMVT = MVT::getVT(Ty: FromType);
754
755 switch (Opcode) {
756 case TargetOpcode::G_FPEXT:
757 return RTLIB::getFPEXT(OpVT: FromMVT, RetVT: ToMVT);
758 case TargetOpcode::G_FPTRUNC:
759 return RTLIB::getFPROUND(OpVT: FromMVT, RetVT: ToMVT);
760 case TargetOpcode::G_FPTOSI:
761 return RTLIB::getFPTOSINT(OpVT: FromMVT, RetVT: ToMVT);
762 case TargetOpcode::G_FPTOUI:
763 return RTLIB::getFPTOUINT(OpVT: FromMVT, RetVT: ToMVT);
764 case TargetOpcode::G_SITOFP:
765 return RTLIB::getSINTTOFP(OpVT: FromMVT, RetVT: ToMVT);
766 case TargetOpcode::G_UITOFP:
767 return RTLIB::getUINTTOFP(OpVT: FromMVT, RetVT: ToMVT);
768 }
769 llvm_unreachable("Unsupported libcall function");
770}
771
772LegalizerHelper::LegalizeResult LegalizerHelper::conversionLibcall(
773 MachineInstr &MI, Type *ToType, Type *FromType,
774 LostDebugLocObserver &LocObserver, bool IsSigned) const {
775 CallLowering::ArgInfo Arg = {MI.getOperand(i: 1).getReg(), FromType, 0};
776 if (FromType->isIntegerTy()) {
777 if (TLI.shouldSignExtendTypeInLibCall(Ty: FromType, IsSigned))
778 Arg.Flags[0].setSExt();
779 else
780 Arg.Flags[0].setZExt();
781 }
782
783 RTLIB::Libcall Libcall = getConvRTLibDesc(Opcode: MI.getOpcode(), ToType, FromType);
784 return createLibcall(Libcall, Result: {MI.getOperand(i: 0).getReg(), ToType, 0}, Args: Arg,
785 LocObserver, MI: &MI);
786}
787
788LegalizerHelper::LegalizeResult
789LegalizerHelper::createMemLibcall(MachineRegisterInfo &MRI, MachineInstr &MI,
790 LostDebugLocObserver &LocObserver) const {
791 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
792
793 SmallVector<CallLowering::ArgInfo, 3> Args;
794 // Add all the args, except for the last which is an imm denoting 'tail'.
795 for (unsigned i = 0; i < MI.getNumOperands() - 1; ++i) {
796 Register Reg = MI.getOperand(i).getReg();
797
798 // Need derive an IR type for call lowering.
799 LLT OpLLT = MRI.getType(Reg);
800 Type *OpTy = nullptr;
801 if (OpLLT.isPointer())
802 OpTy = PointerType::get(C&: Ctx, AddressSpace: OpLLT.getAddressSpace());
803 else
804 OpTy = IntegerType::get(C&: Ctx, NumBits: OpLLT.getSizeInBits());
805 Args.push_back(Elt: {Reg, OpTy, 0});
806 }
807
808 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
809 RTLIB::Libcall RTLibcall;
810 unsigned Opc = MI.getOpcode();
811 switch (Opc) {
812 case TargetOpcode::G_BZERO:
813 RTLibcall = RTLIB::BZERO;
814 break;
815 case TargetOpcode::G_MEMCPY:
816 RTLibcall = RTLIB::MEMCPY;
817 Args[0].Flags[0].setReturned();
818 break;
819 case TargetOpcode::G_MEMMOVE:
820 RTLibcall = RTLIB::MEMMOVE;
821 Args[0].Flags[0].setReturned();
822 break;
823 case TargetOpcode::G_MEMSET:
824 RTLibcall = RTLIB::MEMSET;
825 Args[0].Flags[0].setReturned();
826 break;
827 default:
828 llvm_unreachable("unsupported opcode");
829 }
830
831 if (!Libcalls) // FIXME: Should be mandatory
832 return LegalizerHelper::UnableToLegalize;
833
834 RTLIB::LibcallImpl RTLibcallImpl = Libcalls->getLibcallImpl(Call: RTLibcall);
835
836 // Unsupported libcall on the target.
837 if (RTLibcallImpl == RTLIB::Unsupported) {
838 LLVM_DEBUG(dbgs() << ".. .. Could not find libcall name for "
839 << MIRBuilder.getTII().getName(Opc) << "\n");
840 return LegalizerHelper::UnableToLegalize;
841 }
842
843 CallLowering::CallLoweringInfo Info;
844 Info.CallConv = Libcalls->getLibcallImplCallingConv(Call: RTLibcallImpl);
845
846 StringRef LibcallName =
847 RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: RTLibcallImpl);
848 Info.Callee = MachineOperand::CreateES(SymName: LibcallName.data());
849 Info.OrigRet = CallLowering::ArgInfo({0}, Type::getVoidTy(C&: Ctx), 0);
850 Info.IsTailCall =
851 MI.getOperand(i: MI.getNumOperands() - 1).getImm() &&
852 isLibCallInTailPosition(Result: Info.OrigRet, MI, TII: MIRBuilder.getTII(), MRI);
853
854 llvm::append_range(C&: Info.OrigArgs, R&: Args);
855 if (!CLI.lowerCall(MIRBuilder, Info))
856 return LegalizerHelper::UnableToLegalize;
857
858 if (Info.LoweredTailCall) {
859 assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?");
860
861 // Check debug locations before removing the return.
862 LocObserver.checkpoint(CheckDebugLocs: true);
863
864 // We must have a return following the call (or debug insts) to get past
865 // isLibCallInTailPosition.
866 do {
867 MachineInstr *Next = MI.getNextNode();
868 assert(Next &&
869 (Next->isCopy() || Next->isReturn() || Next->isDebugInstr()) &&
870 "Expected instr following MI to be return or debug inst?");
871 // We lowered a tail call, so the call is now the return from the block.
872 // Delete the old return.
873 Next->eraseFromParent();
874 } while (MI.getNextNode());
875
876 // We expect to lose the debug location from the return.
877 LocObserver.checkpoint(CheckDebugLocs: false);
878 }
879
880 return LegalizerHelper::Legalized;
881}
882
883static RTLIB::Libcall getOutlineAtomicLibcall(MachineInstr &MI) {
884 unsigned Opc = MI.getOpcode();
885 auto &AtomicMI = cast<GMemOperation>(Val&: MI);
886 auto &MMO = AtomicMI.getMMO();
887 auto Ordering = MMO.getMergedOrdering();
888 LLT MemType = MMO.getMemoryType();
889 uint64_t MemSize = MemType.getSizeInBytes();
890 if (MemType.isVector())
891 return RTLIB::UNKNOWN_LIBCALL;
892
893#define LCALLS(A, B) {A##B##_RELAX, A##B##_ACQ, A##B##_REL, A##B##_ACQ_REL}
894#define LCALL5(A) \
895 LCALLS(A, 1), LCALLS(A, 2), LCALLS(A, 4), LCALLS(A, 8), LCALLS(A, 16)
896 switch (Opc) {
897 case TargetOpcode::G_ATOMIC_CMPXCHG:
898 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
899 const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_CAS)};
900 return getOutlineAtomicHelper(LC, Order: Ordering, MemSize);
901 }
902 case TargetOpcode::G_ATOMICRMW_XCHG: {
903 const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_SWP)};
904 return getOutlineAtomicHelper(LC, Order: Ordering, MemSize);
905 }
906 case TargetOpcode::G_ATOMICRMW_ADD:
907 case TargetOpcode::G_ATOMICRMW_SUB: {
908 const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDADD)};
909 return getOutlineAtomicHelper(LC, Order: Ordering, MemSize);
910 }
911 case TargetOpcode::G_ATOMICRMW_AND: {
912 const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDCLR)};
913 return getOutlineAtomicHelper(LC, Order: Ordering, MemSize);
914 }
915 case TargetOpcode::G_ATOMICRMW_OR: {
916 const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDSET)};
917 return getOutlineAtomicHelper(LC, Order: Ordering, MemSize);
918 }
919 case TargetOpcode::G_ATOMICRMW_XOR: {
920 const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDEOR)};
921 return getOutlineAtomicHelper(LC, Order: Ordering, MemSize);
922 }
923 default:
924 return RTLIB::UNKNOWN_LIBCALL;
925 }
926#undef LCALLS
927#undef LCALL5
928}
929
930LegalizerHelper::LegalizeResult
931LegalizerHelper::createAtomicLibcall(MachineInstr &MI) const {
932 auto &Ctx = MIRBuilder.getContext();
933
934 Type *RetTy;
935 SmallVector<Register> RetRegs;
936 SmallVector<CallLowering::ArgInfo, 3> Args;
937 unsigned Opc = MI.getOpcode();
938 switch (Opc) {
939 case TargetOpcode::G_ATOMIC_CMPXCHG:
940 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
941 Register Success;
942 LLT SuccessLLT;
943 auto [Ret, RetLLT, Mem, MemLLT, Cmp, CmpLLT, New, NewLLT] =
944 MI.getFirst4RegLLTs();
945 RetRegs.push_back(Elt: Ret);
946 RetTy = IntegerType::get(C&: Ctx, NumBits: RetLLT.getSizeInBits());
947 if (Opc == TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS) {
948 std::tie(args&: Ret, args&: RetLLT, args&: Success, args&: SuccessLLT, args&: Mem, args&: MemLLT, args&: Cmp, args&: CmpLLT, args&: New,
949 args&: NewLLT) = MI.getFirst5RegLLTs();
950 RetRegs.push_back(Elt: Success);
951 RetTy = StructType::get(
952 Context&: Ctx, Elements: {RetTy, IntegerType::get(C&: Ctx, NumBits: SuccessLLT.getSizeInBits())});
953 }
954 Args.push_back(Elt: {Cmp, IntegerType::get(C&: Ctx, NumBits: CmpLLT.getSizeInBits()), 0});
955 Args.push_back(Elt: {New, IntegerType::get(C&: Ctx, NumBits: NewLLT.getSizeInBits()), 0});
956 Args.push_back(Elt: {Mem, PointerType::get(C&: Ctx, AddressSpace: MemLLT.getAddressSpace()), 0});
957 break;
958 }
959 case TargetOpcode::G_ATOMICRMW_XCHG:
960 case TargetOpcode::G_ATOMICRMW_ADD:
961 case TargetOpcode::G_ATOMICRMW_SUB:
962 case TargetOpcode::G_ATOMICRMW_AND:
963 case TargetOpcode::G_ATOMICRMW_OR:
964 case TargetOpcode::G_ATOMICRMW_XOR: {
965 auto [Ret, RetLLT, Mem, MemLLT, Val, ValLLT] = MI.getFirst3RegLLTs();
966 RetRegs.push_back(Elt: Ret);
967 RetTy = IntegerType::get(C&: Ctx, NumBits: RetLLT.getSizeInBits());
968 if (Opc == TargetOpcode::G_ATOMICRMW_AND)
969 Val =
970 MIRBuilder.buildXor(Dst: ValLLT, Src0: MIRBuilder.buildConstant(Res: ValLLT, Val: -1), Src1: Val)
971 .getReg(Idx: 0);
972 else if (Opc == TargetOpcode::G_ATOMICRMW_SUB)
973 Val =
974 MIRBuilder.buildSub(Dst: ValLLT, Src0: MIRBuilder.buildConstant(Res: ValLLT, Val: 0), Src1: Val)
975 .getReg(Idx: 0);
976 Args.push_back(Elt: {Val, IntegerType::get(C&: Ctx, NumBits: ValLLT.getSizeInBits()), 0});
977 Args.push_back(Elt: {Mem, PointerType::get(C&: Ctx, AddressSpace: MemLLT.getAddressSpace()), 0});
978 break;
979 }
980 default:
981 llvm_unreachable("unsupported opcode");
982 }
983
984 if (!Libcalls) // FIXME: Should be mandatory
985 return LegalizerHelper::UnableToLegalize;
986
987 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
988 RTLIB::Libcall RTLibcall = getOutlineAtomicLibcall(MI);
989 RTLIB::LibcallImpl RTLibcallImpl = Libcalls->getLibcallImpl(Call: RTLibcall);
990
991 // Unsupported libcall on the target.
992 if (RTLibcallImpl == RTLIB::Unsupported) {
993 LLVM_DEBUG(dbgs() << ".. .. Could not find libcall name for "
994 << MIRBuilder.getTII().getName(Opc) << "\n");
995 return LegalizerHelper::UnableToLegalize;
996 }
997
998 CallLowering::CallLoweringInfo Info;
999 Info.CallConv = Libcalls->getLibcallImplCallingConv(Call: RTLibcallImpl);
1000
1001 StringRef LibcallName =
1002 RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: RTLibcallImpl);
1003 Info.Callee = MachineOperand::CreateES(SymName: LibcallName.data());
1004 Info.OrigRet = CallLowering::ArgInfo(RetRegs, RetTy, 0);
1005
1006 llvm::append_range(C&: Info.OrigArgs, R&: Args);
1007 if (!CLI.lowerCall(MIRBuilder, Info))
1008 return LegalizerHelper::UnableToLegalize;
1009
1010 return LegalizerHelper::Legalized;
1011}
1012
1013static RTLIB::Libcall
1014getStateLibraryFunctionFor(MachineInstr &MI, const TargetLowering &TLI) {
1015 RTLIB::Libcall RTLibcall;
1016 switch (MI.getOpcode()) {
1017 case TargetOpcode::G_GET_FPENV:
1018 RTLibcall = RTLIB::FEGETENV;
1019 break;
1020 case TargetOpcode::G_SET_FPENV:
1021 case TargetOpcode::G_RESET_FPENV:
1022 RTLibcall = RTLIB::FESETENV;
1023 break;
1024 case TargetOpcode::G_GET_FPMODE:
1025 RTLibcall = RTLIB::FEGETMODE;
1026 break;
1027 case TargetOpcode::G_SET_FPMODE:
1028 case TargetOpcode::G_RESET_FPMODE:
1029 RTLibcall = RTLIB::FESETMODE;
1030 break;
1031 default:
1032 llvm_unreachable("Unexpected opcode");
1033 }
1034 return RTLibcall;
1035}
1036
1037// Some library functions that read FP state (fegetmode, fegetenv) write the
1038// state into a region in memory. IR intrinsics that do the same operations
1039// (get_fpmode, get_fpenv) return the state as integer value. To implement these
1040// intrinsics via the library functions, we need to use temporary variable,
1041// for example:
1042//
1043// %0:_(s32) = G_GET_FPMODE
1044//
1045// is transformed to:
1046//
1047// %1:_(p0) = G_FRAME_INDEX %stack.0
1048// BL &fegetmode
1049// %0:_(s32) = G_LOAD % 1
1050//
1051LegalizerHelper::LegalizeResult
1052LegalizerHelper::createGetStateLibcall(MachineInstr &MI,
1053 LostDebugLocObserver &LocObserver) {
1054 const DataLayout &DL = MIRBuilder.getDataLayout();
1055 auto &MF = MIRBuilder.getMF();
1056 auto &MRI = *MIRBuilder.getMRI();
1057 auto &Ctx = MF.getFunction().getContext();
1058
1059 // Create temporary, where library function will put the read state.
1060 Register Dst = MI.getOperand(i: 0).getReg();
1061 LLT StateTy = MRI.getType(Reg: Dst);
1062 TypeSize StateSize = StateTy.getSizeInBytes();
1063 Align TempAlign = getStackTemporaryAlignment(Type: StateTy);
1064 MachinePointerInfo TempPtrInfo;
1065 auto Temp = createStackTemporary(Bytes: StateSize, Alignment: TempAlign, PtrInfo&: TempPtrInfo);
1066
1067 // Create a call to library function, with the temporary as an argument.
1068 unsigned TempAddrSpace = DL.getAllocaAddrSpace();
1069 Type *StatePtrTy = PointerType::get(C&: Ctx, AddressSpace: TempAddrSpace);
1070 RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI);
1071 auto Res = createLibcall(
1072 Libcall: RTLibcall, Result: CallLowering::ArgInfo({0}, Type::getVoidTy(C&: Ctx), 0),
1073 Args: CallLowering::ArgInfo({Temp.getReg(Idx: 0), StatePtrTy, 0}), LocObserver,
1074 MI: nullptr);
1075 if (Res != LegalizerHelper::Legalized)
1076 return Res;
1077
1078 // Create a load from the temporary.
1079 MachineMemOperand *MMO = MF.getMachineMemOperand(
1080 PtrInfo: TempPtrInfo, f: MachineMemOperand::MOLoad, MemTy: StateTy, base_alignment: TempAlign);
1081 MIRBuilder.buildLoadInstr(Opcode: TargetOpcode::G_LOAD, Res: Dst, Addr: Temp, MMO&: *MMO);
1082
1083 return LegalizerHelper::Legalized;
1084}
1085
1086// Similar to `createGetStateLibcall` the function calls a library function
1087// using transient space in stack. In this case the library function reads
1088// content of memory region.
1089LegalizerHelper::LegalizeResult
1090LegalizerHelper::createSetStateLibcall(MachineInstr &MI,
1091 LostDebugLocObserver &LocObserver) {
1092 const DataLayout &DL = MIRBuilder.getDataLayout();
1093 auto &MF = MIRBuilder.getMF();
1094 auto &MRI = *MIRBuilder.getMRI();
1095 auto &Ctx = MF.getFunction().getContext();
1096
1097 // Create temporary, where library function will get the new state.
1098 Register Src = MI.getOperand(i: 0).getReg();
1099 LLT StateTy = MRI.getType(Reg: Src);
1100 TypeSize StateSize = StateTy.getSizeInBytes();
1101 Align TempAlign = getStackTemporaryAlignment(Type: StateTy);
1102 MachinePointerInfo TempPtrInfo;
1103 auto Temp = createStackTemporary(Bytes: StateSize, Alignment: TempAlign, PtrInfo&: TempPtrInfo);
1104
1105 // Put the new state into the temporary.
1106 MachineMemOperand *MMO = MF.getMachineMemOperand(
1107 PtrInfo: TempPtrInfo, f: MachineMemOperand::MOStore, MemTy: StateTy, base_alignment: TempAlign);
1108 MIRBuilder.buildStore(Val: Src, Addr: Temp, MMO&: *MMO);
1109
1110 // Create a call to library function, with the temporary as an argument.
1111 unsigned TempAddrSpace = DL.getAllocaAddrSpace();
1112 Type *StatePtrTy = PointerType::get(C&: Ctx, AddressSpace: TempAddrSpace);
1113 RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI);
1114 return createLibcall(Libcall: RTLibcall,
1115 Result: CallLowering::ArgInfo({0}, Type::getVoidTy(C&: Ctx), 0),
1116 Args: CallLowering::ArgInfo({Temp.getReg(Idx: 0), StatePtrTy, 0}),
1117 LocObserver, MI: nullptr);
1118}
1119
1120/// Returns the corresponding libcall for the given Pred and
1121/// the ICMP predicate that should be generated to compare with #0
1122/// after the libcall.
1123static std::pair<RTLIB::Libcall, CmpInst::Predicate>
1124getFCMPLibcallDesc(const CmpInst::Predicate Pred, unsigned Size) {
1125#define RTLIBCASE_CMP(LibcallPrefix, ICmpPred) \
1126 do { \
1127 switch (Size) { \
1128 case 32: \
1129 return {RTLIB::LibcallPrefix##32, ICmpPred}; \
1130 case 64: \
1131 return {RTLIB::LibcallPrefix##64, ICmpPred}; \
1132 case 128: \
1133 return {RTLIB::LibcallPrefix##128, ICmpPred}; \
1134 default: \
1135 llvm_unreachable("unexpected size"); \
1136 } \
1137 } while (0)
1138
1139 switch (Pred) {
1140 case CmpInst::FCMP_OEQ:
1141 RTLIBCASE_CMP(OEQ_F, CmpInst::ICMP_EQ);
1142 case CmpInst::FCMP_UNE:
1143 RTLIBCASE_CMP(UNE_F, CmpInst::ICMP_NE);
1144 case CmpInst::FCMP_OGE:
1145 RTLIBCASE_CMP(OGE_F, CmpInst::ICMP_SGE);
1146 case CmpInst::FCMP_OLT:
1147 RTLIBCASE_CMP(OLT_F, CmpInst::ICMP_SLT);
1148 case CmpInst::FCMP_OLE:
1149 RTLIBCASE_CMP(OLE_F, CmpInst::ICMP_SLE);
1150 case CmpInst::FCMP_OGT:
1151 RTLIBCASE_CMP(OGT_F, CmpInst::ICMP_SGT);
1152 case CmpInst::FCMP_UNO:
1153 RTLIBCASE_CMP(UO_F, CmpInst::ICMP_NE);
1154 default:
1155 return {RTLIB::UNKNOWN_LIBCALL, CmpInst::BAD_ICMP_PREDICATE};
1156 }
1157}
1158
1159LegalizerHelper::LegalizeResult
1160LegalizerHelper::createFCMPLibcall(MachineInstr &MI,
1161 LostDebugLocObserver &LocObserver) {
1162 auto &MF = MIRBuilder.getMF();
1163 auto &Ctx = MF.getFunction().getContext();
1164 const GFCmp *Cmp = cast<GFCmp>(Val: &MI);
1165
1166 LLT OpLLT = MRI.getType(Reg: Cmp->getLHSReg());
1167 unsigned Size = OpLLT.getSizeInBits();
1168 if ((Size != 32 && Size != 64 && Size != 128) ||
1169 OpLLT != MRI.getType(Reg: Cmp->getRHSReg()))
1170 return UnableToLegalize;
1171
1172 Type *OpType = getFloatTypeForLLT(Ctx, Ty: OpLLT);
1173
1174 // DstReg type is s32
1175 const Register DstReg = Cmp->getReg(Idx: 0);
1176 LLT DstTy = MRI.getType(Reg: DstReg);
1177 const auto Cond = Cmp->getCond();
1178
1179 // Reference:
1180 // https://gcc.gnu.org/onlinedocs/gccint/Soft-float-library-routines.html#Comparison-functions-1
1181 // Generates a libcall followed by ICMP.
1182 const auto BuildLibcall = [&](const RTLIB::Libcall Libcall,
1183 const CmpInst::Predicate ICmpPred,
1184 const DstOp &Res) -> Register {
1185 // FCMP libcall always returns an i32, and needs an ICMP with #0.
1186 LLT TempLLT = LLT::integer(SizeInBits: 32);
1187 Register Temp = MRI.createGenericVirtualRegister(Ty: TempLLT);
1188 // Generate libcall, holding result in Temp
1189 const auto Status = createLibcall(
1190 Libcall, Result: {Temp, Type::getInt32Ty(C&: Ctx), 0},
1191 Args: {{Cmp->getLHSReg(), OpType, 0}, {Cmp->getRHSReg(), OpType, 1}},
1192 LocObserver, MI: &MI);
1193 if (!Status)
1194 return {};
1195
1196 // Compare temp with #0 to get the final result.
1197 return MIRBuilder
1198 .buildICmp(Pred: ICmpPred, Res, Op0: Temp, Op1: MIRBuilder.buildConstant(Res: TempLLT, Val: 0))
1199 .getReg(Idx: 0);
1200 };
1201
1202 // Simple case if we have a direct mapping from predicate to libcall
1203 if (const auto [Libcall, ICmpPred] = getFCMPLibcallDesc(Pred: Cond, Size);
1204 Libcall != RTLIB::UNKNOWN_LIBCALL &&
1205 ICmpPred != CmpInst::BAD_ICMP_PREDICATE) {
1206 if (BuildLibcall(Libcall, ICmpPred, DstReg)) {
1207 return Legalized;
1208 }
1209 return UnableToLegalize;
1210 }
1211
1212 // No direct mapping found, should be generated as combination of libcalls.
1213
1214 switch (Cond) {
1215 case CmpInst::FCMP_UEQ: {
1216 // FCMP_UEQ: unordered or equal
1217 // Convert into (FCMP_OEQ || FCMP_UNO).
1218
1219 const auto [OeqLibcall, OeqPred] =
1220 getFCMPLibcallDesc(Pred: CmpInst::FCMP_OEQ, Size);
1221 const auto Oeq = BuildLibcall(OeqLibcall, OeqPred, DstTy);
1222
1223 const auto [UnoLibcall, UnoPred] =
1224 getFCMPLibcallDesc(Pred: CmpInst::FCMP_UNO, Size);
1225 const auto Uno = BuildLibcall(UnoLibcall, UnoPred, DstTy);
1226 if (Oeq && Uno)
1227 MIRBuilder.buildOr(Dst: DstReg, Src0: Oeq, Src1: Uno);
1228 else
1229 return UnableToLegalize;
1230
1231 break;
1232 }
1233 case CmpInst::FCMP_ONE: {
1234 // FCMP_ONE: ordered and operands are unequal
1235 // Convert into (!FCMP_OEQ && !FCMP_UNO).
1236
1237 // We inverse the predicate instead of generating a NOT
1238 // to save one instruction.
1239 // On AArch64 isel can even select two cmp into a single ccmp.
1240 const auto [OeqLibcall, OeqPred] =
1241 getFCMPLibcallDesc(Pred: CmpInst::FCMP_OEQ, Size);
1242 const auto NotOeq =
1243 BuildLibcall(OeqLibcall, CmpInst::getInversePredicate(pred: OeqPred), DstTy);
1244
1245 const auto [UnoLibcall, UnoPred] =
1246 getFCMPLibcallDesc(Pred: CmpInst::FCMP_UNO, Size);
1247 const auto NotUno =
1248 BuildLibcall(UnoLibcall, CmpInst::getInversePredicate(pred: UnoPred), DstTy);
1249
1250 if (NotOeq && NotUno)
1251 MIRBuilder.buildAnd(Dst: DstReg, Src0: NotOeq, Src1: NotUno);
1252 else
1253 return UnableToLegalize;
1254
1255 break;
1256 }
1257 case CmpInst::FCMP_ULT:
1258 case CmpInst::FCMP_UGE:
1259 case CmpInst::FCMP_UGT:
1260 case CmpInst::FCMP_ULE:
1261 case CmpInst::FCMP_ORD: {
1262 // Convert into: !(inverse(Pred))
1263 // E.g. FCMP_ULT becomes !FCMP_OGE
1264 // This is equivalent to the following, but saves some instructions.
1265 // MIRBuilder.buildNot(
1266 // PredTy,
1267 // MIRBuilder.buildFCmp(CmpInst::getInversePredicate(Pred), PredTy,
1268 // Op1, Op2));
1269 const auto [InversedLibcall, InversedPred] =
1270 getFCMPLibcallDesc(Pred: CmpInst::getInversePredicate(pred: Cond), Size);
1271 if (!BuildLibcall(InversedLibcall,
1272 CmpInst::getInversePredicate(pred: InversedPred), DstReg))
1273 return UnableToLegalize;
1274 break;
1275 }
1276 default:
1277 return UnableToLegalize;
1278 }
1279
1280 return Legalized;
1281}
1282
1283// The function is used to legalize operations that set default environment
1284// state. In C library a call like `fesetmode(FE_DFL_MODE)` is used for that.
1285// On most targets supported in glibc FE_DFL_MODE is defined as
1286// `((const femode_t *) -1)`. Such assumption is used here. If for some target
1287// it is not true, the target must provide custom lowering.
1288LegalizerHelper::LegalizeResult
1289LegalizerHelper::createResetStateLibcall(MachineInstr &MI,
1290 LostDebugLocObserver &LocObserver) {
1291 const DataLayout &DL = MIRBuilder.getDataLayout();
1292 auto &MF = MIRBuilder.getMF();
1293 auto &Ctx = MF.getFunction().getContext();
1294
1295 // Create an argument for the library function.
1296 unsigned AddrSpace = DL.getDefaultGlobalsAddressSpace();
1297 Type *StatePtrTy = PointerType::get(C&: Ctx, AddressSpace: AddrSpace);
1298 unsigned PtrSize = DL.getPointerSizeInBits(AS: AddrSpace);
1299 LLT MemTy = LLT::pointer(AddressSpace: AddrSpace, SizeInBits: PtrSize);
1300 auto DefValue = MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: PtrSize), Val: -1LL);
1301 DstOp Dest(MRI.createGenericVirtualRegister(Ty: MemTy));
1302 MIRBuilder.buildIntToPtr(Dst: Dest, Src: DefValue);
1303
1304 RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI);
1305 return createLibcall(
1306 Libcall: RTLibcall, Result: CallLowering::ArgInfo({0}, Type::getVoidTy(C&: Ctx), 0),
1307 Args: CallLowering::ArgInfo({Dest.getReg(), StatePtrTy, 0}), LocObserver, MI: &MI);
1308}
1309
1310LegalizerHelper::LegalizeResult
1311LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
1312 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
1313
1314 switch (MI.getOpcode()) {
1315 default:
1316 return UnableToLegalize;
1317 case TargetOpcode::G_MUL:
1318 case TargetOpcode::G_SDIV:
1319 case TargetOpcode::G_UDIV:
1320 case TargetOpcode::G_SREM:
1321 case TargetOpcode::G_UREM:
1322 case TargetOpcode::G_CTLZ_ZERO_POISON: {
1323 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1324 unsigned Size = LLTy.getSizeInBits();
1325 Type *HLTy = IntegerType::get(C&: Ctx, NumBits: Size);
1326 auto Status = simpleLibcall(MI, MIRBuilder, Size, OpType: HLTy, LocObserver);
1327 if (Status != Legalized)
1328 return Status;
1329 break;
1330 }
1331 case TargetOpcode::G_FADD:
1332 case TargetOpcode::G_FSUB:
1333 case TargetOpcode::G_FMUL:
1334 case TargetOpcode::G_FDIV:
1335 case TargetOpcode::G_FMA:
1336 case TargetOpcode::G_FPOW:
1337 case TargetOpcode::G_FREM:
1338 case TargetOpcode::G_FCOS:
1339 case TargetOpcode::G_FSIN:
1340 case TargetOpcode::G_FTAN:
1341 case TargetOpcode::G_FACOS:
1342 case TargetOpcode::G_FASIN:
1343 case TargetOpcode::G_FATAN:
1344 case TargetOpcode::G_FATAN2:
1345 case TargetOpcode::G_FCOSH:
1346 case TargetOpcode::G_FSINH:
1347 case TargetOpcode::G_FTANH:
1348 case TargetOpcode::G_FLOG10:
1349 case TargetOpcode::G_FLOG:
1350 case TargetOpcode::G_FLOG2:
1351 case TargetOpcode::G_FEXP:
1352 case TargetOpcode::G_FEXP2:
1353 case TargetOpcode::G_FEXP10:
1354 case TargetOpcode::G_FCEIL:
1355 case TargetOpcode::G_FFLOOR:
1356 case TargetOpcode::G_FMINNUM:
1357 case TargetOpcode::G_FMAXNUM:
1358 case TargetOpcode::G_FMINIMUMNUM:
1359 case TargetOpcode::G_FMAXIMUMNUM:
1360 case TargetOpcode::G_FSQRT:
1361 case TargetOpcode::G_FRINT:
1362 case TargetOpcode::G_FNEARBYINT:
1363 case TargetOpcode::G_INTRINSIC_TRUNC:
1364 case TargetOpcode::G_INTRINSIC_ROUND:
1365 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: {
1366 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1367 unsigned Size = LLTy.getSizeInBits();
1368 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1369 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1370 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1371 return UnableToLegalize;
1372 }
1373 auto Status = simpleLibcall(MI, MIRBuilder, Size, OpType: HLTy, LocObserver);
1374 if (Status != Legalized)
1375 return Status;
1376 break;
1377 }
1378 case TargetOpcode::G_FSINCOS: {
1379 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1380 unsigned Size = LLTy.getSizeInBits();
1381 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1382 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1383 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1384 return UnableToLegalize;
1385 }
1386 return emitSincosLibcall(MI, MIRBuilder, Size, OpType: HLTy, LocObserver);
1387 }
1388 case TargetOpcode::G_FMODF: {
1389 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1390 unsigned Size = LLTy.getSizeInBits();
1391 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1392 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1393 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1394 return UnableToLegalize;
1395 }
1396 return emitModfLibcall(MI, MIRBuilder, Size, OpType: HLTy, LocObserver);
1397 }
1398 case TargetOpcode::G_LROUND:
1399 case TargetOpcode::G_LLROUND:
1400 case TargetOpcode::G_INTRINSIC_LRINT:
1401 case TargetOpcode::G_INTRINSIC_LLRINT: {
1402 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
1403 unsigned Size = LLTy.getSizeInBits();
1404 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1405 Type *ITy = IntegerType::get(
1406 C&: Ctx, NumBits: MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits());
1407 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1408 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1409 return UnableToLegalize;
1410 }
1411 auto Libcall = getRTLibDesc(Opcode: MI.getOpcode(), Size);
1412 LegalizeResult Status =
1413 createLibcall(Libcall, Result: {MI.getOperand(i: 0).getReg(), ITy, 0},
1414 Args: {{MI.getOperand(i: 1).getReg(), HLTy, 0}}, LocObserver, MI: &MI);
1415 if (Status != Legalized)
1416 return Status;
1417 MI.eraseFromParent();
1418 return Legalized;
1419 }
1420 case TargetOpcode::G_FPOWI:
1421 case TargetOpcode::G_FLDEXP: {
1422 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1423 unsigned Size = LLTy.getSizeInBits();
1424 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1425 Type *ITy = IntegerType::get(
1426 C&: Ctx, NumBits: MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits());
1427 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1428 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1429 return UnableToLegalize;
1430 }
1431 auto Libcall = getRTLibDesc(Opcode: MI.getOpcode(), Size);
1432 SmallVector<CallLowering::ArgInfo, 2> Args = {
1433 {MI.getOperand(i: 1).getReg(), HLTy, 0},
1434 {MI.getOperand(i: 2).getReg(), ITy, 1}};
1435 Args[1].Flags[0].setSExt();
1436 LegalizeResult Status = createLibcall(
1437 Libcall, Result: {MI.getOperand(i: 0).getReg(), HLTy, 0}, Args, LocObserver, MI: &MI);
1438 if (Status != Legalized)
1439 return Status;
1440 break;
1441 }
1442 case TargetOpcode::G_FPEXT:
1443 case TargetOpcode::G_FPTRUNC: {
1444 Type *FromTy = getFloatTypeForLLT(Ctx, Ty: MRI.getType(Reg: MI.getOperand(i: 1).getReg()));
1445 Type *ToTy = getFloatTypeForLLT(Ctx, Ty: MRI.getType(Reg: MI.getOperand(i: 0).getReg()));
1446 if (!FromTy || !ToTy)
1447 return UnableToLegalize;
1448 LegalizeResult Status = conversionLibcall(MI, ToType: ToTy, FromType: FromTy, LocObserver);
1449 if (Status != Legalized)
1450 return Status;
1451 break;
1452 }
1453 case TargetOpcode::G_FCMP: {
1454 LegalizeResult Status = createFCMPLibcall(MI, LocObserver);
1455 if (Status != Legalized)
1456 return Status;
1457 MI.eraseFromParent();
1458 return Status;
1459 }
1460 case TargetOpcode::G_FPTOSI:
1461 case TargetOpcode::G_FPTOUI: {
1462 // FIXME: Support other types
1463 Type *FromTy =
1464 getFloatTypeForLLT(Ctx, Ty: MRI.getType(Reg: MI.getOperand(i: 1).getReg()));
1465 unsigned ToSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
1466 if ((ToSize != 32 && ToSize != 64 && ToSize != 128) || !FromTy)
1467 return UnableToLegalize;
1468 LegalizeResult Status = conversionLibcall(MI, ToType: Type::getIntNTy(C&: Ctx, N: ToSize),
1469 FromType: FromTy, LocObserver);
1470 if (Status != Legalized)
1471 return Status;
1472 break;
1473 }
1474 case TargetOpcode::G_SITOFP:
1475 case TargetOpcode::G_UITOFP: {
1476 unsigned FromSize = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
1477 Type *ToTy =
1478 getFloatTypeForLLT(Ctx, Ty: MRI.getType(Reg: MI.getOperand(i: 0).getReg()));
1479 if ((FromSize != 32 && FromSize != 64 && FromSize != 128) || !ToTy)
1480 return UnableToLegalize;
1481 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SITOFP;
1482 LegalizeResult Status = conversionLibcall(
1483 MI, ToType: ToTy, FromType: Type::getIntNTy(C&: Ctx, N: FromSize), LocObserver, IsSigned);
1484 if (Status != Legalized)
1485 return Status;
1486 break;
1487 }
1488 case TargetOpcode::G_ATOMICRMW_XCHG:
1489 case TargetOpcode::G_ATOMICRMW_ADD:
1490 case TargetOpcode::G_ATOMICRMW_SUB:
1491 case TargetOpcode::G_ATOMICRMW_AND:
1492 case TargetOpcode::G_ATOMICRMW_OR:
1493 case TargetOpcode::G_ATOMICRMW_XOR:
1494 case TargetOpcode::G_ATOMIC_CMPXCHG:
1495 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
1496 auto Status = createAtomicLibcall(MI);
1497 if (Status != Legalized)
1498 return Status;
1499 break;
1500 }
1501 case TargetOpcode::G_BZERO:
1502 case TargetOpcode::G_MEMCPY:
1503 case TargetOpcode::G_MEMMOVE:
1504 case TargetOpcode::G_MEMSET: {
1505 LegalizeResult Result =
1506 createMemLibcall(MRI&: *MIRBuilder.getMRI(), MI, LocObserver);
1507 if (Result != Legalized)
1508 return Result;
1509 MI.eraseFromParent();
1510 return Result;
1511 }
1512 case TargetOpcode::G_GET_FPENV:
1513 case TargetOpcode::G_GET_FPMODE: {
1514 LegalizeResult Result = createGetStateLibcall(MI, LocObserver);
1515 if (Result != Legalized)
1516 return Result;
1517 break;
1518 }
1519 case TargetOpcode::G_SET_FPENV:
1520 case TargetOpcode::G_SET_FPMODE: {
1521 LegalizeResult Result = createSetStateLibcall(MI, LocObserver);
1522 if (Result != Legalized)
1523 return Result;
1524 break;
1525 }
1526 case TargetOpcode::G_RESET_FPENV:
1527 case TargetOpcode::G_RESET_FPMODE: {
1528 LegalizeResult Result = createResetStateLibcall(MI, LocObserver);
1529 if (Result != Legalized)
1530 return Result;
1531 break;
1532 }
1533 }
1534
1535 MI.eraseFromParent();
1536 return Legalized;
1537}
1538
1539LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
1540 unsigned TypeIdx,
1541 LLT NarrowTy) {
1542 uint64_t SizeOp0 = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
1543 uint64_t NarrowSize = NarrowTy.getSizeInBits();
1544
1545 switch (MI.getOpcode()) {
1546 default:
1547 return UnableToLegalize;
1548 case TargetOpcode::G_IMPLICIT_DEF: {
1549 Register DstReg = MI.getOperand(i: 0).getReg();
1550 LLT DstTy = MRI.getType(Reg: DstReg);
1551
1552 // If SizeOp0 is not an exact multiple of NarrowSize, emit
1553 // G_ANYEXT(G_IMPLICIT_DEF). Cast result to vector if needed.
1554 // FIXME: Although this would also be legal for the general case, it causes
1555 // a lot of regressions in the emitted code (superfluous COPYs, artifact
1556 // combines not being hit). This seems to be a problem related to the
1557 // artifact combiner.
1558 if (SizeOp0 % NarrowSize != 0) {
1559 LLT ImplicitTy = DstTy.changeElementType(NewEltTy: NarrowTy);
1560 Register ImplicitReg = MIRBuilder.buildUndef(Res: ImplicitTy).getReg(Idx: 0);
1561 MIRBuilder.buildAnyExt(Res: DstReg, Op: ImplicitReg);
1562
1563 MI.eraseFromParent();
1564 return Legalized;
1565 }
1566
1567 int NumParts = SizeOp0 / NarrowSize;
1568
1569 SmallVector<Register, 2> DstRegs;
1570 for (int i = 0; i < NumParts; ++i)
1571 DstRegs.push_back(Elt: MIRBuilder.buildUndef(Res: NarrowTy).getReg(Idx: 0));
1572
1573 if (DstTy.isVector())
1574 MIRBuilder.buildBuildVector(Res: DstReg, Ops: DstRegs);
1575 else
1576 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
1577 MI.eraseFromParent();
1578 return Legalized;
1579 }
1580 case TargetOpcode::G_CONSTANT: {
1581 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1582 const APInt &Val = MI.getOperand(i: 1).getCImm()->getValue();
1583 unsigned TotalSize = Ty.getSizeInBits();
1584 unsigned NarrowSize = NarrowTy.getSizeInBits();
1585 int NumParts = TotalSize / NarrowSize;
1586
1587 SmallVector<Register, 4> PartRegs;
1588 for (int I = 0; I != NumParts; ++I) {
1589 unsigned Offset = I * NarrowSize;
1590 auto K = MIRBuilder.buildConstant(Res: NarrowTy,
1591 Val: Val.lshr(shiftAmt: Offset).trunc(width: NarrowSize));
1592 PartRegs.push_back(Elt: K.getReg(Idx: 0));
1593 }
1594
1595 LLT LeftoverTy;
1596 unsigned LeftoverBits = TotalSize - NumParts * NarrowSize;
1597 SmallVector<Register, 1> LeftoverRegs;
1598 if (LeftoverBits != 0) {
1599 LeftoverTy = LLT::scalar(SizeInBits: LeftoverBits);
1600 auto K = MIRBuilder.buildConstant(
1601 Res: LeftoverTy,
1602 Val: Val.lshr(shiftAmt: NumParts * NarrowSize).trunc(width: LeftoverBits));
1603 LeftoverRegs.push_back(Elt: K.getReg(Idx: 0));
1604 }
1605
1606 insertParts(DstReg: MI.getOperand(i: 0).getReg(),
1607 ResultTy: Ty, PartTy: NarrowTy, PartRegs, LeftoverTy, LeftoverRegs);
1608
1609 MI.eraseFromParent();
1610 return Legalized;
1611 }
1612 case TargetOpcode::G_SEXT:
1613 case TargetOpcode::G_ZEXT:
1614 case TargetOpcode::G_ANYEXT:
1615 return narrowScalarExt(MI, TypeIdx, Ty: NarrowTy);
1616 case TargetOpcode::G_TRUNC: {
1617 if (TypeIdx != 1)
1618 return UnableToLegalize;
1619
1620 uint64_t SizeOp1 = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
1621 if (NarrowTy.getSizeInBits() * 2 != SizeOp1) {
1622 LLVM_DEBUG(dbgs() << "Can't narrow trunc to type " << NarrowTy << "\n");
1623 return UnableToLegalize;
1624 }
1625
1626 auto Unmerge = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: MI.getOperand(i: 1));
1627 MIRBuilder.buildCopy(Res: MI.getOperand(i: 0), Op: Unmerge.getReg(Idx: 0));
1628 MI.eraseFromParent();
1629 return Legalized;
1630 }
1631 case TargetOpcode::G_CONSTANT_FOLD_BARRIER:
1632 case TargetOpcode::G_FREEZE: {
1633 if (TypeIdx != 0)
1634 return UnableToLegalize;
1635
1636 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1637 // Should widen scalar first
1638 if (Ty.getSizeInBits() % NarrowTy.getSizeInBits() != 0)
1639 return UnableToLegalize;
1640
1641 auto Unmerge = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: MI.getOperand(i: 1).getReg());
1642 SmallVector<Register, 8> Parts;
1643 for (unsigned i = 0; i < Unmerge->getNumDefs(); ++i) {
1644 Parts.push_back(
1645 Elt: MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {NarrowTy}, SrcOps: {Unmerge.getReg(Idx: i)})
1646 .getReg(Idx: 0));
1647 }
1648
1649 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0).getReg(), Ops: Parts);
1650 MI.eraseFromParent();
1651 return Legalized;
1652 }
1653 case TargetOpcode::G_ADD:
1654 case TargetOpcode::G_SUB:
1655 case TargetOpcode::G_SADDO:
1656 case TargetOpcode::G_SSUBO:
1657 case TargetOpcode::G_SADDE:
1658 case TargetOpcode::G_SSUBE:
1659 case TargetOpcode::G_UADDO:
1660 case TargetOpcode::G_USUBO:
1661 case TargetOpcode::G_UADDE:
1662 case TargetOpcode::G_USUBE:
1663 return narrowScalarAddSub(MI, TypeIdx, NarrowTy);
1664 case TargetOpcode::G_MUL:
1665 case TargetOpcode::G_UMULH:
1666 return narrowScalarMul(MI, Ty: NarrowTy);
1667 case TargetOpcode::G_EXTRACT:
1668 return narrowScalarExtract(MI, TypeIdx, Ty: NarrowTy);
1669 case TargetOpcode::G_INSERT:
1670 return narrowScalarInsert(MI, TypeIdx, Ty: NarrowTy);
1671 case TargetOpcode::G_LOAD: {
1672 auto &LoadMI = cast<GLoad>(Val&: MI);
1673 Register DstReg = LoadMI.getDstReg();
1674 LLT DstTy = MRI.getType(Reg: DstReg);
1675 if (DstTy.isVector())
1676 return UnableToLegalize;
1677
1678 if (8 * LoadMI.getMemSize().getValue() != DstTy.getSizeInBits()) {
1679 Register TmpReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
1680 MIRBuilder.buildLoad(Res: TmpReg, Addr: LoadMI.getPointerReg(), MMO&: LoadMI.getMMO());
1681 MIRBuilder.buildAnyExt(Res: DstReg, Op: TmpReg);
1682 LoadMI.eraseFromParent();
1683 return Legalized;
1684 }
1685
1686 return reduceLoadStoreWidth(MI&: LoadMI, TypeIdx, NarrowTy);
1687 }
1688 case TargetOpcode::G_ZEXTLOAD:
1689 case TargetOpcode::G_SEXTLOAD:
1690 case TargetOpcode::G_FPEXTLOAD: {
1691 auto &LoadMI = cast<GExtLoad>(Val&: MI);
1692 Register DstReg = LoadMI.getDstReg();
1693 Register PtrReg = LoadMI.getPointerReg();
1694
1695 Register TmpReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
1696 auto &MMO = LoadMI.getMMO();
1697 unsigned MemSize = MMO.getSizeInBits().getValue();
1698
1699 if (MemSize == NarrowSize) {
1700 MIRBuilder.buildLoad(Res: TmpReg, Addr: PtrReg, MMO);
1701 } else if (MemSize < NarrowSize) {
1702 MIRBuilder.buildLoadInstr(Opcode: LoadMI.getOpcode(), Res: TmpReg, Addr: PtrReg, MMO);
1703 } else if (MemSize > NarrowSize) {
1704 // FIXME: Need to split the load.
1705 return UnableToLegalize;
1706 }
1707
1708 if (isa<GZExtLoad>(Val: LoadMI))
1709 MIRBuilder.buildZExt(Res: DstReg, Op: TmpReg);
1710 else if (isa<GSExtLoad>(Val: LoadMI))
1711 MIRBuilder.buildSExt(Res: DstReg, Op: TmpReg);
1712 else
1713 MIRBuilder.buildFPExt(Res: DstReg, Op: TmpReg);
1714
1715 LoadMI.eraseFromParent();
1716 return Legalized;
1717 }
1718 case TargetOpcode::G_STORE: {
1719 auto &StoreMI = cast<GStore>(Val&: MI);
1720
1721 Register SrcReg = StoreMI.getValueReg();
1722 LLT SrcTy = MRI.getType(Reg: SrcReg);
1723 if (SrcTy.isVector())
1724 return UnableToLegalize;
1725
1726 int NumParts = SizeOp0 / NarrowSize;
1727 unsigned HandledSize = NumParts * NarrowTy.getSizeInBits();
1728 unsigned LeftoverBits = SrcTy.getSizeInBits() - HandledSize;
1729 if (SrcTy.isVector() && LeftoverBits != 0)
1730 return UnableToLegalize;
1731
1732 if (8 * StoreMI.getMemSize().getValue() != SrcTy.getSizeInBits()) {
1733 Register TmpReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
1734 MIRBuilder.buildTrunc(Res: TmpReg, Op: SrcReg);
1735 MIRBuilder.buildStore(Val: TmpReg, Addr: StoreMI.getPointerReg(), MMO&: StoreMI.getMMO());
1736 StoreMI.eraseFromParent();
1737 return Legalized;
1738 }
1739
1740 return reduceLoadStoreWidth(MI&: StoreMI, TypeIdx: 0, NarrowTy);
1741 }
1742 case TargetOpcode::G_FPTRUNCSTORE: {
1743 auto &StoreMI = cast<GFPTruncStore>(Val&: MI);
1744 Register SrcReg = StoreMI.getValueReg();
1745 Register PtrReg = StoreMI.getPointerReg();
1746
1747 auto &MMO = StoreMI.getMMO();
1748 unsigned MemSize = MMO.getSizeInBits().getValue();
1749 if (MemSize > NarrowSize) {
1750 return UnableToLegalize;
1751 }
1752
1753 auto TmpReg = MIRBuilder.buildFPTrunc(Res: NarrowTy, Op: SrcReg);
1754 if (MemSize == NarrowSize) {
1755 MIRBuilder.buildStore(Val: TmpReg, Addr: PtrReg, MMO);
1756 } else if (MemSize < NarrowSize) {
1757 MIRBuilder.buildStoreInstr(Opcode: TargetOpcode::G_FPTRUNCSTORE, Val: TmpReg, Addr: PtrReg,
1758 MMO);
1759 }
1760
1761 StoreMI.eraseFromParent();
1762 return Legalized;
1763 }
1764 case TargetOpcode::G_SELECT:
1765 return narrowScalarSelect(MI, TypeIdx, Ty: NarrowTy);
1766 case TargetOpcode::G_AND:
1767 case TargetOpcode::G_OR:
1768 case TargetOpcode::G_XOR: {
1769 // Legalize bitwise operation:
1770 // A = BinOp<Ty> B, C
1771 // into:
1772 // B1, ..., BN = G_UNMERGE_VALUES B
1773 // C1, ..., CN = G_UNMERGE_VALUES C
1774 // A1 = BinOp<Ty/N> B1, C2
1775 // ...
1776 // AN = BinOp<Ty/N> BN, CN
1777 // A = G_MERGE_VALUES A1, ..., AN
1778 return narrowScalarBasic(MI, TypeIdx, Ty: NarrowTy);
1779 }
1780 case TargetOpcode::G_SHL:
1781 case TargetOpcode::G_LSHR:
1782 case TargetOpcode::G_ASHR:
1783 return narrowScalarShift(MI, TypeIdx, Ty: NarrowTy);
1784 case TargetOpcode::G_CTLZ:
1785 case TargetOpcode::G_CTLZ_ZERO_POISON:
1786 case TargetOpcode::G_CTTZ:
1787 case TargetOpcode::G_CTTZ_ZERO_POISON:
1788 case TargetOpcode::G_CTLS:
1789 case TargetOpcode::G_CTPOP:
1790 if (TypeIdx == 1)
1791 switch (MI.getOpcode()) {
1792 case TargetOpcode::G_CTLZ:
1793 case TargetOpcode::G_CTLZ_ZERO_POISON:
1794 return narrowScalarCTLZ(MI, TypeIdx, Ty: NarrowTy);
1795 case TargetOpcode::G_CTTZ:
1796 case TargetOpcode::G_CTTZ_ZERO_POISON:
1797 return narrowScalarCTTZ(MI, TypeIdx, Ty: NarrowTy);
1798 case TargetOpcode::G_CTPOP:
1799 return narrowScalarCTPOP(MI, TypeIdx, Ty: NarrowTy);
1800 case TargetOpcode::G_CTLS:
1801 return narrowScalarCTLS(MI, TypeIdx, Ty: NarrowTy);
1802 default:
1803 return UnableToLegalize;
1804 }
1805
1806 Observer.changingInstr(MI);
1807 narrowScalarDst(MI, NarrowTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_ZEXT);
1808 Observer.changedInstr(MI);
1809 return Legalized;
1810 case TargetOpcode::G_INTTOPTR:
1811 if (TypeIdx != 1)
1812 return UnableToLegalize;
1813
1814 Observer.changingInstr(MI);
1815 narrowScalarSrc(MI, NarrowTy, OpIdx: 1);
1816 Observer.changedInstr(MI);
1817 return Legalized;
1818 case TargetOpcode::G_PTRTOINT:
1819 if (TypeIdx != 0)
1820 return UnableToLegalize;
1821
1822 Observer.changingInstr(MI);
1823 narrowScalarDst(MI, NarrowTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_ZEXT);
1824 Observer.changedInstr(MI);
1825 return Legalized;
1826 case TargetOpcode::G_PHI: {
1827 // FIXME: add support for when SizeOp0 isn't an exact multiple of
1828 // NarrowSize.
1829 if (SizeOp0 % NarrowSize != 0)
1830 return UnableToLegalize;
1831
1832 unsigned NumParts = SizeOp0 / NarrowSize;
1833 SmallVector<Register, 2> DstRegs(NumParts);
1834 SmallVector<SmallVector<Register, 2>, 2> SrcRegs(MI.getNumOperands() / 2);
1835 Observer.changingInstr(MI);
1836 for (unsigned i = 1; i < MI.getNumOperands(); i += 2) {
1837 MachineBasicBlock &OpMBB = *MI.getOperand(i: i + 1).getMBB();
1838 MIRBuilder.setInsertPt(MBB&: OpMBB, II: OpMBB.getFirstTerminatorForward());
1839 extractParts(Reg: MI.getOperand(i).getReg(), Ty: NarrowTy, NumParts,
1840 VRegs&: SrcRegs[i / 2], MIRBuilder, MRI);
1841 }
1842 MachineBasicBlock &MBB = *MI.getParent();
1843 MIRBuilder.setInsertPt(MBB, II: MI);
1844 for (unsigned i = 0; i < NumParts; ++i) {
1845 DstRegs[i] = MRI.createGenericVirtualRegister(Ty: NarrowTy);
1846 MachineInstrBuilder MIB =
1847 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_PHI).addDef(RegNo: DstRegs[i]);
1848 for (unsigned j = 1; j < MI.getNumOperands(); j += 2)
1849 MIB.addUse(RegNo: SrcRegs[j / 2][i]).add(MO: MI.getOperand(i: j + 1));
1850 }
1851 MIRBuilder.setInsertPt(MBB, II: MBB.getFirstNonPHI());
1852 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0), Ops: DstRegs);
1853 Observer.changedInstr(MI);
1854 MI.eraseFromParent();
1855 return Legalized;
1856 }
1857 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1858 case TargetOpcode::G_INSERT_VECTOR_ELT: {
1859 if (TypeIdx != 2)
1860 return UnableToLegalize;
1861
1862 int OpIdx = MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
1863 Observer.changingInstr(MI);
1864 narrowScalarSrc(MI, NarrowTy, OpIdx);
1865 Observer.changedInstr(MI);
1866 return Legalized;
1867 }
1868 case TargetOpcode::G_ICMP: {
1869 Register LHS = MI.getOperand(i: 2).getReg();
1870 LLT SrcTy = MRI.getType(Reg: LHS);
1871 CmpInst::Predicate Pred =
1872 static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
1873
1874 LLT LeftoverTy; // Example: s88 -> s64 (NarrowTy) + s24 (leftover)
1875 SmallVector<Register, 4> LHSPartRegs, LHSLeftoverRegs;
1876 if (!extractParts(Reg: LHS, RegTy: SrcTy, MainTy: NarrowTy, LeftoverTy, VRegs&: LHSPartRegs,
1877 LeftoverVRegs&: LHSLeftoverRegs, MIRBuilder, MRI))
1878 return UnableToLegalize;
1879
1880 LLT Unused; // Matches LeftoverTy; G_ICMP LHS and RHS are the same type.
1881 SmallVector<Register, 4> RHSPartRegs, RHSLeftoverRegs;
1882 if (!extractParts(Reg: MI.getOperand(i: 3).getReg(), RegTy: SrcTy, MainTy: NarrowTy, LeftoverTy&: Unused,
1883 VRegs&: RHSPartRegs, LeftoverVRegs&: RHSLeftoverRegs, MIRBuilder, MRI))
1884 return UnableToLegalize;
1885
1886 // We now have the LHS and RHS of the compare split into narrow-type
1887 // registers, plus potentially some leftover type.
1888 Register Dst = MI.getOperand(i: 0).getReg();
1889 LLT ResTy = MRI.getType(Reg: Dst);
1890 if (ICmpInst::isEquality(P: Pred)) {
1891 // For each part on the LHS and RHS, keep track of the result of XOR-ing
1892 // them together. For each equal part, the result should be all 0s. For
1893 // each non-equal part, we'll get at least one 1.
1894 auto Zero = MIRBuilder.buildConstant(Res: NarrowTy, Val: 0);
1895 SmallVector<Register, 4> Xors;
1896 for (auto LHSAndRHS : zip(t&: LHSPartRegs, u&: RHSPartRegs)) {
1897 auto LHS = std::get<0>(t&: LHSAndRHS);
1898 auto RHS = std::get<1>(t&: LHSAndRHS);
1899 auto Xor = MIRBuilder.buildXor(Dst: NarrowTy, Src0: LHS, Src1: RHS).getReg(Idx: 0);
1900 Xors.push_back(Elt: Xor);
1901 }
1902
1903 // Build a G_XOR for each leftover register. Each G_XOR must be widened
1904 // to the desired narrow type so that we can OR them together later.
1905 SmallVector<Register, 4> WidenedXors;
1906 for (auto LHSAndRHS : zip(t&: LHSLeftoverRegs, u&: RHSLeftoverRegs)) {
1907 auto LHS = std::get<0>(t&: LHSAndRHS);
1908 auto RHS = std::get<1>(t&: LHSAndRHS);
1909 auto Xor = MIRBuilder.buildXor(Dst: LeftoverTy, Src0: LHS, Src1: RHS).getReg(Idx: 0);
1910 LLT GCDTy = extractGCDType(Parts&: WidenedXors, DstTy: NarrowTy, NarrowTy: LeftoverTy, SrcReg: Xor);
1911 buildLCMMergePieces(DstTy: LeftoverTy, NarrowTy, GCDTy, VRegs&: WidenedXors,
1912 /* PadStrategy = */ TargetOpcode::G_ZEXT);
1913 llvm::append_range(C&: Xors, R&: WidenedXors);
1914 }
1915
1916 // Now, for each part we broke up, we know if they are equal/not equal
1917 // based off the G_XOR. We can OR these all together and compare against
1918 // 0 to get the result.
1919 assert(Xors.size() >= 2 && "Should have gotten at least two Xors?");
1920 auto Or = MIRBuilder.buildOr(Dst: NarrowTy, Src0: Xors[0], Src1: Xors[1]);
1921 for (unsigned I = 2, E = Xors.size(); I < E; ++I)
1922 Or = MIRBuilder.buildOr(Dst: NarrowTy, Src0: Or, Src1: Xors[I]);
1923 MIRBuilder.buildICmp(Pred, Res: Dst, Op0: Or, Op1: Zero);
1924 } else {
1925 Register CmpIn;
1926 for (unsigned I = 0, E = LHSPartRegs.size(); I != E; ++I) {
1927 Register CmpOut;
1928 CmpInst::Predicate PartPred;
1929
1930 if (I == E - 1 && LHSLeftoverRegs.empty()) {
1931 PartPred = Pred;
1932 CmpOut = Dst;
1933 } else {
1934 PartPred = ICmpInst::getUnsignedPredicate(Pred);
1935 CmpOut = MRI.createGenericVirtualRegister(Ty: ResTy);
1936 }
1937
1938 if (!CmpIn) {
1939 MIRBuilder.buildICmp(Pred: PartPred, Res: CmpOut, Op0: LHSPartRegs[I],
1940 Op1: RHSPartRegs[I]);
1941 } else {
1942 auto Cmp = MIRBuilder.buildICmp(Pred: PartPred, Res: ResTy, Op0: LHSPartRegs[I],
1943 Op1: RHSPartRegs[I]);
1944 auto CmpEq = MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: ResTy,
1945 Op0: LHSPartRegs[I], Op1: RHSPartRegs[I]);
1946 MIRBuilder.buildSelect(Res: CmpOut, Tst: CmpEq, Op0: CmpIn, Op1: Cmp);
1947 }
1948
1949 CmpIn = CmpOut;
1950 }
1951
1952 for (unsigned I = 0, E = LHSLeftoverRegs.size(); I != E; ++I) {
1953 Register CmpOut;
1954 CmpInst::Predicate PartPred;
1955
1956 if (I == E - 1) {
1957 PartPred = Pred;
1958 CmpOut = Dst;
1959 } else {
1960 PartPred = ICmpInst::getUnsignedPredicate(Pred);
1961 CmpOut = MRI.createGenericVirtualRegister(Ty: ResTy);
1962 }
1963
1964 if (!CmpIn) {
1965 MIRBuilder.buildICmp(Pred: PartPred, Res: CmpOut, Op0: LHSLeftoverRegs[I],
1966 Op1: RHSLeftoverRegs[I]);
1967 } else {
1968 auto Cmp = MIRBuilder.buildICmp(Pred: PartPred, Res: ResTy, Op0: LHSLeftoverRegs[I],
1969 Op1: RHSLeftoverRegs[I]);
1970 auto CmpEq =
1971 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: ResTy,
1972 Op0: LHSLeftoverRegs[I], Op1: RHSLeftoverRegs[I]);
1973 MIRBuilder.buildSelect(Res: CmpOut, Tst: CmpEq, Op0: CmpIn, Op1: Cmp);
1974 }
1975
1976 CmpIn = CmpOut;
1977 }
1978 }
1979 MI.eraseFromParent();
1980 return Legalized;
1981 }
1982 case TargetOpcode::G_FCMP:
1983 if (TypeIdx != 0)
1984 return UnableToLegalize;
1985
1986 Observer.changingInstr(MI);
1987 narrowScalarDst(MI, NarrowTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_ZEXT);
1988 Observer.changedInstr(MI);
1989 return Legalized;
1990
1991 case TargetOpcode::G_SEXT_INREG: {
1992 if (TypeIdx != 0)
1993 return UnableToLegalize;
1994
1995 int64_t SizeInBits = MI.getOperand(i: 2).getImm();
1996
1997 // So long as the new type has more bits than the bits we're extending we
1998 // don't need to break it apart.
1999 if (NarrowTy.getScalarSizeInBits() > SizeInBits) {
2000 Observer.changingInstr(MI);
2001 // We don't lose any non-extension bits by truncating the src and
2002 // sign-extending the dst.
2003 MachineOperand &MO1 = MI.getOperand(i: 1);
2004 auto TruncMIB = MIRBuilder.buildTrunc(Res: NarrowTy, Op: MO1);
2005 MO1.setReg(TruncMIB.getReg(Idx: 0));
2006
2007 MachineOperand &MO2 = MI.getOperand(i: 0);
2008 Register DstExt = MRI.createGenericVirtualRegister(Ty: NarrowTy);
2009 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2010 MIRBuilder.buildSExt(Res: MO2, Op: DstExt);
2011 MO2.setReg(DstExt);
2012 Observer.changedInstr(MI);
2013 return Legalized;
2014 }
2015
2016 // Break it apart. Components below the extension point are unmodified. The
2017 // component containing the extension point becomes a narrower SEXT_INREG.
2018 // Components above it are ashr'd from the component containing the
2019 // extension point.
2020 if (SizeOp0 % NarrowSize != 0)
2021 return UnableToLegalize;
2022 int NumParts = SizeOp0 / NarrowSize;
2023
2024 // List the registers where the destination will be scattered.
2025 SmallVector<Register, 2> DstRegs;
2026 // List the registers where the source will be split.
2027 SmallVector<Register, 2> SrcRegs;
2028
2029 // Create all the temporary registers.
2030 for (int i = 0; i < NumParts; ++i) {
2031 Register SrcReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
2032
2033 SrcRegs.push_back(Elt: SrcReg);
2034 }
2035
2036 // Explode the big arguments into smaller chunks.
2037 MIRBuilder.buildUnmerge(Res: SrcRegs, Op: MI.getOperand(i: 1));
2038
2039 Register AshrCstReg =
2040 MIRBuilder.buildConstant(Res: NarrowTy, Val: NarrowTy.getScalarSizeInBits() - 1)
2041 .getReg(Idx: 0);
2042 Register FullExtensionReg;
2043 Register PartialExtensionReg;
2044
2045 // Do the operation on each small part.
2046 for (int i = 0; i < NumParts; ++i) {
2047 if ((i + 1) * NarrowTy.getScalarSizeInBits() <= SizeInBits) {
2048 DstRegs.push_back(Elt: SrcRegs[i]);
2049 PartialExtensionReg = DstRegs.back();
2050 } else if (i * NarrowTy.getScalarSizeInBits() >= SizeInBits) {
2051 assert(PartialExtensionReg &&
2052 "Expected to visit partial extension before full");
2053 if (FullExtensionReg) {
2054 DstRegs.push_back(Elt: FullExtensionReg);
2055 continue;
2056 }
2057 DstRegs.push_back(
2058 Elt: MIRBuilder.buildAShr(Dst: NarrowTy, Src0: PartialExtensionReg, Src1: AshrCstReg)
2059 .getReg(Idx: 0));
2060 FullExtensionReg = DstRegs.back();
2061 } else {
2062 DstRegs.push_back(
2063 Elt: MIRBuilder
2064 .buildInstr(
2065 Opc: TargetOpcode::G_SEXT_INREG, DstOps: {NarrowTy},
2066 SrcOps: {SrcRegs[i], SizeInBits % NarrowTy.getScalarSizeInBits()})
2067 .getReg(Idx: 0));
2068 PartialExtensionReg = DstRegs.back();
2069 }
2070 }
2071
2072 // Gather the destination registers into the final destination.
2073 Register DstReg = MI.getOperand(i: 0).getReg();
2074 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
2075 MI.eraseFromParent();
2076 return Legalized;
2077 }
2078 case TargetOpcode::G_BSWAP:
2079 case TargetOpcode::G_BITREVERSE: {
2080 if (SizeOp0 % NarrowSize != 0)
2081 return UnableToLegalize;
2082
2083 Observer.changingInstr(MI);
2084 SmallVector<Register, 2> SrcRegs, DstRegs;
2085 unsigned NumParts = SizeOp0 / NarrowSize;
2086 extractParts(Reg: MI.getOperand(i: 1).getReg(), Ty: NarrowTy, NumParts, VRegs&: SrcRegs,
2087 MIRBuilder, MRI);
2088
2089 for (unsigned i = 0; i < NumParts; ++i) {
2090 auto DstPart = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {NarrowTy},
2091 SrcOps: {SrcRegs[NumParts - 1 - i]});
2092 DstRegs.push_back(Elt: DstPart.getReg(Idx: 0));
2093 }
2094
2095 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0), Ops: DstRegs);
2096
2097 Observer.changedInstr(MI);
2098 MI.eraseFromParent();
2099 return Legalized;
2100 }
2101 case TargetOpcode::G_PTR_ADD:
2102 case TargetOpcode::G_PTRMASK: {
2103 if (TypeIdx != 1)
2104 return UnableToLegalize;
2105 Observer.changingInstr(MI);
2106 narrowScalarSrc(MI, NarrowTy, OpIdx: 2);
2107 Observer.changedInstr(MI);
2108 return Legalized;
2109 }
2110 case TargetOpcode::G_FPTOUI:
2111 case TargetOpcode::G_FPTOSI:
2112 case TargetOpcode::G_FPTOUI_SAT:
2113 case TargetOpcode::G_FPTOSI_SAT:
2114 return narrowScalarFPTOI(MI, TypeIdx, Ty: NarrowTy);
2115 case TargetOpcode::G_FPEXT:
2116 if (TypeIdx != 0)
2117 return UnableToLegalize;
2118 Observer.changingInstr(MI);
2119 narrowScalarDst(MI, NarrowTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_FPEXT);
2120 Observer.changedInstr(MI);
2121 return Legalized;
2122 case TargetOpcode::G_FLDEXP:
2123 case TargetOpcode::G_STRICT_FLDEXP:
2124 return narrowScalarFLDEXP(MI, TypeIdx, Ty: NarrowTy);
2125 case TargetOpcode::G_VSCALE: {
2126 Register Dst = MI.getOperand(i: 0).getReg();
2127 LLT Ty = MRI.getType(Reg: Dst);
2128
2129 // Assume VSCALE(1) fits into a legal integer
2130 const APInt One(NarrowTy.getSizeInBits(), 1);
2131 auto VScaleBase = MIRBuilder.buildVScale(Res: NarrowTy, MinElts: One);
2132 auto ZExt = MIRBuilder.buildZExt(Res: Ty, Op: VScaleBase);
2133 auto C = MIRBuilder.buildConstant(Res: Ty, Val: *MI.getOperand(i: 1).getCImm());
2134 MIRBuilder.buildMul(Dst, Src0: ZExt, Src1: C);
2135
2136 MI.eraseFromParent();
2137 return Legalized;
2138 }
2139 }
2140}
2141
2142Register LegalizerHelper::coerceToInteger(Register Val) {
2143 LLT Ty = MRI.getType(Reg: Val);
2144 if (Ty.isScalar())
2145 return Val;
2146
2147 const DataLayout &DL = MIRBuilder.getDataLayout();
2148 LLT NewTy = LLT::integer(SizeInBits: Ty.getSizeInBits());
2149 if (Ty.isPointer()) {
2150 if (DL.isNonIntegralAddressSpace(AddrSpace: Ty.getAddressSpace()))
2151 return Register();
2152 return MIRBuilder.buildPtrToInt(Dst: NewTy, Src: Val).getReg(Idx: 0);
2153 }
2154
2155 Register NewVal = Val;
2156
2157 assert(Ty.isVector());
2158 if (Ty.isPointerVector())
2159 NewVal = MIRBuilder.buildPtrToInt(Dst: NewTy, Src: NewVal).getReg(Idx: 0);
2160 return MIRBuilder.buildBitcast(Dst: NewTy, Src: NewVal).getReg(Idx: 0);
2161}
2162
2163void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy,
2164 unsigned OpIdx, unsigned ExtOpcode) {
2165 MachineOperand &MO = MI.getOperand(i: OpIdx);
2166 auto ExtB = MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {WideTy}, SrcOps: {MO});
2167 MO.setReg(ExtB.getReg(Idx: 0));
2168}
2169
2170void LegalizerHelper::widenScalarSrcUsingFPExt(MachineInstr &MI, LLT WideTy,
2171 unsigned OpIdx) {
2172 MachineOperand &MO = MI.getOperand(i: OpIdx);
2173 auto ExtB = MIRBuilder.buildInstr(Opc: TargetOpcode::G_FPEXT, DstOps: {WideTy}, SrcOps: {MO},
2174 Flags: MI.getFlags());
2175 MO.setReg(ExtB.getReg(Idx: 0));
2176}
2177
2178void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy,
2179 unsigned OpIdx) {
2180 MachineOperand &MO = MI.getOperand(i: OpIdx);
2181 auto ExtB = MIRBuilder.buildTrunc(Res: NarrowTy, Op: MO);
2182 MO.setReg(ExtB.getReg(Idx: 0));
2183}
2184
2185void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy,
2186 unsigned OpIdx, unsigned TruncOpcode) {
2187 MachineOperand &MO = MI.getOperand(i: OpIdx);
2188 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2189 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2190 MIRBuilder.buildInstr(Opc: TruncOpcode, DstOps: {MO}, SrcOps: {DstExt});
2191 MO.setReg(DstExt);
2192}
2193
2194void LegalizerHelper::widenScalarDstUsingFPTrunc(MachineInstr &MI, LLT WideTy,
2195 unsigned OpIdx) {
2196 MachineOperand &MO = MI.getOperand(i: OpIdx);
2197 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2198 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2199 MIRBuilder.buildInstr(Opc: TargetOpcode::G_FPTRUNC, DstOps: {MO}, SrcOps: {DstExt}, Flags: MI.getFlags());
2200 MO.setReg(DstExt);
2201}
2202
2203void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy,
2204 unsigned OpIdx, unsigned ExtOpcode) {
2205 MachineOperand &MO = MI.getOperand(i: OpIdx);
2206 Register DstTrunc = MRI.createGenericVirtualRegister(Ty: NarrowTy);
2207 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2208 MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {MO}, SrcOps: {DstTrunc});
2209 MO.setReg(DstTrunc);
2210}
2211
2212void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy,
2213 unsigned OpIdx) {
2214 MachineOperand &MO = MI.getOperand(i: OpIdx);
2215 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2216 Register Dst = MO.getReg();
2217 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2218 MO.setReg(DstExt);
2219 MIRBuilder.buildDeleteTrailingVectorElements(Res: Dst, Op0: DstExt);
2220}
2221
2222void LegalizerHelper::moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy,
2223 unsigned OpIdx) {
2224 MachineOperand &MO = MI.getOperand(i: OpIdx);
2225 MO.setReg(MIRBuilder.buildPadVectorWithUndefElements(Res: MoreTy, Op0: MO).getReg(Idx: 0));
2226}
2227
2228void LegalizerHelper::bitcastSrc(MachineInstr &MI, LLT CastTy, unsigned OpIdx) {
2229 MachineOperand &Op = MI.getOperand(i: OpIdx);
2230 Op.setReg(MIRBuilder.buildBitcast(Dst: CastTy, Src: Op).getReg(Idx: 0));
2231}
2232
2233void LegalizerHelper::bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx) {
2234 MachineOperand &MO = MI.getOperand(i: OpIdx);
2235 Register CastDst = MRI.createGenericVirtualRegister(Ty: CastTy);
2236 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2237 MIRBuilder.buildBitcast(Dst: MO, Src: CastDst);
2238 MO.setReg(CastDst);
2239}
2240
2241LegalizerHelper::LegalizeResult
2242LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx,
2243 LLT WideTy) {
2244 if (TypeIdx != 1)
2245 return UnableToLegalize;
2246
2247 auto [DstReg, DstTy, Src1Reg, Src1Ty] = MI.getFirst2RegLLTs();
2248 if (DstTy.isVector())
2249 return UnableToLegalize;
2250
2251 LLT SrcTy = MRI.getType(Reg: Src1Reg);
2252 const int DstSize = DstTy.getSizeInBits();
2253 const int SrcSize = SrcTy.getSizeInBits();
2254 const int WideSize = WideTy.getSizeInBits();
2255 const int NumMerge = (DstSize + WideSize - 1) / WideSize;
2256
2257 unsigned NumOps = MI.getNumOperands();
2258 unsigned NumSrc = MI.getNumOperands() - 1;
2259 unsigned PartSize = DstTy.getSizeInBits() / NumSrc;
2260
2261 if (WideSize >= DstSize) {
2262 // Directly pack the bits in the target type.
2263 Register ResultReg = MIRBuilder.buildZExt(Res: WideTy, Op: Src1Reg).getReg(Idx: 0);
2264
2265 for (unsigned I = 2; I != NumOps; ++I) {
2266 const unsigned Offset = (I - 1) * PartSize;
2267
2268 Register SrcReg = MI.getOperand(i: I).getReg();
2269 assert(MRI.getType(SrcReg) == LLT::scalar(PartSize));
2270
2271 auto ZextInput = MIRBuilder.buildZExt(Res: WideTy, Op: SrcReg);
2272
2273 Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg :
2274 MRI.createGenericVirtualRegister(Ty: WideTy);
2275
2276 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: Offset);
2277 auto Shl = MIRBuilder.buildShl(Dst: WideTy, Src0: ZextInput, Src1: ShiftAmt);
2278 MIRBuilder.buildOr(Dst: NextResult, Src0: ResultReg, Src1: Shl);
2279 ResultReg = NextResult;
2280 }
2281
2282 if (WideSize > DstSize)
2283 MIRBuilder.buildTrunc(Res: DstReg, Op: ResultReg);
2284 else if (DstTy.isPointer())
2285 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: ResultReg);
2286
2287 MI.eraseFromParent();
2288 return Legalized;
2289 }
2290
2291 // Unmerge the original values to the GCD type, and recombine to the next
2292 // multiple greater than the original type.
2293 //
2294 // %3:_(s12) = G_MERGE_VALUES %0:_(s4), %1:_(s4), %2:_(s4) -> s6
2295 // %4:_(s2), %5:_(s2) = G_UNMERGE_VALUES %0
2296 // %6:_(s2), %7:_(s2) = G_UNMERGE_VALUES %1
2297 // %8:_(s2), %9:_(s2) = G_UNMERGE_VALUES %2
2298 // %10:_(s6) = G_MERGE_VALUES %4, %5, %6
2299 // %11:_(s6) = G_MERGE_VALUES %7, %8, %9
2300 // %12:_(s12) = G_MERGE_VALUES %10, %11
2301 //
2302 // Padding with undef if necessary:
2303 //
2304 // %2:_(s8) = G_MERGE_VALUES %0:_(s4), %1:_(s4) -> s6
2305 // %3:_(s2), %4:_(s2) = G_UNMERGE_VALUES %0
2306 // %5:_(s2), %6:_(s2) = G_UNMERGE_VALUES %1
2307 // %7:_(s2) = G_IMPLICIT_DEF
2308 // %8:_(s6) = G_MERGE_VALUES %3, %4, %5
2309 // %9:_(s6) = G_MERGE_VALUES %6, %7, %7
2310 // %10:_(s12) = G_MERGE_VALUES %8, %9
2311
2312 const int GCD = std::gcd(m: SrcSize, n: WideSize);
2313 LLT GCDTy = WideTy.changeElementSize(NewEltSize: GCD);
2314
2315 SmallVector<Register, 8> NewMergeRegs;
2316 SmallVector<Register, 8> Unmerges;
2317 LLT WideDstTy = WideTy.changeElementSize(NewEltSize: NumMerge * WideSize);
2318
2319 // Decompose the original operands if they don't evenly divide.
2320 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands())) {
2321 Register SrcReg = MO.getReg();
2322 if (GCD == SrcSize) {
2323 Unmerges.push_back(Elt: SrcReg);
2324 } else {
2325 auto Unmerge = MIRBuilder.buildUnmerge(Res: GCDTy, Op: SrcReg);
2326 for (int J = 0, JE = Unmerge->getNumOperands() - 1; J != JE; ++J)
2327 Unmerges.push_back(Elt: Unmerge.getReg(Idx: J));
2328 }
2329 }
2330
2331 // Pad with undef to the next size that is a multiple of the requested size.
2332 if (static_cast<int>(Unmerges.size()) != NumMerge * WideSize) {
2333 Register UndefReg = MIRBuilder.buildUndef(Res: GCDTy).getReg(Idx: 0);
2334 for (int I = Unmerges.size(); I != NumMerge * WideSize; ++I)
2335 Unmerges.push_back(Elt: UndefReg);
2336 }
2337
2338 const int PartsPerGCD = WideSize / GCD;
2339
2340 // Build merges of each piece.
2341 ArrayRef<Register> Slicer(Unmerges);
2342 for (int I = 0; I != NumMerge; ++I, Slicer = Slicer.drop_front(N: PartsPerGCD)) {
2343 auto Merge =
2344 MIRBuilder.buildMergeLikeInstr(Res: WideTy, Ops: Slicer.take_front(N: PartsPerGCD));
2345 NewMergeRegs.push_back(Elt: Merge.getReg(Idx: 0));
2346 }
2347
2348 // A truncate may be necessary if the requested type doesn't evenly divide the
2349 // original result type.
2350 if (DstTy.getSizeInBits() == WideDstTy.getSizeInBits()) {
2351 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: NewMergeRegs);
2352 } else {
2353 auto FinalMerge = MIRBuilder.buildMergeLikeInstr(Res: WideDstTy, Ops: NewMergeRegs);
2354 MIRBuilder.buildTrunc(Res: DstReg, Op: FinalMerge.getReg(Idx: 0));
2355 }
2356
2357 MI.eraseFromParent();
2358 return Legalized;
2359}
2360
2361LegalizerHelper::LegalizeResult
2362LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx,
2363 LLT WideTy) {
2364 if (TypeIdx != 0)
2365 return UnableToLegalize;
2366
2367 int NumDst = MI.getNumOperands() - 1;
2368 Register SrcReg = MI.getOperand(i: NumDst).getReg();
2369 LLT SrcTy = MRI.getType(Reg: SrcReg);
2370 if (SrcTy.isVector())
2371 return UnableToLegalize;
2372
2373 Register Dst0Reg = MI.getOperand(i: 0).getReg();
2374 LLT DstTy = MRI.getType(Reg: Dst0Reg);
2375 if (!DstTy.isScalar())
2376 return UnableToLegalize;
2377
2378 if (WideTy.getSizeInBits() >= SrcTy.getSizeInBits()) {
2379 if (SrcTy.isPointer()) {
2380 const DataLayout &DL = MIRBuilder.getDataLayout();
2381 if (DL.isNonIntegralAddressSpace(AddrSpace: SrcTy.getAddressSpace())) {
2382 LLVM_DEBUG(
2383 dbgs() << "Not casting non-integral address space integer\n");
2384 return UnableToLegalize;
2385 }
2386
2387 SrcTy = LLT::scalar(SizeInBits: SrcTy.getSizeInBits());
2388 SrcReg = MIRBuilder.buildPtrToInt(Dst: SrcTy, Src: SrcReg).getReg(Idx: 0);
2389 }
2390
2391 // Widen SrcTy to WideTy. This does not affect the result, but since the
2392 // user requested this size, it is probably better handled than SrcTy and
2393 // should reduce the total number of legalization artifacts.
2394 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
2395 SrcTy = WideTy;
2396 SrcReg = MIRBuilder.buildAnyExt(Res: WideTy, Op: SrcReg).getReg(Idx: 0);
2397 }
2398
2399 // Theres no unmerge type to target. Directly extract the bits from the
2400 // source type
2401 unsigned DstSize = DstTy.getSizeInBits();
2402
2403 MIRBuilder.buildTrunc(Res: Dst0Reg, Op: SrcReg);
2404 for (int I = 1; I != NumDst; ++I) {
2405 auto ShiftAmt = MIRBuilder.buildConstant(Res: SrcTy, Val: DstSize * I);
2406 auto Shr = MIRBuilder.buildLShr(Dst: SrcTy, Src0: SrcReg, Src1: ShiftAmt);
2407 MIRBuilder.buildTrunc(Res: MI.getOperand(i: I), Op: Shr);
2408 }
2409
2410 MI.eraseFromParent();
2411 return Legalized;
2412 }
2413
2414 // Extend the source to a wider type.
2415 LLT LCMTy = getLCMType(OrigTy: SrcTy, TargetTy: WideTy);
2416
2417 Register WideSrc = SrcReg;
2418 if (LCMTy.getSizeInBits() != SrcTy.getSizeInBits()) {
2419 // TODO: If this is an integral address space, cast to integer and anyext.
2420 if (SrcTy.isPointer()) {
2421 LLVM_DEBUG(dbgs() << "Widening pointer source types not implemented\n");
2422 return UnableToLegalize;
2423 }
2424
2425 WideSrc = MIRBuilder.buildAnyExt(Res: LCMTy, Op: WideSrc).getReg(Idx: 0);
2426 }
2427
2428 auto Unmerge = MIRBuilder.buildUnmerge(Res: WideTy, Op: WideSrc);
2429
2430 // Create a sequence of unmerges and merges to the original results. Since we
2431 // may have widened the source, we will need to pad the results with dead defs
2432 // to cover the source register.
2433 // e.g. widen s48 to s64:
2434 // %1:_(s48), %2:_(s48) = G_UNMERGE_VALUES %0:_(s96)
2435 //
2436 // =>
2437 // %4:_(s192) = G_ANYEXT %0:_(s96)
2438 // %5:_(s64), %6, %7 = G_UNMERGE_VALUES %4 ; Requested unmerge
2439 // ; unpack to GCD type, with extra dead defs
2440 // %8:_(s16), %9, %10, %11 = G_UNMERGE_VALUES %5:_(s64)
2441 // %12:_(s16), %13, dead %14, dead %15 = G_UNMERGE_VALUES %6:_(s64)
2442 // dead %16:_(s16), dead %17, dead %18, dead %18 = G_UNMERGE_VALUES %7:_(s64)
2443 // %1:_(s48) = G_MERGE_VALUES %8:_(s16), %9, %10 ; Remerge to destination
2444 // %2:_(s48) = G_MERGE_VALUES %11:_(s16), %12, %13 ; Remerge to destination
2445 const LLT GCDTy = getGCDType(OrigTy: WideTy, TargetTy: DstTy);
2446 const int NumUnmerge = Unmerge->getNumOperands() - 1;
2447 const int PartsPerRemerge = DstTy.getSizeInBits() / GCDTy.getSizeInBits();
2448
2449 // Directly unmerge to the destination without going through a GCD type
2450 // if possible
2451 if (PartsPerRemerge == 1) {
2452 const int PartsPerUnmerge = WideTy.getSizeInBits() / DstTy.getSizeInBits();
2453
2454 for (int I = 0; I != NumUnmerge; ++I) {
2455 auto MIB = MIRBuilder.buildInstr(Opcode: TargetOpcode::G_UNMERGE_VALUES);
2456
2457 for (int J = 0; J != PartsPerUnmerge; ++J) {
2458 int Idx = I * PartsPerUnmerge + J;
2459 if (Idx < NumDst)
2460 MIB.addDef(RegNo: MI.getOperand(i: Idx).getReg());
2461 else {
2462 // Create dead def for excess components.
2463 MIB.addDef(RegNo: MRI.createGenericVirtualRegister(Ty: DstTy));
2464 }
2465 }
2466
2467 MIB.addUse(RegNo: Unmerge.getReg(Idx: I));
2468 }
2469 } else {
2470 SmallVector<Register, 16> Parts;
2471 for (int J = 0; J != NumUnmerge; ++J)
2472 extractGCDType(Parts, GCDTy, SrcReg: Unmerge.getReg(Idx: J));
2473
2474 SmallVector<Register, 8> RemergeParts;
2475 for (int I = 0; I != NumDst; ++I) {
2476 for (int J = 0; J < PartsPerRemerge; ++J) {
2477 const int Idx = I * PartsPerRemerge + J;
2478 RemergeParts.emplace_back(Args&: Parts[Idx]);
2479 }
2480
2481 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: I).getReg(), Ops: RemergeParts);
2482 RemergeParts.clear();
2483 }
2484 }
2485
2486 MI.eraseFromParent();
2487 return Legalized;
2488}
2489
2490LegalizerHelper::LegalizeResult
2491LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx,
2492 LLT WideTy) {
2493 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
2494 unsigned Offset = MI.getOperand(i: 2).getImm();
2495
2496 if (TypeIdx == 0) {
2497 if (SrcTy.isVector() || DstTy.isVector())
2498 return UnableToLegalize;
2499
2500 SrcOp Src(SrcReg);
2501 if (SrcTy.isPointer()) {
2502 // Extracts from pointers can be handled only if they are really just
2503 // simple integers.
2504 const DataLayout &DL = MIRBuilder.getDataLayout();
2505 if (DL.isNonIntegralAddressSpace(AddrSpace: SrcTy.getAddressSpace()))
2506 return UnableToLegalize;
2507
2508 LLT SrcAsIntTy = LLT::scalar(SizeInBits: SrcTy.getSizeInBits());
2509 Src = MIRBuilder.buildPtrToInt(Dst: SrcAsIntTy, Src);
2510 SrcTy = SrcAsIntTy;
2511 }
2512
2513 if (DstTy.isPointer())
2514 return UnableToLegalize;
2515
2516 if (Offset == 0) {
2517 // Avoid a shift in the degenerate case.
2518 MIRBuilder.buildTrunc(Res: DstReg,
2519 Op: MIRBuilder.buildAnyExtOrTrunc(Res: WideTy, Op: Src));
2520 MI.eraseFromParent();
2521 return Legalized;
2522 }
2523
2524 // Do a shift in the source type.
2525 LLT ShiftTy = SrcTy;
2526 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
2527 Src = MIRBuilder.buildAnyExt(Res: WideTy, Op: Src);
2528 ShiftTy = WideTy;
2529 }
2530
2531 auto LShr = MIRBuilder.buildLShr(
2532 Dst: ShiftTy, Src0: Src, Src1: MIRBuilder.buildConstant(Res: ShiftTy, Val: Offset));
2533 MIRBuilder.buildTrunc(Res: DstReg, Op: LShr);
2534 MI.eraseFromParent();
2535 return Legalized;
2536 }
2537
2538 if (SrcTy.isScalar()) {
2539 Observer.changingInstr(MI);
2540 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2541 Observer.changedInstr(MI);
2542 return Legalized;
2543 }
2544
2545 if (!SrcTy.isVector())
2546 return UnableToLegalize;
2547
2548 if (DstTy != SrcTy.getElementType())
2549 return UnableToLegalize;
2550
2551 if (Offset % SrcTy.getScalarSizeInBits() != 0)
2552 return UnableToLegalize;
2553
2554 Observer.changingInstr(MI);
2555 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2556
2557 MI.getOperand(i: 2).setImm((WideTy.getSizeInBits() / SrcTy.getSizeInBits()) *
2558 Offset);
2559 widenScalarDst(MI, WideTy: WideTy.getScalarType(), OpIdx: 0);
2560 Observer.changedInstr(MI);
2561 return Legalized;
2562}
2563
2564LegalizerHelper::LegalizeResult
2565LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx,
2566 LLT WideTy) {
2567 if (TypeIdx != 0 || WideTy.isVector())
2568 return UnableToLegalize;
2569 Observer.changingInstr(MI);
2570 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2571 widenScalarDst(MI, WideTy);
2572 Observer.changedInstr(MI);
2573 return Legalized;
2574}
2575
2576LegalizerHelper::LegalizeResult
2577LegalizerHelper::widenScalarAddSubOverflow(MachineInstr &MI, unsigned TypeIdx,
2578 LLT WideTy) {
2579 unsigned Opcode;
2580 unsigned ExtOpcode;
2581 std::optional<Register> CarryIn;
2582 switch (MI.getOpcode()) {
2583 default:
2584 llvm_unreachable("Unexpected opcode!");
2585 case TargetOpcode::G_SADDO:
2586 Opcode = TargetOpcode::G_ADD;
2587 ExtOpcode = TargetOpcode::G_SEXT;
2588 break;
2589 case TargetOpcode::G_SSUBO:
2590 Opcode = TargetOpcode::G_SUB;
2591 ExtOpcode = TargetOpcode::G_SEXT;
2592 break;
2593 case TargetOpcode::G_UADDO:
2594 Opcode = TargetOpcode::G_ADD;
2595 ExtOpcode = TargetOpcode::G_ZEXT;
2596 break;
2597 case TargetOpcode::G_USUBO:
2598 Opcode = TargetOpcode::G_SUB;
2599 ExtOpcode = TargetOpcode::G_ZEXT;
2600 break;
2601 case TargetOpcode::G_SADDE:
2602 Opcode = TargetOpcode::G_UADDE;
2603 ExtOpcode = TargetOpcode::G_SEXT;
2604 CarryIn = MI.getOperand(i: 4).getReg();
2605 break;
2606 case TargetOpcode::G_SSUBE:
2607 Opcode = TargetOpcode::G_USUBE;
2608 ExtOpcode = TargetOpcode::G_SEXT;
2609 CarryIn = MI.getOperand(i: 4).getReg();
2610 break;
2611 case TargetOpcode::G_UADDE:
2612 Opcode = TargetOpcode::G_UADDE;
2613 ExtOpcode = TargetOpcode::G_ZEXT;
2614 CarryIn = MI.getOperand(i: 4).getReg();
2615 break;
2616 case TargetOpcode::G_USUBE:
2617 Opcode = TargetOpcode::G_USUBE;
2618 ExtOpcode = TargetOpcode::G_ZEXT;
2619 CarryIn = MI.getOperand(i: 4).getReg();
2620 break;
2621 }
2622
2623 if (TypeIdx == 1) {
2624 unsigned BoolExtOp = MIRBuilder.getBoolExtOp(IsVec: WideTy.isVector(), IsFP: false);
2625
2626 Observer.changingInstr(MI);
2627 if (CarryIn)
2628 widenScalarSrc(MI, WideTy, OpIdx: 4, ExtOpcode: BoolExtOp);
2629 widenScalarDst(MI, WideTy, OpIdx: 1);
2630
2631 Observer.changedInstr(MI);
2632 return Legalized;
2633 }
2634
2635 auto LHSExt = MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {WideTy}, SrcOps: {MI.getOperand(i: 2)});
2636 auto RHSExt = MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {WideTy}, SrcOps: {MI.getOperand(i: 3)});
2637 // Do the arithmetic in the larger type.
2638 Register NewOp;
2639 if (CarryIn) {
2640 LLT CarryOutTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
2641 NewOp = MIRBuilder
2642 .buildInstr(Opc: Opcode, DstOps: {WideTy, CarryOutTy},
2643 SrcOps: {LHSExt, RHSExt, *CarryIn})
2644 .getReg(Idx: 0);
2645 } else {
2646 NewOp = MIRBuilder.buildInstr(Opc: Opcode, DstOps: {WideTy}, SrcOps: {LHSExt, RHSExt}).getReg(Idx: 0);
2647 }
2648 LLT OrigTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2649 auto TruncOp = MIRBuilder.buildTrunc(Res: OrigTy, Op: NewOp);
2650 auto ExtOp = MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {WideTy}, SrcOps: {TruncOp});
2651 // There is no overflow if the ExtOp is the same as NewOp.
2652 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: MI.getOperand(i: 1), Op0: NewOp, Op1: ExtOp);
2653 // Now trunc the NewOp to the original result.
2654 MIRBuilder.buildTrunc(Res: MI.getOperand(i: 0), Op: NewOp);
2655 MI.eraseFromParent();
2656 return Legalized;
2657}
2658
2659LegalizerHelper::LegalizeResult
2660LegalizerHelper::widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx,
2661 LLT WideTy) {
2662 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SADDSAT ||
2663 MI.getOpcode() == TargetOpcode::G_SSUBSAT ||
2664 MI.getOpcode() == TargetOpcode::G_SSHLSAT;
2665 bool IsShift = MI.getOpcode() == TargetOpcode::G_SSHLSAT ||
2666 MI.getOpcode() == TargetOpcode::G_USHLSAT;
2667 // We can convert this to:
2668 // 1. Any extend iN to iM
2669 // 2. SHL by M-N
2670 // 3. [US][ADD|SUB|SHL]SAT
2671 // 4. L/ASHR by M-N
2672 //
2673 // It may be more efficient to lower this to a min and a max operation in
2674 // the higher precision arithmetic if the promoted operation isn't legal,
2675 // but this decision is up to the target's lowering request.
2676 Register DstReg = MI.getOperand(i: 0).getReg();
2677
2678 unsigned NewBits = WideTy.getScalarSizeInBits();
2679 unsigned SHLAmount = NewBits - MRI.getType(Reg: DstReg).getScalarSizeInBits();
2680
2681 // Shifts must zero-extend the RHS to preserve the unsigned quantity, and
2682 // must not left shift the RHS to preserve the shift amount.
2683 auto LHS = MIRBuilder.buildAnyExt(Res: WideTy, Op: MI.getOperand(i: 1));
2684 auto RHS = IsShift ? MIRBuilder.buildZExt(Res: WideTy, Op: MI.getOperand(i: 2))
2685 : MIRBuilder.buildAnyExt(Res: WideTy, Op: MI.getOperand(i: 2));
2686 auto ShiftK = MIRBuilder.buildConstant(Res: WideTy, Val: SHLAmount);
2687 auto ShiftL = MIRBuilder.buildShl(Dst: WideTy, Src0: LHS, Src1: ShiftK);
2688 auto ShiftR = IsShift ? RHS : MIRBuilder.buildShl(Dst: WideTy, Src0: RHS, Src1: ShiftK);
2689
2690 auto WideInst = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {WideTy},
2691 SrcOps: {ShiftL, ShiftR}, Flags: MI.getFlags());
2692
2693 // Use a shift that will preserve the number of sign bits when the trunc is
2694 // folded away.
2695 auto Result = IsSigned ? MIRBuilder.buildAShr(Dst: WideTy, Src0: WideInst, Src1: ShiftK)
2696 : MIRBuilder.buildLShr(Dst: WideTy, Src0: WideInst, Src1: ShiftK);
2697
2698 MIRBuilder.buildTrunc(Res: DstReg, Op: Result);
2699 MI.eraseFromParent();
2700 return Legalized;
2701}
2702
2703LegalizerHelper::LegalizeResult
2704LegalizerHelper::widenScalarMulo(MachineInstr &MI, unsigned TypeIdx,
2705 LLT WideTy) {
2706 if (TypeIdx == 1) {
2707 Observer.changingInstr(MI);
2708 widenScalarDst(MI, WideTy, OpIdx: 1);
2709 Observer.changedInstr(MI);
2710 return Legalized;
2711 }
2712
2713 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SMULO;
2714 auto [Result, OriginalOverflow, LHS, RHS] = MI.getFirst4Regs();
2715 LLT SrcTy = MRI.getType(Reg: LHS);
2716 LLT OverflowTy = MRI.getType(Reg: OriginalOverflow);
2717 unsigned SrcBitWidth = SrcTy.getScalarSizeInBits();
2718
2719 // To determine if the result overflowed in the larger type, we extend the
2720 // input to the larger type, do the multiply (checking if it overflows),
2721 // then also check the high bits of the result to see if overflow happened
2722 // there.
2723 unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
2724 auto LeftOperand = MIRBuilder.buildInstr(Opc: ExtOp, DstOps: {WideTy}, SrcOps: {LHS});
2725 auto RightOperand = MIRBuilder.buildInstr(Opc: ExtOp, DstOps: {WideTy}, SrcOps: {RHS});
2726
2727 // Multiplication cannot overflow if the WideTy is >= 2 * original width,
2728 // so we don't need to check the overflow result of larger type Mulo.
2729 bool WideMulCanOverflow = WideTy.getScalarSizeInBits() < 2 * SrcBitWidth;
2730
2731 unsigned MulOpc =
2732 WideMulCanOverflow ? MI.getOpcode() : (unsigned)TargetOpcode::G_MUL;
2733
2734 MachineInstrBuilder Mulo;
2735 if (WideMulCanOverflow)
2736 Mulo = MIRBuilder.buildInstr(Opc: MulOpc, DstOps: {WideTy, OverflowTy},
2737 SrcOps: {LeftOperand, RightOperand});
2738 else
2739 Mulo = MIRBuilder.buildInstr(Opc: MulOpc, DstOps: {WideTy}, SrcOps: {LeftOperand, RightOperand});
2740
2741 auto Mul = Mulo->getOperand(i: 0);
2742 MIRBuilder.buildTrunc(Res: Result, Op: Mul);
2743
2744 MachineInstrBuilder ExtResult;
2745 // Overflow occurred if it occurred in the larger type, or if the high part
2746 // of the result does not zero/sign-extend the low part. Check this second
2747 // possibility first.
2748 if (IsSigned) {
2749 // For signed, overflow occurred when the high part does not sign-extend
2750 // the low part.
2751 ExtResult = MIRBuilder.buildSExtInReg(Res: WideTy, Op: Mul, ImmOp: SrcBitWidth);
2752 } else {
2753 // Unsigned overflow occurred when the high part does not zero-extend the
2754 // low part.
2755 ExtResult = MIRBuilder.buildZExtInReg(Res: WideTy, Op: Mul, ImmOp: SrcBitWidth);
2756 }
2757
2758 if (WideMulCanOverflow) {
2759 auto Overflow =
2760 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: OverflowTy, Op0: Mul, Op1: ExtResult);
2761 // Finally check if the multiplication in the larger type itself overflowed.
2762 MIRBuilder.buildOr(Dst: OriginalOverflow, Src0: Mulo->getOperand(i: 1), Src1: Overflow);
2763 } else {
2764 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: OriginalOverflow, Op0: Mul, Op1: ExtResult);
2765 }
2766 MI.eraseFromParent();
2767 return Legalized;
2768}
2769
2770LegalizerHelper::LegalizeResult
2771LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
2772 unsigned Opcode = MI.getOpcode();
2773 switch (Opcode) {
2774 default:
2775 return UnableToLegalize;
2776 case TargetOpcode::G_ATOMICRMW_XCHG:
2777 case TargetOpcode::G_ATOMICRMW_ADD:
2778 case TargetOpcode::G_ATOMICRMW_SUB:
2779 case TargetOpcode::G_ATOMICRMW_AND:
2780 case TargetOpcode::G_ATOMICRMW_OR:
2781 case TargetOpcode::G_ATOMICRMW_XOR:
2782 case TargetOpcode::G_ATOMICRMW_MIN:
2783 case TargetOpcode::G_ATOMICRMW_MAX:
2784 case TargetOpcode::G_ATOMICRMW_UMIN:
2785 case TargetOpcode::G_ATOMICRMW_UMAX:
2786 assert(TypeIdx == 0 && "atomicrmw with second scalar type");
2787 Observer.changingInstr(MI);
2788 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
2789 widenScalarDst(MI, WideTy, OpIdx: 0);
2790 Observer.changedInstr(MI);
2791 return Legalized;
2792 case TargetOpcode::G_ATOMIC_CMPXCHG:
2793 assert(TypeIdx == 0 && "G_ATOMIC_CMPXCHG with second scalar type");
2794 Observer.changingInstr(MI);
2795 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
2796 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ANYEXT);
2797 widenScalarDst(MI, WideTy, OpIdx: 0);
2798 Observer.changedInstr(MI);
2799 return Legalized;
2800 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS:
2801 if (TypeIdx == 0) {
2802 Observer.changingInstr(MI);
2803 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ANYEXT);
2804 widenScalarSrc(MI, WideTy, OpIdx: 4, ExtOpcode: TargetOpcode::G_ANYEXT);
2805 widenScalarDst(MI, WideTy, OpIdx: 0);
2806 Observer.changedInstr(MI);
2807 return Legalized;
2808 }
2809 assert(TypeIdx == 1 &&
2810 "G_ATOMIC_CMPXCHG_WITH_SUCCESS with third scalar type");
2811 Observer.changingInstr(MI);
2812 widenScalarDst(MI, WideTy, OpIdx: 1);
2813 Observer.changedInstr(MI);
2814 return Legalized;
2815 case TargetOpcode::G_EXTRACT:
2816 return widenScalarExtract(MI, TypeIdx, WideTy);
2817 case TargetOpcode::G_INSERT:
2818 return widenScalarInsert(MI, TypeIdx, WideTy);
2819 case TargetOpcode::G_MERGE_VALUES:
2820 return widenScalarMergeValues(MI, TypeIdx, WideTy);
2821 case TargetOpcode::G_UNMERGE_VALUES:
2822 return widenScalarUnmergeValues(MI, TypeIdx, WideTy);
2823 case TargetOpcode::G_SADDO:
2824 case TargetOpcode::G_SSUBO:
2825 case TargetOpcode::G_UADDO:
2826 case TargetOpcode::G_USUBO:
2827 case TargetOpcode::G_SADDE:
2828 case TargetOpcode::G_SSUBE:
2829 case TargetOpcode::G_UADDE:
2830 case TargetOpcode::G_USUBE:
2831 return widenScalarAddSubOverflow(MI, TypeIdx, WideTy);
2832 case TargetOpcode::G_UMULO:
2833 case TargetOpcode::G_SMULO:
2834 return widenScalarMulo(MI, TypeIdx, WideTy);
2835 case TargetOpcode::G_SADDSAT:
2836 case TargetOpcode::G_SSUBSAT:
2837 case TargetOpcode::G_SSHLSAT:
2838 case TargetOpcode::G_UADDSAT:
2839 case TargetOpcode::G_USUBSAT:
2840 case TargetOpcode::G_USHLSAT:
2841 return widenScalarAddSubShlSat(MI, TypeIdx, WideTy);
2842 case TargetOpcode::G_CTTZ:
2843 case TargetOpcode::G_CTTZ_ZERO_POISON:
2844 case TargetOpcode::G_CTLZ:
2845 case TargetOpcode::G_CTLZ_ZERO_POISON:
2846 case TargetOpcode::G_CTLS:
2847 case TargetOpcode::G_CTPOP: {
2848 if (TypeIdx == 0) {
2849 Observer.changingInstr(MI);
2850 widenScalarDst(MI, WideTy, OpIdx: 0);
2851 Observer.changedInstr(MI);
2852 return Legalized;
2853 }
2854
2855 Register SrcReg = MI.getOperand(i: 1).getReg();
2856
2857 // First extend the input.
2858 unsigned ExtOpc;
2859 switch (Opcode) {
2860 case TargetOpcode::G_CTTZ:
2861 case TargetOpcode::G_CTTZ_ZERO_POISON:
2862 case TargetOpcode::G_CTLZ_ZERO_POISON: // poison shifted out below
2863 ExtOpc = TargetOpcode::G_ANYEXT;
2864 break;
2865 case TargetOpcode::G_CTLS:
2866 ExtOpc = TargetOpcode::G_SEXT;
2867 break;
2868 default:
2869 ExtOpc = TargetOpcode::G_ZEXT;
2870 }
2871
2872 auto MIBSrc = MIRBuilder.buildInstr(Opc: ExtOpc, DstOps: {WideTy}, SrcOps: {SrcReg});
2873 LLT CurTy = MRI.getType(Reg: SrcReg);
2874 unsigned NewOpc = Opcode;
2875 if (NewOpc == TargetOpcode::G_CTTZ) {
2876 // The count is the same in the larger type except if the original
2877 // value was zero. This can be handled by setting the bit just off
2878 // the top of the original type.
2879 auto TopBit = APInt::getOneBitSet(numBits: WideTy.getScalarSizeInBits(),
2880 BitNo: CurTy.getScalarSizeInBits());
2881 MIBSrc = MIRBuilder.buildOr(
2882 Dst: WideTy, Src0: MIBSrc, Src1: MIRBuilder.buildConstant(Res: WideTy, Val: TopBit));
2883 // Now we know the operand is non-zero, use the more relaxed opcode.
2884 NewOpc = TargetOpcode::G_CTTZ_ZERO_POISON;
2885 }
2886
2887 unsigned SizeDiff =
2888 WideTy.getScalarSizeInBits() - CurTy.getScalarSizeInBits();
2889
2890 if (Opcode == TargetOpcode::G_CTLZ_ZERO_POISON) {
2891 // An optimization where the result is the CTLZ after the left shift by
2892 // (Difference in widety and current ty), that is,
2893 // MIBSrc = MIBSrc << (sizeinbits(WideTy) - sizeinbits(CurTy))
2894 // Result = ctlz MIBSrc
2895 MIBSrc = MIRBuilder.buildShl(Dst: WideTy, Src0: MIBSrc,
2896 Src1: MIRBuilder.buildConstant(Res: WideTy, Val: SizeDiff));
2897 }
2898
2899 // Perform the operation at the larger size.
2900 auto MIBNewOp = MIRBuilder.buildInstr(Opc: NewOpc, DstOps: {WideTy}, SrcOps: {MIBSrc});
2901 // This is already the correct result for CTPOP and CTTZs
2902 if (Opcode == TargetOpcode::G_CTLZ || Opcode == TargetOpcode::G_CTLS) {
2903 // The correct result is NewOp - (Difference in widety and current ty).
2904 // At this stage SUB is guaranteed to be positive no-wrap,
2905 // that to be used in further KnownBits optimizations for CTLZ.
2906 MIBNewOp = MIRBuilder.buildSub(
2907 Dst: WideTy, Src0: MIBNewOp, Src1: MIRBuilder.buildConstant(Res: WideTy, Val: SizeDiff),
2908 Flags: Opcode == TargetOpcode::G_CTLZ
2909 ? std::optional<unsigned>(MachineInstr::NoUWrap)
2910 : std::nullopt);
2911 }
2912
2913 MIRBuilder.buildZExtOrTrunc(Res: MI.getOperand(i: 0), Op: MIBNewOp);
2914 MI.eraseFromParent();
2915 return Legalized;
2916 }
2917 case TargetOpcode::G_BSWAP: {
2918 Observer.changingInstr(MI);
2919 Register DstReg = MI.getOperand(i: 0).getReg();
2920
2921 Register ShrReg = MRI.createGenericVirtualRegister(Ty: WideTy);
2922 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2923 Register ShiftAmtReg = MRI.createGenericVirtualRegister(Ty: WideTy);
2924 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2925
2926 MI.getOperand(i: 0).setReg(DstExt);
2927
2928 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2929
2930 LLT Ty = MRI.getType(Reg: DstReg);
2931 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
2932 MIRBuilder.buildConstant(Res: ShiftAmtReg, Val: DiffBits);
2933 MIRBuilder.buildLShr(Dst: ShrReg, Src0: DstExt, Src1: ShiftAmtReg);
2934
2935 MIRBuilder.buildTrunc(Res: DstReg, Op: ShrReg);
2936 Observer.changedInstr(MI);
2937 return Legalized;
2938 }
2939 case TargetOpcode::G_BITREVERSE: {
2940 Observer.changingInstr(MI);
2941
2942 Register DstReg = MI.getOperand(i: 0).getReg();
2943 LLT Ty = MRI.getType(Reg: DstReg);
2944 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
2945
2946 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2947 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2948 MI.getOperand(i: 0).setReg(DstExt);
2949 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2950
2951 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: DiffBits);
2952 auto Shift = MIRBuilder.buildLShr(Dst: WideTy, Src0: DstExt, Src1: ShiftAmt);
2953 MIRBuilder.buildTrunc(Res: DstReg, Op: Shift);
2954 Observer.changedInstr(MI);
2955 return Legalized;
2956 }
2957 case TargetOpcode::G_FREEZE:
2958 case TargetOpcode::G_CONSTANT_FOLD_BARRIER:
2959 Observer.changingInstr(MI);
2960 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2961 widenScalarDst(MI, WideTy);
2962 Observer.changedInstr(MI);
2963 return Legalized;
2964
2965 case TargetOpcode::G_ABS:
2966 Observer.changingInstr(MI);
2967 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_SEXT);
2968 widenScalarDst(MI, WideTy);
2969 Observer.changedInstr(MI);
2970 return Legalized;
2971
2972 case TargetOpcode::G_ADD:
2973 case TargetOpcode::G_AND:
2974 case TargetOpcode::G_MUL:
2975 case TargetOpcode::G_OR:
2976 case TargetOpcode::G_XOR:
2977 case TargetOpcode::G_SUB:
2978 case TargetOpcode::G_SHUFFLE_VECTOR:
2979 // Perform operation at larger width (any extension is fines here, high bits
2980 // don't affect the result) and then truncate the result back to the
2981 // original type.
2982 Observer.changingInstr(MI);
2983 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2984 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
2985 widenScalarDst(MI, WideTy);
2986 Observer.changedInstr(MI);
2987 return Legalized;
2988
2989 case TargetOpcode::G_SBFX:
2990 case TargetOpcode::G_UBFX:
2991 Observer.changingInstr(MI);
2992
2993 if (TypeIdx == 0) {
2994 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2995 widenScalarDst(MI, WideTy);
2996 } else {
2997 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
2998 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ZEXT);
2999 }
3000
3001 Observer.changedInstr(MI);
3002 return Legalized;
3003
3004 case TargetOpcode::G_SHL:
3005 Observer.changingInstr(MI);
3006
3007 if (TypeIdx == 0) {
3008 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3009 widenScalarDst(MI, WideTy);
3010 } else {
3011 assert(TypeIdx == 1);
3012 // The "number of bits to shift" operand must preserve its value as an
3013 // unsigned integer:
3014 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3015 }
3016
3017 Observer.changedInstr(MI);
3018 return Legalized;
3019
3020 case TargetOpcode::G_ROTR:
3021 case TargetOpcode::G_ROTL:
3022 if (TypeIdx != 1)
3023 return UnableToLegalize;
3024
3025 Observer.changingInstr(MI);
3026 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3027 Observer.changedInstr(MI);
3028 return Legalized;
3029
3030 case TargetOpcode::G_SDIV:
3031 case TargetOpcode::G_SREM:
3032 case TargetOpcode::G_SMIN:
3033 case TargetOpcode::G_SMAX:
3034 case TargetOpcode::G_ABDS:
3035 Observer.changingInstr(MI);
3036 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_SEXT);
3037 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_SEXT);
3038 widenScalarDst(MI, WideTy);
3039 Observer.changedInstr(MI);
3040 return Legalized;
3041
3042 case TargetOpcode::G_SDIVREM:
3043 Observer.changingInstr(MI);
3044 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_SEXT);
3045 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_SEXT);
3046 widenScalarDst(MI, WideTy);
3047 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: --MIRBuilder.getInsertPt());
3048 widenScalarDst(MI, WideTy, OpIdx: 1);
3049 Observer.changedInstr(MI);
3050 return Legalized;
3051
3052 case TargetOpcode::G_ASHR:
3053 case TargetOpcode::G_LSHR:
3054 Observer.changingInstr(MI);
3055
3056 if (TypeIdx == 0) {
3057 unsigned CvtOp = Opcode == TargetOpcode::G_ASHR ? TargetOpcode::G_SEXT
3058 : TargetOpcode::G_ZEXT;
3059
3060 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: CvtOp);
3061 widenScalarDst(MI, WideTy);
3062 } else {
3063 assert(TypeIdx == 1);
3064 // The "number of bits to shift" operand must preserve its value as an
3065 // unsigned integer:
3066 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3067 }
3068
3069 Observer.changedInstr(MI);
3070 return Legalized;
3071 case TargetOpcode::G_UDIV:
3072 case TargetOpcode::G_UREM:
3073 case TargetOpcode::G_ABDU:
3074 Observer.changingInstr(MI);
3075 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ZEXT);
3076 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3077 widenScalarDst(MI, WideTy);
3078 Observer.changedInstr(MI);
3079 return Legalized;
3080 case TargetOpcode::G_UDIVREM:
3081 Observer.changingInstr(MI);
3082 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3083 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ZEXT);
3084 widenScalarDst(MI, WideTy);
3085 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: --MIRBuilder.getInsertPt());
3086 widenScalarDst(MI, WideTy, OpIdx: 1);
3087 Observer.changedInstr(MI);
3088 return Legalized;
3089 case TargetOpcode::G_UMIN:
3090 case TargetOpcode::G_UMAX: {
3091 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3092
3093 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
3094 unsigned ExtOpc =
3095 TLI.isSExtCheaperThanZExt(FromTy: getApproximateEVTForLLT(Ty, Ctx),
3096 ToTy: getApproximateEVTForLLT(Ty: WideTy, Ctx))
3097 ? TargetOpcode::G_SEXT
3098 : TargetOpcode::G_ZEXT;
3099
3100 Observer.changingInstr(MI);
3101 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: ExtOpc);
3102 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: ExtOpc);
3103 widenScalarDst(MI, WideTy);
3104 Observer.changedInstr(MI);
3105 return Legalized;
3106 }
3107
3108 case TargetOpcode::G_SELECT:
3109 Observer.changingInstr(MI);
3110 if (TypeIdx == 0) {
3111 // Perform operation at larger width (any extension is fine here, high
3112 // bits don't affect the result) and then truncate the result back to the
3113 // original type.
3114 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
3115 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ANYEXT);
3116 widenScalarDst(MI, WideTy);
3117 } else {
3118 bool IsVec = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).isVector();
3119 // Explicit extension is required here since high bits affect the result.
3120 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: MIRBuilder.getBoolExtOp(IsVec, IsFP: false));
3121 }
3122 Observer.changedInstr(MI);
3123 return Legalized;
3124
3125 case TargetOpcode::G_FPEXT:
3126 if (TypeIdx != 1)
3127 return UnableToLegalize;
3128
3129 Observer.changingInstr(MI);
3130 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3131 Observer.changedInstr(MI);
3132 return Legalized;
3133 case TargetOpcode::G_FPTOSI:
3134 case TargetOpcode::G_FPTOUI:
3135 case TargetOpcode::G_INTRINSIC_LRINT:
3136 case TargetOpcode::G_INTRINSIC_LLRINT:
3137 case TargetOpcode::G_IS_FPCLASS:
3138 Observer.changingInstr(MI);
3139
3140 if (TypeIdx == 0)
3141 widenScalarDst(MI, WideTy);
3142 else
3143 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3144
3145 Observer.changedInstr(MI);
3146 return Legalized;
3147 case TargetOpcode::G_SITOFP:
3148 Observer.changingInstr(MI);
3149
3150 if (TypeIdx == 0)
3151 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3152 else
3153 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_SEXT);
3154
3155 Observer.changedInstr(MI);
3156 return Legalized;
3157 case TargetOpcode::G_UITOFP:
3158 Observer.changingInstr(MI);
3159
3160 if (TypeIdx == 0)
3161 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3162 else
3163 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ZEXT);
3164
3165 Observer.changedInstr(MI);
3166 return Legalized;
3167 case TargetOpcode::G_FPTOSI_SAT:
3168 case TargetOpcode::G_FPTOUI_SAT:
3169 Observer.changingInstr(MI);
3170
3171 if (TypeIdx == 0) {
3172 Register OldDst = MI.getOperand(i: 0).getReg();
3173 LLT Ty = MRI.getType(Reg: OldDst);
3174 Register ExtReg = MRI.createGenericVirtualRegister(Ty: WideTy);
3175 Register NewDst;
3176 MI.getOperand(i: 0).setReg(ExtReg);
3177 uint64_t ShortBits = Ty.getScalarSizeInBits();
3178 uint64_t WideBits = WideTy.getScalarSizeInBits();
3179 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
3180 if (Opcode == TargetOpcode::G_FPTOSI_SAT) {
3181 // z = i16 fptosi_sat(a)
3182 // ->
3183 // x = i32 fptosi_sat(a)
3184 // y = smin(x, 32767)
3185 // z = smax(y, -32768)
3186 auto MaxVal = MIRBuilder.buildConstant(
3187 Res: WideTy, Val: APInt::getSignedMaxValue(numBits: ShortBits).sext(width: WideBits));
3188 auto MinVal = MIRBuilder.buildConstant(
3189 Res: WideTy, Val: APInt::getSignedMinValue(numBits: ShortBits).sext(width: WideBits));
3190 Register MidReg =
3191 MIRBuilder.buildSMin(Dst: WideTy, Src0: ExtReg, Src1: MaxVal).getReg(Idx: 0);
3192 NewDst = MIRBuilder.buildSMax(Dst: WideTy, Src0: MidReg, Src1: MinVal).getReg(Idx: 0);
3193 } else {
3194 // z = i16 fptoui_sat(a)
3195 // ->
3196 // x = i32 fptoui_sat(a)
3197 // y = smin(x, 65535)
3198 auto MaxVal = MIRBuilder.buildConstant(
3199 Res: WideTy, Val: APInt::getAllOnes(numBits: ShortBits).zext(width: WideBits));
3200 NewDst = MIRBuilder.buildUMin(Dst: WideTy, Src0: ExtReg, Src1: MaxVal).getReg(Idx: 0);
3201 }
3202 MIRBuilder.buildTrunc(Res: OldDst, Op: NewDst);
3203 } else
3204 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3205
3206 Observer.changedInstr(MI);
3207 return Legalized;
3208 case TargetOpcode::G_LOAD:
3209 case TargetOpcode::G_SEXTLOAD:
3210 case TargetOpcode::G_ZEXTLOAD:
3211 case TargetOpcode::G_FPEXTLOAD:
3212 Observer.changingInstr(MI);
3213 widenScalarDst(MI, WideTy);
3214 Observer.changedInstr(MI);
3215 return Legalized;
3216
3217 case TargetOpcode::G_STORE: {
3218 if (TypeIdx != 0)
3219 return UnableToLegalize;
3220
3221 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3222 assert(!Ty.isPointerOrPointerVector() && "Can't widen type");
3223 if (!Ty.isScalar()) {
3224 // We need to widen the vector element type.
3225 Observer.changingInstr(MI);
3226 widenScalarSrc(MI, WideTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_ANYEXT);
3227 // We also need to adjust the MMO to turn this into a truncating store.
3228 MachineMemOperand &MMO = **MI.memoperands_begin();
3229 MachineFunction &MF = MIRBuilder.getMF();
3230 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo: MMO.getPointerInfo(), Ty);
3231 MI.setMemRefs(MF, MemRefs: {NewMMO});
3232 Observer.changedInstr(MI);
3233 return Legalized;
3234 }
3235
3236 Observer.changingInstr(MI);
3237
3238 unsigned ExtType = Ty.getScalarSizeInBits() == 1 ?
3239 TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT;
3240 widenScalarSrc(MI, WideTy, OpIdx: 0, ExtOpcode: ExtType);
3241
3242 Observer.changedInstr(MI);
3243 return Legalized;
3244 }
3245 case TargetOpcode::G_FPTRUNCSTORE:
3246 if (TypeIdx != 0)
3247 return UnableToLegalize;
3248 Observer.changingInstr(MI);
3249 widenScalarSrc(MI, WideTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_FPEXT);
3250 Observer.changedInstr(MI);
3251 return Legalized;
3252 case TargetOpcode::G_CONSTANT: {
3253 MachineOperand &SrcMO = MI.getOperand(i: 1);
3254 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
3255 unsigned ExtOpc = LI.getExtOpcodeForWideningConstant(
3256 SmallTy: MRI.getType(Reg: MI.getOperand(i: 0).getReg()));
3257 assert((ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::G_SEXT ||
3258 ExtOpc == TargetOpcode::G_ANYEXT) &&
3259 "Illegal Extend");
3260 const APInt &SrcVal = SrcMO.getCImm()->getValue();
3261 const APInt &Val = (ExtOpc == TargetOpcode::G_SEXT)
3262 ? SrcVal.sext(width: WideTy.getSizeInBits())
3263 : SrcVal.zext(width: WideTy.getSizeInBits());
3264 Observer.changingInstr(MI);
3265 SrcMO.setCImm(ConstantInt::get(Context&: Ctx, V: Val));
3266
3267 widenScalarDst(MI, WideTy);
3268 Observer.changedInstr(MI);
3269 return Legalized;
3270 }
3271 case TargetOpcode::G_FCONSTANT: {
3272 // To avoid changing the bits of the constant due to extension to a larger
3273 // type and then using G_FPTRUNC, we simply convert to a G_CONSTANT.
3274 MachineOperand &SrcMO = MI.getOperand(i: 1);
3275 APInt Val = SrcMO.getFPImm()->getValueAPF().bitcastToAPInt();
3276 MIRBuilder.setInstrAndDebugLoc(MI);
3277 auto IntCst = MIRBuilder.buildConstant(Res: MI.getOperand(i: 0).getReg(), Val);
3278 widenScalarDst(MI&: *IntCst, WideTy, OpIdx: 0, TruncOpcode: TargetOpcode::G_TRUNC);
3279 MI.eraseFromParent();
3280 return Legalized;
3281 }
3282 case TargetOpcode::G_IMPLICIT_DEF: {
3283 Observer.changingInstr(MI);
3284 widenScalarDst(MI, WideTy);
3285 Observer.changedInstr(MI);
3286 return Legalized;
3287 }
3288 case TargetOpcode::G_BRCOND:
3289 Observer.changingInstr(MI);
3290 widenScalarSrc(MI, WideTy, OpIdx: 0, ExtOpcode: MIRBuilder.getBoolExtOp(IsVec: false, IsFP: false));
3291 Observer.changedInstr(MI);
3292 return Legalized;
3293
3294 case TargetOpcode::G_FCMP:
3295 Observer.changingInstr(MI);
3296 if (TypeIdx == 0)
3297 widenScalarDst(MI, WideTy);
3298 else {
3299 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 2);
3300 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 3);
3301 }
3302 Observer.changedInstr(MI);
3303 return Legalized;
3304
3305 case TargetOpcode::G_ICMP:
3306 Observer.changingInstr(MI);
3307 if (TypeIdx == 0)
3308 widenScalarDst(MI, WideTy);
3309 else {
3310 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
3311 CmpInst::Predicate Pred =
3312 static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
3313
3314 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
3315 unsigned ExtOpcode =
3316 (CmpInst::isSigned(Pred) ||
3317 TLI.isSExtCheaperThanZExt(FromTy: getApproximateEVTForLLT(Ty: SrcTy, Ctx),
3318 ToTy: getApproximateEVTForLLT(Ty: WideTy, Ctx)))
3319 ? TargetOpcode::G_SEXT
3320 : TargetOpcode::G_ZEXT;
3321 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode);
3322 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode);
3323 }
3324 Observer.changedInstr(MI);
3325 return Legalized;
3326
3327 case TargetOpcode::G_PTR_ADD:
3328 assert(TypeIdx == 1 && "unable to legalize pointer of G_PTR_ADD");
3329 Observer.changingInstr(MI);
3330 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_SEXT);
3331 Observer.changedInstr(MI);
3332 return Legalized;
3333
3334 case TargetOpcode::G_PHI: {
3335 assert(TypeIdx == 0 && "Expecting only Idx 0");
3336
3337 Observer.changingInstr(MI);
3338 for (unsigned I = 1; I < MI.getNumOperands(); I += 2) {
3339 MachineBasicBlock &OpMBB = *MI.getOperand(i: I + 1).getMBB();
3340 MIRBuilder.setInsertPt(MBB&: OpMBB, II: OpMBB.getFirstTerminatorForward());
3341 widenScalarSrc(MI, WideTy, OpIdx: I, ExtOpcode: TargetOpcode::G_ANYEXT);
3342 }
3343
3344 MachineBasicBlock &MBB = *MI.getParent();
3345 MIRBuilder.setInsertPt(MBB, II: --MBB.getFirstNonPHI());
3346 widenScalarDst(MI, WideTy);
3347 Observer.changedInstr(MI);
3348 return Legalized;
3349 }
3350 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
3351 if (TypeIdx == 0) {
3352 Register VecReg = MI.getOperand(i: 1).getReg();
3353 LLT VecTy = MRI.getType(Reg: VecReg);
3354 Observer.changingInstr(MI);
3355
3356 widenScalarSrc(MI, WideTy: LLT::vector(EC: VecTy.getElementCount(), ScalarTy: WideTy), OpIdx: 1,
3357 ExtOpcode: TargetOpcode::G_ANYEXT);
3358
3359 widenScalarDst(MI, WideTy, OpIdx: 0);
3360 Observer.changedInstr(MI);
3361 return Legalized;
3362 }
3363
3364 if (TypeIdx != 2)
3365 return UnableToLegalize;
3366 Observer.changingInstr(MI);
3367 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3368 Observer.changedInstr(MI);
3369 return Legalized;
3370 }
3371 case TargetOpcode::G_INSERT_VECTOR_ELT: {
3372 if (TypeIdx == 0) {
3373 Observer.changingInstr(MI);
3374 const LLT WideEltTy = WideTy.getElementType();
3375
3376 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3377 widenScalarSrc(MI, WideTy: WideEltTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
3378 widenScalarDst(MI, WideTy, OpIdx: 0);
3379 Observer.changedInstr(MI);
3380 return Legalized;
3381 }
3382
3383 if (TypeIdx == 1) {
3384 Observer.changingInstr(MI);
3385
3386 Register VecReg = MI.getOperand(i: 1).getReg();
3387 LLT VecTy = MRI.getType(Reg: VecReg);
3388 LLT WideVecTy = VecTy.changeVectorElementType(NewEltTy: WideTy);
3389
3390 widenScalarSrc(MI, WideTy: WideVecTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3391 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
3392 widenScalarDst(MI, WideTy: WideVecTy, OpIdx: 0);
3393 Observer.changedInstr(MI);
3394 return Legalized;
3395 }
3396
3397 if (TypeIdx == 2) {
3398 Observer.changingInstr(MI);
3399 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ZEXT);
3400 Observer.changedInstr(MI);
3401 return Legalized;
3402 }
3403
3404 return UnableToLegalize;
3405 }
3406 case TargetOpcode::G_FADD:
3407 case TargetOpcode::G_FMUL:
3408 case TargetOpcode::G_FSUB:
3409 case TargetOpcode::G_FMA:
3410 case TargetOpcode::G_FMAD:
3411 case TargetOpcode::G_FNEG:
3412 case TargetOpcode::G_FABS:
3413 case TargetOpcode::G_FCANONICALIZE:
3414 case TargetOpcode::G_FMINNUM:
3415 case TargetOpcode::G_FMAXNUM:
3416 case TargetOpcode::G_FMINNUM_IEEE:
3417 case TargetOpcode::G_FMAXNUM_IEEE:
3418 case TargetOpcode::G_FMINIMUM:
3419 case TargetOpcode::G_FMAXIMUM:
3420 case TargetOpcode::G_FMINIMUMNUM:
3421 case TargetOpcode::G_FMAXIMUMNUM:
3422 case TargetOpcode::G_FDIV:
3423 case TargetOpcode::G_FREM:
3424 case TargetOpcode::G_FCEIL:
3425 case TargetOpcode::G_FFLOOR:
3426 case TargetOpcode::G_FCOS:
3427 case TargetOpcode::G_FSIN:
3428 case TargetOpcode::G_FTAN:
3429 case TargetOpcode::G_FACOS:
3430 case TargetOpcode::G_FASIN:
3431 case TargetOpcode::G_FATAN:
3432 case TargetOpcode::G_FATAN2:
3433 case TargetOpcode::G_FCOSH:
3434 case TargetOpcode::G_FSINH:
3435 case TargetOpcode::G_FTANH:
3436 case TargetOpcode::G_FLOG10:
3437 case TargetOpcode::G_FLOG:
3438 case TargetOpcode::G_FLOG2:
3439 case TargetOpcode::G_FRINT:
3440 case TargetOpcode::G_FNEARBYINT:
3441 case TargetOpcode::G_FSQRT:
3442 case TargetOpcode::G_FEXP:
3443 case TargetOpcode::G_FEXP2:
3444 case TargetOpcode::G_FEXP10:
3445 case TargetOpcode::G_FPOW:
3446 case TargetOpcode::G_INTRINSIC_TRUNC:
3447 case TargetOpcode::G_INTRINSIC_ROUND:
3448 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
3449 assert(TypeIdx == 0);
3450 Observer.changingInstr(MI);
3451
3452 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I)
3453 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: I);
3454
3455 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3456 Observer.changedInstr(MI);
3457 return Legalized;
3458 case TargetOpcode::G_FMODF: {
3459 Observer.changingInstr(MI);
3460 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 2);
3461
3462 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 1);
3463 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: --MIRBuilder.getInsertPt());
3464 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3465 Observer.changedInstr(MI);
3466 return Legalized;
3467 }
3468 case TargetOpcode::G_FPOWI:
3469 case TargetOpcode::G_FLDEXP:
3470 case TargetOpcode::G_STRICT_FLDEXP: {
3471 if (TypeIdx == 0) {
3472 if (Opcode == TargetOpcode::G_STRICT_FLDEXP)
3473 return UnableToLegalize;
3474
3475 Observer.changingInstr(MI);
3476 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3477 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3478 Observer.changedInstr(MI);
3479 return Legalized;
3480 }
3481
3482 if (TypeIdx == 1) {
3483 // For some reason SelectionDAG tries to promote to a libcall without
3484 // actually changing the integer type for promotion.
3485 Observer.changingInstr(MI);
3486 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_SEXT);
3487 Observer.changedInstr(MI);
3488 return Legalized;
3489 }
3490
3491 return UnableToLegalize;
3492 }
3493 case TargetOpcode::G_FFREXP: {
3494 Observer.changingInstr(MI);
3495
3496 if (TypeIdx == 0) {
3497 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 2);
3498 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3499 } else {
3500 widenScalarDst(MI, WideTy, OpIdx: 1);
3501 }
3502
3503 Observer.changedInstr(MI);
3504 return Legalized;
3505 }
3506 case TargetOpcode::G_LROUND:
3507 case TargetOpcode::G_LLROUND:
3508 Observer.changingInstr(MI);
3509
3510 if (TypeIdx == 0)
3511 widenScalarDst(MI, WideTy);
3512 else
3513 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3514
3515 Observer.changedInstr(MI);
3516 return Legalized;
3517
3518 case TargetOpcode::G_INTTOPTR:
3519 if (TypeIdx != 1)
3520 return UnableToLegalize;
3521
3522 Observer.changingInstr(MI);
3523 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ZEXT);
3524 Observer.changedInstr(MI);
3525 return Legalized;
3526 case TargetOpcode::G_PTRTOINT:
3527 if (TypeIdx != 0)
3528 return UnableToLegalize;
3529
3530 Observer.changingInstr(MI);
3531 widenScalarDst(MI, WideTy, OpIdx: 0);
3532 Observer.changedInstr(MI);
3533 return Legalized;
3534 case TargetOpcode::G_BUILD_VECTOR: {
3535 Observer.changingInstr(MI);
3536
3537 const LLT WideEltTy = TypeIdx == 1 ? WideTy : WideTy.getElementType();
3538 for (int I = 1, E = MI.getNumOperands(); I != E; ++I)
3539 widenScalarSrc(MI, WideTy: WideEltTy, OpIdx: I, ExtOpcode: TargetOpcode::G_ANYEXT);
3540
3541 // Avoid changing the result vector type if the source element type was
3542 // requested.
3543 if (TypeIdx == 1) {
3544 MI.setDesc(MIRBuilder.getTII().get(Opcode: TargetOpcode::G_BUILD_VECTOR_TRUNC));
3545 } else {
3546 widenScalarDst(MI, WideTy, OpIdx: 0);
3547 }
3548
3549 Observer.changedInstr(MI);
3550 return Legalized;
3551 }
3552 case TargetOpcode::G_SEXT_INREG:
3553 if (TypeIdx != 0)
3554 return UnableToLegalize;
3555
3556 Observer.changingInstr(MI);
3557 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3558 widenScalarDst(MI, WideTy, OpIdx: 0, TruncOpcode: TargetOpcode::G_TRUNC);
3559 Observer.changedInstr(MI);
3560 return Legalized;
3561 case TargetOpcode::G_PTRMASK: {
3562 if (TypeIdx != 1)
3563 return UnableToLegalize;
3564 Observer.changingInstr(MI);
3565 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3566 Observer.changedInstr(MI);
3567 return Legalized;
3568 }
3569 case TargetOpcode::G_VECREDUCE_ADD: {
3570 if (TypeIdx != 1)
3571 return UnableToLegalize;
3572 Observer.changingInstr(MI);
3573 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3574 widenScalarDst(MI, WideTy: WideTy.getScalarType(), OpIdx: 0, TruncOpcode: TargetOpcode::G_TRUNC);
3575 Observer.changedInstr(MI);
3576 return Legalized;
3577 }
3578 case TargetOpcode::G_VECREDUCE_FADD:
3579 case TargetOpcode::G_VECREDUCE_FMUL:
3580 case TargetOpcode::G_VECREDUCE_FMIN:
3581 case TargetOpcode::G_VECREDUCE_FMAX:
3582 case TargetOpcode::G_VECREDUCE_FMINIMUM:
3583 case TargetOpcode::G_VECREDUCE_FMAXIMUM: {
3584 if (TypeIdx != 0)
3585 return UnableToLegalize;
3586 Observer.changingInstr(MI);
3587 Register VecReg = MI.getOperand(i: 1).getReg();
3588 LLT VecTy = MRI.getType(Reg: VecReg);
3589 LLT WideVecTy = VecTy.changeElementType(NewEltTy: WideTy);
3590 widenScalarSrcUsingFPExt(MI, WideTy: WideVecTy, OpIdx: 1);
3591 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3592 Observer.changedInstr(MI);
3593 return Legalized;
3594 }
3595 case TargetOpcode::G_VSCALE: {
3596 MachineOperand &SrcMO = MI.getOperand(i: 1);
3597 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
3598 const APInt &SrcVal = SrcMO.getCImm()->getValue();
3599 // The CImm is always a signed value
3600 const APInt Val = SrcVal.sext(width: WideTy.getSizeInBits());
3601 Observer.changingInstr(MI);
3602 SrcMO.setCImm(ConstantInt::get(Context&: Ctx, V: Val));
3603 widenScalarDst(MI, WideTy);
3604 Observer.changedInstr(MI);
3605 return Legalized;
3606 }
3607 case TargetOpcode::G_SPLAT_VECTOR: {
3608 if (TypeIdx != 1)
3609 return UnableToLegalize;
3610
3611 Observer.changingInstr(MI);
3612 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3613 Observer.changedInstr(MI);
3614 return Legalized;
3615 }
3616 case TargetOpcode::G_INSERT_SUBVECTOR: {
3617 if (TypeIdx != 0)
3618 return UnableToLegalize;
3619
3620 GInsertSubvector &IS = cast<GInsertSubvector>(Val&: MI);
3621 Register BigVec = IS.getBigVec();
3622 Register SubVec = IS.getSubVec();
3623
3624 LLT SubVecTy = MRI.getType(Reg: SubVec);
3625 LLT SubVecWideTy = SubVecTy.changeElementType(NewEltTy: WideTy.getElementType());
3626
3627 // Widen the G_INSERT_SUBVECTOR
3628 auto BigZExt = MIRBuilder.buildZExt(Res: WideTy, Op: BigVec);
3629 auto SubZExt = MIRBuilder.buildZExt(Res: SubVecWideTy, Op: SubVec);
3630 auto WideInsert = MIRBuilder.buildInsertSubvector(Res: WideTy, Src0: BigZExt, Src1: SubZExt,
3631 Index: IS.getIndexImm());
3632
3633 // Truncate back down
3634 auto SplatZero = MIRBuilder.buildSplatVector(
3635 Res: WideTy, Val: MIRBuilder.buildConstant(Res: WideTy.getElementType(), Val: 0));
3636 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_NE, Res: IS.getReg(Idx: 0), Op0: WideInsert,
3637 Op1: SplatZero);
3638
3639 MI.eraseFromParent();
3640
3641 return Legalized;
3642 }
3643 case TargetOpcode::G_BITCAST:
3644 if (WideTy.isVector())
3645 return UnableToLegalize;
3646 Observer.changingInstr(MI);
3647 if (TypeIdx == 0)
3648 widenScalarDst(MI, WideTy, OpIdx: 0, TruncOpcode: TargetOpcode::G_TRUNC);
3649 else
3650 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3651 Observer.changedInstr(MI);
3652
3653 Register Dst = MI.getOperand(i: 0).getReg();
3654 Register Src = MI.getOperand(i: 1).getReg();
3655 if (MRI.getType(Reg: Dst) == MRI.getType(Reg: Src)) {
3656 Observer.changingAllUsesOfReg(MRI, Reg: Dst);
3657 MRI.replaceRegWith(FromReg: Dst, ToReg: Src);
3658 Observer.finishedChangingAllUsesOfReg();
3659 MI.eraseFromParent();
3660 }
3661
3662 return Legalized;
3663 }
3664}
3665
3666static void getUnmergePieces(SmallVectorImpl<Register> &Pieces,
3667 MachineIRBuilder &B, Register Src, LLT Ty) {
3668 auto Unmerge = B.buildUnmerge(Res: Ty, Op: Src);
3669 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
3670 Pieces.push_back(Elt: Unmerge.getReg(Idx: I));
3671}
3672
3673static void emitLoadFromConstantPool(Register DstReg, const Constant *ConstVal,
3674 MachineIRBuilder &MIRBuilder) {
3675 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
3676 MachineFunction &MF = MIRBuilder.getMF();
3677 const DataLayout &DL = MIRBuilder.getDataLayout();
3678 unsigned AddrSpace = DL.getDefaultGlobalsAddressSpace();
3679 LLT AddrPtrTy = LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL.getPointerSizeInBits(AS: AddrSpace));
3680 LLT DstLLT = MRI.getType(Reg: DstReg);
3681
3682 Align Alignment(DL.getABITypeAlign(Ty: ConstVal->getType()));
3683
3684 auto Addr = MIRBuilder.buildConstantPool(
3685 Res: AddrPtrTy,
3686 Idx: MF.getConstantPool()->getConstantPoolIndex(C: ConstVal, Alignment));
3687
3688 MachineMemOperand *MMO =
3689 MF.getMachineMemOperand(PtrInfo: MachinePointerInfo::getConstantPool(MF),
3690 f: MachineMemOperand::MOLoad, MemTy: DstLLT, base_alignment: Alignment);
3691
3692 MIRBuilder.buildLoadInstr(Opcode: TargetOpcode::G_LOAD, Res: DstReg, Addr, MMO&: *MMO);
3693}
3694
3695LegalizerHelper::LegalizeResult
3696LegalizerHelper::lowerConstant(MachineInstr &MI) {
3697 const MachineOperand &ConstOperand = MI.getOperand(i: 1);
3698 const Constant *ConstantVal = ConstOperand.getCImm();
3699
3700 emitLoadFromConstantPool(DstReg: MI.getOperand(i: 0).getReg(), ConstVal: ConstantVal, MIRBuilder);
3701 MI.eraseFromParent();
3702
3703 return Legalized;
3704}
3705
3706LegalizerHelper::LegalizeResult
3707LegalizerHelper::lowerFConstant(MachineInstr &MI) {
3708 const MachineOperand &ConstOperand = MI.getOperand(i: 1);
3709 const Constant *ConstantVal = ConstOperand.getFPImm();
3710
3711 emitLoadFromConstantPool(DstReg: MI.getOperand(i: 0).getReg(), ConstVal: ConstantVal, MIRBuilder);
3712 MI.eraseFromParent();
3713
3714 return Legalized;
3715}
3716
3717LegalizerHelper::LegalizeResult
3718LegalizerHelper::lowerBitcast(MachineInstr &MI) {
3719 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
3720 if (SrcTy.isVector()) {
3721 LLT SrcEltTy = SrcTy.getElementType();
3722 SmallVector<Register, 8> SrcRegs;
3723
3724 if (DstTy.isVector()) {
3725 int NumDstElt = DstTy.getNumElements();
3726 int NumSrcElt = SrcTy.getNumElements();
3727
3728 LLT DstEltTy = DstTy.getElementType();
3729 LLT DstCastTy = DstEltTy; // Intermediate bitcast result type
3730 LLT SrcPartTy = SrcEltTy; // Original unmerge result type.
3731
3732 // If there's an element size mismatch, insert intermediate casts to match
3733 // the result element type.
3734 if (NumSrcElt < NumDstElt) { // Source element type is larger.
3735 // %1:_(<4 x s8>) = G_BITCAST %0:_(<2 x s16>)
3736 //
3737 // =>
3738 //
3739 // %2:_(s16), %3:_(s16) = G_UNMERGE_VALUES %0
3740 // %3:_(<2 x s8>) = G_BITCAST %2
3741 // %4:_(<2 x s8>) = G_BITCAST %3
3742 // %1:_(<4 x s16>) = G_CONCAT_VECTORS %3, %4
3743 DstCastTy = DstTy.changeVectorElementCount(
3744 EC: ElementCount::getFixed(MinVal: NumDstElt / NumSrcElt));
3745 SrcPartTy = SrcEltTy;
3746 } else if (NumSrcElt > NumDstElt) { // Source element type is smaller.
3747 //
3748 // %1:_(<2 x s16>) = G_BITCAST %0:_(<4 x s8>)
3749 //
3750 // =>
3751 //
3752 // %2:_(<2 x s8>), %3:_(<2 x s8>) = G_UNMERGE_VALUES %0
3753 // %3:_(s16) = G_BITCAST %2
3754 // %4:_(s16) = G_BITCAST %3
3755 // %1:_(<2 x s16>) = G_BUILD_VECTOR %3, %4
3756 SrcPartTy = SrcTy.changeVectorElementCount(
3757 EC: ElementCount::getFixed(MinVal: NumSrcElt / NumDstElt));
3758 DstCastTy = DstEltTy;
3759 }
3760
3761 getUnmergePieces(Pieces&: SrcRegs, B&: MIRBuilder, Src, Ty: SrcPartTy);
3762 for (Register &SrcReg : SrcRegs)
3763 SrcReg = MIRBuilder.buildBitcast(Dst: DstCastTy, Src: SrcReg).getReg(Idx: 0);
3764 } else
3765 getUnmergePieces(Pieces&: SrcRegs, B&: MIRBuilder, Src, Ty: SrcEltTy);
3766
3767 MIRBuilder.buildMergeLikeInstr(Res: Dst, Ops: SrcRegs);
3768 MI.eraseFromParent();
3769 return Legalized;
3770 }
3771
3772 if (DstTy.isVector()) {
3773 SmallVector<Register, 8> SrcRegs;
3774 getUnmergePieces(Pieces&: SrcRegs, B&: MIRBuilder, Src, Ty: DstTy.getElementType());
3775 MIRBuilder.buildMergeLikeInstr(Res: Dst, Ops: SrcRegs);
3776 MI.eraseFromParent();
3777 return Legalized;
3778 }
3779
3780 return UnableToLegalize;
3781}
3782
3783/// Figure out the bit offset into a register when coercing a vector index for
3784/// the wide element type. This is only for the case when promoting vector to
3785/// one with larger elements.
3786//
3787///
3788/// %offset_idx = G_AND %idx, ~(-1 << Log2(DstEltSize / SrcEltSize))
3789/// %offset_bits = G_SHL %offset_idx, Log2(SrcEltSize)
3790static Register getBitcastWiderVectorElementOffset(MachineIRBuilder &B,
3791 Register Idx,
3792 unsigned NewEltSize,
3793 unsigned OldEltSize) {
3794 const unsigned Log2EltRatio = Log2_32(Value: NewEltSize / OldEltSize);
3795 LLT IdxTy = B.getMRI()->getType(Reg: Idx);
3796
3797 // Now figure out the amount we need to shift to get the target bits.
3798 auto OffsetMask = B.buildConstant(
3799 Res: IdxTy, Val: ~(APInt::getAllOnes(numBits: IdxTy.getSizeInBits()) << Log2EltRatio));
3800 auto OffsetIdx = B.buildAnd(Dst: IdxTy, Src0: Idx, Src1: OffsetMask);
3801 return B.buildShl(Dst: IdxTy, Src0: OffsetIdx,
3802 Src1: B.buildConstant(Res: IdxTy, Val: Log2_32(Value: OldEltSize))).getReg(Idx: 0);
3803}
3804
3805/// Perform a G_EXTRACT_VECTOR_ELT in a different sized vector element. If this
3806/// is casting to a vector with a smaller element size, perform multiple element
3807/// extracts and merge the results. If this is coercing to a vector with larger
3808/// elements, index the bitcasted vector and extract the target element with bit
3809/// operations. This is intended to force the indexing in the native register
3810/// size for architectures that can dynamically index the register file.
3811LegalizerHelper::LegalizeResult
3812LegalizerHelper::bitcastExtractVectorElt(MachineInstr &MI, unsigned TypeIdx,
3813 LLT CastTy) {
3814 if (TypeIdx != 1)
3815 return UnableToLegalize;
3816
3817 auto [Dst, DstTy, SrcVec, SrcVecTy, Idx, IdxTy] = MI.getFirst3RegLLTs();
3818
3819 LLT SrcEltTy = SrcVecTy.getElementType();
3820 unsigned NewNumElts = CastTy.isVector() ? CastTy.getNumElements() : 1;
3821 unsigned OldNumElts = SrcVecTy.getNumElements();
3822
3823 LLT NewEltTy = CastTy.getScalarType();
3824 Register CastVec = MIRBuilder.buildBitcast(Dst: CastTy, Src: SrcVec).getReg(Idx: 0);
3825
3826 const unsigned NewEltSize = NewEltTy.getSizeInBits();
3827 const unsigned OldEltSize = SrcEltTy.getSizeInBits();
3828 if (NewNumElts > OldNumElts) {
3829 // Decreasing the vector element size
3830 //
3831 // e.g. i64 = extract_vector_elt x:v2i64, y:i32
3832 // =>
3833 // v4i32:castx = bitcast x:v2i64
3834 //
3835 // i64 = bitcast
3836 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
3837 // (i32 (extract_vector_elt castx, (2 * y + 1)))
3838 //
3839 if (NewNumElts % OldNumElts != 0)
3840 return UnableToLegalize;
3841
3842 // Type of the intermediate result vector.
3843 const unsigned NewEltsPerOldElt = NewNumElts / OldNumElts;
3844 LLT MidTy =
3845 CastTy.changeElementCount(EC: ElementCount::getFixed(MinVal: NewEltsPerOldElt));
3846
3847 auto NewEltsPerOldEltK = MIRBuilder.buildConstant(Res: IdxTy, Val: NewEltsPerOldElt);
3848
3849 SmallVector<Register, 8> NewOps(NewEltsPerOldElt);
3850 auto NewBaseIdx = MIRBuilder.buildMul(Dst: IdxTy, Src0: Idx, Src1: NewEltsPerOldEltK);
3851
3852 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
3853 auto IdxOffset = MIRBuilder.buildConstant(Res: IdxTy, Val: I);
3854 auto TmpIdx = MIRBuilder.buildAdd(Dst: IdxTy, Src0: NewBaseIdx, Src1: IdxOffset);
3855 auto Elt = MIRBuilder.buildExtractVectorElement(Res: NewEltTy, Val: CastVec, Idx: TmpIdx);
3856 NewOps[I] = Elt.getReg(Idx: 0);
3857 }
3858
3859 auto NewVec = MIRBuilder.buildBuildVector(Res: MidTy, Ops: NewOps);
3860 MIRBuilder.buildBitcast(Dst, Src: NewVec);
3861 MI.eraseFromParent();
3862 return Legalized;
3863 }
3864
3865 if (NewNumElts < OldNumElts) {
3866 if (NewEltSize % OldEltSize != 0)
3867 return UnableToLegalize;
3868
3869 // This only depends on powers of 2 because we use bit tricks to figure out
3870 // the bit offset we need to shift to get the target element. A general
3871 // expansion could emit division/multiply.
3872 if (!isPowerOf2_32(Value: NewEltSize / OldEltSize))
3873 return UnableToLegalize;
3874
3875 // Increasing the vector element size.
3876 // %elt:_(small_elt) = G_EXTRACT_VECTOR_ELT %vec:_(<N x small_elt>), %idx
3877 //
3878 // =>
3879 //
3880 // %cast = G_BITCAST %vec
3881 // %scaled_idx = G_LSHR %idx, Log2(DstEltSize / SrcEltSize)
3882 // %wide_elt = G_EXTRACT_VECTOR_ELT %cast, %scaled_idx
3883 // %offset_idx = G_AND %idx, ~(-1 << Log2(DstEltSize / SrcEltSize))
3884 // %offset_bits = G_SHL %offset_idx, Log2(SrcEltSize)
3885 // %elt_bits = G_LSHR %wide_elt, %offset_bits
3886 // %elt = G_TRUNC %elt_bits
3887
3888 const unsigned Log2EltRatio = Log2_32(Value: NewEltSize / OldEltSize);
3889 auto Log2Ratio = MIRBuilder.buildConstant(Res: IdxTy, Val: Log2EltRatio);
3890
3891 // Divide to get the index in the wider element type.
3892 auto ScaledIdx = MIRBuilder.buildLShr(Dst: IdxTy, Src0: Idx, Src1: Log2Ratio);
3893
3894 Register WideElt = CastVec;
3895 if (CastTy.isVector()) {
3896 WideElt = MIRBuilder.buildExtractVectorElement(Res: NewEltTy, Val: CastVec,
3897 Idx: ScaledIdx).getReg(Idx: 0);
3898 }
3899
3900 // Compute the bit offset into the register of the target element.
3901 Register OffsetBits = getBitcastWiderVectorElementOffset(
3902 B&: MIRBuilder, Idx, NewEltSize, OldEltSize);
3903
3904 // Shift the wide element to get the target element.
3905 auto ExtractedBits = MIRBuilder.buildLShr(Dst: NewEltTy, Src0: WideElt, Src1: OffsetBits);
3906 MIRBuilder.buildTrunc(Res: Dst, Op: ExtractedBits);
3907 MI.eraseFromParent();
3908 return Legalized;
3909 }
3910
3911 return UnableToLegalize;
3912}
3913
3914/// Emit code to insert \p InsertReg into \p TargetRet at \p OffsetBits in \p
3915/// TargetReg, while preserving other bits in \p TargetReg.
3916///
3917/// (InsertReg << Offset) | (TargetReg & ~(-1 >> InsertReg.size()) << Offset)
3918static Register buildBitFieldInsert(MachineIRBuilder &B,
3919 Register TargetReg, Register InsertReg,
3920 Register OffsetBits) {
3921 LLT TargetTy = B.getMRI()->getType(Reg: TargetReg);
3922 LLT InsertTy = B.getMRI()->getType(Reg: InsertReg);
3923 auto ZextVal = B.buildZExt(Res: TargetTy, Op: InsertReg);
3924 auto ShiftedInsertVal = B.buildShl(Dst: TargetTy, Src0: ZextVal, Src1: OffsetBits);
3925
3926 // Produce a bitmask of the value to insert
3927 auto EltMask = B.buildConstant(
3928 Res: TargetTy, Val: APInt::getLowBitsSet(numBits: TargetTy.getSizeInBits(),
3929 loBitsSet: InsertTy.getSizeInBits()));
3930 // Shift it into position
3931 auto ShiftedMask = B.buildShl(Dst: TargetTy, Src0: EltMask, Src1: OffsetBits);
3932 auto InvShiftedMask = B.buildNot(Dst: TargetTy, Src0: ShiftedMask);
3933
3934 // Clear out the bits in the wide element
3935 auto MaskedOldElt = B.buildAnd(Dst: TargetTy, Src0: TargetReg, Src1: InvShiftedMask);
3936
3937 // The value to insert has all zeros already, so stick it into the masked
3938 // wide element.
3939 return B.buildOr(Dst: TargetTy, Src0: MaskedOldElt, Src1: ShiftedInsertVal).getReg(Idx: 0);
3940}
3941
3942/// Perform a G_INSERT_VECTOR_ELT in a different sized vector element. If this
3943/// is increasing the element size, perform the indexing in the target element
3944/// type, and use bit operations to insert at the element position. This is
3945/// intended for architectures that can dynamically index the register file and
3946/// want to force indexing in the native register size.
3947LegalizerHelper::LegalizeResult
3948LegalizerHelper::bitcastInsertVectorElt(MachineInstr &MI, unsigned TypeIdx,
3949 LLT CastTy) {
3950 if (TypeIdx != 0)
3951 return UnableToLegalize;
3952
3953 auto [Dst, DstTy, SrcVec, SrcVecTy, Val, ValTy, Idx, IdxTy] =
3954 MI.getFirst4RegLLTs();
3955 LLT VecTy = DstTy;
3956
3957 LLT VecEltTy = VecTy.getElementType();
3958 LLT NewEltTy = CastTy.isVector() ? CastTy.getElementType() : CastTy;
3959 const unsigned NewEltSize = NewEltTy.getSizeInBits();
3960 const unsigned OldEltSize = VecEltTy.getSizeInBits();
3961
3962 unsigned NewNumElts = CastTy.isVector() ? CastTy.getNumElements() : 1;
3963 unsigned OldNumElts = VecTy.getNumElements();
3964
3965 Register CastVec = MIRBuilder.buildBitcast(Dst: CastTy, Src: SrcVec).getReg(Idx: 0);
3966 if (NewNumElts < OldNumElts) {
3967 if (NewEltSize % OldEltSize != 0)
3968 return UnableToLegalize;
3969
3970 // This only depends on powers of 2 because we use bit tricks to figure out
3971 // the bit offset we need to shift to get the target element. A general
3972 // expansion could emit division/multiply.
3973 if (!isPowerOf2_32(Value: NewEltSize / OldEltSize))
3974 return UnableToLegalize;
3975
3976 const unsigned Log2EltRatio = Log2_32(Value: NewEltSize / OldEltSize);
3977 auto Log2Ratio = MIRBuilder.buildConstant(Res: IdxTy, Val: Log2EltRatio);
3978
3979 // Divide to get the index in the wider element type.
3980 auto ScaledIdx = MIRBuilder.buildLShr(Dst: IdxTy, Src0: Idx, Src1: Log2Ratio);
3981
3982 Register ExtractedElt = CastVec;
3983 if (CastTy.isVector()) {
3984 ExtractedElt = MIRBuilder.buildExtractVectorElement(Res: NewEltTy, Val: CastVec,
3985 Idx: ScaledIdx).getReg(Idx: 0);
3986 }
3987
3988 // Compute the bit offset into the register of the target element.
3989 Register OffsetBits = getBitcastWiderVectorElementOffset(
3990 B&: MIRBuilder, Idx, NewEltSize, OldEltSize);
3991
3992 Register InsertedElt = buildBitFieldInsert(B&: MIRBuilder, TargetReg: ExtractedElt,
3993 InsertReg: Val, OffsetBits);
3994 if (CastTy.isVector()) {
3995 InsertedElt = MIRBuilder.buildInsertVectorElement(
3996 Res: CastTy, Val: CastVec, Elt: InsertedElt, Idx: ScaledIdx).getReg(Idx: 0);
3997 }
3998
3999 MIRBuilder.buildBitcast(Dst, Src: InsertedElt);
4000 MI.eraseFromParent();
4001 return Legalized;
4002 }
4003
4004 return UnableToLegalize;
4005}
4006
4007// This attempts to handle G_CONCAT_VECTORS with illegal operands, particularly
4008// those that have smaller than legal operands.
4009//
4010// <16 x s8> = G_CONCAT_VECTORS <4 x s8>, <4 x s8>, <4 x s8>, <4 x s8>
4011//
4012// ===>
4013//
4014// s32 = G_BITCAST <4 x s8>
4015// s32 = G_BITCAST <4 x s8>
4016// s32 = G_BITCAST <4 x s8>
4017// s32 = G_BITCAST <4 x s8>
4018// <4 x s32> = G_BUILD_VECTOR s32, s32, s32, s32
4019// <16 x s8> = G_BITCAST <4 x s32>
4020LegalizerHelper::LegalizeResult
4021LegalizerHelper::bitcastConcatVector(MachineInstr &MI, unsigned TypeIdx,
4022 LLT CastTy) {
4023 // Convert it to CONCAT instruction
4024 auto ConcatMI = dyn_cast<GConcatVectors>(Val: &MI);
4025 if (!ConcatMI) {
4026 return UnableToLegalize;
4027 }
4028
4029 // Check if bitcast is Legal
4030 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
4031 LLT SrcScalTy = CastTy.getScalarType();
4032
4033 // Check if the build vector is Legal
4034 if (!LI.isLegal(Query: {TargetOpcode::G_BUILD_VECTOR, {CastTy, SrcScalTy}})) {
4035 return UnableToLegalize;
4036 }
4037
4038 // Bitcast the sources
4039 SmallVector<Register> BitcastRegs;
4040 for (unsigned i = 0; i < ConcatMI->getNumSources(); i++) {
4041 BitcastRegs.push_back(
4042 Elt: MIRBuilder.buildBitcast(Dst: SrcScalTy, Src: ConcatMI->getSourceReg(I: i))
4043 .getReg(Idx: 0));
4044 }
4045
4046 // Build the scalar values into a vector
4047 Register BuildReg =
4048 MIRBuilder.buildBuildVector(Res: CastTy, Ops: BitcastRegs).getReg(Idx: 0);
4049 MIRBuilder.buildBitcast(Dst: DstReg, Src: BuildReg);
4050
4051 MI.eraseFromParent();
4052 return Legalized;
4053}
4054
4055// This bitcasts a shuffle vector to a different type currently of the same
4056// element size. Mostly used to legalize ptr vectors, where ptrtoint/inttoptr
4057// will be used instead.
4058//
4059// <16 x p0> = G_CONCAT_VECTORS <4 x p0>, <4 x p0>, mask
4060// ===>
4061// <4 x s64> = G_PTRTOINT <4 x p0>
4062// <4 x s64> = G_PTRTOINT <4 x p0>
4063// <16 x s64> = G_CONCAT_VECTORS <4 x s64>, <4 x s64>, mask
4064// <16 x p0> = G_INTTOPTR <16 x s64>
4065LegalizerHelper::LegalizeResult
4066LegalizerHelper::bitcastShuffleVector(MachineInstr &MI, unsigned TypeIdx,
4067 LLT CastTy) {
4068 auto ShuffleMI = cast<GShuffleVector>(Val: &MI);
4069 LLT DstTy = MRI.getType(Reg: ShuffleMI->getReg(Idx: 0));
4070 LLT SrcTy = MRI.getType(Reg: ShuffleMI->getReg(Idx: 1));
4071
4072 // We currently only handle vectors of the same size.
4073 if (TypeIdx != 0 ||
4074 CastTy.getScalarSizeInBits() != DstTy.getScalarSizeInBits() ||
4075 CastTy.getElementCount() != DstTy.getElementCount())
4076 return UnableToLegalize;
4077
4078 LLT NewSrcTy = SrcTy.changeElementType(NewEltTy: CastTy.getScalarType());
4079
4080 auto Inp1 = MIRBuilder.buildCast(Dst: NewSrcTy, Src: ShuffleMI->getReg(Idx: 1));
4081 auto Inp2 = MIRBuilder.buildCast(Dst: NewSrcTy, Src: ShuffleMI->getReg(Idx: 2));
4082 auto Shuf =
4083 MIRBuilder.buildShuffleVector(Res: CastTy, Src1: Inp1, Src2: Inp2, Mask: ShuffleMI->getMask());
4084 MIRBuilder.buildCast(Dst: ShuffleMI->getReg(Idx: 0), Src: Shuf);
4085
4086 MI.eraseFromParent();
4087 return Legalized;
4088}
4089
4090/// This attempts to bitcast G_EXTRACT_SUBVECTOR to CastTy.
4091///
4092/// <vscale x 8 x i1> = G_EXTRACT_SUBVECTOR <vscale x 16 x i1>, N
4093///
4094/// ===>
4095///
4096/// <vscale x 2 x i1> = G_BITCAST <vscale x 16 x i1>
4097/// <vscale x 1 x i8> = G_EXTRACT_SUBVECTOR <vscale x 2 x i1>, N / 8
4098/// <vscale x 8 x i1> = G_BITCAST <vscale x 1 x i8>
4099LegalizerHelper::LegalizeResult
4100LegalizerHelper::bitcastExtractSubvector(MachineInstr &MI, unsigned TypeIdx,
4101 LLT CastTy) {
4102 auto ES = cast<GExtractSubvector>(Val: &MI);
4103
4104 if (!CastTy.isVector())
4105 return UnableToLegalize;
4106
4107 if (TypeIdx != 0)
4108 return UnableToLegalize;
4109
4110 Register Dst = ES->getReg(Idx: 0);
4111 Register Src = ES->getSrcVec();
4112 uint64_t Idx = ES->getIndexImm();
4113
4114 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4115
4116 LLT DstTy = MRI.getType(Reg: Dst);
4117 LLT SrcTy = MRI.getType(Reg: Src);
4118 ElementCount DstTyEC = DstTy.getElementCount();
4119 ElementCount SrcTyEC = SrcTy.getElementCount();
4120 auto DstTyMinElts = DstTyEC.getKnownMinValue();
4121 auto SrcTyMinElts = SrcTyEC.getKnownMinValue();
4122
4123 if (DstTy == CastTy)
4124 return Legalized;
4125
4126 if (DstTy.getSizeInBits() != CastTy.getSizeInBits())
4127 return UnableToLegalize;
4128
4129 unsigned CastEltSize = CastTy.getElementType().getSizeInBits();
4130 unsigned DstEltSize = DstTy.getElementType().getSizeInBits();
4131 if (CastEltSize < DstEltSize)
4132 return UnableToLegalize;
4133
4134 auto AdjustAmt = CastEltSize / DstEltSize;
4135 if (Idx % AdjustAmt != 0 || DstTyMinElts % AdjustAmt != 0 ||
4136 SrcTyMinElts % AdjustAmt != 0)
4137 return UnableToLegalize;
4138
4139 Idx /= AdjustAmt;
4140 SrcTy = LLT::vector(EC: SrcTyEC.divideCoefficientBy(RHS: AdjustAmt), ScalarSizeInBits: AdjustAmt);
4141 auto CastVec = MIRBuilder.buildBitcast(Dst: SrcTy, Src);
4142 auto PromotedES = MIRBuilder.buildExtractSubvector(Res: CastTy, Src: CastVec, Index: Idx);
4143 MIRBuilder.buildBitcast(Dst, Src: PromotedES);
4144
4145 ES->eraseFromParent();
4146 return Legalized;
4147}
4148
4149/// This attempts to bitcast G_INSERT_SUBVECTOR to CastTy.
4150///
4151/// <vscale x 16 x i1> = G_INSERT_SUBVECTOR <vscale x 16 x i1>,
4152/// <vscale x 8 x i1>,
4153/// N
4154///
4155/// ===>
4156///
4157/// <vscale x 2 x i8> = G_BITCAST <vscale x 16 x i1>
4158/// <vscale x 1 x i8> = G_BITCAST <vscale x 8 x i1>
4159/// <vscale x 2 x i8> = G_INSERT_SUBVECTOR <vscale x 2 x i8>,
4160/// <vscale x 1 x i8>, N / 8
4161/// <vscale x 16 x i1> = G_BITCAST <vscale x 2 x i8>
4162LegalizerHelper::LegalizeResult
4163LegalizerHelper::bitcastInsertSubvector(MachineInstr &MI, unsigned TypeIdx,
4164 LLT CastTy) {
4165 auto ES = cast<GInsertSubvector>(Val: &MI);
4166
4167 if (!CastTy.isVector())
4168 return UnableToLegalize;
4169
4170 if (TypeIdx != 0)
4171 return UnableToLegalize;
4172
4173 Register Dst = ES->getReg(Idx: 0);
4174 Register BigVec = ES->getBigVec();
4175 Register SubVec = ES->getSubVec();
4176 uint64_t Idx = ES->getIndexImm();
4177
4178 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4179
4180 LLT DstTy = MRI.getType(Reg: Dst);
4181 LLT BigVecTy = MRI.getType(Reg: BigVec);
4182 LLT SubVecTy = MRI.getType(Reg: SubVec);
4183
4184 if (DstTy == CastTy)
4185 return Legalized;
4186
4187 if (DstTy.getSizeInBits() != CastTy.getSizeInBits())
4188 return UnableToLegalize;
4189
4190 ElementCount DstTyEC = DstTy.getElementCount();
4191 ElementCount BigVecTyEC = BigVecTy.getElementCount();
4192 ElementCount SubVecTyEC = SubVecTy.getElementCount();
4193 auto DstTyMinElts = DstTyEC.getKnownMinValue();
4194 auto BigVecTyMinElts = BigVecTyEC.getKnownMinValue();
4195 auto SubVecTyMinElts = SubVecTyEC.getKnownMinValue();
4196
4197 unsigned CastEltSize = CastTy.getElementType().getSizeInBits();
4198 unsigned DstEltSize = DstTy.getElementType().getSizeInBits();
4199 if (CastEltSize < DstEltSize)
4200 return UnableToLegalize;
4201
4202 auto AdjustAmt = CastEltSize / DstEltSize;
4203 if (Idx % AdjustAmt != 0 || DstTyMinElts % AdjustAmt != 0 ||
4204 BigVecTyMinElts % AdjustAmt != 0 || SubVecTyMinElts % AdjustAmt != 0)
4205 return UnableToLegalize;
4206
4207 Idx /= AdjustAmt;
4208 BigVecTy = LLT::vector(EC: BigVecTyEC.divideCoefficientBy(RHS: AdjustAmt), ScalarSizeInBits: AdjustAmt);
4209 SubVecTy = LLT::vector(EC: SubVecTyEC.divideCoefficientBy(RHS: AdjustAmt), ScalarSizeInBits: AdjustAmt);
4210 auto CastBigVec = MIRBuilder.buildBitcast(Dst: BigVecTy, Src: BigVec);
4211 auto CastSubVec = MIRBuilder.buildBitcast(Dst: SubVecTy, Src: SubVec);
4212 auto PromotedIS =
4213 MIRBuilder.buildInsertSubvector(Res: CastTy, Src0: CastBigVec, Src1: CastSubVec, Index: Idx);
4214 MIRBuilder.buildBitcast(Dst, Src: PromotedIS);
4215
4216 ES->eraseFromParent();
4217 return Legalized;
4218}
4219
4220LegalizerHelper::LegalizeResult LegalizerHelper::lowerLoad(GAnyLoad &LoadMI) {
4221 // Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT
4222 Register DstReg = LoadMI.getDstReg();
4223 Register PtrReg = LoadMI.getPointerReg();
4224 LLT DstTy = MRI.getType(Reg: DstReg);
4225 MachineMemOperand &MMO = LoadMI.getMMO();
4226 LLT MemTy = MMO.getMemoryType();
4227 MachineFunction &MF = MIRBuilder.getMF();
4228
4229 LLT EltTy = MemTy.getScalarType();
4230
4231 unsigned MemSizeInBits = MemTy.getSizeInBits();
4232 unsigned MemStoreSizeInBits = 8 * MemTy.getSizeInBytes();
4233
4234 if (MemSizeInBits != MemStoreSizeInBits) {
4235 if (MemTy.isVector())
4236 return UnableToLegalize;
4237
4238 // Promote to a byte-sized load if not loading an integral number of
4239 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
4240 LLT WideMemTy = EltTy.changeElementSize(NewEltSize: MemStoreSizeInBits);
4241 MachineMemOperand *NewMMO =
4242 MF.getMachineMemOperand(MMO: &MMO, PtrInfo: MMO.getPointerInfo(), Ty: WideMemTy);
4243
4244 Register LoadReg = DstReg;
4245 LLT LoadTy = DstTy;
4246
4247 // If this wasn't already an extending load, we need to widen the result
4248 // register to avoid creating a load with a narrower result than the source.
4249 if (MemStoreSizeInBits > DstTy.getSizeInBits()) {
4250 LoadTy = WideMemTy;
4251 LoadReg = MRI.createGenericVirtualRegister(Ty: WideMemTy);
4252 }
4253
4254 if (isa<GSExtLoad>(Val: LoadMI)) {
4255 auto NewLoad = MIRBuilder.buildLoad(Res: LoadTy, Addr: PtrReg, MMO&: *NewMMO);
4256 MIRBuilder.buildSExtInReg(Res: LoadReg, Op: NewLoad, ImmOp: MemSizeInBits);
4257 } else if (isa<GZExtLoad>(Val: LoadMI) || WideMemTy == LoadTy) {
4258 auto NewLoad = MIRBuilder.buildLoad(Res: LoadTy, Addr: PtrReg, MMO&: *NewMMO);
4259 // The extra bits are guaranteed to be zero, since we stored them that
4260 // way. A zext load from Wide thus automatically gives zext from MemVT.
4261 MIRBuilder.buildAssertZExt(Res: LoadReg, Op: NewLoad, Size: MemSizeInBits);
4262 } else {
4263 MIRBuilder.buildLoad(Res: LoadReg, Addr: PtrReg, MMO&: *NewMMO);
4264 }
4265
4266 if (DstTy != LoadTy)
4267 MIRBuilder.buildTrunc(Res: DstReg, Op: LoadReg);
4268
4269 LoadMI.eraseFromParent();
4270 return Legalized;
4271 }
4272
4273 // Big endian lowering not implemented.
4274 if (MIRBuilder.getDataLayout().isBigEndian())
4275 return UnableToLegalize;
4276
4277 // This load needs splitting into power of 2 sized loads.
4278 //
4279 // Our strategy here is to generate anyextending loads for the smaller
4280 // types up to next power-2 result type, and then combine the two larger
4281 // result values together, before truncating back down to the non-pow-2
4282 // type.
4283 // E.g. v1 = i24 load =>
4284 // v2 = i32 zextload (2 byte)
4285 // v3 = i32 load (1 byte)
4286 // v4 = i32 shl v3, 16
4287 // v5 = i32 or v4, v2
4288 // v1 = i24 trunc v5
4289 // By doing this we generate the correct truncate which should get
4290 // combined away as an artifact with a matching extend.
4291
4292 uint64_t LargeSplitSize, SmallSplitSize;
4293
4294 if (!isPowerOf2_32(Value: MemSizeInBits)) {
4295 // This load needs splitting into power of 2 sized loads.
4296 LargeSplitSize = llvm::bit_floor(Value: MemSizeInBits);
4297 SmallSplitSize = MemSizeInBits - LargeSplitSize;
4298 } else {
4299 // This is already a power of 2, but we still need to split this in half.
4300 //
4301 // Assume we're being asked to decompose an unaligned load.
4302 // TODO: If this requires multiple splits, handle them all at once.
4303 auto &Ctx = MF.getFunction().getContext();
4304 if (TLI.allowsMemoryAccess(Context&: Ctx, DL: MIRBuilder.getDataLayout(), Ty: MemTy, MMO))
4305 return UnableToLegalize;
4306
4307 SmallSplitSize = LargeSplitSize = MemSizeInBits / 2;
4308 }
4309
4310 if (MemTy.isVector()) {
4311 // TODO: Handle vector extloads
4312 if (MemTy != DstTy)
4313 return UnableToLegalize;
4314
4315 Align Alignment = LoadMI.getAlign();
4316 // Given an alignment larger than the size of the memory, we can increase
4317 // the size of the load without needing to scalarize it.
4318 if (Alignment.value() * 8 > MemSizeInBits &&
4319 isPowerOf2_64(Value: DstTy.getScalarSizeInBits())) {
4320 LLT MoreTy = DstTy.changeVectorElementCount(
4321 EC: ElementCount::getFixed(MinVal: NextPowerOf2(A: DstTy.getNumElements())));
4322 MachineMemOperand *NewMMO = MF.getMachineMemOperand(MMO: &MMO, Offset: 0, Ty: MoreTy);
4323 auto NewLoad = MIRBuilder.buildLoad(Res: MoreTy, Addr: PtrReg, MMO&: *NewMMO);
4324 MIRBuilder.buildDeleteTrailingVectorElements(Res: LoadMI.getReg(Idx: 0),
4325 Op0: NewLoad.getReg(Idx: 0));
4326 LoadMI.eraseFromParent();
4327 return Legalized;
4328 }
4329
4330 // TODO: We can do better than scalarizing the vector and at least split it
4331 // in half.
4332 return reduceLoadStoreWidth(MI&: LoadMI, TypeIdx: 0, NarrowTy: DstTy.getElementType());
4333 }
4334
4335 MachineMemOperand *LargeMMO =
4336 MF.getMachineMemOperand(MMO: &MMO, Offset: 0, Size: LargeSplitSize / 8);
4337 MachineMemOperand *SmallMMO =
4338 MF.getMachineMemOperand(MMO: &MMO, Offset: LargeSplitSize / 8, Size: SmallSplitSize / 8);
4339
4340 LLT PtrTy = MRI.getType(Reg: PtrReg);
4341 unsigned AnyExtSize = PowerOf2Ceil(A: DstTy.getSizeInBits());
4342
4343 LLT AnyExtTy;
4344 LLT OffsetCstRes;
4345 if (EltTy.isPointer()) {
4346 AnyExtTy = LLT::scalar(SizeInBits: AnyExtSize);
4347 OffsetCstRes = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
4348 } else {
4349 AnyExtTy = DstTy.changeElementSize(NewEltSize: AnyExtSize);
4350 OffsetCstRes = DstTy.changeElementSize(NewEltSize: PtrTy.getSizeInBits());
4351 }
4352
4353 auto LargeLoad = MIRBuilder.buildLoadInstr(Opcode: TargetOpcode::G_ZEXTLOAD, Res: AnyExtTy,
4354 Addr: PtrReg, MMO&: *LargeMMO);
4355
4356 auto OffsetCst = MIRBuilder.buildConstant(Res: OffsetCstRes, Val: LargeSplitSize / 8);
4357 Register PtrAddReg = MRI.createGenericVirtualRegister(Ty: PtrTy);
4358 auto SmallPtr = MIRBuilder.buildObjectPtrOffset(Res: PtrAddReg, Op0: PtrReg, Op1: OffsetCst);
4359 auto SmallLoad = MIRBuilder.buildLoadInstr(Opcode: LoadMI.getOpcode(), Res: AnyExtTy,
4360 Addr: SmallPtr, MMO&: *SmallMMO);
4361
4362 auto ShiftAmt = MIRBuilder.buildConstant(Res: AnyExtTy, Val: LargeSplitSize);
4363 auto Shift = MIRBuilder.buildShl(Dst: AnyExtTy, Src0: SmallLoad, Src1: ShiftAmt);
4364
4365 if (AnyExtTy == DstTy)
4366 MIRBuilder.buildOr(Dst: DstReg, Src0: Shift, Src1: LargeLoad);
4367 else if (AnyExtTy.getSizeInBits() != DstTy.getSizeInBits()) {
4368 auto Or = MIRBuilder.buildOr(Dst: AnyExtTy, Src0: Shift, Src1: LargeLoad);
4369 MIRBuilder.buildTrunc(Res: DstReg, Op: {Or});
4370 } else {
4371 assert(DstTy.isPointer() && "expected pointer");
4372 auto Or = MIRBuilder.buildOr(Dst: AnyExtTy, Src0: Shift, Src1: LargeLoad);
4373
4374 // FIXME: We currently consider this to be illegal for non-integral address
4375 // spaces, but we need still need a way to reinterpret the bits.
4376 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: Or);
4377 }
4378
4379 LoadMI.eraseFromParent();
4380 return Legalized;
4381}
4382
4383LegalizerHelper::LegalizeResult LegalizerHelper::lowerStore(GStore &StoreMI) {
4384 // Lower a non-power of 2 store into multiple pow-2 stores.
4385 // E.g. split an i24 store into an i16 store + i8 store.
4386 // We do this by first extending the stored value to the next largest power
4387 // of 2 type, and then using truncating stores to store the components.
4388 // By doing this, likewise with G_LOAD, generate an extend that can be
4389 // artifact-combined away instead of leaving behind extracts.
4390 Register SrcReg = StoreMI.getValueReg();
4391 Register PtrReg = StoreMI.getPointerReg();
4392 LLT SrcTy = MRI.getType(Reg: SrcReg);
4393 MachineFunction &MF = MIRBuilder.getMF();
4394 MachineMemOperand &MMO = **StoreMI.memoperands_begin();
4395 LLT MemTy = MMO.getMemoryType();
4396
4397 unsigned StoreWidth = MemTy.getSizeInBits();
4398 unsigned StoreSizeInBits = 8 * MemTy.getSizeInBytes();
4399
4400 if (StoreWidth != StoreSizeInBits && !SrcTy.isVector()) {
4401 // Promote to a byte-sized store with upper bits zero if not
4402 // storing an integral number of bytes. For example, promote
4403 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
4404 LLT WideTy = LLT::integer(SizeInBits: StoreSizeInBits);
4405
4406 if (StoreSizeInBits > SrcTy.getSizeInBits()) {
4407 // Avoid creating a store with a narrower source than result.
4408 SrcReg = MIRBuilder.buildAnyExt(Res: WideTy, Op: SrcReg).getReg(Idx: 0);
4409 SrcTy = WideTy;
4410 }
4411
4412 auto ZextInReg = MIRBuilder.buildZExtInReg(Res: SrcTy, Op: SrcReg, ImmOp: StoreWidth);
4413
4414 MachineMemOperand *NewMMO =
4415 MF.getMachineMemOperand(MMO: &MMO, PtrInfo: MMO.getPointerInfo(), Ty: WideTy);
4416 MIRBuilder.buildStore(Val: ZextInReg, Addr: PtrReg, MMO&: *NewMMO);
4417 StoreMI.eraseFromParent();
4418 return Legalized;
4419 }
4420
4421 if (MemTy.isVector()) {
4422 if (MemTy != SrcTy)
4423 return scalarizeVectorBooleanStore(MI&: StoreMI);
4424
4425 // TODO: We can do better than scalarizing the vector and at least split it
4426 // in half.
4427 return reduceLoadStoreWidth(MI&: StoreMI, TypeIdx: 0, NarrowTy: SrcTy.getElementType());
4428 }
4429
4430 unsigned MemSizeInBits = MemTy.getSizeInBits();
4431 uint64_t LargeSplitSize, SmallSplitSize;
4432
4433 if (!isPowerOf2_32(Value: MemSizeInBits)) {
4434 LargeSplitSize = llvm::bit_floor<uint64_t>(Value: MemTy.getSizeInBits());
4435 SmallSplitSize = MemTy.getSizeInBits() - LargeSplitSize;
4436 } else {
4437 auto &Ctx = MF.getFunction().getContext();
4438 if (TLI.allowsMemoryAccess(Context&: Ctx, DL: MIRBuilder.getDataLayout(), Ty: MemTy, MMO))
4439 return UnableToLegalize; // Don't know what we're being asked to do.
4440
4441 SmallSplitSize = LargeSplitSize = MemSizeInBits / 2;
4442 }
4443
4444 // Extend to the next pow-2. If this store was itself the result of lowering,
4445 // e.g. an s56 store being broken into s32 + s24, we might have a stored type
4446 // that's wider than the stored size.
4447 unsigned AnyExtSize = PowerOf2Ceil(A: MemTy.getSizeInBits());
4448 const LLT NewSrcTy = LLT::integer(SizeInBits: AnyExtSize);
4449
4450 if (SrcTy.isPointer()) {
4451 const LLT IntPtrTy = LLT::integer(SizeInBits: SrcTy.getSizeInBits());
4452 SrcReg = MIRBuilder.buildPtrToInt(Dst: IntPtrTy, Src: SrcReg).getReg(Idx: 0);
4453 }
4454
4455 auto ExtVal = MIRBuilder.buildAnyExtOrTrunc(Res: NewSrcTy, Op: SrcReg);
4456
4457 // Obtain the smaller value by shifting away the larger value.
4458 auto ShiftAmt = MIRBuilder.buildConstant(Res: NewSrcTy, Val: LargeSplitSize);
4459 auto SmallVal = MIRBuilder.buildLShr(Dst: NewSrcTy, Src0: ExtVal, Src1: ShiftAmt);
4460
4461 // Generate the PtrAdd and truncating stores.
4462 LLT PtrTy = MRI.getType(Reg: PtrReg);
4463 auto OffsetCst = MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: PtrTy.getSizeInBits()),
4464 Val: LargeSplitSize / 8);
4465 auto SmallPtr = MIRBuilder.buildObjectPtrOffset(Res: PtrTy, Op0: PtrReg, Op1: OffsetCst);
4466
4467 MachineMemOperand *LargeMMO =
4468 MF.getMachineMemOperand(MMO: &MMO, Offset: 0, Size: LargeSplitSize / 8);
4469 MachineMemOperand *SmallMMO =
4470 MF.getMachineMemOperand(MMO: &MMO, Offset: LargeSplitSize / 8, Size: SmallSplitSize / 8);
4471 MIRBuilder.buildStore(Val: ExtVal, Addr: PtrReg, MMO&: *LargeMMO);
4472 MIRBuilder.buildStore(Val: SmallVal, Addr: SmallPtr, MMO&: *SmallMMO);
4473 StoreMI.eraseFromParent();
4474 return Legalized;
4475}
4476
4477LegalizerHelper::LegalizeResult
4478LegalizerHelper::scalarizeVectorBooleanStore(GStore &StoreMI) {
4479 Register SrcReg = StoreMI.getValueReg();
4480 Register PtrReg = StoreMI.getPointerReg();
4481 LLT SrcTy = MRI.getType(Reg: SrcReg);
4482 MachineMemOperand &MMO = **StoreMI.memoperands_begin();
4483 LLT MemTy = MMO.getMemoryType();
4484 LLT MemScalarTy = MemTy.getElementType();
4485 MachineFunction &MF = MIRBuilder.getMF();
4486
4487 assert(SrcTy.isVector() && "Expect a vector store type");
4488
4489 if (!MemScalarTy.isByteSized()) {
4490 // We need to build an integer scalar of the vector bit pattern.
4491 // It's not legal for us to add padding when storing a vector.
4492 unsigned NumBits = MemTy.getSizeInBits();
4493 LLT IntTy = LLT::integer(SizeInBits: NumBits);
4494 auto CurrVal = MIRBuilder.buildConstant(Res: IntTy, Val: 0);
4495 LLT IdxTy = TLI.getVectorIdxLLT(DL: MF.getDataLayout());
4496
4497 for (unsigned I = 0, E = MemTy.getNumElements(); I < E; ++I) {
4498 auto Elt = MIRBuilder.buildExtractVectorElement(
4499 Res: SrcTy.getElementType(), Val: SrcReg, Idx: MIRBuilder.buildConstant(Res: IdxTy, Val: I));
4500 auto Trunc = MIRBuilder.buildTrunc(Res: MemScalarTy, Op: Elt);
4501 auto ZExt = MIRBuilder.buildZExt(Res: IntTy, Op: Trunc);
4502 unsigned ShiftIntoIdx = MF.getDataLayout().isBigEndian()
4503 ? (MemTy.getNumElements() - 1) - I
4504 : I;
4505 auto ShiftAmt = MIRBuilder.buildConstant(
4506 Res: IntTy, Val: ShiftIntoIdx * MemScalarTy.getSizeInBits());
4507 auto Shifted = MIRBuilder.buildShl(Dst: IntTy, Src0: ZExt, Src1: ShiftAmt);
4508 CurrVal = MIRBuilder.buildOr(Dst: IntTy, Src0: CurrVal, Src1: Shifted);
4509 }
4510 auto PtrInfo = MMO.getPointerInfo();
4511 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo, Ty: IntTy);
4512 MIRBuilder.buildStore(Val: CurrVal, Addr: PtrReg, MMO&: *NewMMO);
4513 StoreMI.eraseFromParent();
4514 return Legalized;
4515 }
4516
4517 // TODO: implement simple scalarization.
4518 return UnableToLegalize;
4519}
4520
4521LegalizerHelper::LegalizeResult
4522LegalizerHelper::bitcast(MachineInstr &MI, unsigned TypeIdx, LLT CastTy) {
4523 switch (MI.getOpcode()) {
4524 case TargetOpcode::G_LOAD: {
4525 if (TypeIdx != 0)
4526 return UnableToLegalize;
4527 MachineMemOperand &MMO = **MI.memoperands_begin();
4528
4529 // Not sure how to interpret a bitcast of an extending load.
4530 if (MMO.getMemoryType().getSizeInBits() != CastTy.getSizeInBits())
4531 return UnableToLegalize;
4532
4533 Observer.changingInstr(MI);
4534 bitcastDst(MI, CastTy, OpIdx: 0);
4535 MMO.setType(CastTy);
4536 // The range metadata is no longer valid when reinterpreted as a different
4537 // type.
4538 MMO.clearRanges();
4539 Observer.changedInstr(MI);
4540 return Legalized;
4541 }
4542 case TargetOpcode::G_STORE: {
4543 if (TypeIdx != 0)
4544 return UnableToLegalize;
4545
4546 MachineMemOperand &MMO = **MI.memoperands_begin();
4547
4548 // Not sure how to interpret a bitcast of a truncating store.
4549 if (MMO.getMemoryType().getSizeInBits() != CastTy.getSizeInBits())
4550 return UnableToLegalize;
4551
4552 Observer.changingInstr(MI);
4553 bitcastSrc(MI, CastTy, OpIdx: 0);
4554 MMO.setType(CastTy);
4555 Observer.changedInstr(MI);
4556 return Legalized;
4557 }
4558 case TargetOpcode::G_SELECT: {
4559 if (TypeIdx != 0)
4560 return UnableToLegalize;
4561
4562 if (MRI.getType(Reg: MI.getOperand(i: 1).getReg()).isVector()) {
4563 LLVM_DEBUG(
4564 dbgs() << "bitcast action not implemented for vector select\n");
4565 return UnableToLegalize;
4566 }
4567
4568 Observer.changingInstr(MI);
4569 bitcastSrc(MI, CastTy, OpIdx: 2);
4570 bitcastSrc(MI, CastTy, OpIdx: 3);
4571 bitcastDst(MI, CastTy, OpIdx: 0);
4572 Observer.changedInstr(MI);
4573 return Legalized;
4574 }
4575 case TargetOpcode::G_AND:
4576 case TargetOpcode::G_OR:
4577 case TargetOpcode::G_XOR: {
4578 Observer.changingInstr(MI);
4579 bitcastSrc(MI, CastTy, OpIdx: 1);
4580 bitcastSrc(MI, CastTy, OpIdx: 2);
4581 bitcastDst(MI, CastTy, OpIdx: 0);
4582 Observer.changedInstr(MI);
4583 return Legalized;
4584 }
4585 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
4586 return bitcastExtractVectorElt(MI, TypeIdx, CastTy);
4587 case TargetOpcode::G_INSERT_VECTOR_ELT:
4588 return bitcastInsertVectorElt(MI, TypeIdx, CastTy);
4589 case TargetOpcode::G_CONCAT_VECTORS:
4590 return bitcastConcatVector(MI, TypeIdx, CastTy);
4591 case TargetOpcode::G_SHUFFLE_VECTOR:
4592 return bitcastShuffleVector(MI, TypeIdx, CastTy);
4593 case TargetOpcode::G_EXTRACT_SUBVECTOR:
4594 return bitcastExtractSubvector(MI, TypeIdx, CastTy);
4595 case TargetOpcode::G_INSERT_SUBVECTOR:
4596 return bitcastInsertSubvector(MI, TypeIdx, CastTy);
4597 default:
4598 return UnableToLegalize;
4599 }
4600}
4601
4602// Legalize an instruction by changing the opcode in place.
4603void LegalizerHelper::changeOpcode(MachineInstr &MI, unsigned NewOpcode) {
4604 Observer.changingInstr(MI);
4605 MI.setDesc(MIRBuilder.getTII().get(Opcode: NewOpcode));
4606 Observer.changedInstr(MI);
4607}
4608
4609LegalizerHelper::LegalizeResult
4610LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) {
4611 using namespace TargetOpcode;
4612 switch(MI.getOpcode()) {
4613 default:
4614 return UnableToLegalize;
4615 case TargetOpcode::G_FCONSTANT:
4616 return lowerFConstant(MI);
4617 case TargetOpcode::G_BITCAST:
4618 return lowerBitcast(MI);
4619 case TargetOpcode::G_SREM:
4620 case TargetOpcode::G_UREM: {
4621 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4622 auto Quot =
4623 MIRBuilder.buildInstr(Opc: MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV, DstOps: {Ty},
4624 SrcOps: {MI.getOperand(i: 1), MI.getOperand(i: 2)});
4625
4626 auto Prod = MIRBuilder.buildMul(Dst: Ty, Src0: Quot, Src1: MI.getOperand(i: 2));
4627 MIRBuilder.buildSub(Dst: MI.getOperand(i: 0), Src0: MI.getOperand(i: 1), Src1: Prod);
4628 MI.eraseFromParent();
4629 return Legalized;
4630 }
4631 case TargetOpcode::G_SADDO:
4632 case TargetOpcode::G_SSUBO:
4633 return lowerSADDO_SSUBO(MI);
4634 case TargetOpcode::G_SADDE:
4635 return lowerSADDE(MI);
4636 case TargetOpcode::G_SSUBE:
4637 return lowerSSUBE(MI);
4638 case TargetOpcode::G_UMULH:
4639 case TargetOpcode::G_SMULH:
4640 return lowerSMULH_UMULH(MI);
4641 case TargetOpcode::G_SMULO:
4642 case TargetOpcode::G_UMULO: {
4643 // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the
4644 // result.
4645 auto [Res, Overflow, LHS, RHS] = MI.getFirst4Regs();
4646 LLT Ty = MRI.getType(Reg: Res);
4647
4648 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO
4649 ? TargetOpcode::G_SMULH
4650 : TargetOpcode::G_UMULH;
4651
4652 Observer.changingInstr(MI);
4653 const auto &TII = MIRBuilder.getTII();
4654 MI.setDesc(TII.get(Opcode: TargetOpcode::G_MUL));
4655 MI.removeOperand(OpNo: 1);
4656 Observer.changedInstr(MI);
4657
4658 auto HiPart = MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Ty}, SrcOps: {LHS, RHS});
4659 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
4660
4661 // Move insert point forward so we can use the Res register if needed.
4662 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
4663
4664 // For *signed* multiply, overflow is detected by checking:
4665 // (hi != (lo >> bitwidth-1))
4666 if (Opcode == TargetOpcode::G_SMULH) {
4667 auto ShiftAmt = MIRBuilder.buildConstant(Res: Ty, Val: Ty.getSizeInBits() - 1);
4668 auto Shifted = MIRBuilder.buildAShr(Dst: Ty, Src0: Res, Src1: ShiftAmt);
4669 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: Overflow, Op0: HiPart, Op1: Shifted);
4670 } else {
4671 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: Overflow, Op0: HiPart, Op1: Zero);
4672 }
4673 return Legalized;
4674 }
4675 case TargetOpcode::G_FNEG: {
4676 auto [Res, ResTy, SubByReg, SubByRegTy] = MI.getFirst2RegLLTs();
4677 LLT TyInt =
4678 ResTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: ResTy.getScalarSizeInBits()));
4679 Register CastedSubByReg = SubByReg;
4680
4681 if (!SubByRegTy.getScalarType().isAnyScalar() &&
4682 !SubByRegTy.getScalarType().isInteger()) {
4683 auto BitcastDst = SubByRegTy.changeElementType(
4684 NewEltTy: LLT::integer(SizeInBits: SubByRegTy.getScalarSizeInBits()));
4685 CastedSubByReg = MIRBuilder.buildBitcast(Dst: BitcastDst, Src: SubByReg).getReg(Idx: 0);
4686 }
4687
4688 auto SignMask = MIRBuilder.buildConstant(
4689 Res: TyInt, Val: APInt::getSignMask(BitWidth: TyInt.getScalarSizeInBits()));
4690
4691 if (ResTy != TyInt) {
4692 Register NewDst =
4693 MIRBuilder.buildXor(Dst: TyInt, Src0: CastedSubByReg, Src1: SignMask).getReg(Idx: 0);
4694 MIRBuilder.buildBitcast(Dst: Res, Src: NewDst);
4695 } else
4696 MIRBuilder.buildXor(Dst: Res, Src0: CastedSubByReg, Src1: SignMask).getReg(Idx: 0);
4697
4698 MI.eraseFromParent();
4699 return Legalized;
4700 }
4701 case TargetOpcode::G_FSUB:
4702 case TargetOpcode::G_STRICT_FSUB: {
4703 auto [Res, LHS, RHS] = MI.getFirst3Regs();
4704 LLT Ty = MRI.getType(Reg: Res);
4705
4706 // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)).
4707 auto Neg = MIRBuilder.buildFNeg(Dst: Ty, Src0: RHS);
4708
4709 if (MI.getOpcode() == TargetOpcode::G_STRICT_FSUB)
4710 MIRBuilder.buildStrictFAdd(Dst: Res, Src0: LHS, Src1: Neg, Flags: MI.getFlags());
4711 else
4712 MIRBuilder.buildFAdd(Dst: Res, Src0: LHS, Src1: Neg, Flags: MI.getFlags());
4713
4714 MI.eraseFromParent();
4715 return Legalized;
4716 }
4717 case TargetOpcode::G_FMAD:
4718 return lowerFMad(MI);
4719 case TargetOpcode::G_FFLOOR:
4720 return lowerFFloor(MI);
4721 case TargetOpcode::G_LROUND:
4722 case TargetOpcode::G_LLROUND: {
4723 Register DstReg = MI.getOperand(i: 0).getReg();
4724 Register SrcReg = MI.getOperand(i: 1).getReg();
4725 LLT SrcTy = MRI.getType(Reg: SrcReg);
4726 auto Round = MIRBuilder.buildInstr(Opc: TargetOpcode::G_INTRINSIC_ROUND, DstOps: {SrcTy},
4727 SrcOps: {SrcReg});
4728 MIRBuilder.buildFPTOSI(Dst: DstReg, Src0: Round);
4729 MI.eraseFromParent();
4730 return Legalized;
4731 }
4732 case TargetOpcode::G_INTRINSIC_ROUND:
4733 return lowerIntrinsicRound(MI);
4734 case TargetOpcode::G_FRINT: {
4735 // Since round even is the assumed rounding mode for unconstrained FP
4736 // operations, rint and roundeven are the same operation.
4737 changeOpcode(MI, NewOpcode: TargetOpcode::G_INTRINSIC_ROUNDEVEN);
4738 return Legalized;
4739 }
4740 case TargetOpcode::G_INTRINSIC_LRINT:
4741 case TargetOpcode::G_INTRINSIC_LLRINT: {
4742 Register DstReg = MI.getOperand(i: 0).getReg();
4743 Register SrcReg = MI.getOperand(i: 1).getReg();
4744 LLT SrcTy = MRI.getType(Reg: SrcReg);
4745 auto Round =
4746 MIRBuilder.buildInstr(Opc: TargetOpcode::G_FRINT, DstOps: {SrcTy}, SrcOps: {SrcReg});
4747 MIRBuilder.buildFPTOSI(Dst: DstReg, Src0: Round);
4748 MI.eraseFromParent();
4749 return Legalized;
4750 }
4751 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
4752 auto [OldValRes, SuccessRes, Addr, CmpVal, NewVal] = MI.getFirst5Regs();
4753 Register NewOldValRes = MRI.cloneVirtualRegister(VReg: OldValRes);
4754 MIRBuilder.buildAtomicCmpXchg(OldValRes: NewOldValRes, Addr, CmpVal, NewVal,
4755 MMO&: **MI.memoperands_begin());
4756 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: SuccessRes, Op0: NewOldValRes, Op1: CmpVal);
4757 MIRBuilder.buildCopy(Res: OldValRes, Op: NewOldValRes);
4758 MI.eraseFromParent();
4759 return Legalized;
4760 }
4761 case TargetOpcode::G_LOAD:
4762 case TargetOpcode::G_SEXTLOAD:
4763 case TargetOpcode::G_ZEXTLOAD:
4764 return lowerLoad(LoadMI&: cast<GAnyLoad>(Val&: MI));
4765 case TargetOpcode::G_STORE:
4766 return lowerStore(StoreMI&: cast<GStore>(Val&: MI));
4767 case TargetOpcode::G_CTLZ_ZERO_POISON:
4768 case TargetOpcode::G_CTTZ_ZERO_POISON:
4769 case TargetOpcode::G_CTLZ:
4770 case TargetOpcode::G_CTTZ:
4771 case TargetOpcode::G_CTPOP:
4772 case TargetOpcode::G_CTLS:
4773 return lowerBitCount(MI);
4774 case G_UADDO: {
4775 auto [Res, CarryOut, LHS, RHS] = MI.getFirst4Regs();
4776
4777 Register NewRes = MRI.cloneVirtualRegister(VReg: Res);
4778
4779 MIRBuilder.buildAdd(Dst: NewRes, Src0: LHS, Src1: RHS);
4780 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_ULT, Res: CarryOut, Op0: NewRes, Op1: RHS);
4781
4782 MIRBuilder.buildCopy(Res, Op: NewRes);
4783
4784 MI.eraseFromParent();
4785 return Legalized;
4786 }
4787 case G_UADDE: {
4788 auto [Res, CarryOut, LHS, RHS, CarryIn] = MI.getFirst5Regs();
4789 const LLT CondTy = MRI.getType(Reg: CarryOut);
4790 const LLT Ty = MRI.getType(Reg: Res);
4791
4792 Register NewRes = MRI.cloneVirtualRegister(VReg: Res);
4793
4794 // Initial add of the two operands.
4795 auto TmpRes = MIRBuilder.buildAdd(Dst: Ty, Src0: LHS, Src1: RHS);
4796
4797 // Initial check for carry.
4798 auto Carry = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_ULT, Res: CondTy, Op0: TmpRes, Op1: LHS);
4799
4800 // Add the sum and the carry.
4801 auto ZExtCarryIn = MIRBuilder.buildZExt(Res: Ty, Op: CarryIn);
4802 MIRBuilder.buildAdd(Dst: NewRes, Src0: TmpRes, Src1: ZExtCarryIn);
4803
4804 // Second check for carry. We can only carry if the initial sum is all 1s
4805 // and the carry is set, resulting in a new sum of 0.
4806 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
4807 auto ResEqZero =
4808 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CondTy, Op0: NewRes, Op1: Zero);
4809 auto Carry2 = MIRBuilder.buildAnd(Dst: CondTy, Src0: ResEqZero, Src1: CarryIn);
4810 MIRBuilder.buildOr(Dst: CarryOut, Src0: Carry, Src1: Carry2);
4811
4812 MIRBuilder.buildCopy(Res, Op: NewRes);
4813
4814 MI.eraseFromParent();
4815 return Legalized;
4816 }
4817 case G_USUBO: {
4818 auto [Res, BorrowOut, LHS, RHS] = MI.getFirst4Regs();
4819
4820 MIRBuilder.buildSub(Dst: Res, Src0: LHS, Src1: RHS);
4821 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_ULT, Res: BorrowOut, Op0: LHS, Op1: RHS);
4822
4823 MI.eraseFromParent();
4824 return Legalized;
4825 }
4826 case G_USUBE: {
4827 auto [Res, BorrowOut, LHS, RHS, BorrowIn] = MI.getFirst5Regs();
4828 const LLT CondTy = MRI.getType(Reg: BorrowOut);
4829 const LLT Ty = MRI.getType(Reg: Res);
4830
4831 // Initial subtract of the two operands.
4832 auto TmpRes = MIRBuilder.buildSub(Dst: Ty, Src0: LHS, Src1: RHS);
4833
4834 // Initial check for borrow.
4835 auto Borrow = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_UGT, Res: CondTy, Op0: TmpRes, Op1: LHS);
4836
4837 // Subtract the borrow from the first subtract.
4838 auto ZExtBorrowIn = MIRBuilder.buildZExt(Res: Ty, Op: BorrowIn);
4839 MIRBuilder.buildSub(Dst: Res, Src0: TmpRes, Src1: ZExtBorrowIn);
4840
4841 // Second check for borrow. We can only borrow if the initial difference is
4842 // 0 and the borrow is set, resulting in a new difference of all 1s.
4843 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
4844 auto TmpResEqZero =
4845 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CondTy, Op0: TmpRes, Op1: Zero);
4846 auto Borrow2 = MIRBuilder.buildAnd(Dst: CondTy, Src0: TmpResEqZero, Src1: BorrowIn);
4847 MIRBuilder.buildOr(Dst: BorrowOut, Src0: Borrow, Src1: Borrow2);
4848
4849 MI.eraseFromParent();
4850 return Legalized;
4851 }
4852 case G_UITOFP:
4853 return lowerUITOFP(MI);
4854 case G_SITOFP:
4855 return lowerSITOFP(MI);
4856 case G_FPTOUI:
4857 return lowerFPTOUI(MI);
4858 case G_FPTOSI:
4859 return lowerFPTOSI(MI);
4860 case G_FPTOUI_SAT:
4861 case G_FPTOSI_SAT:
4862 return lowerFPTOINT_SAT(MI);
4863 case G_FPEXT:
4864 return lowerFPExtAndTruncMem(MI);
4865 case G_FPTRUNC:
4866 return lowerFPTRUNC(MI);
4867 case G_FPOWI:
4868 return lowerFPOWI(MI);
4869 case G_FMODF:
4870 return lowerFMODF(MI);
4871 case G_SMIN:
4872 case G_SMAX:
4873 case G_UMIN:
4874 case G_UMAX:
4875 return lowerMinMax(MI);
4876 case G_SCMP:
4877 case G_UCMP:
4878 return lowerThreewayCompare(MI);
4879 case G_FCOPYSIGN:
4880 return lowerFCopySign(MI);
4881 case G_FMINNUM:
4882 case G_FMAXNUM:
4883 case G_FMINIMUMNUM:
4884 case G_FMAXIMUMNUM:
4885 return lowerFMinNumMaxNum(MI);
4886 case G_FMINIMUM:
4887 case G_FMAXIMUM:
4888 return lowerFMinimumMaximum(MI);
4889 case G_MERGE_VALUES:
4890 return lowerMergeValues(MI);
4891 case G_UNMERGE_VALUES:
4892 return lowerUnmergeValues(MI);
4893 case TargetOpcode::G_SEXT_INREG: {
4894 assert(MI.getOperand(2).isImm() && "Expected immediate");
4895 int64_t SizeInBits = MI.getOperand(i: 2).getImm();
4896
4897 auto [DstReg, SrcReg] = MI.getFirst2Regs();
4898 LLT DstTy = MRI.getType(Reg: DstReg);
4899 Register TmpRes = MRI.createGenericVirtualRegister(Ty: DstTy);
4900
4901 auto MIBSz = MIRBuilder.buildConstant(Res: DstTy, Val: DstTy.getScalarSizeInBits() - SizeInBits);
4902 MIRBuilder.buildShl(Dst: TmpRes, Src0: SrcReg, Src1: MIBSz->getOperand(i: 0));
4903 MIRBuilder.buildAShr(Dst: DstReg, Src0: TmpRes, Src1: MIBSz->getOperand(i: 0));
4904 MI.eraseFromParent();
4905 return Legalized;
4906 }
4907 case G_EXTRACT_VECTOR_ELT:
4908 case G_INSERT_VECTOR_ELT:
4909 return lowerExtractInsertVectorElt(MI);
4910 case G_SHUFFLE_VECTOR:
4911 return lowerShuffleVector(MI);
4912 case G_VECTOR_COMPRESS:
4913 return lowerVECTOR_COMPRESS(MI);
4914 case G_DYN_STACKALLOC:
4915 return lowerDynStackAlloc(MI);
4916 case G_INSERT_SUBVECTOR: {
4917 if (MRI.getType(Reg: MI.getOperand(i: 1).getReg()).isScalable() ||
4918 MRI.getType(Reg: MI.getOperand(i: 2).getReg()).isScalable())
4919 return UnableToLegalize;
4920
4921 // Check that subvector is half size of main vector
4922 Register Vector = MI.getOperand(i: 1).getReg();
4923 Register Subvector = MI.getOperand(i: 2).getReg();
4924 auto InsertionPointImm = MI.getOperand(i: 3).getImm();
4925
4926 LLT VectorTy = MRI.getType(Reg: Vector);
4927 LLT SubvectorTy = MRI.getType(Reg: Subvector);
4928 // If so, -> concat(subvector, extract(half of vector))
4929 // (Operands can be either way round depending on insertion point
4930 if (VectorTy.getSizeInBits() == SubvectorTy.getSizeInBits() * 2) {
4931 bool InsertInLowHalf = InsertionPointImm == 0;
4932 auto Extract = MIRBuilder.buildExtractSubvector(
4933 Res: SubvectorTy, Src: Vector,
4934 Index: (uint64_t)(InsertInLowHalf ? VectorTy.getNumElements() / 2 : 0));
4935
4936 auto LowHalf = InsertInLowHalf ? Subvector : Extract.getReg(Idx: 0);
4937 auto HighHalf = InsertInLowHalf ? Extract.getReg(Idx: 0) : Subvector;
4938
4939 MIRBuilder.buildInstr(Opc: TargetOpcode::G_CONCAT_VECTORS, DstOps: {MI.getOperand(i: 0)},
4940 SrcOps: {LowHalf, HighHalf});
4941 MI.eraseFromParent();
4942 return Legalized;
4943 }
4944 // Else -> shuffle(vector, extend(subvector, size(vector)), mask)
4945 else {
4946 // Extend subvector to same size as vector
4947 Register ExtendedSubvector = MRI.createGenericVirtualRegister(Ty: VectorTy);
4948 MIRBuilder.buildPadVectorWithUndefElements(Res: ExtendedSubvector, Op0: Subvector);
4949
4950 // Calculate mask required for this shuffle
4951 SmallVector<int> Mask;
4952 for (int i = 0; i < VectorTy.getNumElements(); i++) {
4953 // If this index is within bounds, put subvector's index into mask
4954 if (i >= InsertionPointImm &&
4955 i < InsertionPointImm + SubvectorTy.getNumElements())
4956 Mask.push_back(Elt: VectorTy.getNumElements() + i - InsertionPointImm);
4957 else
4958 Mask.push_back(Elt: i);
4959 }
4960
4961 // Build shuffle
4962 MIRBuilder.buildShuffleVector(Res: MI.getOperand(i: 0), Src1: Vector, Src2: ExtendedSubvector,
4963 Mask);
4964 MI.eraseFromParent();
4965 return Legalized;
4966 }
4967 }
4968 case G_STACKSAVE:
4969 return lowerStackSave(MI);
4970 case G_STACKRESTORE:
4971 return lowerStackRestore(MI);
4972 case G_EXTRACT:
4973 return lowerExtract(MI);
4974 case G_INSERT:
4975 return lowerInsert(MI);
4976 case G_BSWAP:
4977 return lowerBswap(MI);
4978 case G_BITREVERSE:
4979 return lowerBitreverse(MI);
4980 case G_READ_REGISTER:
4981 case G_WRITE_REGISTER:
4982 return lowerReadWriteRegister(MI);
4983 case G_UADDSAT:
4984 case G_USUBSAT: {
4985 // Try to make a reasonable guess about which lowering strategy to use. The
4986 // target can override this with custom lowering and calling the
4987 // implementation functions.
4988 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4989 if (LI.isLegalOrCustom(Query: {G_UMIN, Ty}))
4990 return lowerAddSubSatToMinMax(MI);
4991 return lowerAddSubSatToAddoSubo(MI);
4992 }
4993 case G_SADDSAT:
4994 case G_SSUBSAT: {
4995 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4996
4997 // FIXME: It would probably make more sense to see if G_SADDO is preferred,
4998 // since it's a shorter expansion. However, we would need to figure out the
4999 // preferred boolean type for the carry out for the query.
5000 if (LI.isLegalOrCustom(Query: {G_SMIN, Ty}) && LI.isLegalOrCustom(Query: {G_SMAX, Ty}))
5001 return lowerAddSubSatToMinMax(MI);
5002 return lowerAddSubSatToAddoSubo(MI);
5003 }
5004 case G_SSHLSAT:
5005 case G_USHLSAT:
5006 return lowerShlSat(MI);
5007 case G_ABS:
5008 return lowerAbsToAddXor(MI);
5009 case G_ABDS:
5010 case G_ABDU: {
5011 bool IsSigned = MI.getOpcode() == G_ABDS;
5012 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5013 if ((IsSigned && LI.isLegal(Query: {G_SMIN, Ty}) && LI.isLegal(Query: {G_SMAX, Ty})) ||
5014 (!IsSigned && LI.isLegal(Query: {G_UMIN, Ty}) && LI.isLegal(Query: {G_UMAX, Ty}))) {
5015 return lowerAbsDiffToMinMax(MI);
5016 }
5017 return lowerAbsDiffToSelect(MI);
5018 }
5019 case G_FABS:
5020 return lowerFAbs(MI);
5021 case G_SELECT:
5022 return lowerSelect(MI);
5023 case G_IS_FPCLASS:
5024 return lowerISFPCLASS(MI);
5025 case G_SDIVREM:
5026 case G_UDIVREM:
5027 return lowerDIVREM(MI);
5028 case G_FSHL:
5029 case G_FSHR:
5030 return lowerFunnelShift(MI);
5031 case G_ROTL:
5032 case G_ROTR:
5033 return lowerRotate(MI);
5034 case G_MEMSET:
5035 case G_MEMCPY:
5036 case G_MEMMOVE:
5037 case G_MEMCPY_INLINE:
5038 case G_MEMSET_INLINE:
5039 return lowerMemCpyFamily(MI);
5040 case G_ZEXT:
5041 case G_SEXT:
5042 case G_ANYEXT:
5043 return lowerEXT(MI);
5044 case G_TRUNC:
5045 return lowerTRUNC(MI);
5046 GISEL_VECREDUCE_CASES_NONSEQ
5047 return lowerVectorReduction(MI);
5048 case G_VAARG:
5049 return lowerVAArg(MI);
5050 case G_ATOMICRMW_SUB: {
5051 auto [Ret, Mem, Val] = MI.getFirst3Regs();
5052 const LLT ValTy = MRI.getType(Reg: Val);
5053 MachineMemOperand *MMO = *MI.memoperands_begin();
5054
5055 auto VNeg = MIRBuilder.buildNeg(Dst: ValTy, Src0: Val);
5056 MIRBuilder.buildAtomicRMW(Opcode: G_ATOMICRMW_ADD, OldValRes: Ret, Addr: Mem, Val: VNeg, MMO&: *MMO);
5057 MI.eraseFromParent();
5058 return Legalized;
5059 }
5060 case G_SMULFIX:
5061 case G_UMULFIX:
5062 return lowerMulfix(MI);
5063 }
5064}
5065
5066Align LegalizerHelper::getStackTemporaryAlignment(LLT Ty,
5067 Align MinAlign) const {
5068 // FIXME: We're missing a way to go back from LLT to llvm::Type to query the
5069 // datalayout for the preferred alignment. Also there should be a target hook
5070 // for this to allow targets to reduce the alignment and ignore the
5071 // datalayout. e.g. AMDGPU should always use a 4-byte alignment, regardless of
5072 // the type.
5073 return std::max(a: Align(PowerOf2Ceil(A: Ty.getSizeInBytes())), b: MinAlign);
5074}
5075
5076MachineInstrBuilder
5077LegalizerHelper::createStackTemporary(TypeSize Bytes, Align Alignment,
5078 MachinePointerInfo &PtrInfo) {
5079 MachineFunction &MF = MIRBuilder.getMF();
5080 const DataLayout &DL = MIRBuilder.getDataLayout();
5081 int FrameIdx = MF.getFrameInfo().CreateStackObject(Size: Bytes, Alignment, isSpillSlot: false);
5082
5083 unsigned AddrSpace = DL.getAllocaAddrSpace();
5084 LLT FramePtrTy = LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL.getPointerSizeInBits(AS: AddrSpace));
5085
5086 PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIdx);
5087 return MIRBuilder.buildFrameIndex(Res: FramePtrTy, Idx: FrameIdx);
5088}
5089
5090MachineInstrBuilder LegalizerHelper::createStackStoreLoad(const DstOp &Res,
5091 const SrcOp &Val) {
5092 LLT SrcTy = Val.getLLTTy(MRI);
5093 Align StackTypeAlign =
5094 std::max(a: getStackTemporaryAlignment(Ty: SrcTy),
5095 b: getStackTemporaryAlignment(Ty: Res.getLLTTy(MRI)));
5096 MachinePointerInfo PtrInfo;
5097 auto StackTemp =
5098 createStackTemporary(Bytes: SrcTy.getSizeInBytes(), Alignment: StackTypeAlign, PtrInfo);
5099
5100 MIRBuilder.buildStore(Val, Addr: StackTemp, PtrInfo, Alignment: StackTypeAlign);
5101 return MIRBuilder.buildLoad(Res, Addr: StackTemp, PtrInfo, Alignment: StackTypeAlign);
5102}
5103
5104static Register clampVectorIndex(MachineIRBuilder &B, Register IdxReg,
5105 LLT VecTy) {
5106 LLT IdxTy = B.getMRI()->getType(Reg: IdxReg);
5107 unsigned NElts = VecTy.getNumElements();
5108
5109 int64_t IdxVal;
5110 if (mi_match(R: IdxReg, MRI: *B.getMRI(), P: m_ICst(Cst&: IdxVal))) {
5111 if (IdxVal < VecTy.getNumElements())
5112 return IdxReg;
5113 // If a constant index would be out of bounds, clamp it as well.
5114 }
5115
5116 if (isPowerOf2_32(Value: NElts)) {
5117 APInt Imm = APInt::getLowBitsSet(numBits: IdxTy.getSizeInBits(), loBitsSet: Log2_32(Value: NElts));
5118 return B.buildAnd(Dst: IdxTy, Src0: IdxReg, Src1: B.buildConstant(Res: IdxTy, Val: Imm)).getReg(Idx: 0);
5119 }
5120
5121 return B.buildUMin(Dst: IdxTy, Src0: IdxReg, Src1: B.buildConstant(Res: IdxTy, Val: NElts - 1))
5122 .getReg(Idx: 0);
5123}
5124
5125Register LegalizerHelper::getVectorElementPointer(Register VecPtr, LLT VecTy,
5126 Register Index) {
5127 LLT EltTy = VecTy.getElementType();
5128
5129 // Calculate the element offset and add it to the pointer.
5130 unsigned EltSize = EltTy.getSizeInBits() / 8; // FIXME: should be ABI size.
5131 assert(EltSize * 8 == EltTy.getSizeInBits() &&
5132 "Converting bits to bytes lost precision");
5133
5134 Index = clampVectorIndex(B&: MIRBuilder, IdxReg: Index, VecTy);
5135
5136 // Convert index to the correct size for the address space.
5137 const DataLayout &DL = MIRBuilder.getDataLayout();
5138 unsigned AS = MRI.getType(Reg: VecPtr).getAddressSpace();
5139 unsigned IndexSizeInBits = DL.getIndexSize(AS) * 8;
5140 LLT IdxTy = MRI.getType(Reg: Index).changeElementSize(NewEltSize: IndexSizeInBits);
5141 if (IdxTy != MRI.getType(Reg: Index))
5142 Index = MIRBuilder.buildSExtOrTrunc(Res: IdxTy, Op: Index).getReg(Idx: 0);
5143
5144 auto Mul = MIRBuilder.buildMul(Dst: IdxTy, Src0: Index,
5145 Src1: MIRBuilder.buildConstant(Res: IdxTy, Val: EltSize));
5146
5147 LLT PtrTy = MRI.getType(Reg: VecPtr);
5148 return MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: VecPtr, Op1: Mul).getReg(Idx: 0);
5149}
5150
5151#ifndef NDEBUG
5152/// Check that all vector operands have same number of elements. Other operands
5153/// should be listed in NonVecOp.
5154static bool hasSameNumEltsOnAllVectorOperands(
5155 GenericMachineInstr &MI, MachineRegisterInfo &MRI,
5156 std::initializer_list<unsigned> NonVecOpIndices) {
5157 if (MI.getNumMemOperands() != 0)
5158 return false;
5159
5160 LLT VecTy = MRI.getType(MI.getReg(0));
5161 if (!VecTy.isVector())
5162 return false;
5163 unsigned NumElts = VecTy.getNumElements();
5164
5165 for (unsigned OpIdx = 1; OpIdx < MI.getNumOperands(); ++OpIdx) {
5166 MachineOperand &Op = MI.getOperand(OpIdx);
5167 if (!Op.isReg()) {
5168 if (!is_contained(NonVecOpIndices, OpIdx))
5169 return false;
5170 continue;
5171 }
5172
5173 LLT Ty = MRI.getType(Op.getReg());
5174 if (!Ty.isVector()) {
5175 if (!is_contained(NonVecOpIndices, OpIdx))
5176 return false;
5177 continue;
5178 }
5179
5180 if (Ty.getNumElements() != NumElts)
5181 return false;
5182 }
5183
5184 return true;
5185}
5186#endif
5187
5188/// Fill \p DstOps with DstOps that have same number of elements combined as
5189/// the Ty. These DstOps have either scalar type when \p NumElts = 1 or are
5190/// vectors with \p NumElts elements. When Ty.getNumElements() is not multiple
5191/// of \p NumElts last DstOp (leftover) has fewer then \p NumElts elements.
5192static void makeDstOps(SmallVectorImpl<DstOp> &DstOps, LLT Ty,
5193 unsigned NumElts) {
5194 LLT LeftoverTy;
5195 assert(Ty.isVector() && "Expected vector type");
5196 LLT NarrowTy = Ty.changeElementCount(EC: ElementCount::getFixed(MinVal: NumElts));
5197 int NumParts, NumLeftover;
5198 std::tie(args&: NumParts, args&: NumLeftover) =
5199 getNarrowTypeBreakDown(OrigTy: Ty, NarrowTy, LeftoverTy);
5200
5201 assert(NumParts > 0 && "Error in getNarrowTypeBreakDown");
5202 for (int i = 0; i < NumParts; ++i) {
5203 DstOps.push_back(Elt: NarrowTy);
5204 }
5205
5206 if (LeftoverTy.isValid()) {
5207 assert(NumLeftover == 1 && "expected exactly one leftover");
5208 DstOps.push_back(Elt: LeftoverTy);
5209 }
5210}
5211
5212/// Operand \p Op is used on \p N sub-instructions. Fill \p Ops with \p N SrcOps
5213/// made from \p Op depending on operand type.
5214static void broadcastSrcOp(SmallVectorImpl<SrcOp> &Ops, unsigned N,
5215 MachineOperand &Op) {
5216 for (unsigned i = 0; i < N; ++i) {
5217 if (Op.isReg())
5218 Ops.push_back(Elt: Op.getReg());
5219 else if (Op.isImm())
5220 Ops.push_back(Elt: Op.getImm());
5221 else if (Op.isPredicate())
5222 Ops.push_back(Elt: static_cast<CmpInst::Predicate>(Op.getPredicate()));
5223 else
5224 llvm_unreachable("Unsupported type");
5225 }
5226}
5227
5228// Handle splitting vector operations which need to have the same number of
5229// elements in each type index, but each type index may have a different element
5230// type.
5231//
5232// e.g. <4 x s64> = G_SHL <4 x s64>, <4 x s32> ->
5233// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
5234// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
5235//
5236// Also handles some irregular breakdown cases, e.g.
5237// e.g. <3 x s64> = G_SHL <3 x s64>, <3 x s32> ->
5238// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
5239// s64 = G_SHL s64, s32
5240LegalizerHelper::LegalizeResult
5241LegalizerHelper::fewerElementsVectorMultiEltType(
5242 GenericMachineInstr &MI, unsigned NumElts,
5243 std::initializer_list<unsigned> NonVecOpIndices) {
5244 assert(hasSameNumEltsOnAllVectorOperands(MI, MRI, NonVecOpIndices) &&
5245 "Non-compatible opcode or not specified non-vector operands");
5246 unsigned OrigNumElts = MRI.getType(Reg: MI.getReg(Idx: 0)).getNumElements();
5247
5248 unsigned NumInputs = MI.getNumOperands() - MI.getNumDefs();
5249 unsigned NumDefs = MI.getNumDefs();
5250
5251 // Create DstOps (sub-vectors with NumElts elts + Leftover) for each output.
5252 // Build instructions with DstOps to use instruction found by CSE directly.
5253 // CSE copies found instruction into given vreg when building with vreg dest.
5254 SmallVector<SmallVector<DstOp, 8>, 2> OutputOpsPieces(NumDefs);
5255 // Output registers will be taken from created instructions.
5256 SmallVector<SmallVector<Register, 8>, 2> OutputRegs(NumDefs);
5257 for (unsigned i = 0; i < NumDefs; ++i) {
5258 makeDstOps(DstOps&: OutputOpsPieces[i], Ty: MRI.getType(Reg: MI.getReg(Idx: i)), NumElts);
5259 }
5260
5261 // Split vector input operands into sub-vectors with NumElts elts + Leftover.
5262 // Operands listed in NonVecOpIndices will be used as is without splitting;
5263 // examples: compare predicate in icmp and fcmp (op 1), vector select with i1
5264 // scalar condition (op 1), immediate in sext_inreg (op 2).
5265 SmallVector<SmallVector<SrcOp, 8>, 3> InputOpsPieces(NumInputs);
5266 for (unsigned UseIdx = NumDefs, UseNo = 0; UseIdx < MI.getNumOperands();
5267 ++UseIdx, ++UseNo) {
5268 if (is_contained(Set: NonVecOpIndices, Element: UseIdx)) {
5269 broadcastSrcOp(Ops&: InputOpsPieces[UseNo], N: OutputOpsPieces[0].size(),
5270 Op&: MI.getOperand(i: UseIdx));
5271 } else {
5272 SmallVector<Register, 8> SplitPieces;
5273 extractVectorParts(Reg: MI.getReg(Idx: UseIdx), NumElts, VRegs&: SplitPieces, MIRBuilder,
5274 MRI);
5275 llvm::append_range(C&: InputOpsPieces[UseNo], R&: SplitPieces);
5276 }
5277 }
5278
5279 unsigned NumLeftovers = OrigNumElts % NumElts ? 1 : 0;
5280
5281 // Take i-th piece of each input operand split and build sub-vector/scalar
5282 // instruction. Set i-th DstOp(s) from OutputOpsPieces as destination(s).
5283 for (unsigned i = 0; i < OrigNumElts / NumElts + NumLeftovers; ++i) {
5284 SmallVector<DstOp, 2> Defs;
5285 for (unsigned DstNo = 0; DstNo < NumDefs; ++DstNo)
5286 Defs.push_back(Elt: OutputOpsPieces[DstNo][i]);
5287
5288 SmallVector<SrcOp, 3> Uses;
5289 for (unsigned InputNo = 0; InputNo < NumInputs; ++InputNo)
5290 Uses.push_back(Elt: InputOpsPieces[InputNo][i]);
5291
5292 auto I = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: Defs, SrcOps: Uses, Flags: MI.getFlags());
5293 for (unsigned DstNo = 0; DstNo < NumDefs; ++DstNo)
5294 OutputRegs[DstNo].push_back(Elt: I.getReg(Idx: DstNo));
5295 }
5296
5297 // Merge small outputs into MI's output for each def operand.
5298 if (NumLeftovers) {
5299 for (unsigned i = 0; i < NumDefs; ++i)
5300 mergeMixedSubvectors(DstReg: MI.getReg(Idx: i), PartRegs: OutputRegs[i]);
5301 } else {
5302 for (unsigned i = 0; i < NumDefs; ++i)
5303 MIRBuilder.buildMergeLikeInstr(Res: MI.getReg(Idx: i), Ops: OutputRegs[i]);
5304 }
5305
5306 MI.eraseFromParent();
5307 return Legalized;
5308}
5309
5310LegalizerHelper::LegalizeResult
5311LegalizerHelper::fewerElementsVectorPhi(GenericMachineInstr &MI,
5312 unsigned NumElts) {
5313 unsigned OrigNumElts = MRI.getType(Reg: MI.getReg(Idx: 0)).getNumElements();
5314
5315 unsigned NumInputs = MI.getNumOperands() - MI.getNumDefs();
5316 unsigned NumDefs = MI.getNumDefs();
5317
5318 SmallVector<DstOp, 8> OutputOpsPieces;
5319 SmallVector<Register, 8> OutputRegs;
5320 makeDstOps(DstOps&: OutputOpsPieces, Ty: MRI.getType(Reg: MI.getReg(Idx: 0)), NumElts);
5321
5322 // Instructions that perform register split will be inserted in basic block
5323 // where register is defined (basic block is in the next operand).
5324 SmallVector<SmallVector<Register, 8>, 3> InputOpsPieces(NumInputs / 2);
5325 for (unsigned UseIdx = NumDefs, UseNo = 0; UseIdx < MI.getNumOperands();
5326 UseIdx += 2, ++UseNo) {
5327 MachineBasicBlock &OpMBB = *MI.getOperand(i: UseIdx + 1).getMBB();
5328 MIRBuilder.setInsertPt(MBB&: OpMBB, II: OpMBB.getFirstTerminatorForward());
5329 extractVectorParts(Reg: MI.getReg(Idx: UseIdx), NumElts, VRegs&: InputOpsPieces[UseNo],
5330 MIRBuilder, MRI);
5331 }
5332
5333 // Build PHIs with fewer elements.
5334 unsigned NumLeftovers = OrigNumElts % NumElts ? 1 : 0;
5335 MIRBuilder.setInsertPt(MBB&: *MI.getParent(), II: MI);
5336 for (unsigned i = 0; i < OrigNumElts / NumElts + NumLeftovers; ++i) {
5337 auto Phi = MIRBuilder.buildInstr(Opcode: TargetOpcode::G_PHI);
5338 Phi.addDef(
5339 RegNo: MRI.createGenericVirtualRegister(Ty: OutputOpsPieces[i].getLLTTy(MRI)));
5340 OutputRegs.push_back(Elt: Phi.getReg(Idx: 0));
5341
5342 for (unsigned j = 0; j < NumInputs / 2; ++j) {
5343 Phi.addUse(RegNo: InputOpsPieces[j][i]);
5344 Phi.add(MO: MI.getOperand(i: 1 + j * 2 + 1));
5345 }
5346 }
5347
5348 // Set the insert point after the existing PHIs
5349 MachineBasicBlock &MBB = *MI.getParent();
5350 MIRBuilder.setInsertPt(MBB, II: MBB.getFirstNonPHI());
5351
5352 // Merge small outputs into MI's def.
5353 if (NumLeftovers) {
5354 mergeMixedSubvectors(DstReg: MI.getReg(Idx: 0), PartRegs: OutputRegs);
5355 } else {
5356 MIRBuilder.buildMergeLikeInstr(Res: MI.getReg(Idx: 0), Ops: OutputRegs);
5357 }
5358
5359 MI.eraseFromParent();
5360 return Legalized;
5361}
5362
5363LegalizerHelper::LegalizeResult
5364LegalizerHelper::fewerElementsVectorUnmergeValues(MachineInstr &MI,
5365 unsigned TypeIdx,
5366 LLT NarrowTy) {
5367 const int NumDst = MI.getNumOperands() - 1;
5368 const Register SrcReg = MI.getOperand(i: NumDst).getReg();
5369 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5370 LLT SrcTy = MRI.getType(Reg: SrcReg);
5371
5372 if (TypeIdx != 1 || NarrowTy == DstTy)
5373 return UnableToLegalize;
5374
5375 // Requires compatible types. Otherwise SrcReg should have been defined by
5376 // merge-like instruction that would get artifact combined. Most likely
5377 // instruction that defines SrcReg has to perform more/fewer elements
5378 // legalization compatible with NarrowTy.
5379 assert(SrcTy.isVector() && NarrowTy.isVector() && "Expected vector types");
5380 assert((SrcTy.getScalarType() == NarrowTy.getScalarType()) && "bad type");
5381
5382 if ((SrcTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0) ||
5383 (NarrowTy.getSizeInBits() % DstTy.getSizeInBits() != 0))
5384 return UnableToLegalize;
5385
5386 // This is most likely DstTy (smaller then register size) packed in SrcTy
5387 // (larger then register size) and since unmerge was not combined it will be
5388 // lowered to bit sequence extracts from register. Unpack SrcTy to NarrowTy
5389 // (register size) pieces first. Then unpack each of NarrowTy pieces to DstTy.
5390
5391 // %1:_(DstTy), %2, %3, %4 = G_UNMERGE_VALUES %0:_(SrcTy)
5392 //
5393 // %5:_(NarrowTy), %6 = G_UNMERGE_VALUES %0:_(SrcTy) - reg sequence
5394 // %1:_(DstTy), %2 = G_UNMERGE_VALUES %5:_(NarrowTy) - sequence of bits in reg
5395 // %3:_(DstTy), %4 = G_UNMERGE_VALUES %6:_(NarrowTy)
5396 auto Unmerge = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: SrcReg);
5397 const int NumUnmerge = Unmerge->getNumOperands() - 1;
5398 const int PartsPerUnmerge = NumDst / NumUnmerge;
5399
5400 for (int I = 0; I != NumUnmerge; ++I) {
5401 auto MIB = MIRBuilder.buildInstr(Opcode: TargetOpcode::G_UNMERGE_VALUES);
5402
5403 for (int J = 0; J != PartsPerUnmerge; ++J)
5404 MIB.addDef(RegNo: MI.getOperand(i: I * PartsPerUnmerge + J).getReg());
5405 MIB.addUse(RegNo: Unmerge.getReg(Idx: I));
5406 }
5407
5408 MI.eraseFromParent();
5409 return Legalized;
5410}
5411
5412LegalizerHelper::LegalizeResult
5413LegalizerHelper::fewerElementsVectorMerge(MachineInstr &MI, unsigned TypeIdx,
5414 LLT NarrowTy) {
5415 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
5416 // Requires compatible types. Otherwise user of DstReg did not perform unmerge
5417 // that should have been artifact combined. Most likely instruction that uses
5418 // DstReg has to do more/fewer elements legalization compatible with NarrowTy.
5419 assert(DstTy.isVector() && NarrowTy.isVector() && "Expected vector types");
5420 assert((DstTy.getScalarType() == NarrowTy.getScalarType()) && "bad type");
5421 if (NarrowTy == SrcTy)
5422 return UnableToLegalize;
5423
5424 // This attempts to lower part of LCMTy merge/unmerge sequence. Intended use
5425 // is for old mir tests. Since the changes to more/fewer elements it should no
5426 // longer be possible to generate MIR like this when starting from llvm-ir
5427 // because LCMTy approach was replaced with merge/unmerge to vector elements.
5428 if (TypeIdx == 1) {
5429 assert(SrcTy.isVector() && "Expected vector types");
5430 assert((SrcTy.getScalarType() == NarrowTy.getScalarType()) && "bad type");
5431 if ((DstTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0) ||
5432 (NarrowTy.getNumElements() >= SrcTy.getNumElements()))
5433 return UnableToLegalize;
5434 // %2:_(DstTy) = G_CONCAT_VECTORS %0:_(SrcTy), %1:_(SrcTy)
5435 //
5436 // %3:_(EltTy), %4, %5 = G_UNMERGE_VALUES %0:_(SrcTy)
5437 // %6:_(EltTy), %7, %8 = G_UNMERGE_VALUES %1:_(SrcTy)
5438 // %9:_(NarrowTy) = G_BUILD_VECTOR %3:_(EltTy), %4
5439 // %10:_(NarrowTy) = G_BUILD_VECTOR %5:_(EltTy), %6
5440 // %11:_(NarrowTy) = G_BUILD_VECTOR %7:_(EltTy), %8
5441 // %2:_(DstTy) = G_CONCAT_VECTORS %9:_(NarrowTy), %10, %11
5442
5443 SmallVector<Register, 8> Elts;
5444 LLT EltTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getScalarType();
5445 for (unsigned i = 1; i < MI.getNumOperands(); ++i) {
5446 auto Unmerge = MIRBuilder.buildUnmerge(Res: EltTy, Op: MI.getOperand(i).getReg());
5447 for (unsigned j = 0; j < Unmerge->getNumDefs(); ++j)
5448 Elts.push_back(Elt: Unmerge.getReg(Idx: j));
5449 }
5450
5451 SmallVector<Register, 8> NarrowTyElts;
5452 unsigned NumNarrowTyElts = NarrowTy.getNumElements();
5453 unsigned NumNarrowTyPieces = DstTy.getNumElements() / NumNarrowTyElts;
5454 for (unsigned i = 0, Offset = 0; i < NumNarrowTyPieces;
5455 ++i, Offset += NumNarrowTyElts) {
5456 ArrayRef<Register> Pieces(&Elts[Offset], NumNarrowTyElts);
5457 NarrowTyElts.push_back(
5458 Elt: MIRBuilder.buildMergeLikeInstr(Res: NarrowTy, Ops: Pieces).getReg(Idx: 0));
5459 }
5460
5461 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: NarrowTyElts);
5462 MI.eraseFromParent();
5463 return Legalized;
5464 }
5465
5466 assert(TypeIdx == 0 && "Bad type index");
5467 if ((NarrowTy.getSizeInBits() % SrcTy.getSizeInBits() != 0) ||
5468 (DstTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0))
5469 return UnableToLegalize;
5470
5471 // This is most likely SrcTy (smaller then register size) packed in DstTy
5472 // (larger then register size) and since merge was not combined it will be
5473 // lowered to bit sequence packing into register. Merge SrcTy to NarrowTy
5474 // (register size) pieces first. Then merge each of NarrowTy pieces to DstTy.
5475
5476 // %0:_(DstTy) = G_MERGE_VALUES %1:_(SrcTy), %2, %3, %4
5477 //
5478 // %5:_(NarrowTy) = G_MERGE_VALUES %1:_(SrcTy), %2 - sequence of bits in reg
5479 // %6:_(NarrowTy) = G_MERGE_VALUES %3:_(SrcTy), %4
5480 // %0:_(DstTy) = G_MERGE_VALUES %5:_(NarrowTy), %6 - reg sequence
5481 SmallVector<Register, 8> NarrowTyElts;
5482 unsigned NumParts = DstTy.getNumElements() / NarrowTy.getNumElements();
5483 unsigned NumSrcElts = SrcTy.isVector() ? SrcTy.getNumElements() : 1;
5484 unsigned NumElts = NarrowTy.getNumElements() / NumSrcElts;
5485 for (unsigned i = 0; i < NumParts; ++i) {
5486 SmallVector<Register, 8> Sources;
5487 for (unsigned j = 0; j < NumElts; ++j)
5488 Sources.push_back(Elt: MI.getOperand(i: 1 + i * NumElts + j).getReg());
5489 NarrowTyElts.push_back(
5490 Elt: MIRBuilder.buildMergeLikeInstr(Res: NarrowTy, Ops: Sources).getReg(Idx: 0));
5491 }
5492
5493 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: NarrowTyElts);
5494 MI.eraseFromParent();
5495 return Legalized;
5496}
5497
5498LegalizerHelper::LegalizeResult
5499LegalizerHelper::fewerElementsVectorExtractInsertVectorElt(MachineInstr &MI,
5500 unsigned TypeIdx,
5501 LLT NarrowVecTy) {
5502 auto [DstReg, SrcVec] = MI.getFirst2Regs();
5503 Register InsertVal;
5504 bool IsInsert = MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT;
5505
5506 assert((IsInsert ? TypeIdx == 0 : TypeIdx == 1) && "not a vector type index");
5507 if (IsInsert)
5508 InsertVal = MI.getOperand(i: 2).getReg();
5509
5510 Register Idx = MI.getOperand(i: MI.getNumOperands() - 1).getReg();
5511 LLT VecTy = MRI.getType(Reg: SrcVec);
5512
5513 // If the index is a constant, we can really break this down as you would
5514 // expect, and index into the target size pieces.
5515 auto MaybeCst = getIConstantVRegValWithLookThrough(VReg: Idx, MRI);
5516 if (MaybeCst) {
5517 uint64_t IdxVal = MaybeCst->Value.getZExtValue();
5518 // Avoid out of bounds indexing the pieces.
5519 if (IdxVal >= VecTy.getNumElements()) {
5520 MIRBuilder.buildUndef(Res: DstReg);
5521 MI.eraseFromParent();
5522 return Legalized;
5523 }
5524
5525 if (!NarrowVecTy.isVector()) {
5526 SmallVector<Register, 8> SplitPieces;
5527 extractParts(Reg: MI.getOperand(i: 1).getReg(), Ty: NarrowVecTy,
5528 NumParts: VecTy.getNumElements(), VRegs&: SplitPieces, MIRBuilder, MRI);
5529 if (IsInsert) {
5530 SplitPieces[IdxVal] = InsertVal;
5531 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0).getReg(), Ops: SplitPieces);
5532 } else {
5533 MIRBuilder.buildCopy(Res: MI.getOperand(i: 0).getReg(), Op: SplitPieces[IdxVal]);
5534 }
5535 } else {
5536 SmallVector<Register, 8> VecParts;
5537 LLT GCDTy = extractGCDType(Parts&: VecParts, DstTy: VecTy, NarrowTy: NarrowVecTy, SrcReg: SrcVec);
5538
5539 // Build a sequence of NarrowTy pieces in VecParts for this operand.
5540 LLT LCMTy = buildLCMMergePieces(DstTy: VecTy, NarrowTy: NarrowVecTy, GCDTy, VRegs&: VecParts,
5541 PadStrategy: TargetOpcode::G_ANYEXT);
5542
5543 unsigned NewNumElts = NarrowVecTy.getNumElements();
5544
5545 LLT IdxTy = MRI.getType(Reg: Idx);
5546 int64_t PartIdx = IdxVal / NewNumElts;
5547 auto NewIdx =
5548 MIRBuilder.buildConstant(Res: IdxTy, Val: IdxVal - NewNumElts * PartIdx);
5549
5550 if (IsInsert) {
5551 LLT PartTy = MRI.getType(Reg: VecParts[PartIdx]);
5552
5553 // Use the adjusted index to insert into one of the subvectors.
5554 auto InsertPart = MIRBuilder.buildInsertVectorElement(
5555 Res: PartTy, Val: VecParts[PartIdx], Elt: InsertVal, Idx: NewIdx);
5556 VecParts[PartIdx] = InsertPart.getReg(Idx: 0);
5557
5558 // Recombine the inserted subvector with the others to reform the result
5559 // vector.
5560 buildWidenedRemergeToDst(DstReg, LCMTy, RemergeRegs: VecParts);
5561 } else {
5562 MIRBuilder.buildExtractVectorElement(Res: DstReg, Val: VecParts[PartIdx], Idx: NewIdx);
5563 }
5564 }
5565
5566 MI.eraseFromParent();
5567 return Legalized;
5568 }
5569
5570 // With a variable index, we can't perform the operation in a smaller type, so
5571 // we're forced to expand this.
5572 //
5573 // TODO: We could emit a chain of compare/select to figure out which piece to
5574 // index.
5575 return lowerExtractInsertVectorElt(MI);
5576}
5577
5578LegalizerHelper::LegalizeResult
5579LegalizerHelper::reduceLoadStoreWidth(GLoadStore &LdStMI, unsigned TypeIdx,
5580 LLT NarrowTy) {
5581 // FIXME: Don't know how to handle secondary types yet.
5582 if (TypeIdx != 0)
5583 return UnableToLegalize;
5584
5585 if (!NarrowTy.isByteSized()) {
5586 LLVM_DEBUG(dbgs() << "Can't narrow load/store to non-byte-sized type\n");
5587 return UnableToLegalize;
5588 }
5589
5590 // This implementation doesn't work for atomics. Give up instead of doing
5591 // something invalid.
5592 if (LdStMI.isAtomic())
5593 return UnableToLegalize;
5594
5595 bool IsLoad = isa<GLoad>(Val: LdStMI);
5596 Register ValReg = LdStMI.getReg(Idx: 0);
5597 Register AddrReg = LdStMI.getPointerReg();
5598 LLT ValTy = MRI.getType(Reg: ValReg);
5599
5600 // FIXME: Do we need a distinct NarrowMemory legalize action?
5601 if (ValTy.getSizeInBits() != 8 * LdStMI.getMemSize().getValue()) {
5602 LLVM_DEBUG(dbgs() << "Can't narrow extload/truncstore\n");
5603 return UnableToLegalize;
5604 }
5605
5606 int NumParts = -1;
5607 int NumLeftover = -1;
5608 LLT LeftoverTy;
5609 SmallVector<Register, 8> NarrowRegs, NarrowLeftoverRegs;
5610 if (IsLoad) {
5611 std::tie(args&: NumParts, args&: NumLeftover) = getNarrowTypeBreakDown(OrigTy: ValTy, NarrowTy, LeftoverTy);
5612 } else {
5613 if (extractParts(Reg: ValReg, RegTy: ValTy, MainTy: NarrowTy, LeftoverTy, VRegs&: NarrowRegs,
5614 LeftoverVRegs&: NarrowLeftoverRegs, MIRBuilder, MRI)) {
5615 NumParts = NarrowRegs.size();
5616 NumLeftover = NarrowLeftoverRegs.size();
5617 }
5618 }
5619
5620 if (NumParts == -1)
5621 return UnableToLegalize;
5622
5623 LLT PtrTy = MRI.getType(Reg: AddrReg);
5624 const LLT OffsetTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
5625
5626 unsigned TotalSize = ValTy.getSizeInBits();
5627
5628 // Split the load/store into PartTy sized pieces starting at Offset. If this
5629 // is a load, return the new registers in ValRegs. For a store, each elements
5630 // of ValRegs should be PartTy. Returns the next offset that needs to be
5631 // handled.
5632 bool isBigEndian = MIRBuilder.getDataLayout().isBigEndian();
5633 auto MMO = LdStMI.getMMO();
5634 auto splitTypePieces = [=](LLT PartTy, SmallVectorImpl<Register> &ValRegs,
5635 unsigned NumParts, unsigned Offset) -> unsigned {
5636 MachineFunction &MF = MIRBuilder.getMF();
5637 unsigned PartSize = PartTy.getSizeInBits();
5638 for (unsigned Idx = 0, E = NumParts; Idx != E && Offset < TotalSize;
5639 ++Idx) {
5640 unsigned ByteOffset = Offset / 8;
5641 Register NewAddrReg;
5642
5643 MIRBuilder.materializeObjectPtrOffset(Res&: NewAddrReg, Op0: AddrReg, ValueTy: OffsetTy,
5644 Value: ByteOffset);
5645
5646 MachineMemOperand *NewMMO =
5647 MF.getMachineMemOperand(MMO: &MMO, Offset: ByteOffset, Ty: PartTy);
5648
5649 if (IsLoad) {
5650 Register Dst = MRI.createGenericVirtualRegister(Ty: PartTy);
5651 ValRegs.push_back(Elt: Dst);
5652 MIRBuilder.buildLoad(Res: Dst, Addr: NewAddrReg, MMO&: *NewMMO);
5653 } else {
5654 MIRBuilder.buildStore(Val: ValRegs[Idx], Addr: NewAddrReg, MMO&: *NewMMO);
5655 }
5656 Offset = isBigEndian ? Offset - PartSize : Offset + PartSize;
5657 }
5658
5659 return Offset;
5660 };
5661
5662 unsigned Offset = isBigEndian ? TotalSize - NarrowTy.getSizeInBits() : 0;
5663 unsigned HandledOffset =
5664 splitTypePieces(NarrowTy, NarrowRegs, NumParts, Offset);
5665
5666 // Handle the rest of the register if this isn't an even type breakdown.
5667 if (LeftoverTy.isValid())
5668 splitTypePieces(LeftoverTy, NarrowLeftoverRegs, NumLeftover, HandledOffset);
5669
5670 if (IsLoad) {
5671 insertParts(DstReg: ValReg, ResultTy: ValTy, PartTy: NarrowTy, PartRegs: NarrowRegs,
5672 LeftoverTy, LeftoverRegs: NarrowLeftoverRegs);
5673 }
5674
5675 LdStMI.eraseFromParent();
5676 return Legalized;
5677}
5678
5679LegalizerHelper::LegalizeResult
5680LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
5681 LLT NarrowTy) {
5682 using namespace TargetOpcode;
5683 GenericMachineInstr &GMI = cast<GenericMachineInstr>(Val&: MI);
5684 unsigned NumElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
5685
5686 switch (MI.getOpcode()) {
5687 case G_IMPLICIT_DEF:
5688 case G_TRUNC:
5689 case G_AND:
5690 case G_OR:
5691 case G_XOR:
5692 case G_ADD:
5693 case G_SUB:
5694 case G_MUL:
5695 case G_PTR_ADD:
5696 case G_SMULH:
5697 case G_UMULH:
5698 case G_FADD:
5699 case G_FMUL:
5700 case G_FSUB:
5701 case G_FNEG:
5702 case G_FABS:
5703 case G_FCANONICALIZE:
5704 case G_FDIV:
5705 case G_FREM:
5706 case G_FMA:
5707 case G_FMAD:
5708 case G_FPOW:
5709 case G_FEXP:
5710 case G_FEXP2:
5711 case G_FEXP10:
5712 case G_FLOG:
5713 case G_FLOG2:
5714 case G_FLOG10:
5715 case G_FLDEXP:
5716 case G_FNEARBYINT:
5717 case G_FCEIL:
5718 case G_FFLOOR:
5719 case G_FRINT:
5720 case G_INTRINSIC_LRINT:
5721 case G_INTRINSIC_LLRINT:
5722 case G_INTRINSIC_ROUND:
5723 case G_INTRINSIC_ROUNDEVEN:
5724 case G_LROUND:
5725 case G_LLROUND:
5726 case G_INTRINSIC_TRUNC:
5727 case G_FMODF:
5728 case G_FCOS:
5729 case G_FSIN:
5730 case G_FTAN:
5731 case G_FACOS:
5732 case G_FASIN:
5733 case G_FATAN:
5734 case G_FATAN2:
5735 case G_FCOSH:
5736 case G_FSINH:
5737 case G_FTANH:
5738 case G_FSQRT:
5739 case G_BSWAP:
5740 case G_BITREVERSE:
5741 case G_SDIV:
5742 case G_UDIV:
5743 case G_SREM:
5744 case G_UREM:
5745 case G_SDIVREM:
5746 case G_UDIVREM:
5747 case G_SMIN:
5748 case G_SMAX:
5749 case G_UMIN:
5750 case G_UMAX:
5751 case G_ABS:
5752 case G_FMINNUM:
5753 case G_FMAXNUM:
5754 case G_FMINNUM_IEEE:
5755 case G_FMAXNUM_IEEE:
5756 case G_FMINIMUM:
5757 case G_FMAXIMUM:
5758 case G_FMINIMUMNUM:
5759 case G_FMAXIMUMNUM:
5760 case G_FSHL:
5761 case G_FSHR:
5762 case G_ROTL:
5763 case G_ROTR:
5764 case G_FREEZE:
5765 case G_SADDSAT:
5766 case G_SSUBSAT:
5767 case G_UADDSAT:
5768 case G_USUBSAT:
5769 case G_UMULO:
5770 case G_SMULO:
5771 case G_SHL:
5772 case G_LSHR:
5773 case G_ASHR:
5774 case G_SSHLSAT:
5775 case G_USHLSAT:
5776 case G_CTLZ:
5777 case G_CTLZ_ZERO_POISON:
5778 case G_CTTZ:
5779 case G_CTTZ_ZERO_POISON:
5780 case G_CTPOP:
5781 case G_CTLS:
5782 case G_FCOPYSIGN:
5783 case G_ZEXT:
5784 case G_SEXT:
5785 case G_ANYEXT:
5786 case G_FPEXT:
5787 case G_FPTRUNC:
5788 case G_SITOFP:
5789 case G_UITOFP:
5790 case G_FPTOSI:
5791 case G_FPTOUI:
5792 case G_FPTOSI_SAT:
5793 case G_FPTOUI_SAT:
5794 case G_INTTOPTR:
5795 case G_PTRTOINT:
5796 case G_ADDRSPACE_CAST:
5797 case G_UADDO:
5798 case G_USUBO:
5799 case G_UADDE:
5800 case G_USUBE:
5801 case G_SADDO:
5802 case G_SSUBO:
5803 case G_SADDE:
5804 case G_SSUBE:
5805 case G_STRICT_FADD:
5806 case G_STRICT_FSUB:
5807 case G_STRICT_FMUL:
5808 case G_STRICT_FMA:
5809 case G_STRICT_FLDEXP:
5810 case G_FFREXP:
5811 case G_TRUNC_SSAT_S:
5812 case G_TRUNC_SSAT_U:
5813 case G_TRUNC_USAT_U:
5814 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts);
5815 case G_ICMP:
5816 case G_FCMP:
5817 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {1 /*cpm predicate*/});
5818 case G_IS_FPCLASS:
5819 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {2, 3 /*mask,fpsem*/});
5820 case G_SELECT:
5821 if (MRI.getType(Reg: MI.getOperand(i: 1).getReg()).isVector())
5822 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts);
5823 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {1 /*scalar cond*/});
5824 case G_PHI:
5825 return fewerElementsVectorPhi(MI&: GMI, NumElts);
5826 case G_UNMERGE_VALUES:
5827 return fewerElementsVectorUnmergeValues(MI, TypeIdx, NarrowTy);
5828 case G_BUILD_VECTOR:
5829 assert(TypeIdx == 0 && "not a vector type index");
5830 return fewerElementsVectorMerge(MI, TypeIdx, NarrowTy);
5831 case G_CONCAT_VECTORS:
5832 if (TypeIdx != 1) // TODO: This probably does work as expected already.
5833 return UnableToLegalize;
5834 return fewerElementsVectorMerge(MI, TypeIdx, NarrowTy);
5835 case G_EXTRACT_VECTOR_ELT:
5836 case G_INSERT_VECTOR_ELT:
5837 return fewerElementsVectorExtractInsertVectorElt(MI, TypeIdx, NarrowVecTy: NarrowTy);
5838 case G_LOAD:
5839 case G_STORE:
5840 return reduceLoadStoreWidth(LdStMI&: cast<GLoadStore>(Val&: MI), TypeIdx, NarrowTy);
5841 case G_SEXT_INREG:
5842 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {2 /*imm*/});
5843 GISEL_VECREDUCE_CASES_NONSEQ
5844 return fewerElementsVectorReductions(MI, TypeIdx, NarrowTy);
5845 case TargetOpcode::G_VECREDUCE_SEQ_FADD:
5846 case TargetOpcode::G_VECREDUCE_SEQ_FMUL:
5847 return fewerElementsVectorSeqReductions(MI, TypeIdx, NarrowTy);
5848 case G_SHUFFLE_VECTOR:
5849 return fewerElementsVectorShuffle(MI, TypeIdx, NarrowTy);
5850 case G_FPOWI:
5851 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {2 /*pow*/});
5852 case G_BITCAST:
5853 return fewerElementsBitcast(MI, TypeIdx, NarrowTy);
5854 case G_INTRINSIC_FPTRUNC_ROUND:
5855 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {2});
5856 default:
5857 return UnableToLegalize;
5858 }
5859}
5860
5861LegalizerHelper::LegalizeResult
5862LegalizerHelper::fewerElementsBitcast(MachineInstr &MI, unsigned int TypeIdx,
5863 LLT NarrowTy) {
5864 assert(MI.getOpcode() == TargetOpcode::G_BITCAST &&
5865 "Not a bitcast operation");
5866
5867 if (TypeIdx != 0)
5868 return UnableToLegalize;
5869
5870 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
5871
5872 unsigned NewElemCount =
5873 NarrowTy.getSizeInBits() / SrcTy.getScalarSizeInBits();
5874 SmallVector<Register> SrcVRegs, BitcastVRegs;
5875 if (NewElemCount == 1) {
5876 LLT SrcNarrowTy = SrcTy.getElementType();
5877
5878 auto Unmerge = MIRBuilder.buildUnmerge(Res: SrcNarrowTy, Op: SrcReg);
5879 getUnmergeResults(Regs&: SrcVRegs, MI: *Unmerge);
5880 } else {
5881 LLT SrcNarrowTy =
5882 SrcTy.changeVectorElementCount(EC: ElementCount::getFixed(MinVal: NewElemCount));
5883
5884 // Split the Src and Dst Reg into smaller registers
5885 if (extractGCDType(Parts&: SrcVRegs, DstTy, NarrowTy: SrcNarrowTy, SrcReg) != SrcNarrowTy)
5886 return UnableToLegalize;
5887 }
5888
5889 // Build new smaller bitcast instructions
5890 // Not supporting Leftover types for now but will have to
5891 for (Register Reg : SrcVRegs)
5892 BitcastVRegs.push_back(Elt: MIRBuilder.buildBitcast(Dst: NarrowTy, Src: Reg).getReg(Idx: 0));
5893
5894 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: BitcastVRegs);
5895 MI.eraseFromParent();
5896 return Legalized;
5897}
5898
5899LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorShuffle(
5900 MachineInstr &MI, unsigned int TypeIdx, LLT NarrowTy) {
5901 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
5902 if (TypeIdx != 0)
5903 return UnableToLegalize;
5904
5905 auto [DstReg, DstTy, Src1Reg, Src1Ty, Src2Reg, Src2Ty] =
5906 MI.getFirst3RegLLTs();
5907 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
5908 // The shuffle should be canonicalized by now.
5909 if (DstTy != Src1Ty)
5910 return UnableToLegalize;
5911 if (DstTy != Src2Ty)
5912 return UnableToLegalize;
5913
5914 if (!isPowerOf2_32(Value: DstTy.getNumElements()))
5915 return UnableToLegalize;
5916
5917 // We only support splitting a shuffle into 2, so adjust NarrowTy accordingly.
5918 // Further legalization attempts will be needed to do split further.
5919 NarrowTy =
5920 DstTy.changeElementCount(EC: DstTy.getElementCount().divideCoefficientBy(RHS: 2));
5921 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
5922
5923 SmallVector<Register> SplitSrc1Regs, SplitSrc2Regs;
5924 extractParts(Reg: Src1Reg, Ty: NarrowTy, NumParts: 2, VRegs&: SplitSrc1Regs, MIRBuilder, MRI);
5925 extractParts(Reg: Src2Reg, Ty: NarrowTy, NumParts: 2, VRegs&: SplitSrc2Regs, MIRBuilder, MRI);
5926 Register Inputs[4] = {SplitSrc1Regs[0], SplitSrc1Regs[1], SplitSrc2Regs[0],
5927 SplitSrc2Regs[1]};
5928
5929 Register Hi, Lo;
5930
5931 // If Lo or Hi uses elements from at most two of the four input vectors, then
5932 // express it as a vector shuffle of those two inputs. Otherwise extract the
5933 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
5934 SmallVector<int, 16> Ops;
5935 for (unsigned High = 0; High < 2; ++High) {
5936 Register &Output = High ? Hi : Lo;
5937
5938 // Build a shuffle mask for the output, discovering on the fly which
5939 // input vectors to use as shuffle operands (recorded in InputUsed).
5940 // If building a suitable shuffle vector proves too hard, then bail
5941 // out with useBuildVector set.
5942 unsigned InputUsed[2] = {-1U, -1U}; // Not yet discovered.
5943 unsigned FirstMaskIdx = High * NewElts;
5944 bool UseBuildVector = false;
5945 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
5946 // The mask element. This indexes into the input.
5947 int Idx = Mask[FirstMaskIdx + MaskOffset];
5948
5949 // The input vector this mask element indexes into.
5950 unsigned Input = (unsigned)Idx / NewElts;
5951
5952 if (Input >= std::size(Inputs)) {
5953 // The mask element does not index into any input vector.
5954 Ops.push_back(Elt: -1);
5955 continue;
5956 }
5957
5958 // Turn the index into an offset from the start of the input vector.
5959 Idx -= Input * NewElts;
5960
5961 // Find or create a shuffle vector operand to hold this input.
5962 unsigned OpNo;
5963 for (OpNo = 0; OpNo < std::size(InputUsed); ++OpNo) {
5964 if (InputUsed[OpNo] == Input) {
5965 // This input vector is already an operand.
5966 break;
5967 } else if (InputUsed[OpNo] == -1U) {
5968 // Create a new operand for this input vector.
5969 InputUsed[OpNo] = Input;
5970 break;
5971 }
5972 }
5973
5974 if (OpNo >= std::size(InputUsed)) {
5975 // More than two input vectors used! Give up on trying to create a
5976 // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
5977 UseBuildVector = true;
5978 break;
5979 }
5980
5981 // Add the mask index for the new shuffle vector.
5982 Ops.push_back(Elt: Idx + OpNo * NewElts);
5983 }
5984
5985 if (UseBuildVector) {
5986 LLT EltTy = NarrowTy.getElementType();
5987 SmallVector<Register, 16> SVOps;
5988
5989 // Extract the input elements by hand.
5990 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
5991 // The mask element. This indexes into the input.
5992 int Idx = Mask[FirstMaskIdx + MaskOffset];
5993
5994 // The input vector this mask element indexes into.
5995 unsigned Input = (unsigned)Idx / NewElts;
5996
5997 if (Input >= std::size(Inputs)) {
5998 // The mask element is "undef" or indexes off the end of the input.
5999 SVOps.push_back(Elt: MIRBuilder.buildUndef(Res: EltTy).getReg(Idx: 0));
6000 continue;
6001 }
6002
6003 // Turn the index into an offset from the start of the input vector.
6004 Idx -= Input * NewElts;
6005
6006 // Extract the vector element by hand.
6007 SVOps.push_back(Elt: MIRBuilder
6008 .buildExtractVectorElement(
6009 Res: EltTy, Val: Inputs[Input],
6010 Idx: MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: Idx))
6011 .getReg(Idx: 0));
6012 }
6013
6014 // Construct the Lo/Hi output using a G_BUILD_VECTOR.
6015 Output = MIRBuilder.buildBuildVector(Res: NarrowTy, Ops: SVOps).getReg(Idx: 0);
6016 } else if (InputUsed[0] == -1U) {
6017 // No input vectors were used! The result is undefined.
6018 Output = MIRBuilder.buildUndef(Res: NarrowTy).getReg(Idx: 0);
6019 } else if (NewElts == 1) {
6020 Output = MIRBuilder.buildCopy(Res: NarrowTy, Op: Inputs[InputUsed[0]]).getReg(Idx: 0);
6021 } else {
6022 Register Op0 = Inputs[InputUsed[0]];
6023 // If only one input was used, use an undefined vector for the other.
6024 Register Op1 = InputUsed[1] == -1U
6025 ? MIRBuilder.buildUndef(Res: NarrowTy).getReg(Idx: 0)
6026 : Inputs[InputUsed[1]];
6027 // At least one input vector was used. Create a new shuffle vector.
6028 Output = MIRBuilder.buildShuffleVector(Res: NarrowTy, Src1: Op0, Src2: Op1, Mask: Ops).getReg(Idx: 0);
6029 }
6030
6031 Ops.clear();
6032 }
6033
6034 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: {Lo, Hi});
6035 MI.eraseFromParent();
6036 return Legalized;
6037}
6038
6039LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorReductions(
6040 MachineInstr &MI, unsigned int TypeIdx, LLT NarrowTy) {
6041 auto &RdxMI = cast<GVecReduce>(Val&: MI);
6042
6043 if (TypeIdx != 1)
6044 return UnableToLegalize;
6045
6046 // The semantics of the normal non-sequential reductions allow us to freely
6047 // re-associate the operation.
6048 auto [DstReg, DstTy, SrcReg, SrcTy] = RdxMI.getFirst2RegLLTs();
6049
6050 if (NarrowTy.isVector() &&
6051 (SrcTy.getNumElements() % NarrowTy.getNumElements() != 0))
6052 return UnableToLegalize;
6053
6054 unsigned ScalarOpc = RdxMI.getScalarOpcForReduction();
6055 SmallVector<Register> SplitSrcs;
6056 // If NarrowTy is a scalar then we're being asked to scalarize.
6057 const unsigned NumParts =
6058 NarrowTy.isVector() ? SrcTy.getNumElements() / NarrowTy.getNumElements()
6059 : SrcTy.getNumElements();
6060
6061 extractParts(Reg: SrcReg, Ty: NarrowTy, NumParts, VRegs&: SplitSrcs, MIRBuilder, MRI);
6062 if (NarrowTy.isScalar()) {
6063 if (DstTy != NarrowTy)
6064 return UnableToLegalize; // FIXME: handle implicit extensions.
6065
6066 if (isPowerOf2_32(Value: NumParts)) {
6067 // Generate a tree of scalar operations to reduce the critical path.
6068 SmallVector<Register> PartialResults;
6069 unsigned NumPartsLeft = NumParts;
6070 while (NumPartsLeft > 1) {
6071 for (unsigned Idx = 0; Idx < NumPartsLeft - 1; Idx += 2) {
6072 PartialResults.emplace_back(
6073 Args: MIRBuilder
6074 .buildInstr(Opc: ScalarOpc, DstOps: {NarrowTy},
6075 SrcOps: {SplitSrcs[Idx], SplitSrcs[Idx + 1]})
6076 .getReg(Idx: 0));
6077 }
6078 SplitSrcs = PartialResults;
6079 PartialResults.clear();
6080 NumPartsLeft = SplitSrcs.size();
6081 }
6082 assert(SplitSrcs.size() == 1);
6083 MIRBuilder.buildCopy(Res: DstReg, Op: SplitSrcs[0]);
6084 MI.eraseFromParent();
6085 return Legalized;
6086 }
6087 // If we can't generate a tree, then just do sequential operations.
6088 Register Acc = SplitSrcs[0];
6089 for (unsigned Idx = 1; Idx < NumParts; ++Idx)
6090 Acc = MIRBuilder.buildInstr(Opc: ScalarOpc, DstOps: {NarrowTy}, SrcOps: {Acc, SplitSrcs[Idx]})
6091 .getReg(Idx: 0);
6092 MIRBuilder.buildCopy(Res: DstReg, Op: Acc);
6093 MI.eraseFromParent();
6094 return Legalized;
6095 }
6096 SmallVector<Register> PartialReductions;
6097 for (unsigned Part = 0; Part < NumParts; ++Part) {
6098 PartialReductions.push_back(
6099 Elt: MIRBuilder.buildInstr(Opc: RdxMI.getOpcode(), DstOps: {DstTy}, SrcOps: {SplitSrcs[Part]})
6100 .getReg(Idx: 0));
6101 }
6102
6103 // If the types involved are powers of 2, we can generate intermediate vector
6104 // ops, before generating a final reduction operation.
6105 if (isPowerOf2_32(Value: SrcTy.getNumElements()) &&
6106 isPowerOf2_32(Value: NarrowTy.getNumElements())) {
6107 return tryNarrowPow2Reduction(MI, SrcReg, SrcTy, NarrowTy, ScalarOpc);
6108 }
6109
6110 Register Acc = PartialReductions[0];
6111 for (unsigned Part = 1; Part < NumParts; ++Part) {
6112 if (Part == NumParts - 1) {
6113 MIRBuilder.buildInstr(Opc: ScalarOpc, DstOps: {DstReg},
6114 SrcOps: {Acc, PartialReductions[Part]});
6115 } else {
6116 Acc = MIRBuilder
6117 .buildInstr(Opc: ScalarOpc, DstOps: {DstTy}, SrcOps: {Acc, PartialReductions[Part]})
6118 .getReg(Idx: 0);
6119 }
6120 }
6121 MI.eraseFromParent();
6122 return Legalized;
6123}
6124
6125LegalizerHelper::LegalizeResult
6126LegalizerHelper::fewerElementsVectorSeqReductions(MachineInstr &MI,
6127 unsigned int TypeIdx,
6128 LLT NarrowTy) {
6129 auto [DstReg, DstTy, ScalarReg, ScalarTy, SrcReg, SrcTy] =
6130 MI.getFirst3RegLLTs();
6131 if (!NarrowTy.isScalar() || TypeIdx != 2 || DstTy != ScalarTy ||
6132 DstTy != NarrowTy)
6133 return UnableToLegalize;
6134
6135 assert((MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FADD ||
6136 MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FMUL) &&
6137 "Unexpected vecreduce opcode");
6138 unsigned ScalarOpc = MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FADD
6139 ? TargetOpcode::G_FADD
6140 : TargetOpcode::G_FMUL;
6141
6142 SmallVector<Register> SplitSrcs;
6143 unsigned NumParts = SrcTy.getNumElements();
6144 extractParts(Reg: SrcReg, Ty: NarrowTy, NumParts, VRegs&: SplitSrcs, MIRBuilder, MRI);
6145 Register Acc = ScalarReg;
6146 for (unsigned i = 0; i < NumParts; i++)
6147 Acc = MIRBuilder.buildInstr(Opc: ScalarOpc, DstOps: {NarrowTy}, SrcOps: {Acc, SplitSrcs[i]})
6148 .getReg(Idx: 0);
6149
6150 MIRBuilder.buildCopy(Res: DstReg, Op: Acc);
6151 MI.eraseFromParent();
6152 return Legalized;
6153}
6154
6155LegalizerHelper::LegalizeResult
6156LegalizerHelper::tryNarrowPow2Reduction(MachineInstr &MI, Register SrcReg,
6157 LLT SrcTy, LLT NarrowTy,
6158 unsigned ScalarOpc) {
6159 SmallVector<Register> SplitSrcs;
6160 // Split the sources into NarrowTy size pieces.
6161 extractParts(Reg: SrcReg, Ty: NarrowTy,
6162 NumParts: SrcTy.getNumElements() / NarrowTy.getNumElements(), VRegs&: SplitSrcs,
6163 MIRBuilder, MRI);
6164 // We're going to do a tree reduction using vector operations until we have
6165 // one NarrowTy size value left.
6166 while (SplitSrcs.size() > 1) {
6167 SmallVector<Register> PartialRdxs;
6168 for (unsigned Idx = 0; Idx < SplitSrcs.size()-1; Idx += 2) {
6169 Register LHS = SplitSrcs[Idx];
6170 Register RHS = SplitSrcs[Idx + 1];
6171 // Create the intermediate vector op.
6172 Register Res =
6173 MIRBuilder.buildInstr(Opc: ScalarOpc, DstOps: {NarrowTy}, SrcOps: {LHS, RHS}).getReg(Idx: 0);
6174 PartialRdxs.push_back(Elt: Res);
6175 }
6176 SplitSrcs = std::move(PartialRdxs);
6177 }
6178 // Finally generate the requested NarrowTy based reduction.
6179 Observer.changingInstr(MI);
6180 MI.getOperand(i: 1).setReg(SplitSrcs[0]);
6181 Observer.changedInstr(MI);
6182 return Legalized;
6183}
6184
6185LegalizerHelper::LegalizeResult
6186LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt,
6187 const LLT HalfTy, const LLT AmtTy) {
6188
6189 Register InL = MRI.createGenericVirtualRegister(Ty: HalfTy);
6190 Register InH = MRI.createGenericVirtualRegister(Ty: HalfTy);
6191 MIRBuilder.buildUnmerge(Res: {InL, InH}, Op: MI.getOperand(i: 1));
6192
6193 if (Amt.isZero()) {
6194 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0), Ops: {InL, InH});
6195 MI.eraseFromParent();
6196 return Legalized;
6197 }
6198
6199 LLT NVT = HalfTy;
6200 unsigned NVTBits = HalfTy.getSizeInBits();
6201 unsigned VTBits = 2 * NVTBits;
6202
6203 SrcOp Lo(Register(0)), Hi(Register(0));
6204 if (MI.getOpcode() == TargetOpcode::G_SHL) {
6205 if (Amt.ugt(RHS: VTBits)) {
6206 Lo = Hi = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6207 } else if (Amt.ugt(RHS: NVTBits)) {
6208 Lo = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6209 Hi = MIRBuilder.buildShl(Dst: NVT, Src0: InL,
6210 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt - NVTBits));
6211 } else if (Amt == NVTBits) {
6212 Lo = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6213 Hi = InL;
6214 } else {
6215 Lo = MIRBuilder.buildShl(Dst: NVT, Src0: InL, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt));
6216 auto OrLHS =
6217 MIRBuilder.buildShl(Dst: NVT, Src0: InH, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt));
6218 auto OrRHS = MIRBuilder.buildLShr(
6219 Dst: NVT, Src0: InL, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: -Amt + NVTBits));
6220 Hi = MIRBuilder.buildOr(Dst: NVT, Src0: OrLHS, Src1: OrRHS);
6221 }
6222 } else if (MI.getOpcode() == TargetOpcode::G_LSHR) {
6223 if (Amt.ugt(RHS: VTBits)) {
6224 Lo = Hi = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6225 } else if (Amt.ugt(RHS: NVTBits)) {
6226 Lo = MIRBuilder.buildLShr(Dst: NVT, Src0: InH,
6227 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt - NVTBits));
6228 Hi = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6229 } else if (Amt == NVTBits) {
6230 Lo = InH;
6231 Hi = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6232 } else {
6233 auto ShiftAmtConst = MIRBuilder.buildConstant(Res: AmtTy, Val: Amt);
6234
6235 auto OrLHS = MIRBuilder.buildLShr(Dst: NVT, Src0: InL, Src1: ShiftAmtConst);
6236 auto OrRHS = MIRBuilder.buildShl(
6237 Dst: NVT, Src0: InH, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: -Amt + NVTBits));
6238
6239 Lo = MIRBuilder.buildOr(Dst: NVT, Src0: OrLHS, Src1: OrRHS);
6240 Hi = MIRBuilder.buildLShr(Dst: NVT, Src0: InH, Src1: ShiftAmtConst);
6241 }
6242 } else {
6243 if (Amt.ugt(RHS: VTBits)) {
6244 Hi = Lo = MIRBuilder.buildAShr(
6245 Dst: NVT, Src0: InH, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: NVTBits - 1));
6246 } else if (Amt.ugt(RHS: NVTBits)) {
6247 Lo = MIRBuilder.buildAShr(Dst: NVT, Src0: InH,
6248 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt - NVTBits));
6249 Hi = MIRBuilder.buildAShr(Dst: NVT, Src0: InH,
6250 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: NVTBits - 1));
6251 } else if (Amt == NVTBits) {
6252 Lo = InH;
6253 Hi = MIRBuilder.buildAShr(Dst: NVT, Src0: InH,
6254 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: NVTBits - 1));
6255 } else {
6256 auto ShiftAmtConst = MIRBuilder.buildConstant(Res: AmtTy, Val: Amt);
6257
6258 auto OrLHS = MIRBuilder.buildLShr(Dst: NVT, Src0: InL, Src1: ShiftAmtConst);
6259 auto OrRHS = MIRBuilder.buildShl(
6260 Dst: NVT, Src0: InH, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: -Amt + NVTBits));
6261
6262 Lo = MIRBuilder.buildOr(Dst: NVT, Src0: OrLHS, Src1: OrRHS);
6263 Hi = MIRBuilder.buildAShr(Dst: NVT, Src0: InH, Src1: ShiftAmtConst);
6264 }
6265 }
6266
6267 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0), Ops: {Lo, Hi});
6268 MI.eraseFromParent();
6269
6270 return Legalized;
6271}
6272
6273LegalizerHelper::LegalizeResult
6274LegalizerHelper::narrowScalarShift(MachineInstr &MI, unsigned TypeIdx,
6275 LLT RequestedTy) {
6276 if (TypeIdx == 1) {
6277 Observer.changingInstr(MI);
6278 narrowScalarSrc(MI, NarrowTy: RequestedTy, OpIdx: 2);
6279 Observer.changedInstr(MI);
6280 return Legalized;
6281 }
6282
6283 Register DstReg = MI.getOperand(i: 0).getReg();
6284 LLT DstTy = MRI.getType(Reg: DstReg);
6285 if (DstTy.isVector())
6286 return UnableToLegalize;
6287
6288 Register Amt = MI.getOperand(i: 2).getReg();
6289 LLT ShiftAmtTy = MRI.getType(Reg: Amt);
6290 const unsigned DstEltSize = DstTy.getScalarSizeInBits();
6291 if (DstEltSize % 2 != 0)
6292 return UnableToLegalize;
6293
6294 // Check if we should use multi-way splitting instead of recursive binary
6295 // splitting.
6296 //
6297 // Multi-way splitting directly decomposes wide shifts (e.g., 128-bit ->
6298 // 4×32-bit) in a single legalization step, avoiding the recursive overhead
6299 // and dependency chains created by usual binary splitting approach
6300 // (128->64->32).
6301 //
6302 // The >= 8 parts threshold ensures we only use this optimization when binary
6303 // splitting would require multiple recursive passes, avoiding overhead for
6304 // simple 2-way splits where binary approach is sufficient.
6305 if (RequestedTy.isValid() && RequestedTy.isScalar() &&
6306 DstEltSize % RequestedTy.getSizeInBits() == 0) {
6307 const unsigned NumParts = DstEltSize / RequestedTy.getSizeInBits();
6308 // Use multiway if we have 8 or more parts (i.e., would need 3+ recursive
6309 // steps).
6310 if (NumParts >= 8)
6311 return narrowScalarShiftMultiway(MI, TargetTy: RequestedTy);
6312 }
6313
6314 // Fall back to binary splitting:
6315 // Ignore the input type. We can only go to exactly half the size of the
6316 // input. If that isn't small enough, the resulting pieces will be further
6317 // legalized.
6318 const unsigned NewBitSize = DstEltSize / 2;
6319 const LLT HalfTy = DstTy.getScalarType().changeElementSize(NewEltSize: NewBitSize);
6320 const LLT CondTy = LLT::integer(SizeInBits: 1);
6321
6322 if (auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: Amt, MRI)) {
6323 return narrowScalarShiftByConstant(MI, Amt: VRegAndVal->Value, HalfTy,
6324 AmtTy: ShiftAmtTy);
6325 }
6326
6327 // TODO: Expand with known bits.
6328
6329 // Handle the fully general expansion by an unknown amount.
6330 auto NewBits = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: NewBitSize);
6331
6332 Register InL = MRI.createGenericVirtualRegister(Ty: HalfTy);
6333 Register InH = MRI.createGenericVirtualRegister(Ty: HalfTy);
6334 MIRBuilder.buildUnmerge(Res: {InL, InH}, Op: MI.getOperand(i: 1));
6335
6336 auto AmtExcess = MIRBuilder.buildSub(Dst: ShiftAmtTy, Src0: Amt, Src1: NewBits);
6337 auto AmtLack = MIRBuilder.buildSub(Dst: ShiftAmtTy, Src0: NewBits, Src1: Amt);
6338
6339 auto Zero = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: 0);
6340 auto IsShort = MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_ULT, Res: CondTy, Op0: Amt, Op1: NewBits);
6341 auto IsZero = MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: CondTy, Op0: Amt, Op1: Zero);
6342
6343 Register ResultRegs[2];
6344 switch (MI.getOpcode()) {
6345 case TargetOpcode::G_SHL: {
6346 // Short: ShAmt < NewBitSize
6347 auto LoS = MIRBuilder.buildShl(Dst: HalfTy, Src0: InL, Src1: Amt);
6348
6349 auto LoOr = MIRBuilder.buildLShr(Dst: HalfTy, Src0: InL, Src1: AmtLack);
6350 auto HiOr = MIRBuilder.buildShl(Dst: HalfTy, Src0: InH, Src1: Amt);
6351 auto HiS = MIRBuilder.buildOr(Dst: HalfTy, Src0: LoOr, Src1: HiOr);
6352
6353 // Long: ShAmt >= NewBitSize
6354 auto LoL = MIRBuilder.buildConstant(Res: HalfTy, Val: 0); // Lo part is zero.
6355 auto HiL = MIRBuilder.buildShl(Dst: HalfTy, Src0: InL, Src1: AmtExcess); // Hi from Lo part.
6356
6357 auto Lo = MIRBuilder.buildSelect(Res: HalfTy, Tst: IsShort, Op0: LoS, Op1: LoL);
6358 auto Hi = MIRBuilder.buildSelect(
6359 Res: HalfTy, Tst: IsZero, Op0: InH, Op1: MIRBuilder.buildSelect(Res: HalfTy, Tst: IsShort, Op0: HiS, Op1: HiL));
6360
6361 ResultRegs[0] = Lo.getReg(Idx: 0);
6362 ResultRegs[1] = Hi.getReg(Idx: 0);
6363 break;
6364 }
6365 case TargetOpcode::G_LSHR:
6366 case TargetOpcode::G_ASHR: {
6367 // Short: ShAmt < NewBitSize
6368 auto HiS = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {HalfTy}, SrcOps: {InH, Amt});
6369
6370 auto LoOr = MIRBuilder.buildLShr(Dst: HalfTy, Src0: InL, Src1: Amt);
6371 auto HiOr = MIRBuilder.buildShl(Dst: HalfTy, Src0: InH, Src1: AmtLack);
6372 auto LoS = MIRBuilder.buildOr(Dst: HalfTy, Src0: LoOr, Src1: HiOr);
6373
6374 // Long: ShAmt >= NewBitSize
6375 MachineInstrBuilder HiL;
6376 if (MI.getOpcode() == TargetOpcode::G_LSHR) {
6377 HiL = MIRBuilder.buildConstant(Res: HalfTy, Val: 0); // Hi part is zero.
6378 } else {
6379 auto ShiftAmt = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: NewBitSize - 1);
6380 HiL = MIRBuilder.buildAShr(Dst: HalfTy, Src0: InH, Src1: ShiftAmt); // Sign of Hi part.
6381 }
6382 auto LoL = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {HalfTy},
6383 SrcOps: {InH, AmtExcess}); // Lo from Hi part.
6384
6385 auto Lo = MIRBuilder.buildSelect(
6386 Res: HalfTy, Tst: IsZero, Op0: InL, Op1: MIRBuilder.buildSelect(Res: HalfTy, Tst: IsShort, Op0: LoS, Op1: LoL));
6387
6388 auto Hi = MIRBuilder.buildSelect(Res: HalfTy, Tst: IsShort, Op0: HiS, Op1: HiL);
6389
6390 ResultRegs[0] = Lo.getReg(Idx: 0);
6391 ResultRegs[1] = Hi.getReg(Idx: 0);
6392 break;
6393 }
6394 default:
6395 llvm_unreachable("not a shift");
6396 }
6397
6398 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: ResultRegs);
6399 MI.eraseFromParent();
6400 return Legalized;
6401}
6402
6403Register LegalizerHelper::buildConstantShiftPart(unsigned Opcode,
6404 unsigned PartIdx,
6405 unsigned NumParts,
6406 ArrayRef<Register> SrcParts,
6407 const ShiftParams &Params,
6408 LLT TargetTy, LLT ShiftAmtTy) {
6409 auto WordShiftConst = getIConstantVRegVal(VReg: Params.WordShift, MRI);
6410 auto BitShiftConst = getIConstantVRegVal(VReg: Params.BitShift, MRI);
6411 assert(WordShiftConst && BitShiftConst && "Expected constants");
6412
6413 const unsigned ShiftWords = WordShiftConst->getZExtValue();
6414 const unsigned ShiftBits = BitShiftConst->getZExtValue();
6415 const bool NeedsInterWordShift = ShiftBits != 0;
6416
6417 switch (Opcode) {
6418 case TargetOpcode::G_SHL: {
6419 // Data moves from lower indices to higher indices
6420 // If this part would come from a source beyond our range, it's zero
6421 if (PartIdx < ShiftWords)
6422 return Params.Zero;
6423
6424 unsigned SrcIdx = PartIdx - ShiftWords;
6425 if (!NeedsInterWordShift)
6426 return SrcParts[SrcIdx];
6427
6428 // Combine shifted main part with carry from previous part
6429 auto Hi = MIRBuilder.buildShl(Dst: TargetTy, Src0: SrcParts[SrcIdx], Src1: Params.BitShift);
6430 if (SrcIdx > 0) {
6431 auto Lo = MIRBuilder.buildLShr(Dst: TargetTy, Src0: SrcParts[SrcIdx - 1],
6432 Src1: Params.InvBitShift);
6433 return MIRBuilder.buildOr(Dst: TargetTy, Src0: Hi, Src1: Lo).getReg(Idx: 0);
6434 }
6435 return Hi.getReg(Idx: 0);
6436 }
6437
6438 case TargetOpcode::G_LSHR: {
6439 unsigned SrcIdx = PartIdx + ShiftWords;
6440 if (SrcIdx >= NumParts)
6441 return Params.Zero;
6442 if (!NeedsInterWordShift)
6443 return SrcParts[SrcIdx];
6444
6445 // Combine shifted main part with carry from next part
6446 auto Lo = MIRBuilder.buildLShr(Dst: TargetTy, Src0: SrcParts[SrcIdx], Src1: Params.BitShift);
6447 if (SrcIdx + 1 < NumParts) {
6448 auto Hi = MIRBuilder.buildShl(Dst: TargetTy, Src0: SrcParts[SrcIdx + 1],
6449 Src1: Params.InvBitShift);
6450 return MIRBuilder.buildOr(Dst: TargetTy, Src0: Lo, Src1: Hi).getReg(Idx: 0);
6451 }
6452 return Lo.getReg(Idx: 0);
6453 }
6454
6455 case TargetOpcode::G_ASHR: {
6456 // Like LSHR but preserves sign bit
6457 unsigned SrcIdx = PartIdx + ShiftWords;
6458 if (SrcIdx >= NumParts)
6459 return Params.SignBit;
6460 if (!NeedsInterWordShift)
6461 return SrcParts[SrcIdx];
6462
6463 // Only the original MSB part uses arithmetic shift to preserve sign. All
6464 // other parts use logical shift since they're just moving data bits.
6465 auto Lo =
6466 (SrcIdx == NumParts - 1)
6467 ? MIRBuilder.buildAShr(Dst: TargetTy, Src0: SrcParts[SrcIdx], Src1: Params.BitShift)
6468 : MIRBuilder.buildLShr(Dst: TargetTy, Src0: SrcParts[SrcIdx], Src1: Params.BitShift);
6469 Register HiSrc =
6470 (SrcIdx + 1 < NumParts) ? SrcParts[SrcIdx + 1] : Params.SignBit;
6471 auto Hi = MIRBuilder.buildShl(Dst: TargetTy, Src0: HiSrc, Src1: Params.InvBitShift);
6472 return MIRBuilder.buildOr(Dst: TargetTy, Src0: Lo, Src1: Hi).getReg(Idx: 0);
6473 }
6474
6475 default:
6476 llvm_unreachable("not a shift");
6477 }
6478}
6479
6480Register LegalizerHelper::buildVariableShiftPart(unsigned Opcode,
6481 Register MainOperand,
6482 Register ShiftAmt,
6483 LLT TargetTy,
6484 Register CarryOperand) {
6485 // This helper generates a single output part for variable shifts by combining
6486 // the main operand (shifted by BitShift) with carry bits from an adjacent
6487 // part.
6488
6489 // For G_ASHR, individual parts don't have their own sign bit, only the
6490 // complete value does. So we use LSHR for the main operand shift in ASHR
6491 // context.
6492 unsigned MainOpcode = (Opcode == TargetOpcode::G_ASHR)
6493 ? static_cast<unsigned>(TargetOpcode::G_LSHR)
6494 : Opcode;
6495
6496 // Perform the primary shift on the main operand
6497 Register MainShifted =
6498 MIRBuilder.buildInstr(Opc: MainOpcode, DstOps: {TargetTy}, SrcOps: {MainOperand, ShiftAmt})
6499 .getReg(Idx: 0);
6500
6501 // No carry operand available
6502 if (!CarryOperand.isValid())
6503 return MainShifted;
6504
6505 // If BitShift is 0 (word-aligned shift), no inter-word bit movement occurs,
6506 // so carry bits aren't needed.
6507 LLT ShiftAmtTy = MRI.getType(Reg: ShiftAmt);
6508 auto ZeroConst = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: 0);
6509 LLT BoolTy = LLT::scalar(SizeInBits: 1);
6510 auto IsZeroBitShift =
6511 MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: BoolTy, Op0: ShiftAmt, Op1: ZeroConst);
6512
6513 // Extract bits from the adjacent part that will "carry over" into this part.
6514 // The carry direction is opposite to the main shift direction, so we can
6515 // align the two shifted values before combining them with OR.
6516
6517 // Determine the carry shift opcode (opposite direction)
6518 unsigned CarryOpcode = (Opcode == TargetOpcode::G_SHL) ? TargetOpcode::G_LSHR
6519 : TargetOpcode::G_SHL;
6520
6521 // Calculate inverse shift amount: BitWidth - ShiftAmt
6522 auto TargetBitsConst =
6523 MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetTy.getScalarSizeInBits());
6524 auto InvShiftAmt = MIRBuilder.buildSub(Dst: ShiftAmtTy, Src0: TargetBitsConst, Src1: ShiftAmt);
6525
6526 // Shift the carry operand
6527 Register CarryBits =
6528 MIRBuilder
6529 .buildInstr(Opc: CarryOpcode, DstOps: {TargetTy}, SrcOps: {CarryOperand, InvShiftAmt})
6530 .getReg(Idx: 0);
6531
6532 // If BitShift is 0, don't include carry bits (InvShiftAmt would equal
6533 // TargetBits which would be poison for the individual carry shift operation).
6534 auto ZeroReg = MIRBuilder.buildConstant(Res: TargetTy, Val: 0);
6535 Register SafeCarryBits =
6536 MIRBuilder.buildSelect(Res: TargetTy, Tst: IsZeroBitShift, Op0: ZeroReg, Op1: CarryBits)
6537 .getReg(Idx: 0);
6538
6539 // Combine the main shifted part with the carry bits
6540 return MIRBuilder.buildOr(Dst: TargetTy, Src0: MainShifted, Src1: SafeCarryBits).getReg(Idx: 0);
6541}
6542
6543LegalizerHelper::LegalizeResult
6544LegalizerHelper::narrowScalarShiftByConstantMultiway(MachineInstr &MI,
6545 const APInt &Amt,
6546 LLT TargetTy,
6547 LLT ShiftAmtTy) {
6548 // Any wide shift can be decomposed into WordShift + BitShift components.
6549 // When shift amount is known constant, directly compute the decomposition
6550 // values and generate constant registers.
6551 Register DstReg = MI.getOperand(i: 0).getReg();
6552 Register SrcReg = MI.getOperand(i: 1).getReg();
6553 LLT DstTy = MRI.getType(Reg: DstReg);
6554
6555 const unsigned DstBits = DstTy.getScalarSizeInBits();
6556 const unsigned TargetBits = TargetTy.getScalarSizeInBits();
6557 const unsigned NumParts = DstBits / TargetBits;
6558
6559 assert(DstBits % TargetBits == 0 && "Target type must evenly divide source");
6560
6561 // When the shift amount is known at compile time, we just calculate which
6562 // source parts contribute to each output part.
6563
6564 SmallVector<Register, 8> SrcParts;
6565 extractParts(Reg: SrcReg, Ty: TargetTy, NumParts, VRegs&: SrcParts, MIRBuilder, MRI);
6566
6567 if (Amt.isZero()) {
6568 // No shift needed, just copy
6569 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: SrcParts);
6570 MI.eraseFromParent();
6571 return Legalized;
6572 }
6573
6574 ShiftParams Params;
6575 const unsigned ShiftWords = Amt.getZExtValue() / TargetBits;
6576 const unsigned ShiftBits = Amt.getZExtValue() % TargetBits;
6577
6578 // Generate constants and values needed by all shift types
6579 Params.WordShift = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: ShiftWords).getReg(Idx: 0);
6580 Params.BitShift = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: ShiftBits).getReg(Idx: 0);
6581 Params.InvBitShift =
6582 MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBits - ShiftBits).getReg(Idx: 0);
6583 Params.Zero = MIRBuilder.buildConstant(Res: TargetTy, Val: 0).getReg(Idx: 0);
6584
6585 // For ASHR, we need the sign-extended value to fill shifted-out positions
6586 if (MI.getOpcode() == TargetOpcode::G_ASHR)
6587 Params.SignBit =
6588 MIRBuilder
6589 .buildAShr(Dst: TargetTy, Src0: SrcParts[SrcParts.size() - 1],
6590 Src1: MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBits - 1))
6591 .getReg(Idx: 0);
6592
6593 SmallVector<Register, 8> DstParts(NumParts);
6594 for (unsigned I = 0; I < NumParts; ++I)
6595 DstParts[I] = buildConstantShiftPart(Opcode: MI.getOpcode(), PartIdx: I, NumParts, SrcParts,
6596 Params, TargetTy, ShiftAmtTy);
6597
6598 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstParts);
6599 MI.eraseFromParent();
6600 return Legalized;
6601}
6602
6603LegalizerHelper::LegalizeResult
6604LegalizerHelper::narrowScalarShiftMultiway(MachineInstr &MI, LLT TargetTy) {
6605 Register DstReg = MI.getOperand(i: 0).getReg();
6606 Register SrcReg = MI.getOperand(i: 1).getReg();
6607 Register AmtReg = MI.getOperand(i: 2).getReg();
6608 LLT DstTy = MRI.getType(Reg: DstReg);
6609 LLT ShiftAmtTy = MRI.getType(Reg: AmtReg);
6610
6611 const unsigned DstBits = DstTy.getScalarSizeInBits();
6612 const unsigned TargetBits = TargetTy.getScalarSizeInBits();
6613 const unsigned NumParts = DstBits / TargetBits;
6614
6615 assert(DstBits % TargetBits == 0 && "Target type must evenly divide source");
6616 assert(isPowerOf2_32(TargetBits) && "Target bit width must be power of 2");
6617
6618 // If the shift amount is known at compile time, we can use direct indexing
6619 // instead of generating select chains in the general case.
6620 if (auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: AmtReg, MRI))
6621 return narrowScalarShiftByConstantMultiway(MI, Amt: VRegAndVal->Value, TargetTy,
6622 ShiftAmtTy);
6623
6624 // For runtime-variable shift amounts, we must generate a more complex
6625 // sequence that handles all possible shift values using select chains.
6626
6627 // Split the input into target-sized pieces
6628 SmallVector<Register, 8> SrcParts;
6629 extractParts(Reg: SrcReg, Ty: TargetTy, NumParts, VRegs&: SrcParts, MIRBuilder, MRI);
6630
6631 // Shifting by zero should be a no-op.
6632 auto ZeroAmtConst = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: 0);
6633 LLT BoolTy = LLT::scalar(SizeInBits: 1);
6634 auto IsZeroShift =
6635 MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: BoolTy, Op0: AmtReg, Op1: ZeroAmtConst);
6636
6637 // Any wide shift can be decomposed into two components:
6638 // 1. WordShift: number of complete target-sized words to shift
6639 // 2. BitShift: number of bits to shift within each word
6640 //
6641 // Example: 128-bit >> 50 with 32-bit target:
6642 // WordShift = 50 / 32 = 1 (shift right by 1 complete word)
6643 // BitShift = 50 % 32 = 18 (shift each word right by 18 bits)
6644 unsigned TargetBitsLog2 = Log2_32(Value: TargetBits);
6645 auto TargetBitsLog2Const =
6646 MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBitsLog2);
6647 auto TargetBitsMask = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBits - 1);
6648
6649 Register WordShift =
6650 MIRBuilder.buildLShr(Dst: ShiftAmtTy, Src0: AmtReg, Src1: TargetBitsLog2Const).getReg(Idx: 0);
6651 Register BitShift =
6652 MIRBuilder.buildAnd(Dst: ShiftAmtTy, Src0: AmtReg, Src1: TargetBitsMask).getReg(Idx: 0);
6653
6654 // Fill values:
6655 // - SHL/LSHR: fill with zeros
6656 // - ASHR: fill with sign-extended MSB
6657 Register ZeroReg = MIRBuilder.buildConstant(Res: TargetTy, Val: 0).getReg(Idx: 0);
6658
6659 Register FillValue;
6660 if (MI.getOpcode() == TargetOpcode::G_ASHR) {
6661 auto TargetBitsMinusOneConst =
6662 MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBits - 1);
6663 FillValue = MIRBuilder
6664 .buildAShr(Dst: TargetTy, Src0: SrcParts[NumParts - 1],
6665 Src1: TargetBitsMinusOneConst)
6666 .getReg(Idx: 0);
6667 } else {
6668 FillValue = ZeroReg;
6669 }
6670
6671 SmallVector<Register, 8> DstParts(NumParts);
6672
6673 // For each output part, generate a select chain that chooses the correct
6674 // result based on the runtime WordShift value. This handles all possible
6675 // word shift amounts by pre-calculating what each would produce.
6676 for (unsigned I = 0; I < NumParts; ++I) {
6677 // Initialize with appropriate default value for this shift type
6678 Register InBoundsResult = FillValue;
6679
6680 // clang-format off
6681 // Build a branchless select chain by pre-computing results for all possible
6682 // WordShift values (0 to NumParts-1). Each iteration nests a new select:
6683 //
6684 // K=0: select(WordShift==0, result0, FillValue)
6685 // K=1: select(WordShift==1, result1, select(WordShift==0, result0, FillValue))
6686 // K=2: select(WordShift==2, result2, select(WordShift==1, result1, select(...)))
6687 // clang-format on
6688 for (unsigned K = 0; K < NumParts; ++K) {
6689 auto WordShiftKConst = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: K);
6690 auto IsWordShiftK = MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: BoolTy,
6691 Op0: WordShift, Op1: WordShiftKConst);
6692
6693 // Calculate source indices for this word shift
6694 //
6695 // For 4-part 128-bit value with K=1 word shift:
6696 // SHL: [3][2][1][0] << K => [2][1][0][Z]
6697 // -> (MainIdx = I-K, CarryIdx = I-K-1)
6698 // LSHR: [3][2][1][0] >> K => [Z][3][2][1]
6699 // -> (MainIdx = I+K, CarryIdx = I+K+1)
6700 int MainSrcIdx;
6701 int CarrySrcIdx; // Index for the word that provides the carried-in bits.
6702
6703 switch (MI.getOpcode()) {
6704 case TargetOpcode::G_SHL:
6705 MainSrcIdx = (int)I - (int)K;
6706 CarrySrcIdx = MainSrcIdx - 1;
6707 break;
6708 case TargetOpcode::G_LSHR:
6709 case TargetOpcode::G_ASHR:
6710 MainSrcIdx = (int)I + (int)K;
6711 CarrySrcIdx = MainSrcIdx + 1;
6712 break;
6713 default:
6714 llvm_unreachable("Not a shift");
6715 }
6716
6717 // Check bounds and build the result for this word shift
6718 Register ResultForK;
6719 if (MainSrcIdx >= 0 && MainSrcIdx < (int)NumParts) {
6720 Register MainOp = SrcParts[MainSrcIdx];
6721 Register CarryOp;
6722
6723 // Determine carry operand with bounds checking
6724 if (CarrySrcIdx >= 0 && CarrySrcIdx < (int)NumParts)
6725 CarryOp = SrcParts[CarrySrcIdx];
6726 else if (MI.getOpcode() == TargetOpcode::G_ASHR &&
6727 CarrySrcIdx >= (int)NumParts)
6728 CarryOp = FillValue; // Use sign extension
6729
6730 ResultForK = buildVariableShiftPart(Opcode: MI.getOpcode(), MainOperand: MainOp, ShiftAmt: BitShift,
6731 TargetTy, CarryOperand: CarryOp);
6732 } else {
6733 // Out of bounds - use fill value for this k
6734 ResultForK = FillValue;
6735 }
6736
6737 // Select this result if WordShift equals k
6738 InBoundsResult =
6739 MIRBuilder
6740 .buildSelect(Res: TargetTy, Tst: IsWordShiftK, Op0: ResultForK, Op1: InBoundsResult)
6741 .getReg(Idx: 0);
6742 }
6743
6744 // Handle zero-shift special case: if shift is 0, use original input
6745 DstParts[I] =
6746 MIRBuilder
6747 .buildSelect(Res: TargetTy, Tst: IsZeroShift, Op0: SrcParts[I], Op1: InBoundsResult)
6748 .getReg(Idx: 0);
6749 }
6750
6751 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstParts);
6752 MI.eraseFromParent();
6753 return Legalized;
6754}
6755
6756LegalizerHelper::LegalizeResult
6757LegalizerHelper::moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
6758 LLT MoreTy) {
6759 assert(TypeIdx == 0 && "Expecting only Idx 0");
6760
6761 Observer.changingInstr(MI);
6762 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
6763 MachineBasicBlock &OpMBB = *MI.getOperand(i: I + 1).getMBB();
6764 MIRBuilder.setInsertPt(MBB&: OpMBB, II: OpMBB.getFirstTerminator());
6765 moreElementsVectorSrc(MI, MoreTy, OpIdx: I);
6766 }
6767
6768 MachineBasicBlock &MBB = *MI.getParent();
6769 MIRBuilder.setInsertPt(MBB, II: --MBB.getFirstNonPHI());
6770 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
6771 Observer.changedInstr(MI);
6772 return Legalized;
6773}
6774
6775MachineInstrBuilder LegalizerHelper::getNeutralElementForVecReduce(
6776 unsigned Opcode, MachineIRBuilder &MIRBuilder, LLT Ty) {
6777 assert(Ty.isScalar() && "Expected scalar type to make neutral element for");
6778
6779 switch (Opcode) {
6780 default:
6781 llvm_unreachable(
6782 "getNeutralElementForVecReduce called with invalid opcode!");
6783 case TargetOpcode::G_VECREDUCE_ADD:
6784 case TargetOpcode::G_VECREDUCE_OR:
6785 case TargetOpcode::G_VECREDUCE_XOR:
6786 case TargetOpcode::G_VECREDUCE_UMAX:
6787 return MIRBuilder.buildConstant(Res: Ty, Val: 0);
6788 case TargetOpcode::G_VECREDUCE_MUL:
6789 return MIRBuilder.buildConstant(Res: Ty, Val: 1);
6790 case TargetOpcode::G_VECREDUCE_AND:
6791 case TargetOpcode::G_VECREDUCE_UMIN:
6792 return MIRBuilder.buildConstant(
6793 Res: Ty, Val: APInt::getAllOnes(numBits: Ty.getScalarSizeInBits()));
6794 case TargetOpcode::G_VECREDUCE_SMAX:
6795 return MIRBuilder.buildConstant(
6796 Res: Ty, Val: APInt::getSignedMinValue(numBits: Ty.getSizeInBits()));
6797 case TargetOpcode::G_VECREDUCE_SMIN:
6798 return MIRBuilder.buildConstant(
6799 Res: Ty, Val: APInt::getSignedMaxValue(numBits: Ty.getSizeInBits()));
6800 case TargetOpcode::G_VECREDUCE_FADD:
6801 return MIRBuilder.buildFConstant(Res: Ty, Val: -0.0);
6802 case TargetOpcode::G_VECREDUCE_FMUL:
6803 return MIRBuilder.buildFConstant(Res: Ty, Val: 1.0);
6804 case TargetOpcode::G_VECREDUCE_FMINIMUM:
6805 case TargetOpcode::G_VECREDUCE_FMAXIMUM:
6806 assert(false && "getNeutralElementForVecReduce unimplemented for "
6807 "G_VECREDUCE_FMINIMUM and G_VECREDUCE_FMAXIMUM!");
6808 }
6809 llvm_unreachable("switch expected to return!");
6810}
6811
6812LegalizerHelper::LegalizeResult
6813LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
6814 LLT MoreTy) {
6815 unsigned Opc = MI.getOpcode();
6816 switch (Opc) {
6817 case TargetOpcode::G_IMPLICIT_DEF:
6818 case TargetOpcode::G_LOAD: {
6819 if (TypeIdx != 0)
6820 return UnableToLegalize;
6821 Observer.changingInstr(MI);
6822 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
6823 Observer.changedInstr(MI);
6824 return Legalized;
6825 }
6826 case TargetOpcode::G_STORE:
6827 if (TypeIdx != 0)
6828 return UnableToLegalize;
6829 Observer.changingInstr(MI);
6830 moreElementsVectorSrc(MI, MoreTy, OpIdx: 0);
6831 Observer.changedInstr(MI);
6832 return Legalized;
6833 case TargetOpcode::G_AND:
6834 case TargetOpcode::G_OR:
6835 case TargetOpcode::G_XOR:
6836 case TargetOpcode::G_ADD:
6837 case TargetOpcode::G_SUB:
6838 case TargetOpcode::G_MUL:
6839 case TargetOpcode::G_FADD:
6840 case TargetOpcode::G_FSUB:
6841 case TargetOpcode::G_FMUL:
6842 case TargetOpcode::G_FDIV:
6843 case TargetOpcode::G_FCOPYSIGN:
6844 case TargetOpcode::G_UADDSAT:
6845 case TargetOpcode::G_USUBSAT:
6846 case TargetOpcode::G_SADDSAT:
6847 case TargetOpcode::G_SSUBSAT:
6848 case TargetOpcode::G_SMIN:
6849 case TargetOpcode::G_SMAX:
6850 case TargetOpcode::G_UMIN:
6851 case TargetOpcode::G_UMAX:
6852 case TargetOpcode::G_FMINNUM:
6853 case TargetOpcode::G_FMAXNUM:
6854 case TargetOpcode::G_FMINNUM_IEEE:
6855 case TargetOpcode::G_FMAXNUM_IEEE:
6856 case TargetOpcode::G_FMINIMUM:
6857 case TargetOpcode::G_FMAXIMUM:
6858 case TargetOpcode::G_FMINIMUMNUM:
6859 case TargetOpcode::G_FMAXIMUMNUM:
6860 case TargetOpcode::G_STRICT_FADD:
6861 case TargetOpcode::G_STRICT_FSUB:
6862 case TargetOpcode::G_STRICT_FMUL: {
6863 Observer.changingInstr(MI);
6864 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
6865 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
6866 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
6867 Observer.changedInstr(MI);
6868 return Legalized;
6869 }
6870 case TargetOpcode::G_SHL:
6871 case TargetOpcode::G_ASHR:
6872 case TargetOpcode::G_LSHR: {
6873 Observer.changingInstr(MI);
6874 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
6875 // The shift operand may have a different scalar type from the source and
6876 // destination operands.
6877 LLT ShiftMoreTy = MoreTy.changeElementType(
6878 NewEltTy: MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getElementType());
6879 moreElementsVectorSrc(MI, MoreTy: ShiftMoreTy, OpIdx: 2);
6880 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
6881 Observer.changedInstr(MI);
6882 return Legalized;
6883 }
6884 case TargetOpcode::G_FMA:
6885 case TargetOpcode::G_STRICT_FMA:
6886 case TargetOpcode::G_FSHR:
6887 case TargetOpcode::G_FSHL: {
6888 Observer.changingInstr(MI);
6889 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
6890 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
6891 moreElementsVectorSrc(MI, MoreTy, OpIdx: 3);
6892 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
6893 Observer.changedInstr(MI);
6894 return Legalized;
6895 }
6896 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
6897 case TargetOpcode::G_EXTRACT:
6898 if (TypeIdx != 1)
6899 return UnableToLegalize;
6900 Observer.changingInstr(MI);
6901 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
6902 Observer.changedInstr(MI);
6903 return Legalized;
6904 case TargetOpcode::G_INSERT:
6905 case TargetOpcode::G_INSERT_VECTOR_ELT:
6906 case TargetOpcode::G_FREEZE:
6907 case TargetOpcode::G_FNEG:
6908 case TargetOpcode::G_FABS:
6909 case TargetOpcode::G_FSQRT:
6910 case TargetOpcode::G_FCEIL:
6911 case TargetOpcode::G_FFLOOR:
6912 case TargetOpcode::G_FNEARBYINT:
6913 case TargetOpcode::G_FRINT:
6914 case TargetOpcode::G_INTRINSIC_ROUND:
6915 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
6916 case TargetOpcode::G_INTRINSIC_TRUNC:
6917 case TargetOpcode::G_BITREVERSE:
6918 case TargetOpcode::G_BSWAP:
6919 case TargetOpcode::G_FCANONICALIZE:
6920 case TargetOpcode::G_SEXT_INREG:
6921 case TargetOpcode::G_ABS:
6922 case TargetOpcode::G_CTLZ:
6923 case TargetOpcode::G_CTPOP:
6924 if (TypeIdx != 0)
6925 return UnableToLegalize;
6926 Observer.changingInstr(MI);
6927 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
6928 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
6929 Observer.changedInstr(MI);
6930 return Legalized;
6931 case TargetOpcode::G_SELECT: {
6932 auto [DstReg, DstTy, CondReg, CondTy] = MI.getFirst2RegLLTs();
6933 if (TypeIdx == 1) {
6934 if (!CondTy.isScalar() ||
6935 DstTy.getElementCount() != MoreTy.getElementCount())
6936 return UnableToLegalize;
6937
6938 // This is turning a scalar select of vectors into a vector
6939 // select. Broadcast the select condition.
6940 auto ShufSplat = MIRBuilder.buildShuffleSplat(Res: MoreTy, Src: CondReg);
6941 Observer.changingInstr(MI);
6942 MI.getOperand(i: 1).setReg(ShufSplat.getReg(Idx: 0));
6943 Observer.changedInstr(MI);
6944 return Legalized;
6945 }
6946
6947 if (CondTy.isVector())
6948 return UnableToLegalize;
6949
6950 Observer.changingInstr(MI);
6951 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
6952 moreElementsVectorSrc(MI, MoreTy, OpIdx: 3);
6953 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
6954 Observer.changedInstr(MI);
6955 return Legalized;
6956 }
6957 case TargetOpcode::G_UNMERGE_VALUES:
6958 return UnableToLegalize;
6959 case TargetOpcode::G_PHI:
6960 return moreElementsVectorPhi(MI, TypeIdx, MoreTy);
6961 case TargetOpcode::G_SHUFFLE_VECTOR:
6962 return moreElementsVectorShuffle(MI, TypeIdx, MoreTy);
6963 case TargetOpcode::G_BUILD_VECTOR: {
6964 SmallVector<SrcOp, 8> Elts;
6965 for (auto Op : MI.uses()) {
6966 Elts.push_back(Elt: Op.getReg());
6967 }
6968
6969 for (unsigned i = Elts.size(); i < MoreTy.getNumElements(); ++i) {
6970 Elts.push_back(Elt: MIRBuilder.buildUndef(Res: MoreTy.getScalarType()));
6971 }
6972
6973 MIRBuilder.buildDeleteTrailingVectorElements(
6974 Res: MI.getOperand(i: 0).getReg(), Op0: MIRBuilder.buildInstr(Opc, DstOps: {MoreTy}, SrcOps: Elts));
6975 MI.eraseFromParent();
6976 return Legalized;
6977 }
6978 case TargetOpcode::G_SEXT:
6979 case TargetOpcode::G_ZEXT:
6980 case TargetOpcode::G_ANYEXT:
6981 case TargetOpcode::G_TRUNC:
6982 case TargetOpcode::G_FPTRUNC:
6983 case TargetOpcode::G_FPEXT:
6984 case TargetOpcode::G_FPTOSI:
6985 case TargetOpcode::G_FPTOUI:
6986 case TargetOpcode::G_FPTOSI_SAT:
6987 case TargetOpcode::G_FPTOUI_SAT:
6988 case TargetOpcode::G_SITOFP:
6989 case TargetOpcode::G_UITOFP: {
6990 Observer.changingInstr(MI);
6991 LLT SrcExtTy;
6992 LLT DstExtTy;
6993 if (TypeIdx == 0) {
6994 DstExtTy = MoreTy;
6995 SrcExtTy = MoreTy.changeElementType(
6996 NewEltTy: MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getElementType());
6997 } else {
6998 DstExtTy = MoreTy.changeElementType(
6999 NewEltTy: MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getElementType());
7000 SrcExtTy = MoreTy;
7001 }
7002 moreElementsVectorSrc(MI, MoreTy: SrcExtTy, OpIdx: 1);
7003 moreElementsVectorDst(MI, WideTy: DstExtTy, OpIdx: 0);
7004 Observer.changedInstr(MI);
7005 return Legalized;
7006 }
7007 case TargetOpcode::G_ICMP:
7008 case TargetOpcode::G_FCMP: {
7009 if (TypeIdx != 1)
7010 return UnableToLegalize;
7011
7012 Observer.changingInstr(MI);
7013 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
7014 moreElementsVectorSrc(MI, MoreTy, OpIdx: 3);
7015 LLT CondTy = MoreTy.changeVectorElementType(
7016 NewEltTy: MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getElementType());
7017 moreElementsVectorDst(MI, WideTy: CondTy, OpIdx: 0);
7018 Observer.changedInstr(MI);
7019 return Legalized;
7020 }
7021 case TargetOpcode::G_BITCAST: {
7022 if (TypeIdx != 0)
7023 return UnableToLegalize;
7024
7025 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
7026 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7027
7028 unsigned coefficient = SrcTy.getNumElements() * MoreTy.getNumElements();
7029 if (coefficient % DstTy.getNumElements() != 0)
7030 return UnableToLegalize;
7031
7032 coefficient = coefficient / DstTy.getNumElements();
7033
7034 LLT NewTy = SrcTy.changeElementCount(
7035 EC: ElementCount::get(MinVal: coefficient, Scalable: MoreTy.isScalable()));
7036 Observer.changingInstr(MI);
7037 moreElementsVectorSrc(MI, MoreTy: NewTy, OpIdx: 1);
7038 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7039 Observer.changedInstr(MI);
7040 return Legalized;
7041 }
7042 case TargetOpcode::G_VECREDUCE_FADD:
7043 case TargetOpcode::G_VECREDUCE_FMUL:
7044 case TargetOpcode::G_VECREDUCE_ADD:
7045 case TargetOpcode::G_VECREDUCE_MUL:
7046 case TargetOpcode::G_VECREDUCE_AND:
7047 case TargetOpcode::G_VECREDUCE_OR:
7048 case TargetOpcode::G_VECREDUCE_XOR:
7049 case TargetOpcode::G_VECREDUCE_SMAX:
7050 case TargetOpcode::G_VECREDUCE_SMIN:
7051 case TargetOpcode::G_VECREDUCE_UMAX:
7052 case TargetOpcode::G_VECREDUCE_UMIN: {
7053 LLT OrigTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
7054 MachineOperand &MO = MI.getOperand(i: 1);
7055 auto NewVec = MIRBuilder.buildPadVectorWithUndefElements(Res: MoreTy, Op0: MO);
7056 auto NeutralElement = getNeutralElementForVecReduce(
7057 Opcode: MI.getOpcode(), MIRBuilder, Ty: MoreTy.getElementType());
7058
7059 LLT IdxTy(TLI.getVectorIdxLLT(DL: MIRBuilder.getDataLayout()));
7060 for (size_t i = OrigTy.getNumElements(), e = MoreTy.getNumElements();
7061 i != e; i++) {
7062 auto Idx = MIRBuilder.buildConstant(Res: IdxTy, Val: i);
7063 NewVec = MIRBuilder.buildInsertVectorElement(Res: MoreTy, Val: NewVec,
7064 Elt: NeutralElement, Idx);
7065 }
7066
7067 Observer.changingInstr(MI);
7068 MO.setReg(NewVec.getReg(Idx: 0));
7069 Observer.changedInstr(MI);
7070 return Legalized;
7071 }
7072
7073 default:
7074 return UnableToLegalize;
7075 }
7076}
7077
7078LegalizerHelper::LegalizeResult
7079LegalizerHelper::equalizeVectorShuffleLengths(MachineInstr &MI) {
7080 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7081 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
7082 unsigned MaskNumElts = Mask.size();
7083 unsigned SrcNumElts = SrcTy.getNumElements();
7084 LLT DestEltTy = DstTy.getElementType();
7085
7086 if (MaskNumElts == SrcNumElts)
7087 return Legalized;
7088
7089 if (MaskNumElts < SrcNumElts) {
7090 // Extend mask to match new destination vector size with
7091 // undef values.
7092 SmallVector<int, 16> NewMask(SrcNumElts, -1);
7093 llvm::copy(Range&: Mask, Out: NewMask.begin());
7094
7095 moreElementsVectorDst(MI, WideTy: SrcTy, OpIdx: 0);
7096 MIRBuilder.setInstrAndDebugLoc(MI);
7097 MIRBuilder.buildShuffleVector(Res: MI.getOperand(i: 0).getReg(),
7098 Src1: MI.getOperand(i: 1).getReg(),
7099 Src2: MI.getOperand(i: 2).getReg(), Mask: NewMask);
7100 MI.eraseFromParent();
7101
7102 return Legalized;
7103 }
7104
7105 unsigned PaddedMaskNumElts = alignTo(Value: MaskNumElts, Align: SrcNumElts);
7106 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
7107 LLT PaddedTy =
7108 DstTy.changeVectorElementCount(EC: ElementCount::getFixed(MinVal: PaddedMaskNumElts));
7109
7110 // Create new source vectors by concatenating the initial
7111 // source vectors with undefined vectors of the same size.
7112 auto Undef = MIRBuilder.buildUndef(Res: SrcTy);
7113 SmallVector<Register, 8> MOps1(NumConcat, Undef.getReg(Idx: 0));
7114 SmallVector<Register, 8> MOps2(NumConcat, Undef.getReg(Idx: 0));
7115 MOps1[0] = MI.getOperand(i: 1).getReg();
7116 MOps2[0] = MI.getOperand(i: 2).getReg();
7117
7118 auto Src1 = MIRBuilder.buildConcatVectors(Res: PaddedTy, Ops: MOps1);
7119 auto Src2 = MIRBuilder.buildConcatVectors(Res: PaddedTy, Ops: MOps2);
7120
7121 // Readjust mask for new input vector length.
7122 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
7123 for (unsigned I = 0; I != MaskNumElts; ++I) {
7124 int Idx = Mask[I];
7125 if (Idx >= static_cast<int>(SrcNumElts))
7126 Idx += PaddedMaskNumElts - SrcNumElts;
7127 MappedOps[I] = Idx;
7128 }
7129
7130 // If we got more elements than required, extract subvector.
7131 if (MaskNumElts != PaddedMaskNumElts) {
7132 auto Shuffle =
7133 MIRBuilder.buildShuffleVector(Res: PaddedTy, Src1, Src2, Mask: MappedOps);
7134
7135 SmallVector<Register, 16> Elts(MaskNumElts);
7136 for (unsigned I = 0; I < MaskNumElts; ++I) {
7137 Elts[I] =
7138 MIRBuilder.buildExtractVectorElementConstant(Res: DestEltTy, Val: Shuffle, Idx: I)
7139 .getReg(Idx: 0);
7140 }
7141 MIRBuilder.buildBuildVector(Res: DstReg, Ops: Elts);
7142 } else {
7143 MIRBuilder.buildShuffleVector(Res: DstReg, Src1, Src2, Mask: MappedOps);
7144 }
7145
7146 MI.eraseFromParent();
7147 return LegalizerHelper::LegalizeResult::Legalized;
7148}
7149
7150LegalizerHelper::LegalizeResult
7151LegalizerHelper::moreElementsVectorShuffle(MachineInstr &MI,
7152 unsigned int TypeIdx, LLT MoreTy) {
7153 auto [DstTy, Src1Ty, Src2Ty] = MI.getFirst3LLTs();
7154 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
7155 unsigned NumElts = DstTy.getNumElements();
7156 unsigned WidenNumElts = MoreTy.getNumElements();
7157
7158 if (DstTy.isVector() && Src1Ty.isVector() &&
7159 DstTy.getNumElements() != Src1Ty.getNumElements()) {
7160 return equalizeVectorShuffleLengths(MI);
7161 }
7162
7163 if (TypeIdx != 0)
7164 return UnableToLegalize;
7165
7166 // Expect a canonicalized shuffle.
7167 if (DstTy != Src1Ty || DstTy != Src2Ty)
7168 return UnableToLegalize;
7169
7170 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
7171 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
7172
7173 // Adjust mask based on new input vector length.
7174 SmallVector<int, 16> NewMask(WidenNumElts, -1);
7175 for (unsigned I = 0; I != NumElts; ++I) {
7176 int Idx = Mask[I];
7177 if (Idx < static_cast<int>(NumElts))
7178 NewMask[I] = Idx;
7179 else
7180 NewMask[I] = Idx - NumElts + WidenNumElts;
7181 }
7182 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7183 MIRBuilder.setInstrAndDebugLoc(MI);
7184 MIRBuilder.buildShuffleVector(Res: MI.getOperand(i: 0).getReg(),
7185 Src1: MI.getOperand(i: 1).getReg(),
7186 Src2: MI.getOperand(i: 2).getReg(), Mask: NewMask);
7187 MI.eraseFromParent();
7188 return Legalized;
7189}
7190
7191void LegalizerHelper::multiplyRegisters(SmallVectorImpl<Register> &DstRegs,
7192 ArrayRef<Register> Src1Regs,
7193 ArrayRef<Register> Src2Regs,
7194 LLT NarrowTy) {
7195 MachineIRBuilder &B = MIRBuilder;
7196 unsigned SrcParts = Src1Regs.size();
7197 unsigned DstParts = DstRegs.size();
7198
7199 unsigned DstIdx = 0; // Low bits of the result.
7200 Register FactorSum =
7201 B.buildMul(Dst: NarrowTy, Src0: Src1Regs[DstIdx], Src1: Src2Regs[DstIdx]).getReg(Idx: 0);
7202 DstRegs[DstIdx] = FactorSum;
7203
7204 Register CarrySumPrevDstIdx;
7205 SmallVector<Register, 4> Factors;
7206
7207 for (DstIdx = 1; DstIdx < DstParts; DstIdx++) {
7208 // Collect high parts of muls from previous DstIdx.
7209 for (unsigned i = DstIdx < SrcParts ? 0 : DstIdx - SrcParts;
7210 i <= std::min(a: DstIdx - 1, b: SrcParts - 1); ++i) {
7211 MachineInstrBuilder Umulh =
7212 B.buildUMulH(Dst: NarrowTy, Src0: Src1Regs[DstIdx - 1 - i], Src1: Src2Regs[i]);
7213 Factors.push_back(Elt: Umulh.getReg(Idx: 0));
7214 }
7215 // Collect low parts of muls for DstIdx. Visit the diagonal starting with
7216 // the low Src1 part, so multiply-add selectors can use it as the first
7217 // accumulated cross product.
7218 unsigned LowStart = DstIdx + 1 < SrcParts ? 0 : DstIdx - SrcParts + 1;
7219 unsigned LowEnd = std::min(a: DstIdx, b: SrcParts - 1);
7220 for (unsigned RevI = LowEnd + 1; RevI != LowStart; --RevI) {
7221 unsigned i = RevI - 1;
7222 MachineInstrBuilder Mul =
7223 B.buildMul(Dst: NarrowTy, Src0: Src1Regs[DstIdx - i], Src1: Src2Regs[i]);
7224 Factors.push_back(Elt: Mul.getReg(Idx: 0));
7225 }
7226 // Add CarrySum from additions calculated for previous DstIdx.
7227 if (DstIdx != 1) {
7228 Factors.push_back(Elt: CarrySumPrevDstIdx);
7229 }
7230
7231 Register CarrySum;
7232 // Add all factors and accumulate all carries into CarrySum.
7233 if (DstIdx != DstParts - 1) {
7234 MachineInstrBuilder Uaddo =
7235 B.buildUAddo(Res: NarrowTy, CarryOut: LLT::integer(SizeInBits: 1), Op0: Factors[0], Op1: Factors[1]);
7236 FactorSum = Uaddo.getReg(Idx: 0);
7237 CarrySum = B.buildZExt(Res: NarrowTy, Op: Uaddo.getReg(Idx: 1)).getReg(Idx: 0);
7238 for (unsigned i = 2; i < Factors.size(); ++i) {
7239 MachineInstrBuilder Uaddo =
7240 B.buildUAddo(Res: NarrowTy, CarryOut: LLT::integer(SizeInBits: 1), Op0: FactorSum, Op1: Factors[i]);
7241 FactorSum = Uaddo.getReg(Idx: 0);
7242 MachineInstrBuilder Carry = B.buildZExt(Res: NarrowTy, Op: Uaddo.getReg(Idx: 1));
7243 CarrySum = B.buildAdd(Dst: NarrowTy, Src0: CarrySum, Src1: Carry).getReg(Idx: 0);
7244 }
7245 } else {
7246 // Since value for the next index is not calculated, neither is CarrySum.
7247 FactorSum = B.buildAdd(Dst: NarrowTy, Src0: Factors[0], Src1: Factors[1]).getReg(Idx: 0);
7248 for (unsigned i = 2; i < Factors.size(); ++i)
7249 FactorSum = B.buildAdd(Dst: NarrowTy, Src0: FactorSum, Src1: Factors[i]).getReg(Idx: 0);
7250 }
7251
7252 CarrySumPrevDstIdx = CarrySum;
7253 DstRegs[DstIdx] = FactorSum;
7254 Factors.clear();
7255 }
7256}
7257
7258LegalizerHelper::LegalizeResult
7259LegalizerHelper::narrowScalarAddSub(MachineInstr &MI, unsigned TypeIdx,
7260 LLT NarrowTy) {
7261 if (TypeIdx != 0)
7262 return UnableToLegalize;
7263
7264 Register DstReg = MI.getOperand(i: 0).getReg();
7265 LLT DstType = MRI.getType(Reg: DstReg);
7266 // FIXME: add support for vector types
7267 if (DstType.isVector())
7268 return UnableToLegalize;
7269
7270 unsigned Opcode = MI.getOpcode();
7271 unsigned OpO, OpE, OpF;
7272 switch (Opcode) {
7273 case TargetOpcode::G_SADDO:
7274 case TargetOpcode::G_SADDE:
7275 case TargetOpcode::G_UADDO:
7276 case TargetOpcode::G_UADDE:
7277 case TargetOpcode::G_ADD:
7278 OpO = TargetOpcode::G_UADDO;
7279 OpE = TargetOpcode::G_UADDE;
7280 OpF = TargetOpcode::G_UADDE;
7281 if (Opcode == TargetOpcode::G_SADDO || Opcode == TargetOpcode::G_SADDE)
7282 OpF = TargetOpcode::G_SADDE;
7283 break;
7284 case TargetOpcode::G_SSUBO:
7285 case TargetOpcode::G_SSUBE:
7286 case TargetOpcode::G_USUBO:
7287 case TargetOpcode::G_USUBE:
7288 case TargetOpcode::G_SUB:
7289 OpO = TargetOpcode::G_USUBO;
7290 OpE = TargetOpcode::G_USUBE;
7291 OpF = TargetOpcode::G_USUBE;
7292 if (Opcode == TargetOpcode::G_SSUBO || Opcode == TargetOpcode::G_SSUBE)
7293 OpF = TargetOpcode::G_SSUBE;
7294 break;
7295 default:
7296 llvm_unreachable("Unexpected add/sub opcode!");
7297 }
7298
7299 // 1 for a plain add/sub, 2 if this is an operation with a carry-out.
7300 unsigned NumDefs = MI.getNumExplicitDefs();
7301 Register Src1 = MI.getOperand(i: NumDefs).getReg();
7302 Register Src2 = MI.getOperand(i: NumDefs + 1).getReg();
7303 Register CarryDst, CarryIn;
7304 if (NumDefs == 2)
7305 CarryDst = MI.getOperand(i: 1).getReg();
7306 if (MI.getNumOperands() == NumDefs + 3)
7307 CarryIn = MI.getOperand(i: NumDefs + 2).getReg();
7308
7309 LLT RegTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7310 LLT LeftoverTy, DummyTy;
7311 SmallVector<Register, 2> Src1Regs, Src2Regs, Src1Left, Src2Left, DstRegs;
7312 extractParts(Reg: Src1, RegTy, MainTy: NarrowTy, LeftoverTy, VRegs&: Src1Regs, LeftoverVRegs&: Src1Left,
7313 MIRBuilder, MRI);
7314 extractParts(Reg: Src2, RegTy, MainTy: NarrowTy, LeftoverTy&: DummyTy, VRegs&: Src2Regs, LeftoverVRegs&: Src2Left, MIRBuilder,
7315 MRI);
7316
7317 int NarrowParts = Src1Regs.size();
7318 Src1Regs.append(RHS: Src1Left);
7319 Src2Regs.append(RHS: Src2Left);
7320 DstRegs.reserve(N: Src1Regs.size());
7321
7322 for (int i = 0, e = Src1Regs.size(); i != e; ++i) {
7323 Register DstReg =
7324 MRI.createGenericVirtualRegister(Ty: MRI.getType(Reg: Src1Regs[i]));
7325 Register CarryOut;
7326 // Forward the final carry-out to the destination register
7327 if (i == e - 1 && CarryDst)
7328 CarryOut = CarryDst;
7329 else
7330 CarryOut = MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: 1));
7331
7332 if (!CarryIn) {
7333 MIRBuilder.buildInstr(Opc: OpO, DstOps: {DstReg, CarryOut},
7334 SrcOps: {Src1Regs[i], Src2Regs[i]});
7335 } else if (i == e - 1) {
7336 MIRBuilder.buildInstr(Opc: OpF, DstOps: {DstReg, CarryOut},
7337 SrcOps: {Src1Regs[i], Src2Regs[i], CarryIn});
7338 } else {
7339 MIRBuilder.buildInstr(Opc: OpE, DstOps: {DstReg, CarryOut},
7340 SrcOps: {Src1Regs[i], Src2Regs[i], CarryIn});
7341 }
7342
7343 DstRegs.push_back(Elt: DstReg);
7344 CarryIn = CarryOut;
7345 }
7346 insertParts(DstReg: MI.getOperand(i: 0).getReg(), ResultTy: RegTy, PartTy: NarrowTy,
7347 PartRegs: ArrayRef(DstRegs).take_front(N: NarrowParts), LeftoverTy,
7348 LeftoverRegs: ArrayRef(DstRegs).drop_front(N: NarrowParts));
7349
7350 MI.eraseFromParent();
7351 return Legalized;
7352}
7353
7354LegalizerHelper::LegalizeResult
7355LegalizerHelper::narrowScalarMul(MachineInstr &MI, LLT NarrowTy) {
7356 auto [DstReg, Src1, Src2] = MI.getFirst3Regs();
7357
7358 LLT Ty = MRI.getType(Reg: DstReg);
7359 if (Ty.isVector())
7360 return UnableToLegalize;
7361
7362 unsigned Size = Ty.getSizeInBits();
7363 unsigned NarrowSize = NarrowTy.getSizeInBits();
7364 if (Size % NarrowSize != 0)
7365 return UnableToLegalize;
7366
7367 unsigned NumParts = Size / NarrowSize;
7368 bool IsMulHigh = MI.getOpcode() == TargetOpcode::G_UMULH;
7369 unsigned DstTmpParts = NumParts * (IsMulHigh ? 2 : 1);
7370
7371 SmallVector<Register, 2> Src1Parts, Src2Parts;
7372 SmallVector<Register, 2> DstTmpRegs(DstTmpParts);
7373 extractParts(Reg: Src1, Ty: NarrowTy, NumParts, VRegs&: Src1Parts, MIRBuilder, MRI);
7374 extractParts(Reg: Src2, Ty: NarrowTy, NumParts, VRegs&: Src2Parts, MIRBuilder, MRI);
7375 multiplyRegisters(DstRegs&: DstTmpRegs, Src1Regs: Src1Parts, Src2Regs: Src2Parts, NarrowTy);
7376
7377 // Take only high half of registers if this is high mul.
7378 ArrayRef<Register> DstRegs(&DstTmpRegs[DstTmpParts - NumParts], NumParts);
7379 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
7380 MI.eraseFromParent();
7381 return Legalized;
7382}
7383
7384LegalizerHelper::LegalizeResult
7385LegalizerHelper::narrowScalarFPTOI(MachineInstr &MI, unsigned TypeIdx,
7386 LLT NarrowTy) {
7387 if (TypeIdx != 0)
7388 return UnableToLegalize;
7389
7390 bool IsSigned = MI.getOpcode() == TargetOpcode::G_FPTOSI;
7391
7392 Register Src = MI.getOperand(i: 1).getReg();
7393 LLT SrcTy = MRI.getType(Reg: Src);
7394
7395 // If all finite floats fit into the narrowed integer type, we can just swap
7396 // out the result type. This is practically only useful for conversions from
7397 // half to at least 16-bits, so just handle the one case.
7398 if (SrcTy.getScalarType() != LLT::scalar(SizeInBits: 16) ||
7399 NarrowTy.getScalarSizeInBits() < (IsSigned ? 17u : 16u))
7400 return UnableToLegalize;
7401
7402 Observer.changingInstr(MI);
7403 narrowScalarDst(MI, NarrowTy, OpIdx: 0,
7404 ExtOpcode: IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT);
7405 Observer.changedInstr(MI);
7406 return Legalized;
7407}
7408
7409LegalizerHelper::LegalizeResult
7410LegalizerHelper::narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx,
7411 LLT NarrowTy) {
7412 if (TypeIdx != 1)
7413 return UnableToLegalize;
7414
7415 uint64_t NarrowSize = NarrowTy.getSizeInBits();
7416
7417 int64_t SizeOp1 = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
7418 // FIXME: add support for when SizeOp1 isn't an exact multiple of
7419 // NarrowSize.
7420 if (SizeOp1 % NarrowSize != 0)
7421 return UnableToLegalize;
7422 int NumParts = SizeOp1 / NarrowSize;
7423
7424 SmallVector<Register, 2> SrcRegs, DstRegs;
7425 extractParts(Reg: MI.getOperand(i: 1).getReg(), Ty: NarrowTy, NumParts, VRegs&: SrcRegs,
7426 MIRBuilder, MRI);
7427
7428 Register OpReg = MI.getOperand(i: 0).getReg();
7429 uint64_t OpStart = MI.getOperand(i: 2).getImm();
7430 uint64_t OpSize = MRI.getType(Reg: OpReg).getSizeInBits();
7431 for (int i = 0; i < NumParts; ++i) {
7432 unsigned SrcStart = i * NarrowSize;
7433
7434 if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
7435 // No part of the extract uses this subregister, ignore it.
7436 continue;
7437 } else if (SrcStart == OpStart && NarrowTy == MRI.getType(Reg: OpReg)) {
7438 // The entire subregister is extracted, forward the value.
7439 DstRegs.push_back(Elt: SrcRegs[i]);
7440 continue;
7441 }
7442
7443 // OpSegStart is where this destination segment would start in OpReg if it
7444 // extended infinitely in both directions.
7445 int64_t ExtractOffset;
7446 uint64_t SegSize;
7447 if (OpStart < SrcStart) {
7448 ExtractOffset = 0;
7449 SegSize = std::min(a: NarrowSize, b: OpStart + OpSize - SrcStart);
7450 } else {
7451 ExtractOffset = OpStart - SrcStart;
7452 SegSize = std::min(a: SrcStart + NarrowSize - OpStart, b: OpSize);
7453 }
7454
7455 Register SegReg = SrcRegs[i];
7456 if (ExtractOffset != 0 || SegSize != NarrowSize) {
7457 // A genuine extract is needed.
7458 SegReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: SegSize));
7459 MIRBuilder.buildExtract(Res: SegReg, Src: SrcRegs[i], Index: ExtractOffset);
7460 }
7461
7462 DstRegs.push_back(Elt: SegReg);
7463 }
7464
7465 Register DstReg = MI.getOperand(i: 0).getReg();
7466 if (MRI.getType(Reg: DstReg).isVector())
7467 MIRBuilder.buildBuildVector(Res: DstReg, Ops: DstRegs);
7468 else if (DstRegs.size() > 1)
7469 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
7470 else
7471 MIRBuilder.buildCopy(Res: DstReg, Op: DstRegs[0]);
7472 MI.eraseFromParent();
7473 return Legalized;
7474}
7475
7476LegalizerHelper::LegalizeResult
7477LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx,
7478 LLT NarrowTy) {
7479 // FIXME: Don't know how to handle secondary types yet.
7480 if (TypeIdx != 0)
7481 return UnableToLegalize;
7482
7483 SmallVector<Register, 2> SrcRegs, LeftoverRegs, DstRegs;
7484 LLT RegTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7485 LLT LeftoverTy;
7486 extractParts(Reg: MI.getOperand(i: 1).getReg(), RegTy, MainTy: NarrowTy, LeftoverTy, VRegs&: SrcRegs,
7487 LeftoverVRegs&: LeftoverRegs, MIRBuilder, MRI);
7488
7489 SrcRegs.append(RHS: LeftoverRegs);
7490
7491 uint64_t NarrowSize = NarrowTy.getSizeInBits();
7492 Register OpReg = MI.getOperand(i: 2).getReg();
7493 uint64_t OpStart = MI.getOperand(i: 3).getImm();
7494 uint64_t OpSize = MRI.getType(Reg: OpReg).getSizeInBits();
7495 for (int I = 0, E = SrcRegs.size(); I != E; ++I) {
7496 unsigned DstStart = I * NarrowSize;
7497
7498 if (DstStart == OpStart && NarrowTy == MRI.getType(Reg: OpReg)) {
7499 // The entire subregister is defined by this insert, forward the new
7500 // value.
7501 DstRegs.push_back(Elt: OpReg);
7502 continue;
7503 }
7504
7505 Register SrcReg = SrcRegs[I];
7506 if (MRI.getType(Reg: SrcRegs[I]) == LeftoverTy) {
7507 // The leftover reg is smaller than NarrowTy, so we need to extend it.
7508 SrcReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
7509 MIRBuilder.buildAnyExt(Res: SrcReg, Op: SrcRegs[I]);
7510 }
7511
7512 if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
7513 // No part of the insert affects this subregister, forward the original.
7514 DstRegs.push_back(Elt: SrcReg);
7515 continue;
7516 }
7517
7518 // OpSegStart is where this destination segment would start in OpReg if it
7519 // extended infinitely in both directions.
7520 int64_t ExtractOffset, InsertOffset;
7521 uint64_t SegSize;
7522 if (OpStart < DstStart) {
7523 InsertOffset = 0;
7524 ExtractOffset = DstStart - OpStart;
7525 SegSize = std::min(a: NarrowSize, b: OpStart + OpSize - DstStart);
7526 } else {
7527 InsertOffset = OpStart - DstStart;
7528 ExtractOffset = 0;
7529 SegSize =
7530 std::min(a: NarrowSize - InsertOffset, b: OpStart + OpSize - DstStart);
7531 }
7532
7533 Register SegReg = OpReg;
7534 if (ExtractOffset != 0 || SegSize != OpSize) {
7535 // A genuine extract is needed.
7536 SegReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: SegSize));
7537 MIRBuilder.buildExtract(Res: SegReg, Src: OpReg, Index: ExtractOffset);
7538 }
7539
7540 Register DstReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
7541 MIRBuilder.buildInsert(Res: DstReg, Src: SrcReg, Op: SegReg, Index: InsertOffset);
7542 DstRegs.push_back(Elt: DstReg);
7543 }
7544
7545 uint64_t WideSize = DstRegs.size() * NarrowSize;
7546 Register DstReg = MI.getOperand(i: 0).getReg();
7547 if (WideSize > RegTy.getSizeInBits()) {
7548 Register MergeReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: WideSize));
7549 MIRBuilder.buildMergeLikeInstr(Res: MergeReg, Ops: DstRegs);
7550 MIRBuilder.buildTrunc(Res: DstReg, Op: MergeReg);
7551 } else
7552 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
7553
7554 MI.eraseFromParent();
7555 return Legalized;
7556}
7557
7558LegalizerHelper::LegalizeResult
7559LegalizerHelper::narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx,
7560 LLT NarrowTy) {
7561 Register DstReg = MI.getOperand(i: 0).getReg();
7562 LLT DstTy = MRI.getType(Reg: DstReg);
7563
7564 assert(MI.getNumOperands() == 3 && TypeIdx == 0);
7565
7566 SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
7567 SmallVector<Register, 4> Src0Regs, Src0LeftoverRegs;
7568 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
7569 LLT LeftoverTy;
7570 if (!extractParts(Reg: MI.getOperand(i: 1).getReg(), RegTy: DstTy, MainTy: NarrowTy, LeftoverTy,
7571 VRegs&: Src0Regs, LeftoverVRegs&: Src0LeftoverRegs, MIRBuilder, MRI))
7572 return UnableToLegalize;
7573
7574 LLT Unused;
7575 if (!extractParts(Reg: MI.getOperand(i: 2).getReg(), RegTy: DstTy, MainTy: NarrowTy, LeftoverTy&: Unused,
7576 VRegs&: Src1Regs, LeftoverVRegs&: Src1LeftoverRegs, MIRBuilder, MRI))
7577 llvm_unreachable("inconsistent extractParts result");
7578
7579 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
7580 auto Inst = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {NarrowTy},
7581 SrcOps: {Src0Regs[I], Src1Regs[I]});
7582 DstRegs.push_back(Elt: Inst.getReg(Idx: 0));
7583 }
7584
7585 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
7586 auto Inst = MIRBuilder.buildInstr(
7587 Opc: MI.getOpcode(),
7588 DstOps: {LeftoverTy}, SrcOps: {Src0LeftoverRegs[I], Src1LeftoverRegs[I]});
7589 DstLeftoverRegs.push_back(Elt: Inst.getReg(Idx: 0));
7590 }
7591
7592 insertParts(DstReg, ResultTy: DstTy, PartTy: NarrowTy, PartRegs: DstRegs,
7593 LeftoverTy, LeftoverRegs: DstLeftoverRegs);
7594
7595 MI.eraseFromParent();
7596 return Legalized;
7597}
7598
7599LegalizerHelper::LegalizeResult
7600LegalizerHelper::narrowScalarExt(MachineInstr &MI, unsigned TypeIdx,
7601 LLT NarrowTy) {
7602 if (TypeIdx != 0)
7603 return UnableToLegalize;
7604
7605 auto [DstReg, SrcReg] = MI.getFirst2Regs();
7606
7607 LLT DstTy = MRI.getType(Reg: DstReg);
7608 if (DstTy.isVector())
7609 return UnableToLegalize;
7610
7611 SmallVector<Register, 8> Parts;
7612 LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg);
7613 LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, VRegs&: Parts, PadStrategy: MI.getOpcode());
7614 buildWidenedRemergeToDst(DstReg, LCMTy, RemergeRegs: Parts);
7615
7616 MI.eraseFromParent();
7617 return Legalized;
7618}
7619
7620LegalizerHelper::LegalizeResult
7621LegalizerHelper::narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx,
7622 LLT NarrowTy) {
7623 if (TypeIdx != 0)
7624 return UnableToLegalize;
7625
7626 Register CondReg = MI.getOperand(i: 1).getReg();
7627 LLT CondTy = MRI.getType(Reg: CondReg);
7628 if (CondTy.isVector()) // TODO: Handle vselect
7629 return UnableToLegalize;
7630
7631 Register DstReg = MI.getOperand(i: 0).getReg();
7632 LLT DstTy = MRI.getType(Reg: DstReg);
7633
7634 SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
7635 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
7636 SmallVector<Register, 4> Src2Regs, Src2LeftoverRegs;
7637 LLT LeftoverTy;
7638 if (!extractParts(Reg: MI.getOperand(i: 2).getReg(), RegTy: DstTy, MainTy: NarrowTy, LeftoverTy,
7639 VRegs&: Src1Regs, LeftoverVRegs&: Src1LeftoverRegs, MIRBuilder, MRI))
7640 return UnableToLegalize;
7641
7642 LLT Unused;
7643 if (!extractParts(Reg: MI.getOperand(i: 3).getReg(), RegTy: DstTy, MainTy: NarrowTy, LeftoverTy&: Unused,
7644 VRegs&: Src2Regs, LeftoverVRegs&: Src2LeftoverRegs, MIRBuilder, MRI))
7645 llvm_unreachable("inconsistent extractParts result");
7646
7647 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
7648 auto Select = MIRBuilder.buildSelect(Res: NarrowTy,
7649 Tst: CondReg, Op0: Src1Regs[I], Op1: Src2Regs[I]);
7650 DstRegs.push_back(Elt: Select.getReg(Idx: 0));
7651 }
7652
7653 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
7654 auto Select = MIRBuilder.buildSelect(
7655 Res: LeftoverTy, Tst: CondReg, Op0: Src1LeftoverRegs[I], Op1: Src2LeftoverRegs[I]);
7656 DstLeftoverRegs.push_back(Elt: Select.getReg(Idx: 0));
7657 }
7658
7659 insertParts(DstReg, ResultTy: DstTy, PartTy: NarrowTy, PartRegs: DstRegs,
7660 LeftoverTy, LeftoverRegs: DstLeftoverRegs);
7661
7662 MI.eraseFromParent();
7663 return Legalized;
7664}
7665
7666LegalizerHelper::LegalizeResult
7667LegalizerHelper::narrowScalarCTLZ(MachineInstr &MI, unsigned TypeIdx,
7668 LLT NarrowTy) {
7669 if (TypeIdx != 1)
7670 return UnableToLegalize;
7671
7672 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7673 unsigned NarrowSize = NarrowTy.getSizeInBits();
7674
7675 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
7676 const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_POISON;
7677
7678 MachineIRBuilder &B = MIRBuilder;
7679 auto UnmergeSrc = B.buildUnmerge(Res: NarrowTy, Op: SrcReg);
7680 // ctlz(Hi:Lo) -> Hi == 0 ? (NarrowSize + ctlz(Lo)) : ctlz(Hi)
7681 auto C_0 = B.buildConstant(Res: NarrowTy, Val: 0);
7682 auto HiIsZero = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: LLT::integer(SizeInBits: 1),
7683 Op0: UnmergeSrc.getReg(Idx: 1), Op1: C_0);
7684 auto LoCTLZ = IsUndef ? B.buildCTLZ_ZERO_POISON(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 0))
7685 : B.buildCTLZ(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 0));
7686 auto C_NarrowSize = B.buildConstant(Res: DstTy, Val: NarrowSize);
7687 auto HiIsZeroCTLZ = B.buildAdd(Dst: DstTy, Src0: LoCTLZ, Src1: C_NarrowSize);
7688 auto HiCTLZ = B.buildCTLZ_ZERO_POISON(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 1));
7689 B.buildSelect(Res: DstReg, Tst: HiIsZero, Op0: HiIsZeroCTLZ, Op1: HiCTLZ);
7690
7691 MI.eraseFromParent();
7692 return Legalized;
7693 }
7694
7695 return UnableToLegalize;
7696}
7697
7698LegalizerHelper::LegalizeResult
7699LegalizerHelper::narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx,
7700 LLT NarrowTy) {
7701 if (TypeIdx != 1)
7702 return UnableToLegalize;
7703
7704 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7705 unsigned NarrowSize = NarrowTy.getSizeInBits();
7706
7707 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
7708 const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTTZ_ZERO_POISON;
7709
7710 MachineIRBuilder &B = MIRBuilder;
7711 auto UnmergeSrc = B.buildUnmerge(Res: NarrowTy, Op: SrcReg);
7712 // cttz(Hi:Lo) -> Lo == 0 ? (cttz(Hi) + NarrowSize) : cttz(Lo)
7713 auto C_0 = B.buildConstant(Res: NarrowTy, Val: 0);
7714 auto LoIsZero = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: LLT::scalar(SizeInBits: 1),
7715 Op0: UnmergeSrc.getReg(Idx: 0), Op1: C_0);
7716 auto HiCTTZ = IsUndef ? B.buildCTTZ_ZERO_POISON(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 1))
7717 : B.buildCTTZ(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 1));
7718 auto C_NarrowSize = B.buildConstant(Res: DstTy, Val: NarrowSize);
7719 auto LoIsZeroCTTZ = B.buildAdd(Dst: DstTy, Src0: HiCTTZ, Src1: C_NarrowSize);
7720 auto LoCTTZ = B.buildCTTZ_ZERO_POISON(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 0));
7721 B.buildSelect(Res: DstReg, Tst: LoIsZero, Op0: LoIsZeroCTTZ, Op1: LoCTTZ);
7722
7723 MI.eraseFromParent();
7724 return Legalized;
7725 }
7726
7727 return UnableToLegalize;
7728}
7729
7730LegalizerHelper::LegalizeResult
7731LegalizerHelper::narrowScalarCTLS(MachineInstr &MI, unsigned TypeIdx,
7732 LLT NarrowTy) {
7733 if (TypeIdx != 1)
7734 return UnableToLegalize;
7735
7736 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7737 unsigned NarrowSize = NarrowTy.getSizeInBits();
7738
7739 if (!SrcTy.isScalar() || SrcTy.getSizeInBits() != 2 * NarrowSize)
7740 return UnableToLegalize;
7741
7742 MachineIRBuilder &B = MIRBuilder;
7743
7744 auto UnmergeSrc = B.buildUnmerge(Res: NarrowTy, Op: SrcReg);
7745 Register Lo = UnmergeSrc.getReg(Idx: 0);
7746 Register Hi = UnmergeSrc.getReg(Idx: 1);
7747
7748 auto ShAmt = B.buildConstant(Res: NarrowTy, Val: NarrowSize - 1);
7749 auto Sign = B.buildAShr(Dst: NarrowTy, Src0: Hi, Src1: ShAmt);
7750
7751 auto HiIsSign = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: LLT::scalar(SizeInBits: 1), Op0: Hi, Op1: Sign);
7752
7753 // Invert Lo if Hi is negative. Then count the leading zeros. If there are no
7754 // leading zeros, then the MSB of Lo is different than the MSB of Hi.
7755 // Otherwise the leading zeros represent additional sign bits of the original
7756 // value.
7757 auto LoInv = B.buildXor(Dst: DstTy, Src0: Lo, Src1: Sign);
7758 auto LoCTLZ = B.buildCTLZ(Dst: DstTy, Src0: LoInv);
7759
7760 // Add NarrowSize-1 to LoCTLZ. This is the full CTLS if Hi is all sign bits.
7761 auto C_NarrowSizeM1 = B.buildConstant(Res: DstTy, Val: NarrowSize - 1);
7762 auto HiIsSignCTLS = B.buildAdd(Dst: DstTy, Src0: LoCTLZ, Src1: C_NarrowSizeM1);
7763
7764 auto HiCTLS = B.buildCTLS(Dst: DstTy, Src0: Hi);
7765
7766 B.buildSelect(Res: DstReg, Tst: HiIsSign, Op0: HiIsSignCTLS, Op1: HiCTLS);
7767
7768 MI.eraseFromParent();
7769 return Legalized;
7770}
7771
7772LegalizerHelper::LegalizeResult
7773LegalizerHelper::narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx,
7774 LLT NarrowTy) {
7775 if (TypeIdx != 1)
7776 return UnableToLegalize;
7777
7778 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7779 unsigned NarrowSize = NarrowTy.getSizeInBits();
7780
7781 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
7782 auto UnmergeSrc = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: MI.getOperand(i: 1));
7783
7784 auto LoCTPOP = MIRBuilder.buildCTPOP(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 0));
7785 auto HiCTPOP = MIRBuilder.buildCTPOP(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 1));
7786 MIRBuilder.buildAdd(Dst: DstReg, Src0: HiCTPOP, Src1: LoCTPOP);
7787
7788 MI.eraseFromParent();
7789 return Legalized;
7790 }
7791
7792 return UnableToLegalize;
7793}
7794
7795LegalizerHelper::LegalizeResult
7796LegalizerHelper::narrowScalarFLDEXP(MachineInstr &MI, unsigned TypeIdx,
7797 LLT NarrowTy) {
7798 if (TypeIdx != 1)
7799 return UnableToLegalize;
7800
7801 MachineIRBuilder &B = MIRBuilder;
7802 Register ExpReg = MI.getOperand(i: 2).getReg();
7803 LLT ExpTy = MRI.getType(Reg: ExpReg);
7804
7805 unsigned ClampSize = NarrowTy.getScalarSizeInBits();
7806
7807 // Clamp the exponent to the range of the target type.
7808 auto MinExp = B.buildConstant(Res: ExpTy, Val: minIntN(N: ClampSize));
7809 auto ClampMin = B.buildSMax(Dst: ExpTy, Src0: ExpReg, Src1: MinExp);
7810 auto MaxExp = B.buildConstant(Res: ExpTy, Val: maxIntN(N: ClampSize));
7811 auto Clamp = B.buildSMin(Dst: ExpTy, Src0: ClampMin, Src1: MaxExp);
7812
7813 auto Trunc = B.buildTrunc(Res: NarrowTy, Op: Clamp);
7814 Observer.changingInstr(MI);
7815 MI.getOperand(i: 2).setReg(Trunc.getReg(Idx: 0));
7816 Observer.changedInstr(MI);
7817 return Legalized;
7818}
7819
7820LegalizerHelper::LegalizeResult
7821LegalizerHelper::lowerBitCount(MachineInstr &MI) {
7822 unsigned Opc = MI.getOpcode();
7823 const auto &TII = MIRBuilder.getTII();
7824 auto isSupported = [this](const LegalityQuery &Q) {
7825 auto QAction = LI.getAction(Query: Q).Action;
7826 return QAction == Legal || QAction == Libcall || QAction == Custom;
7827 };
7828 switch (Opc) {
7829 default:
7830 return UnableToLegalize;
7831 case TargetOpcode::G_CTLZ_ZERO_POISON: {
7832 // This trivially expands to CTLZ.
7833 Observer.changingInstr(MI);
7834 MI.setDesc(TII.get(Opcode: TargetOpcode::G_CTLZ));
7835 Observer.changedInstr(MI);
7836 return Legalized;
7837 }
7838 case TargetOpcode::G_CTLZ: {
7839 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7840 unsigned Len = SrcTy.getScalarSizeInBits();
7841
7842 if (isSupported({TargetOpcode::G_CTLZ_ZERO_POISON, {DstTy, SrcTy}})) {
7843 // If CTLZ_ZERO_POISON is supported, emit that and a select for zero.
7844 auto CtlzZU = MIRBuilder.buildCTLZ_ZERO_POISON(Dst: DstTy, Src0: SrcReg);
7845 auto ZeroSrc = MIRBuilder.buildConstant(Res: SrcTy, Val: 0);
7846 auto ICmp = MIRBuilder.buildICmp(
7847 Pred: CmpInst::ICMP_EQ, Res: SrcTy.changeElementSize(NewEltSize: 1), Op0: SrcReg, Op1: ZeroSrc);
7848 auto LenConst = MIRBuilder.buildConstant(Res: DstTy, Val: Len);
7849 MIRBuilder.buildSelect(Res: DstReg, Tst: ICmp, Op0: LenConst, Op1: CtlzZU);
7850 MI.eraseFromParent();
7851 return Legalized;
7852 }
7853 // for now, we do this:
7854 // NewLen = NextPowerOf2(Len);
7855 // x = x | (x >> 1);
7856 // x = x | (x >> 2);
7857 // ...
7858 // x = x | (x >>16);
7859 // x = x | (x >>32); // for 64-bit input
7860 // Upto NewLen/2
7861 // return Len - popcount(x);
7862 //
7863 // Ref: "Hacker's Delight" by Henry Warren
7864 Register Op = SrcReg;
7865 unsigned NewLen = PowerOf2Ceil(A: Len);
7866 for (unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) {
7867 auto MIBShiftAmt = MIRBuilder.buildConstant(Res: SrcTy, Val: 1ULL << i);
7868 auto MIBOp = MIRBuilder.buildOr(
7869 Dst: SrcTy, Src0: Op, Src1: MIRBuilder.buildLShr(Dst: SrcTy, Src0: Op, Src1: MIBShiftAmt));
7870 Op = MIBOp.getReg(Idx: 0);
7871 }
7872 auto MIBPop = MIRBuilder.buildCTPOP(Dst: DstTy, Src0: Op);
7873 MIRBuilder.buildSub(Dst: MI.getOperand(i: 0), Src0: MIRBuilder.buildConstant(Res: DstTy, Val: Len),
7874 Src1: MIBPop);
7875 MI.eraseFromParent();
7876 return Legalized;
7877 }
7878 case TargetOpcode::G_CTTZ_ZERO_POISON: {
7879 // This trivially expands to CTTZ.
7880 Observer.changingInstr(MI);
7881 MI.setDesc(TII.get(Opcode: TargetOpcode::G_CTTZ));
7882 Observer.changedInstr(MI);
7883 return Legalized;
7884 }
7885 case TargetOpcode::G_CTTZ: {
7886 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7887
7888 unsigned Len = SrcTy.getScalarSizeInBits();
7889 if (isSupported({TargetOpcode::G_CTTZ_ZERO_POISON, {DstTy, SrcTy}})) {
7890 // If CTTZ_ZERO_POISON is legal or custom, emit that and a select with
7891 // zero.
7892 auto CttzZU = MIRBuilder.buildCTTZ_ZERO_POISON(Dst: DstTy, Src0: SrcReg);
7893 auto Zero = MIRBuilder.buildConstant(Res: SrcTy, Val: 0);
7894 auto ICmp = MIRBuilder.buildICmp(
7895 Pred: CmpInst::ICMP_EQ, Res: DstTy.changeElementSize(NewEltSize: 1), Op0: SrcReg, Op1: Zero);
7896 auto LenConst = MIRBuilder.buildConstant(Res: DstTy, Val: Len);
7897 MIRBuilder.buildSelect(Res: DstReg, Tst: ICmp, Op0: LenConst, Op1: CttzZU);
7898 MI.eraseFromParent();
7899 return Legalized;
7900 }
7901 // for now, we use: { return popcount(~x & (x - 1)); }
7902 // unless the target has ctlz but not ctpop, in which case we use:
7903 // { return 32 - nlz(~x & (x-1)); }
7904 // Ref: "Hacker's Delight" by Henry Warren
7905 auto MIBCstNeg1 = MIRBuilder.buildConstant(Res: SrcTy, Val: -1);
7906 auto MIBNot = MIRBuilder.buildXor(Dst: SrcTy, Src0: SrcReg, Src1: MIBCstNeg1);
7907 auto MIBTmp = MIRBuilder.buildAnd(
7908 Dst: SrcTy, Src0: MIBNot, Src1: MIRBuilder.buildAdd(Dst: SrcTy, Src0: SrcReg, Src1: MIBCstNeg1));
7909 if (!isSupported({TargetOpcode::G_CTPOP, {SrcTy, SrcTy}}) &&
7910 isSupported({TargetOpcode::G_CTLZ, {SrcTy, SrcTy}})) {
7911 auto MIBCstLen = MIRBuilder.buildConstant(Res: SrcTy, Val: Len);
7912 MIRBuilder.buildSub(Dst: MI.getOperand(i: 0), Src0: MIBCstLen,
7913 Src1: MIRBuilder.buildCTLZ(Dst: SrcTy, Src0: MIBTmp));
7914 MI.eraseFromParent();
7915 return Legalized;
7916 }
7917 Observer.changingInstr(MI);
7918 MI.setDesc(TII.get(Opcode: TargetOpcode::G_CTPOP));
7919 MI.getOperand(i: 1).setReg(MIBTmp.getReg(Idx: 0));
7920 Observer.changedInstr(MI);
7921 return Legalized;
7922 }
7923 case TargetOpcode::G_CTPOP: {
7924 Register SrcReg = MI.getOperand(i: 1).getReg();
7925 LLT Ty = MRI.getType(Reg: SrcReg);
7926 unsigned Size = Ty.getScalarSizeInBits();
7927 MachineIRBuilder &B = MIRBuilder;
7928
7929 // Bail out on irregular type lengths.
7930 if (Size > 128 || Size % 8 != 0)
7931 return UnableToLegalize;
7932
7933 // Count set bits in blocks of 2 bits. Default approach would be
7934 // B2Count = { val & 0x55555555 } + { (val >> 1) & 0x55555555 }
7935 // We use following formula instead:
7936 // B2Count = val - { (val >> 1) & 0x55555555 }
7937 // since it gives same result in blocks of 2 with one instruction less.
7938 auto C_1 = B.buildConstant(Res: Ty, Val: 1);
7939 auto B2Set1LoTo1Hi = B.buildLShr(Dst: Ty, Src0: SrcReg, Src1: C_1);
7940 APInt B2Mask1HiTo0 = APInt::getSplat(NewLen: Size, V: APInt(8, 0x55));
7941 auto C_B2Mask1HiTo0 = B.buildConstant(Res: Ty, Val: B2Mask1HiTo0);
7942 auto B2Count1Hi = B.buildAnd(Dst: Ty, Src0: B2Set1LoTo1Hi, Src1: C_B2Mask1HiTo0);
7943 auto B2Count = B.buildSub(Dst: Ty, Src0: SrcReg, Src1: B2Count1Hi);
7944
7945 // In order to get count in blocks of 4 add values from adjacent block of 2.
7946 // B4Count = { B2Count & 0x33333333 } + { (B2Count >> 2) & 0x33333333 }
7947 auto C_2 = B.buildConstant(Res: Ty, Val: 2);
7948 auto B4Set2LoTo2Hi = B.buildLShr(Dst: Ty, Src0: B2Count, Src1: C_2);
7949 APInt B4Mask2HiTo0 = APInt::getSplat(NewLen: Size, V: APInt(8, 0x33));
7950 auto C_B4Mask2HiTo0 = B.buildConstant(Res: Ty, Val: B4Mask2HiTo0);
7951 auto B4HiB2Count = B.buildAnd(Dst: Ty, Src0: B4Set2LoTo2Hi, Src1: C_B4Mask2HiTo0);
7952 auto B4LoB2Count = B.buildAnd(Dst: Ty, Src0: B2Count, Src1: C_B4Mask2HiTo0);
7953 auto B4Count = B.buildAdd(Dst: Ty, Src0: B4HiB2Count, Src1: B4LoB2Count);
7954
7955 // For count in blocks of 8 bits we don't have to mask high 4 bits before
7956 // addition since count value sits in range {0,...,8} and 4 bits are enough
7957 // to hold such binary values. After addition high 4 bits still hold count
7958 // of set bits in high 4 bit block, set them to zero and get 8 bit result.
7959 // B8Count = { B4Count + (B4Count >> 4) } & 0x0F0F0F0F
7960 auto C_4 = B.buildConstant(Res: Ty, Val: 4);
7961 auto B8HiB4Count = B.buildLShr(Dst: Ty, Src0: B4Count, Src1: C_4);
7962 auto B8CountDirty4Hi = B.buildAdd(Dst: Ty, Src0: B8HiB4Count, Src1: B4Count);
7963 APInt B8Mask4HiTo0 = APInt::getSplat(NewLen: Size, V: APInt(8, 0x0F));
7964 auto C_B8Mask4HiTo0 = B.buildConstant(Res: Ty, Val: B8Mask4HiTo0);
7965 auto B8Count = B.buildAnd(Dst: Ty, Src0: B8CountDirty4Hi, Src1: C_B8Mask4HiTo0);
7966
7967 assert(Size <= 128 && "Scalar size is too large for CTPOP lower algorithm");
7968
7969 // Avoid the multiply when shift-add is cheaper.
7970 if (Size == 16 && !Ty.isVector()) {
7971 // v = (v + (v >> 8)) & 0xFF;
7972 auto C_8 = B.buildConstant(Res: Ty, Val: 8);
7973 auto HighSum = B.buildLShr(Dst: Ty, Src0: B8Count, Src1: C_8);
7974 auto Res = B.buildAdd(Dst: Ty, Src0: B8Count, Src1: HighSum);
7975 B.buildAnd(Dst: MI.getOperand(i: 0).getReg(), Src0: Res, Src1: B.buildConstant(Res: Ty, Val: 0xFF));
7976 MI.eraseFromParent();
7977 return Legalized;
7978 }
7979
7980 // 8 bits can hold CTPOP result of 128 bit int or smaller. Mul with this
7981 // bitmask will set 8 msb in ResTmp to sum of all B8Counts in 8 bit blocks.
7982 auto MulMask = B.buildConstant(Res: Ty, Val: APInt::getSplat(NewLen: Size, V: APInt(8, 0x01)));
7983
7984 // Shift count result from 8 high bits to low bits.
7985 auto C_SizeM8 = B.buildConstant(Res: Ty, Val: Size - 8);
7986
7987 auto IsMulSupported = [this](const LLT Ty) {
7988 auto Action = LI.getAction(Query: {TargetOpcode::G_MUL, {Ty}}).Action;
7989 return Action == Legal || Action == WidenScalar || Action == Custom;
7990 };
7991 if (IsMulSupported(Ty)) {
7992 auto ResTmp = B.buildMul(Dst: Ty, Src0: B8Count, Src1: MulMask);
7993 B.buildLShr(Dst: MI.getOperand(i: 0).getReg(), Src0: ResTmp, Src1: C_SizeM8);
7994 } else {
7995 auto ResTmp = B8Count;
7996 for (unsigned Shift = 8; Shift < Size; Shift *= 2) {
7997 auto ShiftC = B.buildConstant(Res: Ty, Val: Shift);
7998 auto Shl = B.buildShl(Dst: Ty, Src0: ResTmp, Src1: ShiftC);
7999 ResTmp = B.buildAdd(Dst: Ty, Src0: ResTmp, Src1: Shl);
8000 }
8001 B.buildLShr(Dst: MI.getOperand(i: 0).getReg(), Src0: ResTmp, Src1: C_SizeM8);
8002 }
8003 MI.eraseFromParent();
8004 return Legalized;
8005 }
8006 case TargetOpcode::G_CTLS: {
8007 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
8008
8009 // ctls(x) -> ctlz(x ^ (x >> (N - 1))) - 1
8010 auto SignIdxC =
8011 MIRBuilder.buildConstant(Res: SrcTy, Val: SrcTy.getScalarSizeInBits() - 1);
8012 auto OneC = MIRBuilder.buildConstant(Res: DstTy, Val: 1);
8013
8014 auto Shr = MIRBuilder.buildAShr(Dst: SrcTy, Src0: SrcReg, Src1: SignIdxC);
8015
8016 auto Xor = MIRBuilder.buildXor(Dst: SrcTy, Src0: SrcReg, Src1: Shr);
8017 auto Ctlz = MIRBuilder.buildCTLZ(Dst: DstTy, Src0: Xor);
8018
8019 MIRBuilder.buildSub(Dst: DstReg, Src0: Ctlz, Src1: OneC);
8020 MI.eraseFromParent();
8021 return Legalized;
8022 }
8023 }
8024}
8025
8026// Check that (every element of) Reg is undef or not an exact multiple of BW.
8027static bool isNonZeroModBitWidthOrUndef(const MachineRegisterInfo &MRI,
8028 Register Reg, unsigned BW) {
8029 return matchUnaryPredicate(
8030 MRI, Reg,
8031 Match: [=](const Constant *C) {
8032 // Null constant here means an undef.
8033 const ConstantInt *CI = dyn_cast_or_null<ConstantInt>(Val: C);
8034 return !CI || CI->getValue().urem(RHS: BW) != 0;
8035 },
8036 /*AllowUndefs*/ true);
8037}
8038
8039LegalizerHelper::LegalizeResult
8040LegalizerHelper::lowerFunnelShiftWithInverse(MachineInstr &MI) {
8041 auto [Dst, X, Y, Z] = MI.getFirst4Regs();
8042 LLT Ty = MRI.getType(Reg: Dst);
8043 LLT ShTy = MRI.getType(Reg: Z);
8044
8045 unsigned BW = Ty.getScalarSizeInBits();
8046
8047 if (!isPowerOf2_32(Value: BW))
8048 return UnableToLegalize;
8049
8050 const bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL;
8051 unsigned RevOpcode = IsFSHL ? TargetOpcode::G_FSHR : TargetOpcode::G_FSHL;
8052
8053 if (isNonZeroModBitWidthOrUndef(MRI, Reg: Z, BW)) {
8054 // fshl X, Y, Z -> fshr X, Y, -Z
8055 // fshr X, Y, Z -> fshl X, Y, -Z
8056 auto Zero = MIRBuilder.buildConstant(Res: ShTy, Val: 0);
8057 Z = MIRBuilder.buildSub(Dst: Ty, Src0: Zero, Src1: Z).getReg(Idx: 0);
8058 } else {
8059 // fshl X, Y, Z -> fshr (srl X, 1), (fshr X, Y, 1), ~Z
8060 // fshr X, Y, Z -> fshl (fshl X, Y, 1), (shl Y, 1), ~Z
8061 auto One = MIRBuilder.buildConstant(Res: ShTy, Val: 1);
8062 if (IsFSHL) {
8063 Y = MIRBuilder.buildInstr(Opc: RevOpcode, DstOps: {Ty}, SrcOps: {X, Y, One}).getReg(Idx: 0);
8064 X = MIRBuilder.buildLShr(Dst: Ty, Src0: X, Src1: One).getReg(Idx: 0);
8065 } else {
8066 X = MIRBuilder.buildInstr(Opc: RevOpcode, DstOps: {Ty}, SrcOps: {X, Y, One}).getReg(Idx: 0);
8067 Y = MIRBuilder.buildShl(Dst: Ty, Src0: Y, Src1: One).getReg(Idx: 0);
8068 }
8069
8070 Z = MIRBuilder.buildNot(Dst: ShTy, Src0: Z).getReg(Idx: 0);
8071 }
8072
8073 MIRBuilder.buildInstr(Opc: RevOpcode, DstOps: {Dst}, SrcOps: {X, Y, Z});
8074 MI.eraseFromParent();
8075 return Legalized;
8076}
8077
8078LegalizerHelper::LegalizeResult
8079LegalizerHelper::lowerFunnelShiftAsShifts(MachineInstr &MI) {
8080 auto [Dst, X, Y, Z] = MI.getFirst4Regs();
8081 LLT Ty = MRI.getType(Reg: Dst);
8082 LLT ShTy = MRI.getType(Reg: Z);
8083
8084 const unsigned BW = Ty.getScalarSizeInBits();
8085 const bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL;
8086
8087 Register ShX, ShY;
8088 Register ShAmt, InvShAmt;
8089
8090 // FIXME: Emit optimized urem by constant instead of letting it expand later.
8091 if (isNonZeroModBitWidthOrUndef(MRI, Reg: Z, BW)) {
8092 // fshl: X << C | Y >> (BW - C)
8093 // fshr: X << (BW - C) | Y >> C
8094 // where C = Z % BW is not zero
8095 auto BitWidthC = MIRBuilder.buildConstant(Res: ShTy, Val: BW);
8096 ShAmt = MIRBuilder.buildURem(Dst: ShTy, Src0: Z, Src1: BitWidthC).getReg(Idx: 0);
8097 InvShAmt = MIRBuilder.buildSub(Dst: ShTy, Src0: BitWidthC, Src1: ShAmt).getReg(Idx: 0);
8098 ShX = MIRBuilder.buildShl(Dst: Ty, Src0: X, Src1: IsFSHL ? ShAmt : InvShAmt).getReg(Idx: 0);
8099 ShY = MIRBuilder.buildLShr(Dst: Ty, Src0: Y, Src1: IsFSHL ? InvShAmt : ShAmt).getReg(Idx: 0);
8100 } else {
8101 // fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))
8102 // fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)
8103 auto Mask = MIRBuilder.buildConstant(Res: ShTy, Val: BW - 1);
8104 if (isPowerOf2_32(Value: BW)) {
8105 // Z % BW -> Z & (BW - 1)
8106 ShAmt = MIRBuilder.buildAnd(Dst: ShTy, Src0: Z, Src1: Mask).getReg(Idx: 0);
8107 // (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
8108 auto NotZ = MIRBuilder.buildNot(Dst: ShTy, Src0: Z);
8109 InvShAmt = MIRBuilder.buildAnd(Dst: ShTy, Src0: NotZ, Src1: Mask).getReg(Idx: 0);
8110 } else {
8111 auto BitWidthC = MIRBuilder.buildConstant(Res: ShTy, Val: BW);
8112 ShAmt = MIRBuilder.buildURem(Dst: ShTy, Src0: Z, Src1: BitWidthC).getReg(Idx: 0);
8113 InvShAmt = MIRBuilder.buildSub(Dst: ShTy, Src0: Mask, Src1: ShAmt).getReg(Idx: 0);
8114 }
8115
8116 auto One = MIRBuilder.buildConstant(Res: ShTy, Val: 1);
8117 if (IsFSHL) {
8118 ShX = MIRBuilder.buildShl(Dst: Ty, Src0: X, Src1: ShAmt).getReg(Idx: 0);
8119 auto ShY1 = MIRBuilder.buildLShr(Dst: Ty, Src0: Y, Src1: One);
8120 ShY = MIRBuilder.buildLShr(Dst: Ty, Src0: ShY1, Src1: InvShAmt).getReg(Idx: 0);
8121 } else {
8122 auto ShX1 = MIRBuilder.buildShl(Dst: Ty, Src0: X, Src1: One);
8123 ShX = MIRBuilder.buildShl(Dst: Ty, Src0: ShX1, Src1: InvShAmt).getReg(Idx: 0);
8124 ShY = MIRBuilder.buildLShr(Dst: Ty, Src0: Y, Src1: ShAmt).getReg(Idx: 0);
8125 }
8126 }
8127
8128 MIRBuilder.buildOr(Dst, Src0: ShX, Src1: ShY, Flags: MachineInstr::Disjoint);
8129 MI.eraseFromParent();
8130 return Legalized;
8131}
8132
8133LegalizerHelper::LegalizeResult
8134LegalizerHelper::lowerFunnelShift(MachineInstr &MI) {
8135 // These operations approximately do the following (while avoiding undefined
8136 // shifts by BW):
8137 // G_FSHL: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
8138 // G_FSHR: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
8139 Register Dst = MI.getOperand(i: 0).getReg();
8140 LLT Ty = MRI.getType(Reg: Dst);
8141 LLT ShTy = MRI.getType(Reg: MI.getOperand(i: 3).getReg());
8142
8143 bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL;
8144 unsigned RevOpcode = IsFSHL ? TargetOpcode::G_FSHR : TargetOpcode::G_FSHL;
8145
8146 // TODO: Use smarter heuristic that accounts for vector legalization.
8147 if (LI.getAction(Query: {RevOpcode, {Ty, ShTy}}).Action == Lower)
8148 return lowerFunnelShiftAsShifts(MI);
8149
8150 // This only works for powers of 2, fallback to shifts if it fails.
8151 LegalizerHelper::LegalizeResult Result = lowerFunnelShiftWithInverse(MI);
8152 if (Result == UnableToLegalize)
8153 return lowerFunnelShiftAsShifts(MI);
8154 return Result;
8155}
8156
8157LegalizerHelper::LegalizeResult LegalizerHelper::lowerEXT(MachineInstr &MI) {
8158 auto [Dst, Src] = MI.getFirst2Regs();
8159 LLT DstTy = MRI.getType(Reg: Dst);
8160 LLT SrcTy = MRI.getType(Reg: Src);
8161
8162 uint32_t DstTySize = DstTy.getSizeInBits();
8163 uint32_t DstTyScalarSize = DstTy.getScalarSizeInBits();
8164 uint32_t SrcTyScalarSize = SrcTy.getScalarSizeInBits();
8165
8166 if (!isPowerOf2_32(Value: DstTySize) || !isPowerOf2_32(Value: DstTyScalarSize) ||
8167 !isPowerOf2_32(Value: SrcTyScalarSize))
8168 return UnableToLegalize;
8169
8170 // The step between extend is too large, split it by creating an intermediate
8171 // extend instruction
8172 if (SrcTyScalarSize * 2 < DstTyScalarSize) {
8173 LLT MidTy = SrcTy.changeElementSize(NewEltSize: SrcTyScalarSize * 2);
8174 // If the destination type is illegal, split it into multiple statements
8175 // zext x -> zext(merge(zext(unmerge), zext(unmerge)))
8176 auto NewExt = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {MidTy}, SrcOps: {Src});
8177 // Unmerge the vector
8178 LLT EltTy = MidTy.changeElementCount(
8179 EC: MidTy.getElementCount().divideCoefficientBy(RHS: 2));
8180 auto UnmergeSrc = MIRBuilder.buildUnmerge(Res: EltTy, Op: NewExt);
8181
8182 // ZExt the vectors
8183 LLT ZExtResTy = DstTy.changeElementCount(
8184 EC: DstTy.getElementCount().divideCoefficientBy(RHS: 2));
8185 auto ZExtRes1 = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {ZExtResTy},
8186 SrcOps: {UnmergeSrc.getReg(Idx: 0)});
8187 auto ZExtRes2 = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {ZExtResTy},
8188 SrcOps: {UnmergeSrc.getReg(Idx: 1)});
8189
8190 // Merge the ending vectors
8191 MIRBuilder.buildMergeLikeInstr(Res: Dst, Ops: {ZExtRes1, ZExtRes2});
8192
8193 MI.eraseFromParent();
8194 return Legalized;
8195 }
8196 return UnableToLegalize;
8197}
8198
8199LegalizerHelper::LegalizeResult LegalizerHelper::lowerTRUNC(MachineInstr &MI) {
8200 // MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
8201 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
8202 // Similar to how operand splitting is done in SelectiondDAG, we can handle
8203 // %res(v8s8) = G_TRUNC %in(v8s32) by generating:
8204 // %inlo(<4x s32>), %inhi(<4 x s32>) = G_UNMERGE %in(<8 x s32>)
8205 // %lo16(<4 x s16>) = G_TRUNC %inlo
8206 // %hi16(<4 x s16>) = G_TRUNC %inhi
8207 // %in16(<8 x s16>) = G_CONCAT_VECTORS %lo16, %hi16
8208 // %res(<8 x s8>) = G_TRUNC %in16
8209
8210 assert(MI.getOpcode() == TargetOpcode::G_TRUNC);
8211
8212 Register DstReg = MI.getOperand(i: 0).getReg();
8213 Register SrcReg = MI.getOperand(i: 1).getReg();
8214 LLT DstTy = MRI.getType(Reg: DstReg);
8215 LLT SrcTy = MRI.getType(Reg: SrcReg);
8216
8217 if (DstTy.isVector() && isPowerOf2_32(Value: DstTy.getNumElements()) &&
8218 isPowerOf2_32(Value: DstTy.getScalarSizeInBits()) &&
8219 isPowerOf2_32(Value: SrcTy.getNumElements()) &&
8220 isPowerOf2_32(Value: SrcTy.getScalarSizeInBits())) {
8221 // Split input type.
8222 LLT SplitSrcTy = SrcTy.changeElementCount(
8223 EC: SrcTy.getElementCount().divideCoefficientBy(RHS: 2));
8224
8225 // First, split the source into two smaller vectors.
8226 SmallVector<Register, 2> SplitSrcs;
8227 extractParts(Reg: SrcReg, Ty: SplitSrcTy, NumParts: 2, VRegs&: SplitSrcs, MIRBuilder, MRI);
8228
8229 // Truncate the splits into intermediate narrower elements.
8230 LLT InterTy;
8231 if (DstTy.getScalarSizeInBits() * 2 < SrcTy.getScalarSizeInBits())
8232 InterTy = SplitSrcTy.changeElementSize(NewEltSize: DstTy.getScalarSizeInBits() * 2);
8233 else
8234 InterTy = SplitSrcTy.changeElementSize(NewEltSize: DstTy.getScalarSizeInBits());
8235 for (Register &Src : SplitSrcs)
8236 Src = MIRBuilder.buildTrunc(Res: InterTy, Op: Src).getReg(Idx: 0);
8237
8238 // Combine the new truncates into one vector
8239 auto Merge = MIRBuilder.buildMergeLikeInstr(
8240 Res: DstTy.changeElementSize(NewEltSize: InterTy.getScalarSizeInBits()), Ops: SplitSrcs);
8241
8242 // Truncate the new vector to the final result type
8243 if (DstTy.getScalarSizeInBits() * 2 < SrcTy.getScalarSizeInBits())
8244 MIRBuilder.buildTrunc(Res: MI.getOperand(i: 0).getReg(), Op: Merge.getReg(Idx: 0));
8245 else
8246 MIRBuilder.buildCopy(Res: MI.getOperand(i: 0).getReg(), Op: Merge.getReg(Idx: 0));
8247
8248 MI.eraseFromParent();
8249
8250 return Legalized;
8251 }
8252 return UnableToLegalize;
8253}
8254
8255LegalizerHelper::LegalizeResult
8256LegalizerHelper::lowerRotateWithReverseRotate(MachineInstr &MI) {
8257 auto [Dst, DstTy, Src, SrcTy, Amt, AmtTy] = MI.getFirst3RegLLTs();
8258 auto Zero = MIRBuilder.buildConstant(Res: AmtTy, Val: 0);
8259 bool IsLeft = MI.getOpcode() == TargetOpcode::G_ROTL;
8260 unsigned RevRot = IsLeft ? TargetOpcode::G_ROTR : TargetOpcode::G_ROTL;
8261 auto Neg = MIRBuilder.buildSub(Dst: AmtTy, Src0: Zero, Src1: Amt);
8262 MIRBuilder.buildInstr(Opc: RevRot, DstOps: {Dst}, SrcOps: {Src, Neg});
8263 MI.eraseFromParent();
8264 return Legalized;
8265}
8266
8267LegalizerHelper::LegalizeResult LegalizerHelper::lowerRotate(MachineInstr &MI) {
8268 auto [Dst, DstTy, Src, SrcTy, Amt, AmtTy] = MI.getFirst3RegLLTs();
8269
8270 unsigned EltSizeInBits = DstTy.getScalarSizeInBits();
8271 bool IsLeft = MI.getOpcode() == TargetOpcode::G_ROTL;
8272
8273 MIRBuilder.setInstrAndDebugLoc(MI);
8274
8275 // If a rotate in the other direction is supported, use it.
8276 unsigned RevRot = IsLeft ? TargetOpcode::G_ROTR : TargetOpcode::G_ROTL;
8277 if (LI.isLegalOrCustom(Query: {RevRot, {DstTy, SrcTy}}) &&
8278 isPowerOf2_32(Value: EltSizeInBits))
8279 return lowerRotateWithReverseRotate(MI);
8280
8281 // If a funnel shift is supported, use it.
8282 unsigned FShOpc = IsLeft ? TargetOpcode::G_FSHL : TargetOpcode::G_FSHR;
8283 unsigned RevFsh = !IsLeft ? TargetOpcode::G_FSHL : TargetOpcode::G_FSHR;
8284 bool IsFShLegal = false;
8285 if ((IsFShLegal = LI.isLegalOrCustom(Query: {FShOpc, {DstTy, AmtTy}})) ||
8286 LI.isLegalOrCustom(Query: {RevFsh, {DstTy, AmtTy}})) {
8287 auto buildFunnelShift = [&](unsigned Opc, Register R1, Register R2,
8288 Register R3) {
8289 MIRBuilder.buildInstr(Opc, DstOps: {R1}, SrcOps: {R2, R2, R3});
8290 MI.eraseFromParent();
8291 return Legalized;
8292 };
8293 // If a funnel shift in the other direction is supported, use it.
8294 if (IsFShLegal) {
8295 return buildFunnelShift(FShOpc, Dst, Src, Amt);
8296 } else if (isPowerOf2_32(Value: EltSizeInBits)) {
8297 Amt = MIRBuilder.buildNeg(Dst: DstTy, Src0: Amt).getReg(Idx: 0);
8298 return buildFunnelShift(RevFsh, Dst, Src, Amt);
8299 }
8300 }
8301
8302 auto Zero = MIRBuilder.buildConstant(Res: AmtTy, Val: 0);
8303 unsigned ShOpc = IsLeft ? TargetOpcode::G_SHL : TargetOpcode::G_LSHR;
8304 unsigned RevShiftOpc = IsLeft ? TargetOpcode::G_LSHR : TargetOpcode::G_SHL;
8305 auto BitWidthMinusOneC = MIRBuilder.buildConstant(Res: AmtTy, Val: EltSizeInBits - 1);
8306 Register ShVal;
8307 Register RevShiftVal;
8308 if (isPowerOf2_32(Value: EltSizeInBits)) {
8309 // (rotl x, c) -> x << (c & (w - 1)) | x >> (-c & (w - 1))
8310 // (rotr x, c) -> x >> (c & (w - 1)) | x << (-c & (w - 1))
8311 auto NegAmt = MIRBuilder.buildSub(Dst: AmtTy, Src0: Zero, Src1: Amt);
8312 auto ShAmt = MIRBuilder.buildAnd(Dst: AmtTy, Src0: Amt, Src1: BitWidthMinusOneC);
8313 ShVal = MIRBuilder.buildInstr(Opc: ShOpc, DstOps: {DstTy}, SrcOps: {Src, ShAmt}).getReg(Idx: 0);
8314 auto RevAmt = MIRBuilder.buildAnd(Dst: AmtTy, Src0: NegAmt, Src1: BitWidthMinusOneC);
8315 RevShiftVal =
8316 MIRBuilder.buildInstr(Opc: RevShiftOpc, DstOps: {DstTy}, SrcOps: {Src, RevAmt}).getReg(Idx: 0);
8317 } else {
8318 // (rotl x, c) -> x << (c % w) | x >> 1 >> (w - 1 - (c % w))
8319 // (rotr x, c) -> x >> (c % w) | x << 1 << (w - 1 - (c % w))
8320 auto BitWidthC = MIRBuilder.buildConstant(Res: AmtTy, Val: EltSizeInBits);
8321 auto ShAmt = MIRBuilder.buildURem(Dst: AmtTy, Src0: Amt, Src1: BitWidthC);
8322 ShVal = MIRBuilder.buildInstr(Opc: ShOpc, DstOps: {DstTy}, SrcOps: {Src, ShAmt}).getReg(Idx: 0);
8323 auto RevAmt = MIRBuilder.buildSub(Dst: AmtTy, Src0: BitWidthMinusOneC, Src1: ShAmt);
8324 auto One = MIRBuilder.buildConstant(Res: AmtTy, Val: 1);
8325 auto Inner = MIRBuilder.buildInstr(Opc: RevShiftOpc, DstOps: {DstTy}, SrcOps: {Src, One});
8326 RevShiftVal =
8327 MIRBuilder.buildInstr(Opc: RevShiftOpc, DstOps: {DstTy}, SrcOps: {Inner, RevAmt}).getReg(Idx: 0);
8328 }
8329 MIRBuilder.buildOr(Dst, Src0: ShVal, Src1: RevShiftVal, Flags: MachineInstr::Disjoint);
8330 MI.eraseFromParent();
8331 return Legalized;
8332}
8333
8334// Expand s32 = G_UITOFP s64 using bit operations to an IEEE float
8335// representation.
8336LegalizerHelper::LegalizeResult
8337LegalizerHelper::lowerU64ToF32BitOps(MachineInstr &MI) {
8338 auto [Dst, Src] = MI.getFirst2Regs();
8339 const LLT S64 = LLT::scalar(SizeInBits: 64);
8340 const LLT S32 = LLT::scalar(SizeInBits: 32);
8341 const LLT S1 = LLT::scalar(SizeInBits: 1);
8342
8343 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S32);
8344
8345 // unsigned cul2f(ulong u) {
8346 // uint lz = clz(u);
8347 // uint e = (u != 0) ? 127U + 63U - lz : 0;
8348 // u = (u << lz) & 0x7fffffffffffffffUL;
8349 // ulong t = u & 0xffffffffffUL;
8350 // uint v = (e << 23) | (uint)(u >> 40);
8351 // uint r = t > 0x8000000000UL ? 1U : (t == 0x8000000000UL ? v & 1U : 0U);
8352 // return as_float(v + r);
8353 // }
8354
8355 auto Zero32 = MIRBuilder.buildConstant(Res: S32, Val: 0);
8356 auto Zero64 = MIRBuilder.buildConstant(Res: S64, Val: 0);
8357
8358 auto LZ = MIRBuilder.buildCTLZ_ZERO_POISON(Dst: S32, Src0: Src);
8359
8360 auto K = MIRBuilder.buildConstant(Res: S32, Val: 127U + 63U);
8361 auto Sub = MIRBuilder.buildSub(Dst: S32, Src0: K, Src1: LZ);
8362
8363 auto NotZero = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: S1, Op0: Src, Op1: Zero64);
8364 auto E = MIRBuilder.buildSelect(Res: S32, Tst: NotZero, Op0: Sub, Op1: Zero32);
8365
8366 auto Mask0 = MIRBuilder.buildConstant(Res: S64, Val: (-1ULL) >> 1);
8367 auto ShlLZ = MIRBuilder.buildShl(Dst: S64, Src0: Src, Src1: LZ);
8368
8369 auto U = MIRBuilder.buildAnd(Dst: S64, Src0: ShlLZ, Src1: Mask0);
8370
8371 auto Mask1 = MIRBuilder.buildConstant(Res: S64, Val: 0xffffffffffULL);
8372 auto T = MIRBuilder.buildAnd(Dst: S64, Src0: U, Src1: Mask1);
8373
8374 auto UShl = MIRBuilder.buildLShr(Dst: S64, Src0: U, Src1: MIRBuilder.buildConstant(Res: S64, Val: 40));
8375 auto ShlE = MIRBuilder.buildShl(Dst: S32, Src0: E, Src1: MIRBuilder.buildConstant(Res: S32, Val: 23));
8376 auto V = MIRBuilder.buildOr(Dst: S32, Src0: ShlE, Src1: MIRBuilder.buildTrunc(Res: S32, Op: UShl));
8377
8378 auto C = MIRBuilder.buildConstant(Res: S64, Val: 0x8000000000ULL);
8379 auto RCmp = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_UGT, Res: S1, Op0: T, Op1: C);
8380 auto TCmp = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: S1, Op0: T, Op1: C);
8381 auto One = MIRBuilder.buildConstant(Res: S32, Val: 1);
8382
8383 auto VTrunc1 = MIRBuilder.buildAnd(Dst: S32, Src0: V, Src1: One);
8384 auto Select0 = MIRBuilder.buildSelect(Res: S32, Tst: TCmp, Op0: VTrunc1, Op1: Zero32);
8385 auto R = MIRBuilder.buildSelect(Res: S32, Tst: RCmp, Op0: One, Op1: Select0);
8386 MIRBuilder.buildAdd(Dst, Src0: V, Src1: R);
8387
8388 MI.eraseFromParent();
8389 return Legalized;
8390}
8391
8392// Expand s32 = G_UITOFP s64 to an IEEE float representation using bit
8393// operations and G_SITOFP
8394LegalizerHelper::LegalizeResult
8395LegalizerHelper::lowerU64ToF32WithSITOFP(MachineInstr &MI) {
8396 auto [Dst, Src] = MI.getFirst2Regs();
8397 const LLT S64 = LLT::scalar(SizeInBits: 64);
8398 const LLT S32 = LLT::scalar(SizeInBits: 32);
8399 const LLT S1 = LLT::scalar(SizeInBits: 1);
8400
8401 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S32);
8402
8403 // For i64 < INT_MAX we simply reuse SITOFP.
8404 // Otherwise, divide i64 by 2, round result by ORing with the lowest bit
8405 // saved before division, convert to float by SITOFP, multiply the result
8406 // by 2.
8407 auto One = MIRBuilder.buildConstant(Res: S64, Val: 1);
8408 auto Zero = MIRBuilder.buildConstant(Res: S64, Val: 0);
8409 // Result if Src < INT_MAX
8410 auto SmallResult = MIRBuilder.buildSITOFP(Dst: S32, Src0: Src);
8411 // Result if Src >= INT_MAX
8412 auto Halved = MIRBuilder.buildLShr(Dst: S64, Src0: Src, Src1: One);
8413 auto LowerBit = MIRBuilder.buildAnd(Dst: S64, Src0: Src, Src1: One);
8414 auto RoundedHalved = MIRBuilder.buildOr(Dst: S64, Src0: Halved, Src1: LowerBit);
8415 auto HalvedFP = MIRBuilder.buildSITOFP(Dst: S32, Src0: RoundedHalved);
8416 auto LargeResult = MIRBuilder.buildFAdd(Dst: S32, Src0: HalvedFP, Src1: HalvedFP);
8417 // Check if the original value is larger than INT_MAX by comparing with
8418 // zero to pick one of the two conversions.
8419 auto IsLarge =
8420 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_SLT, Res: S1, Op0: Src, Op1: Zero);
8421 MIRBuilder.buildSelect(Res: Dst, Tst: IsLarge, Op0: LargeResult, Op1: SmallResult);
8422
8423 MI.eraseFromParent();
8424 return Legalized;
8425}
8426
8427// Expand s64 = G_UITOFP s64 using bit and float arithmetic operations to an
8428// IEEE double representation.
8429LegalizerHelper::LegalizeResult
8430LegalizerHelper::lowerU64ToF64BitFloatOps(MachineInstr &MI) {
8431 auto [Dst, Src] = MI.getFirst2Regs();
8432 const LLT S64 = LLT::scalar(SizeInBits: 64);
8433 const LLT S32 = LLT::scalar(SizeInBits: 32);
8434
8435 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S64);
8436
8437 // We create double value from 32 bit parts with 32 exponent difference.
8438 // Note that + and - are float operations that adjust the implicit leading
8439 // one, the bases 2^52 and 2^84 are for illustrative purposes.
8440 //
8441 // X = 2^52 * 1.0...LowBits
8442 // Y = 2^84 * 1.0...HighBits
8443 // Scratch = 2^84 * 1.0...HighBits - 2^84 * 1.0 - 2^52 * 1.0
8444 // = - 2^52 * 1.0...HighBits
8445 // Result = - 2^52 * 1.0...HighBits + 2^52 * 1.0...LowBits
8446 auto TwoP52 = MIRBuilder.buildConstant(Res: S64, UINT64_C(0x4330000000000000));
8447 auto TwoP84 = MIRBuilder.buildConstant(Res: S64, UINT64_C(0x4530000000000000));
8448 auto TwoP52P84 = llvm::bit_cast<double>(UINT64_C(0x4530000000100000));
8449 auto TwoP52P84FP = MIRBuilder.buildFConstant(Res: S64, Val: TwoP52P84);
8450 auto HalfWidth = MIRBuilder.buildConstant(Res: S64, Val: 32);
8451
8452 auto LowBits = MIRBuilder.buildTrunc(Res: S32, Op: Src);
8453 LowBits = MIRBuilder.buildZExt(Res: S64, Op: LowBits);
8454 auto LowBitsFP = MIRBuilder.buildOr(Dst: S64, Src0: TwoP52, Src1: LowBits);
8455 auto HighBits = MIRBuilder.buildLShr(Dst: S64, Src0: Src, Src1: HalfWidth);
8456 auto HighBitsFP = MIRBuilder.buildOr(Dst: S64, Src0: TwoP84, Src1: HighBits);
8457 auto Scratch = MIRBuilder.buildFSub(Dst: S64, Src0: HighBitsFP, Src1: TwoP52P84FP);
8458 MIRBuilder.buildFAdd(Dst, Src0: Scratch, Src1: LowBitsFP);
8459
8460 MI.eraseFromParent();
8461 return Legalized;
8462}
8463
8464/// i64->fp16 itofp can be lowered to i64->f64,f64->f32,f32->f16. We cannot
8465/// convert fpround f64->f16 without double-rounding, so we manually perform the
8466/// lowering here where we know it is valid.
8467static LegalizerHelper::LegalizeResult
8468loweri64tof16ITOFP(MachineInstr &MI, Register Dst, LLT DstTy, Register Src,
8469 LLT SrcTy, MachineIRBuilder &MIRBuilder) {
8470 auto DstFpTy =
8471 SrcTy.changeElementType(NewEltTy: LLT::floatIEEE(SizeInBits: SrcTy.getScalarSizeInBits()));
8472 auto M1 = MI.getOpcode() == TargetOpcode::G_UITOFP
8473 ? MIRBuilder.buildUITOFP(Dst: DstFpTy, Src0: Src)
8474 : MIRBuilder.buildSITOFP(Dst: DstFpTy, Src0: Src);
8475 LLT F32Ty = DstFpTy.changeElementSize(NewEltSize: 32);
8476 auto M2 = MIRBuilder.buildFPTrunc(Res: F32Ty, Op: M1);
8477 MIRBuilder.buildFPTrunc(Res: Dst, Op: M2);
8478 MI.eraseFromParent();
8479 return LegalizerHelper::Legalized;
8480}
8481
8482LegalizerHelper::LegalizeResult LegalizerHelper::lowerUITOFP(MachineInstr &MI) {
8483 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8484
8485 if (SrcTy == LLT::scalar(SizeInBits: 1)) {
8486 auto True = MIRBuilder.buildFConstant(Res: DstTy, Val: 1.0);
8487 auto False = MIRBuilder.buildFConstant(Res: DstTy, Val: 0.0);
8488 MIRBuilder.buildSelect(Res: Dst, Tst: Src, Op0: True, Op1: False);
8489 MI.eraseFromParent();
8490 return Legalized;
8491 }
8492
8493 if (DstTy.getScalarSizeInBits() == 16 && SrcTy.getScalarSizeInBits() == 64)
8494 return loweri64tof16ITOFP(MI, Dst, DstTy, Src, SrcTy, MIRBuilder);
8495
8496 if (SrcTy != LLT::scalar(SizeInBits: 64))
8497 return UnableToLegalize;
8498
8499 if (DstTy == LLT::scalar(SizeInBits: 32))
8500 // TODO: SelectionDAG has several alternative expansions to port which may
8501 // be more reasonable depending on the available instructions. We also need
8502 // a more advanced mechanism to choose an optimal version depending on
8503 // target features such as sitofp or CTLZ availability.
8504 return lowerU64ToF32WithSITOFP(MI);
8505
8506 if (DstTy == LLT::scalar(SizeInBits: 64))
8507 return lowerU64ToF64BitFloatOps(MI);
8508
8509 return UnableToLegalize;
8510}
8511
8512LegalizerHelper::LegalizeResult LegalizerHelper::lowerSITOFP(MachineInstr &MI) {
8513 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8514
8515 const LLT I64 = LLT::integer(SizeInBits: 64);
8516 const LLT I32 = LLT::integer(SizeInBits: 32);
8517 const LLT I1 = LLT::integer(SizeInBits: 1);
8518
8519 if (SrcTy == I1) {
8520 auto True = MIRBuilder.buildFConstant(Res: DstTy, Val: -1.0);
8521 auto False = MIRBuilder.buildFConstant(Res: DstTy, Val: 0.0);
8522 MIRBuilder.buildSelect(Res: Dst, Tst: Src, Op0: True, Op1: False);
8523 MI.eraseFromParent();
8524 return Legalized;
8525 }
8526
8527 if (DstTy.getScalarSizeInBits() == 16 && SrcTy.getScalarSizeInBits() == 64)
8528 return loweri64tof16ITOFP(MI, Dst, DstTy, Src, SrcTy, MIRBuilder);
8529
8530 if (SrcTy != I64)
8531 return UnableToLegalize;
8532
8533 if (DstTy.getScalarSizeInBits() == 32) {
8534 // signed cl2f(long l) {
8535 // long s = l >> 63;
8536 // float r = cul2f((l + s) ^ s);
8537 // return s ? -r : r;
8538 // }
8539 Register L = Src;
8540 auto SignBit = MIRBuilder.buildConstant(Res: I64, Val: 63);
8541 auto S = MIRBuilder.buildAShr(Dst: I64, Src0: L, Src1: SignBit);
8542
8543 auto LPlusS = MIRBuilder.buildAdd(Dst: I64, Src0: L, Src1: S);
8544 auto Xor = MIRBuilder.buildXor(Dst: I64, Src0: LPlusS, Src1: S);
8545 auto R = MIRBuilder.buildUITOFP(Dst: I32, Src0: Xor);
8546
8547 auto RNeg = MIRBuilder.buildFNeg(Dst: I32, Src0: R);
8548 auto SignNotZero = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: I1, Op0: S,
8549 Op1: MIRBuilder.buildConstant(Res: I64, Val: 0));
8550 MIRBuilder.buildSelect(Res: Dst, Tst: SignNotZero, Op0: RNeg, Op1: R);
8551 MI.eraseFromParent();
8552 return Legalized;
8553 }
8554
8555 return UnableToLegalize;
8556}
8557
8558LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOUI(MachineInstr &MI) {
8559 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8560 const LLT S64 = LLT::scalar(SizeInBits: 64);
8561 const LLT S32 = LLT::scalar(SizeInBits: 32);
8562
8563 if (SrcTy != S64 && SrcTy != S32)
8564 return UnableToLegalize;
8565 if (DstTy != S32 && DstTy != S64)
8566 return UnableToLegalize;
8567
8568 // FPTOSI gives same result as FPTOUI for positive signed integers.
8569 // FPTOUI needs to deal with fp values that convert to unsigned integers
8570 // greater or equal to 2^31 for float or 2^63 for double. For brevity 2^Exp.
8571
8572 APInt TwoPExpInt = APInt::getSignMask(BitWidth: DstTy.getSizeInBits());
8573 APFloat TwoPExpFP(SrcTy.getSizeInBits() == 32 ? APFloat::IEEEsingle()
8574 : APFloat::IEEEdouble(),
8575 APInt::getZero(numBits: SrcTy.getSizeInBits()));
8576 TwoPExpFP.convertFromAPInt(Input: TwoPExpInt, IsSigned: false, RM: APFloat::rmNearestTiesToEven);
8577
8578 MachineInstrBuilder FPTOSI = MIRBuilder.buildFPTOSI(Dst: DstTy, Src0: Src);
8579
8580 MachineInstrBuilder Threshold = MIRBuilder.buildFConstant(Res: SrcTy, Val: TwoPExpFP);
8581 // For fp Value greater or equal to Threshold(2^Exp), we use FPTOSI on
8582 // (Value - 2^Exp) and add 2^Exp by setting highest bit in result to 1.
8583 MachineInstrBuilder FSub = MIRBuilder.buildFSub(Dst: SrcTy, Src0: Src, Src1: Threshold);
8584 MachineInstrBuilder ResLowBits = MIRBuilder.buildFPTOSI(Dst: DstTy, Src0: FSub);
8585 MachineInstrBuilder ResHighBit = MIRBuilder.buildConstant(Res: DstTy, Val: TwoPExpInt);
8586 MachineInstrBuilder Res = MIRBuilder.buildXor(Dst: DstTy, Src0: ResLowBits, Src1: ResHighBit);
8587
8588 const LLT S1 = LLT::scalar(SizeInBits: 1);
8589
8590 MachineInstrBuilder FCMP =
8591 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_ULT, Res: S1, Op0: Src, Op1: Threshold);
8592 MIRBuilder.buildSelect(Res: Dst, Tst: FCMP, Op0: FPTOSI, Op1: Res);
8593
8594 MI.eraseFromParent();
8595 return Legalized;
8596}
8597
8598LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOSI(MachineInstr &MI) {
8599 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8600 const LLT S64 = LLT::scalar(SizeInBits: 64);
8601 const LLT S32 = LLT::scalar(SizeInBits: 32);
8602
8603 // FIXME: Only f32 to i64 conversions are supported.
8604 if (SrcTy.getScalarType() != S32 || DstTy.getScalarType() != S64)
8605 return UnableToLegalize;
8606
8607 // Expand f32 -> i64 conversion
8608 // This algorithm comes from compiler-rt's implementation of fixsfdi:
8609 // https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/fixsfdi.c
8610
8611 unsigned SrcEltBits = SrcTy.getScalarSizeInBits();
8612
8613 auto ExponentMask = MIRBuilder.buildConstant(Res: SrcTy, Val: 0x7F800000);
8614 auto ExponentLoBit = MIRBuilder.buildConstant(Res: SrcTy, Val: 23);
8615
8616 auto AndExpMask = MIRBuilder.buildAnd(Dst: SrcTy, Src0: Src, Src1: ExponentMask);
8617 auto ExponentBits = MIRBuilder.buildLShr(Dst: SrcTy, Src0: AndExpMask, Src1: ExponentLoBit);
8618
8619 auto SignMask = MIRBuilder.buildConstant(Res: SrcTy,
8620 Val: APInt::getSignMask(BitWidth: SrcEltBits));
8621 auto AndSignMask = MIRBuilder.buildAnd(Dst: SrcTy, Src0: Src, Src1: SignMask);
8622 auto SignLowBit = MIRBuilder.buildConstant(Res: SrcTy, Val: SrcEltBits - 1);
8623 auto Sign = MIRBuilder.buildAShr(Dst: SrcTy, Src0: AndSignMask, Src1: SignLowBit);
8624 Sign = MIRBuilder.buildSExt(Res: DstTy, Op: Sign);
8625
8626 auto MantissaMask = MIRBuilder.buildConstant(Res: SrcTy, Val: 0x007FFFFF);
8627 auto AndMantissaMask = MIRBuilder.buildAnd(Dst: SrcTy, Src0: Src, Src1: MantissaMask);
8628 auto K = MIRBuilder.buildConstant(Res: SrcTy, Val: 0x00800000);
8629
8630 auto R = MIRBuilder.buildOr(Dst: SrcTy, Src0: AndMantissaMask, Src1: K);
8631 R = MIRBuilder.buildZExt(Res: DstTy, Op: R);
8632
8633 auto Bias = MIRBuilder.buildConstant(Res: SrcTy, Val: 127);
8634 auto Exponent = MIRBuilder.buildSub(Dst: SrcTy, Src0: ExponentBits, Src1: Bias);
8635 auto SubExponent = MIRBuilder.buildSub(Dst: SrcTy, Src0: Exponent, Src1: ExponentLoBit);
8636 auto ExponentSub = MIRBuilder.buildSub(Dst: SrcTy, Src0: ExponentLoBit, Src1: Exponent);
8637
8638 auto Shl = MIRBuilder.buildShl(Dst: DstTy, Src0: R, Src1: SubExponent);
8639 auto Srl = MIRBuilder.buildLShr(Dst: DstTy, Src0: R, Src1: ExponentSub);
8640
8641 const LLT S1 = LLT::scalar(SizeInBits: 1);
8642 auto CmpGt = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SGT,
8643 Res: S1, Op0: Exponent, Op1: ExponentLoBit);
8644
8645 R = MIRBuilder.buildSelect(Res: DstTy, Tst: CmpGt, Op0: Shl, Op1: Srl);
8646
8647 auto XorSign = MIRBuilder.buildXor(Dst: DstTy, Src0: R, Src1: Sign);
8648 auto Ret = MIRBuilder.buildSub(Dst: DstTy, Src0: XorSign, Src1: Sign);
8649
8650 auto ZeroSrcTy = MIRBuilder.buildConstant(Res: SrcTy, Val: 0);
8651
8652 auto ExponentLt0 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT,
8653 Res: S1, Op0: Exponent, Op1: ZeroSrcTy);
8654
8655 auto ZeroDstTy = MIRBuilder.buildConstant(Res: DstTy, Val: 0);
8656 MIRBuilder.buildSelect(Res: Dst, Tst: ExponentLt0, Op0: ZeroDstTy, Op1: Ret);
8657
8658 MI.eraseFromParent();
8659 return Legalized;
8660}
8661
8662LegalizerHelper::LegalizeResult
8663LegalizerHelper::lowerFPTOINT_SAT(MachineInstr &MI) {
8664 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8665
8666 bool IsSigned = MI.getOpcode() == TargetOpcode::G_FPTOSI_SAT;
8667 unsigned SatWidth = DstTy.getScalarSizeInBits();
8668
8669 // Determine minimum and maximum integer values and their corresponding
8670 // floating-point values.
8671 APInt MinInt, MaxInt;
8672 if (IsSigned) {
8673 MinInt = APInt::getSignedMinValue(numBits: SatWidth);
8674 MaxInt = APInt::getSignedMaxValue(numBits: SatWidth);
8675 } else {
8676 MinInt = APInt::getMinValue(numBits: SatWidth);
8677 MaxInt = APInt::getMaxValue(numBits: SatWidth);
8678 }
8679
8680 const fltSemantics &Semantics = getFltSemanticForLLT(Ty: SrcTy.getScalarType());
8681 APFloat MinFloat(Semantics);
8682 APFloat MaxFloat(Semantics);
8683
8684 APFloat::opStatus MinStatus =
8685 MinFloat.convertFromAPInt(Input: MinInt, IsSigned, RM: APFloat::rmTowardZero);
8686 APFloat::opStatus MaxStatus =
8687 MaxFloat.convertFromAPInt(Input: MaxInt, IsSigned, RM: APFloat::rmTowardZero);
8688 bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact) &&
8689 !(MaxStatus & APFloat::opStatus::opInexact);
8690
8691 // If the integer bounds are exactly representable as floats, emit a
8692 // min+max+fptoi sequence. Otherwise we have to use a sequence of comparisons
8693 // and selects.
8694 if (AreExactFloatBounds) {
8695 // Clamp Src by MinFloat from below. If Src is NaN the result is MinFloat.
8696 auto MaxC = MIRBuilder.buildFConstant(Res: SrcTy, Val: MinFloat);
8697 auto MaxP =
8698 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OGT, Res: LLT::integer(SizeInBits: 1), Op0: Src, Op1: MaxC);
8699 auto Max = MIRBuilder.buildSelect(Res: SrcTy, Tst: MaxP, Op0: Src, Op1: MaxC);
8700 // Clamp by MaxFloat from above. NaN cannot occur.
8701 auto MinC = MIRBuilder.buildFConstant(Res: SrcTy, Val: MaxFloat);
8702 auto MinP = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OLT, Res: LLT::integer(SizeInBits: 1), Op0: Max,
8703 Op1: MinC, Flags: MachineInstr::FmNoNans);
8704 auto Min =
8705 MIRBuilder.buildSelect(Res: SrcTy, Tst: MinP, Op0: Max, Op1: MinC, Flags: MachineInstr::FmNoNans);
8706 // Convert clamped value to integer. In the unsigned case we're done,
8707 // because we mapped NaN to MinFloat, which will cast to zero.
8708 if (!IsSigned) {
8709 MIRBuilder.buildFPTOUI(Dst, Src0: Min);
8710 MI.eraseFromParent();
8711 return Legalized;
8712 }
8713
8714 // Otherwise, select 0 if Src is NaN.
8715 auto FpToInt = MIRBuilder.buildFPTOSI(Dst: DstTy, Src0: Min);
8716 auto IsZero =
8717 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_UNO, Res: LLT::integer(SizeInBits: 1), Op0: Src, Op1: Src);
8718 MIRBuilder.buildSelect(Res: Dst, Tst: IsZero, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: 0),
8719 Op1: FpToInt);
8720 MI.eraseFromParent();
8721 return Legalized;
8722 }
8723
8724 // Result of direct conversion. The assumption here is that the operation is
8725 // non-trapping and it's fine to apply it to an out-of-range value if we
8726 // select it away later.
8727 auto FpToInt = IsSigned ? MIRBuilder.buildFPTOSI(Dst: DstTy, Src0: Src)
8728 : MIRBuilder.buildFPTOUI(Dst: DstTy, Src0: Src);
8729
8730 // If Src ULT MinFloat, select MinInt. In particular, this also selects
8731 // MinInt if Src is NaN.
8732 auto ULT = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_ULT, Res: LLT::integer(SizeInBits: 1), Op0: Src,
8733 Op1: MIRBuilder.buildFConstant(Res: SrcTy, Val: MinFloat));
8734 auto Max = MIRBuilder.buildSelect(
8735 Res: DstTy, Tst: ULT, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: MinInt), Op1: FpToInt);
8736 // If Src OGT MaxFloat, select MaxInt.
8737 auto OGT = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OGT, Res: LLT::integer(SizeInBits: 1), Op0: Src,
8738 Op1: MIRBuilder.buildFConstant(Res: SrcTy, Val: MaxFloat));
8739
8740 // In the unsigned case we are done, because we mapped NaN to MinInt, which
8741 // is already zero.
8742 if (!IsSigned) {
8743 MIRBuilder.buildSelect(Res: Dst, Tst: OGT, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: MaxInt),
8744 Op1: Max);
8745 MI.eraseFromParent();
8746 return Legalized;
8747 }
8748
8749 // Otherwise, select 0 if Src is NaN.
8750 auto Min = MIRBuilder.buildSelect(
8751 Res: DstTy, Tst: OGT, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: MaxInt), Op1: Max);
8752 auto IsZero =
8753 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_UNO, Res: LLT::integer(SizeInBits: 1), Op0: Src, Op1: Src);
8754 MIRBuilder.buildSelect(Res: Dst, Tst: IsZero, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: 0), Op1: Min);
8755 MI.eraseFromParent();
8756 return Legalized;
8757}
8758
8759// Floating-point conversions using truncating and extending loads and stores.
8760LegalizerHelper::LegalizeResult
8761LegalizerHelper::lowerFPExtAndTruncMem(MachineInstr &MI) {
8762 assert((MI.getOpcode() == TargetOpcode::G_FPEXT ||
8763 MI.getOpcode() == TargetOpcode::G_FPTRUNC) &&
8764 "Only G_FPEXT and G_FPTRUNC are expected");
8765
8766 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
8767 MachinePointerInfo PtrInfo;
8768 unsigned StoreOpc;
8769 unsigned LoadOpc;
8770 LLT StackTy;
8771 if (MI.getOpcode() == TargetOpcode::G_FPEXT) {
8772 StackTy = SrcTy;
8773 StoreOpc = TargetOpcode::G_STORE;
8774 LoadOpc = TargetOpcode::G_FPEXTLOAD;
8775 } else {
8776 StackTy = DstTy;
8777 StoreOpc = TargetOpcode::G_FPTRUNCSTORE;
8778 LoadOpc = TargetOpcode::G_LOAD;
8779 }
8780
8781 Align StackTyAlign = getStackTemporaryAlignment(Ty: StackTy);
8782 auto StackTemp =
8783 createStackTemporary(Bytes: StackTy.getSizeInBytes(), Alignment: StackTyAlign, PtrInfo);
8784
8785 MachineFunction &MF = MIRBuilder.getMF();
8786 auto *StoreMMO = MF.getMachineMemOperand(PtrInfo, f: MachineMemOperand::MOStore,
8787 MemTy: StackTy, base_alignment: StackTyAlign);
8788 MIRBuilder.buildStoreInstr(Opcode: StoreOpc, Val: SrcReg, Addr: StackTemp, MMO&: *StoreMMO);
8789
8790 auto *LoadMMO = MF.getMachineMemOperand(PtrInfo, f: MachineMemOperand::MOLoad,
8791 MemTy: StackTy, base_alignment: StackTyAlign);
8792 MIRBuilder.buildLoadInstr(Opcode: LoadOpc, Res: DstReg, Addr: StackTemp, MMO&: *LoadMMO);
8793
8794 MI.eraseFromParent();
8795 return Legalized;
8796}
8797
8798// f64 -> f16 conversion using round-to-nearest-even rounding mode.
8799LegalizerHelper::LegalizeResult
8800LegalizerHelper::lowerFPTRUNC_F64_TO_F16(MachineInstr &MI) {
8801 const LLT S1 = LLT::scalar(SizeInBits: 1);
8802 const LLT I32 = LLT::integer(SizeInBits: 32);
8803
8804 auto [Dst, Src] = MI.getFirst2Regs();
8805 assert(MRI.getType(Dst).getScalarType() == LLT::float16() &&
8806 MRI.getType(Src).getScalarType() == LLT::float64());
8807
8808 if (MRI.getType(Reg: Src).isVector()) // TODO: Handle vectors directly.
8809 return UnableToLegalize;
8810
8811 if (MI.getFlag(Flag: MachineInstr::FmAfn)) {
8812 unsigned Flags = MI.getFlags();
8813 auto Src32 = MIRBuilder.buildFPTrunc(Res: LLT::float32(), Op: Src, Flags);
8814 MIRBuilder.buildFPTrunc(Res: Dst, Op: Src32, Flags);
8815 MI.eraseFromParent();
8816 return Legalized;
8817 }
8818
8819 const unsigned ExpMask = 0x7ff;
8820 const unsigned ExpBiasf64 = 1023;
8821 const unsigned ExpBiasf16 = 15;
8822
8823 auto Unmerge = MIRBuilder.buildUnmerge(Res: I32, Op: Src);
8824 Register U = Unmerge.getReg(Idx: 0);
8825 Register UH = Unmerge.getReg(Idx: 1);
8826
8827 auto E = MIRBuilder.buildLShr(Dst: I32, Src0: UH, Src1: MIRBuilder.buildConstant(Res: I32, Val: 20));
8828 E = MIRBuilder.buildAnd(Dst: I32, Src0: E, Src1: MIRBuilder.buildConstant(Res: I32, Val: ExpMask));
8829
8830 // Subtract the fp64 exponent bias (1023) to get the real exponent and
8831 // add the f16 bias (15) to get the biased exponent for the f16 format.
8832 E = MIRBuilder.buildAdd(
8833 Dst: I32, Src0: E, Src1: MIRBuilder.buildConstant(Res: I32, Val: -ExpBiasf64 + ExpBiasf16));
8834
8835 auto M = MIRBuilder.buildLShr(Dst: I32, Src0: UH, Src1: MIRBuilder.buildConstant(Res: I32, Val: 8));
8836 M = MIRBuilder.buildAnd(Dst: I32, Src0: M, Src1: MIRBuilder.buildConstant(Res: I32, Val: 0xffe));
8837
8838 auto MaskedSig =
8839 MIRBuilder.buildAnd(Dst: I32, Src0: UH, Src1: MIRBuilder.buildConstant(Res: I32, Val: 0x1ff));
8840 MaskedSig = MIRBuilder.buildOr(Dst: I32, Src0: MaskedSig, Src1: U);
8841
8842 auto Zero = MIRBuilder.buildConstant(Res: I32, Val: 0);
8843 auto SigCmpNE0 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: S1, Op0: MaskedSig, Op1: Zero);
8844 auto Lo40Set = MIRBuilder.buildZExt(Res: I32, Op: SigCmpNE0);
8845 M = MIRBuilder.buildOr(Dst: I32, Src0: M, Src1: Lo40Set);
8846
8847 // (M != 0 ? 0x0200 : 0) | 0x7c00;
8848 auto Bits0x200 = MIRBuilder.buildConstant(Res: I32, Val: 0x0200);
8849 auto CmpM_NE0 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: S1, Op0: M, Op1: Zero);
8850 auto SelectCC = MIRBuilder.buildSelect(Res: I32, Tst: CmpM_NE0, Op0: Bits0x200, Op1: Zero);
8851
8852 auto Bits0x7c00 = MIRBuilder.buildConstant(Res: I32, Val: 0x7c00);
8853 auto I = MIRBuilder.buildOr(Dst: I32, Src0: SelectCC, Src1: Bits0x7c00);
8854
8855 // N = M | (E << 12);
8856 auto EShl12 = MIRBuilder.buildShl(Dst: I32, Src0: E, Src1: MIRBuilder.buildConstant(Res: I32, Val: 12));
8857 auto N = MIRBuilder.buildOr(Dst: I32, Src0: M, Src1: EShl12);
8858
8859 // B = clamp(1-E, 0, 13);
8860 auto One = MIRBuilder.buildConstant(Res: I32, Val: 1);
8861 auto OneSubExp = MIRBuilder.buildSub(Dst: I32, Src0: One, Src1: E);
8862 auto B = MIRBuilder.buildSMax(Dst: I32, Src0: OneSubExp, Src1: Zero);
8863 B = MIRBuilder.buildSMin(Dst: I32, Src0: B, Src1: MIRBuilder.buildConstant(Res: I32, Val: 13));
8864
8865 auto SigSetHigh =
8866 MIRBuilder.buildOr(Dst: I32, Src0: M, Src1: MIRBuilder.buildConstant(Res: I32, Val: 0x1000));
8867
8868 auto D = MIRBuilder.buildLShr(Dst: I32, Src0: SigSetHigh, Src1: B);
8869 auto D0 = MIRBuilder.buildShl(Dst: I32, Src0: D, Src1: B);
8870
8871 auto D0_NE_SigSetHigh = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: S1,
8872 Op0: D0, Op1: SigSetHigh);
8873 auto D1 = MIRBuilder.buildZExt(Res: I32, Op: D0_NE_SigSetHigh);
8874 D = MIRBuilder.buildOr(Dst: I32, Src0: D, Src1: D1);
8875
8876 auto CmpELtOne = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: S1, Op0: E, Op1: One);
8877 auto V = MIRBuilder.buildSelect(Res: I32, Tst: CmpELtOne, Op0: D, Op1: N);
8878
8879 auto VLow3 = MIRBuilder.buildAnd(Dst: I32, Src0: V, Src1: MIRBuilder.buildConstant(Res: I32, Val: 7));
8880 V = MIRBuilder.buildLShr(Dst: I32, Src0: V, Src1: MIRBuilder.buildConstant(Res: I32, Val: 2));
8881
8882 auto VLow3Eq3 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: S1, Op0: VLow3,
8883 Op1: MIRBuilder.buildConstant(Res: I32, Val: 3));
8884 auto V0 = MIRBuilder.buildZExt(Res: I32, Op: VLow3Eq3);
8885
8886 auto VLow3Gt5 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SGT, Res: S1, Op0: VLow3,
8887 Op1: MIRBuilder.buildConstant(Res: I32, Val: 5));
8888 auto V1 = MIRBuilder.buildZExt(Res: I32, Op: VLow3Gt5);
8889
8890 V1 = MIRBuilder.buildOr(Dst: I32, Src0: V0, Src1: V1);
8891 V = MIRBuilder.buildAdd(Dst: I32, Src0: V, Src1: V1);
8892
8893 auto CmpEGt30 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SGT, Res: S1, Op0: E,
8894 Op1: MIRBuilder.buildConstant(Res: I32, Val: 30));
8895 V = MIRBuilder.buildSelect(Res: I32, Tst: CmpEGt30,
8896 Op0: MIRBuilder.buildConstant(Res: I32, Val: 0x7c00), Op1: V);
8897
8898 auto CmpEGt1039 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: S1, Op0: E,
8899 Op1: MIRBuilder.buildConstant(Res: I32, Val: 1039));
8900 V = MIRBuilder.buildSelect(Res: I32, Tst: CmpEGt1039, Op0: I, Op1: V);
8901
8902 // Extract the sign bit.
8903 auto Sign = MIRBuilder.buildLShr(Dst: I32, Src0: UH, Src1: MIRBuilder.buildConstant(Res: I32, Val: 16));
8904 Sign = MIRBuilder.buildAnd(Dst: I32, Src0: Sign, Src1: MIRBuilder.buildConstant(Res: I32, Val: 0x8000));
8905
8906 // Insert the sign bit
8907 V = MIRBuilder.buildOr(Dst: I32, Src0: Sign, Src1: V);
8908
8909 MIRBuilder.buildTrunc(Res: Dst, Op: V);
8910 MI.eraseFromParent();
8911 return Legalized;
8912}
8913
8914// f32 -> bf16 conversion using round-to-nearest-even rounding mode.
8915LegalizerHelper::LegalizeResult
8916LegalizerHelper::lowerFPTRUNC_F32_TO_BF16(MachineInstr &MI) {
8917 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
8918 assert(DstTy.getScalarType() == LLT::bfloat16() &&
8919 SrcTy.getScalarType() == LLT::float32());
8920
8921 LLT I1Ty = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
8922 LLT I16Ty = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 16));
8923 LLT I32Ty = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 32));
8924
8925 auto SrcI = MIRBuilder.buildBitcast(Dst: I32Ty, Src: SrcReg);
8926
8927 // Conversions should set NaN's quiet bit. This also prevents NaNs from
8928 // turning into infinities.
8929 auto NaN = MIRBuilder.buildOr(Dst: I32Ty, Src0: SrcI,
8930 Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 0x400000));
8931
8932 // Factor in the contribution of the low 16 bits.
8933 auto Lsb =
8934 MIRBuilder.buildLShr(Dst: I32Ty, Src0: SrcI, Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 16));
8935 Lsb = MIRBuilder.buildAnd(Dst: I32Ty, Src0: Lsb, Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 1));
8936 auto RoundingBias =
8937 MIRBuilder.buildAdd(Dst: I32Ty, Src0: Lsb, Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 0x7fff));
8938 auto Add = MIRBuilder.buildAdd(Dst: I32Ty, Src0: SrcI, Src1: RoundingBias);
8939
8940 // Don't round if we had a NaN, we don't want to turn 0x7fffffff into
8941 // 0x80000000.
8942 if (!MI.getFlag(Flag: MachineInstr::FmNoNans)) {
8943 auto IsNaN = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_UNO, Res: I1Ty, Op0: SrcReg,
8944 Op1: MIRBuilder.buildFConstant(Res: SrcTy, Val: 0));
8945 Add = MIRBuilder.buildSelect(Res: I32Ty, Tst: IsNaN, Op0: NaN, Op1: Add);
8946 }
8947
8948 // Now that we have rounded, shift the bits into position.
8949 auto Srl =
8950 MIRBuilder.buildLShr(Dst: I32Ty, Src0: Add, Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 16));
8951 auto Trunc = MIRBuilder.buildTrunc(Res: I16Ty, Op: Srl);
8952 MIRBuilder.buildBitcast(Dst: DstReg, Src: Trunc);
8953 MI.eraseFromParent();
8954 return Legalized;
8955}
8956
8957LegalizerHelper::LegalizeResult
8958LegalizerHelper::lowerFPTRUNC(MachineInstr &MI) {
8959 auto [DstTy, SrcTy] = MI.getFirst2LLTs();
8960 if (DstTy.getScalarType().isFloat16() && SrcTy.getScalarType().isFloat64())
8961 return lowerFPTRUNC_F64_TO_F16(MI);
8962
8963 if (DstTy.getScalarType().isBFloat16() && SrcTy.getScalarType().isFloat32())
8964 return lowerFPTRUNC_F32_TO_BF16(MI);
8965
8966 return lowerFPExtAndTruncMem(MI);
8967}
8968
8969LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPOWI(MachineInstr &MI) {
8970 auto [Dst, Src0, Src1] = MI.getFirst3Regs();
8971 LLT Ty = MRI.getType(Reg: Dst);
8972
8973 auto CvtSrc1 = MIRBuilder.buildSITOFP(Dst: Ty, Src0: Src1);
8974 MIRBuilder.buildFPow(Dst, Src0, Src1: CvtSrc1, Flags: MI.getFlags());
8975 MI.eraseFromParent();
8976 return Legalized;
8977}
8978
8979LegalizerHelper::LegalizeResult LegalizerHelper::lowerFMODF(MachineInstr &MI) {
8980 auto [DstFrac, DstInt, Src] = MI.getFirst3Regs();
8981 LLT Ty = MRI.getType(Reg: Src);
8982 auto Flags = MI.getFlags();
8983 const LLT CondTy = Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
8984
8985 auto IntPart = MIRBuilder.buildIntrinsicTrunc(Dst: Ty, Src0: Src, Flags);
8986 auto FracPart = MIRBuilder.buildFSub(Dst: Ty, Src0: Src, Src1: IntPart, Flags);
8987
8988 Register FracToUse;
8989 if (MI.getFlag(Flag: MachineInstr::FmNoInfs)) {
8990 FracToUse = FracPart.getReg(Idx: 0);
8991 } else {
8992 auto Abs = MIRBuilder.buildFAbs(Dst: Ty, Src0: Src, Flags);
8993 const fltSemantics &Semantics = getFltSemanticForLLT(Ty: Ty.getScalarType());
8994 auto Inf = MIRBuilder.buildFConstant(Res: Ty, Val: APFloat::getInf(Sem: Semantics));
8995 auto IsInf = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OEQ, Res: CondTy, Op0: Abs, Op1: Inf);
8996 auto Zero = MIRBuilder.buildFConstant(Res: Ty, Val: 0.0);
8997 auto Select = MIRBuilder.buildSelect(Res: Ty, Tst: IsInf, Op0: Zero, Op1: FracPart);
8998 FracToUse = Select.getReg(Idx: 0);
8999 }
9000
9001 MIRBuilder.buildFCopysign(Dst: DstFrac, Src0: FracToUse, Src1: Src, Flags);
9002 MIRBuilder.buildCopy(Res: DstInt, Op: IntPart.getReg(Idx: 0));
9003
9004 MI.eraseFromParent();
9005 return Legalized;
9006}
9007
9008static CmpInst::Predicate minMaxToCompare(unsigned Opc) {
9009 switch (Opc) {
9010 case TargetOpcode::G_SMIN:
9011 return CmpInst::ICMP_SLT;
9012 case TargetOpcode::G_SMAX:
9013 return CmpInst::ICMP_SGT;
9014 case TargetOpcode::G_UMIN:
9015 return CmpInst::ICMP_ULT;
9016 case TargetOpcode::G_UMAX:
9017 return CmpInst::ICMP_UGT;
9018 default:
9019 llvm_unreachable("not in integer min/max");
9020 }
9021}
9022
9023LegalizerHelper::LegalizeResult LegalizerHelper::lowerMinMax(MachineInstr &MI) {
9024 auto [Dst, Src0, Src1] = MI.getFirst3Regs();
9025
9026 const CmpInst::Predicate Pred = minMaxToCompare(Opc: MI.getOpcode());
9027 LLT CmpType = MRI.getType(Reg: Dst).changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9028
9029 auto Cmp = MIRBuilder.buildICmp(Pred, Res: CmpType, Op0: Src0, Op1: Src1);
9030 MIRBuilder.buildSelect(Res: Dst, Tst: Cmp, Op0: Src0, Op1: Src1);
9031
9032 MI.eraseFromParent();
9033 return Legalized;
9034}
9035
9036LegalizerHelper::LegalizeResult
9037LegalizerHelper::lowerThreewayCompare(MachineInstr &MI) {
9038 GSUCmp *Cmp = cast<GSUCmp>(Val: &MI);
9039
9040 Register Dst = Cmp->getReg(Idx: 0);
9041 LLT DstTy = MRI.getType(Reg: Dst);
9042 LLT SrcTy = MRI.getType(Reg: Cmp->getReg(Idx: 1));
9043 LLT CmpTy = DstTy.changeElementSize(NewEltSize: 1);
9044
9045 CmpInst::Predicate LTPredicate = Cmp->isSigned()
9046 ? CmpInst::Predicate::ICMP_SLT
9047 : CmpInst::Predicate::ICMP_ULT;
9048 CmpInst::Predicate GTPredicate = Cmp->isSigned()
9049 ? CmpInst::Predicate::ICMP_SGT
9050 : CmpInst::Predicate::ICMP_UGT;
9051
9052 auto Zero = MIRBuilder.buildConstant(Res: DstTy, Val: 0);
9053 auto IsGT = MIRBuilder.buildICmp(Pred: GTPredicate, Res: CmpTy, Op0: Cmp->getLHSReg(),
9054 Op1: Cmp->getRHSReg());
9055 auto IsLT = MIRBuilder.buildICmp(Pred: LTPredicate, Res: CmpTy, Op0: Cmp->getLHSReg(),
9056 Op1: Cmp->getRHSReg());
9057
9058 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
9059 auto BC = TLI.getBooleanContents(isVec: DstTy.isVector(), /*isFP=*/isFloat: false);
9060 if (TLI.preferSelectsOverBooleanArithmetic(
9061 VT: getApproximateEVTForLLT(Ty: SrcTy, Ctx)) ||
9062 BC == TargetLowering::UndefinedBooleanContent) {
9063 auto One = MIRBuilder.buildConstant(Res: DstTy, Val: 1);
9064 auto SelectZeroOrOne = MIRBuilder.buildSelect(Res: DstTy, Tst: IsGT, Op0: One, Op1: Zero);
9065
9066 auto MinusOne = MIRBuilder.buildConstant(Res: DstTy, Val: -1);
9067 MIRBuilder.buildSelect(Res: Dst, Tst: IsLT, Op0: MinusOne, Op1: SelectZeroOrOne);
9068 } else {
9069 if (BC == TargetLowering::ZeroOrNegativeOneBooleanContent)
9070 std::swap(a&: IsGT, b&: IsLT);
9071 // Extend boolean results to DstTy, which is at least i2, before subtracting
9072 // them.
9073 unsigned BoolExtOp =
9074 MIRBuilder.getBoolExtOp(IsVec: DstTy.isVector(), /*isFP=*/IsFP: false);
9075 IsGT = MIRBuilder.buildInstr(Opc: BoolExtOp, DstOps: {DstTy}, SrcOps: {IsGT});
9076 IsLT = MIRBuilder.buildInstr(Opc: BoolExtOp, DstOps: {DstTy}, SrcOps: {IsLT});
9077 MIRBuilder.buildSub(Dst, Src0: IsGT, Src1: IsLT);
9078 }
9079
9080 MI.eraseFromParent();
9081 return Legalized;
9082}
9083
9084LegalizerHelper::LegalizeResult
9085LegalizerHelper::lowerFCopySign(MachineInstr &MI) {
9086 auto [Dst, DstTy, Src0, Src0Ty, Src1, Src1Ty] = MI.getFirst3RegLLTs();
9087 const int Src0Size = Src0Ty.getScalarSizeInBits();
9088 const int Src1Size = Src1Ty.getScalarSizeInBits();
9089
9090 LLT DstIntTy =
9091 DstTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: DstTy.getScalarSizeInBits()));
9092 LLT Src0IntTy = Src0Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: Src0Size));
9093 LLT Src1IntTy = Src1Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: Src1Size));
9094
9095 Register Src0Int = Src0;
9096 Register Src1Int = Src1;
9097
9098 if (!(Src0Ty.getScalarType().isAnyScalar() ||
9099 Src0Ty.getScalarType().isInteger()))
9100 Src0Int = MIRBuilder.buildBitcast(Dst: Src0IntTy, Src: Src0).getReg(Idx: 0);
9101
9102 if (!(Src1Ty.getScalarType().isAnyScalar() ||
9103 Src1Ty.getScalarType().isInteger()))
9104 Src1Int = MIRBuilder.buildBitcast(Dst: Src1IntTy, Src: Src1).getReg(Idx: 0);
9105
9106 auto SignBitMask =
9107 MIRBuilder.buildConstant(Res: Src0IntTy, Val: APInt::getSignMask(BitWidth: Src0Size));
9108
9109 auto NotSignBitMask = MIRBuilder.buildConstant(
9110 Res: Src0IntTy, Val: APInt::getLowBitsSet(numBits: Src0Size, loBitsSet: Src0Size - 1));
9111
9112 Register And0 =
9113 MIRBuilder.buildAnd(Dst: Src0IntTy, Src0: Src0Int, Src1: NotSignBitMask).getReg(Idx: 0);
9114 Register And1;
9115 if (Src0Ty == Src1Ty) {
9116 And1 = MIRBuilder.buildAnd(Dst: Src1IntTy, Src0: Src1Int, Src1: SignBitMask).getReg(Idx: 0);
9117 } else if (Src0Size > Src1Size) {
9118 auto ShiftAmt = MIRBuilder.buildConstant(Res: Src0IntTy, Val: Src0Size - Src1Size);
9119 auto Zext = MIRBuilder.buildZExt(Res: Src0IntTy, Op: Src1Int);
9120 auto Shift = MIRBuilder.buildShl(Dst: Src0IntTy, Src0: Zext, Src1: ShiftAmt);
9121 And1 = MIRBuilder.buildAnd(Dst: Src0Ty, Src0: Shift, Src1: SignBitMask).getReg(Idx: 0);
9122 } else {
9123 auto ShiftAmt = MIRBuilder.buildConstant(Res: Src1IntTy, Val: Src1Size - Src0Size);
9124 auto Shift = MIRBuilder.buildLShr(Dst: Src1IntTy, Src0: Src1Int, Src1: ShiftAmt);
9125 auto Trunc = MIRBuilder.buildTrunc(Res: Src0IntTy, Op: Shift);
9126 And1 = MIRBuilder.buildAnd(Dst: Src0IntTy, Src0: Trunc, Src1: SignBitMask).getReg(Idx: 0);
9127 }
9128
9129 // Be careful about setting nsz/nnan/ninf on every instruction, since the
9130 // constants are a nan and -0.0, but the final result should preserve
9131 // everything.
9132 unsigned Flags = MI.getFlags();
9133
9134 // We masked the sign bit and the not-sign bit, so these are disjoint.
9135 Flags |= MachineInstr::Disjoint;
9136
9137 if (DstTy == DstIntTy)
9138 MIRBuilder.buildOr(Dst, Src0: And0, Src1: And1, Flags).getReg(Idx: 0);
9139 else {
9140 Register NewDst = MIRBuilder.buildOr(Dst: DstIntTy, Src0: And0, Src1: And1, Flags).getReg(Idx: 0);
9141 MIRBuilder.buildBitcast(Dst, Src: NewDst);
9142 }
9143
9144 MI.eraseFromParent();
9145 return Legalized;
9146}
9147
9148LegalizerHelper::LegalizeResult
9149LegalizerHelper::lowerFMinNumMaxNum(MachineInstr &MI) {
9150 // FIXME: fminnum/fmaxnum and fminimumnum/fmaximumnum should not have
9151 // identical handling. fminimumnum/fmaximumnum also need a path that do not
9152 // depend on fminnum/fmaxnum.
9153
9154 unsigned NewOp;
9155 switch (MI.getOpcode()) {
9156 case TargetOpcode::G_FMINNUM:
9157 NewOp = TargetOpcode::G_FMINNUM_IEEE;
9158 break;
9159 case TargetOpcode::G_FMINIMUMNUM:
9160 NewOp = TargetOpcode::G_FMINNUM;
9161 break;
9162 case TargetOpcode::G_FMAXNUM:
9163 NewOp = TargetOpcode::G_FMAXNUM_IEEE;
9164 break;
9165 case TargetOpcode::G_FMAXIMUMNUM:
9166 NewOp = TargetOpcode::G_FMAXNUM;
9167 break;
9168 default:
9169 llvm_unreachable("unexpected min/max opcode");
9170 }
9171
9172 auto [Dst, Src0, Src1] = MI.getFirst3Regs();
9173 LLT Ty = MRI.getType(Reg: Dst);
9174
9175 if (!MI.getFlag(Flag: MachineInstr::FmNoNans)) {
9176 // Insert canonicalizes if it's possible we need to quiet to get correct
9177 // sNaN behavior.
9178
9179 // Note this must be done here, and not as an optimization combine in the
9180 // absence of a dedicate quiet-snan instruction as we're using an
9181 // omni-purpose G_FCANONICALIZE.
9182 if (!VT->isKnownNeverSNaN(Val: Src0))
9183 Src0 = MIRBuilder.buildFCanonicalize(Dst: Ty, Src0, Flags: MI.getFlags()).getReg(Idx: 0);
9184
9185 if (!VT->isKnownNeverSNaN(Val: Src1))
9186 Src1 = MIRBuilder.buildFCanonicalize(Dst: Ty, Src0: Src1, Flags: MI.getFlags()).getReg(Idx: 0);
9187 }
9188
9189 // If there are no nans, it's safe to simply replace this with the non-IEEE
9190 // version.
9191 MIRBuilder.buildInstr(Opc: NewOp, DstOps: {Dst}, SrcOps: {Src0, Src1}, Flags: MI.getFlags());
9192 MI.eraseFromParent();
9193 return Legalized;
9194}
9195
9196LegalizerHelper::LegalizeResult
9197LegalizerHelper::lowerFMinimumMaximum(MachineInstr &MI) {
9198 unsigned Opc = MI.getOpcode();
9199 auto [Dst, Src0, Src1] = MI.getFirst3Regs();
9200 LLT Ty = MRI.getType(Reg: Dst);
9201 const LLT CmpTy = Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9202
9203 bool IsMax = (Opc == TargetOpcode::G_FMAXIMUM);
9204 unsigned OpcIeee =
9205 IsMax ? TargetOpcode::G_FMAXNUM_IEEE : TargetOpcode::G_FMINNUM_IEEE;
9206 unsigned OpcNonIeee =
9207 IsMax ? TargetOpcode::G_FMAXNUM : TargetOpcode::G_FMINNUM;
9208 bool MinMaxMustRespectOrderedZero = false;
9209 Register Res;
9210
9211 // IEEE variants don't need canonicalization
9212 if (LI.isLegalOrCustom(Query: {OpcIeee, Ty})) {
9213 Res = MIRBuilder.buildInstr(Opc: OpcIeee, DstOps: {Ty}, SrcOps: {Src0, Src1}).getReg(Idx: 0);
9214 MinMaxMustRespectOrderedZero = true;
9215 } else if (LI.isLegalOrCustom(Query: {OpcNonIeee, Ty})) {
9216 Res = MIRBuilder.buildInstr(Opc: OpcNonIeee, DstOps: {Ty}, SrcOps: {Src0, Src1}).getReg(Idx: 0);
9217 } else {
9218 auto Compare = MIRBuilder.buildFCmp(
9219 Pred: IsMax ? CmpInst::FCMP_OGT : CmpInst::FCMP_OLT, Res: CmpTy, Op0: Src0, Op1: Src1);
9220 Res = MIRBuilder.buildSelect(Res: Ty, Tst: Compare, Op0: Src0, Op1: Src1).getReg(Idx: 0);
9221 }
9222
9223 // Propagate any NaN of both operands
9224 if (!MI.getFlag(Flag: MachineInstr::FmNoNans) &&
9225 (!VT->isKnownNeverNaN(Val: Src0) || !VT->isKnownNeverNaN(Val: Src1))) {
9226 auto IsOrdered = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_ORD, Res: CmpTy, Op0: Src0, Op1: Src1);
9227
9228 LLT ElementTy = Ty.isScalar() ? Ty : Ty.getElementType();
9229 APFloat NaNValue = APFloat::getNaN(Sem: getFltSemanticForLLT(Ty: ElementTy));
9230 Register NaN = MIRBuilder.buildFConstant(Res: ElementTy, Val: NaNValue).getReg(Idx: 0);
9231 if (Ty.isVector())
9232 NaN = MIRBuilder.buildSplatBuildVector(Res: Ty, Src: NaN).getReg(Idx: 0);
9233
9234 Res = MIRBuilder.buildSelect(Res: Ty, Tst: IsOrdered, Op0: Res, Op1: NaN).getReg(Idx: 0);
9235 }
9236
9237 // fminimum/fmaximum requires -0.0 less than +0.0
9238 if (!MinMaxMustRespectOrderedZero && !MI.getFlag(Flag: MachineInstr::FmNsz)) {
9239 GISelValueTracking VT(MIRBuilder.getMF());
9240 KnownFPClass Src0Info = VT.computeKnownFPClass(R: Src0, InterestedClasses: fcZero);
9241 KnownFPClass Src1Info = VT.computeKnownFPClass(R: Src1, InterestedClasses: fcZero);
9242
9243 if (!Src0Info.isKnownNeverZero() && !Src1Info.isKnownNeverZero()) {
9244 const unsigned Flags = MI.getFlags();
9245 Register Zero = MIRBuilder.buildFConstant(Res: Ty, Val: 0.0).getReg(Idx: 0);
9246 auto IsZero = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OEQ, Res: CmpTy, Op0: Res, Op1: Zero);
9247
9248 unsigned TestClass = IsMax ? fcPosZero : fcNegZero;
9249
9250 auto LHSTestZero = MIRBuilder.buildIsFPClass(Res: CmpTy, Src: Src0, Mask: TestClass);
9251 auto LHSSelect =
9252 MIRBuilder.buildSelect(Res: Ty, Tst: LHSTestZero, Op0: Src0, Op1: Res, Flags);
9253
9254 auto RHSTestZero = MIRBuilder.buildIsFPClass(Res: CmpTy, Src: Src1, Mask: TestClass);
9255 auto RHSSelect =
9256 MIRBuilder.buildSelect(Res: Ty, Tst: RHSTestZero, Op0: Src1, Op1: LHSSelect, Flags);
9257
9258 Res = MIRBuilder.buildSelect(Res: Ty, Tst: IsZero, Op0: RHSSelect, Op1: Res, Flags).getReg(Idx: 0);
9259 }
9260 }
9261
9262 MIRBuilder.buildCopy(Res: Dst, Op: Res);
9263 MI.eraseFromParent();
9264 return Legalized;
9265}
9266
9267LegalizerHelper::LegalizeResult LegalizerHelper::lowerFMad(MachineInstr &MI) {
9268 // Expand G_FMAD a, b, c -> G_FADD (G_FMUL a, b), c
9269 Register DstReg = MI.getOperand(i: 0).getReg();
9270 LLT Ty = MRI.getType(Reg: DstReg);
9271 unsigned Flags = MI.getFlags();
9272
9273 auto Mul = MIRBuilder.buildFMul(Dst: Ty, Src0: MI.getOperand(i: 1), Src1: MI.getOperand(i: 2),
9274 Flags);
9275 MIRBuilder.buildFAdd(Dst: DstReg, Src0: Mul, Src1: MI.getOperand(i: 3), Flags);
9276 MI.eraseFromParent();
9277 return Legalized;
9278}
9279
9280LegalizerHelper::LegalizeResult
9281LegalizerHelper::lowerIntrinsicRound(MachineInstr &MI) {
9282 auto [DstReg, X] = MI.getFirst2Regs();
9283 const unsigned Flags = MI.getFlags();
9284 const LLT Ty = MRI.getType(Reg: DstReg);
9285 const LLT CondTy = Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9286
9287 // round(x) =>
9288 // t = trunc(x);
9289 // d = fabs(x - t);
9290 // o = copysign(d >= 0.5 ? 1.0 : 0.0, x);
9291 // return t + o;
9292
9293 auto T = MIRBuilder.buildIntrinsicTrunc(Dst: Ty, Src0: X, Flags);
9294
9295 auto Diff = MIRBuilder.buildFSub(Dst: Ty, Src0: X, Src1: T, Flags);
9296 auto AbsDiff = MIRBuilder.buildFAbs(Dst: Ty, Src0: Diff, Flags);
9297
9298 auto Half = MIRBuilder.buildFConstant(Res: Ty, Val: 0.5);
9299 auto Cmp =
9300 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OGE, Res: CondTy, Op0: AbsDiff, Op1: Half, Flags);
9301
9302 // Could emit G_UITOFP instead
9303 auto One = MIRBuilder.buildFConstant(Res: Ty, Val: 1.0);
9304 auto Zero = MIRBuilder.buildFConstant(Res: Ty, Val: 0.0);
9305 auto BoolFP = MIRBuilder.buildSelect(Res: Ty, Tst: Cmp, Op0: One, Op1: Zero);
9306 auto SignedOffset = MIRBuilder.buildFCopysign(Dst: Ty, Src0: BoolFP, Src1: X);
9307
9308 MIRBuilder.buildFAdd(Dst: DstReg, Src0: T, Src1: SignedOffset, Flags);
9309
9310 MI.eraseFromParent();
9311 return Legalized;
9312}
9313
9314LegalizerHelper::LegalizeResult LegalizerHelper::lowerFFloor(MachineInstr &MI) {
9315 auto [DstReg, SrcReg] = MI.getFirst2Regs();
9316 unsigned Flags = MI.getFlags();
9317 LLT Ty = MRI.getType(Reg: DstReg);
9318 const LLT CondTy = Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9319
9320 // result = trunc(src);
9321 // if (src < 0.0 && src != result)
9322 // result += -1.0.
9323
9324 auto Trunc = MIRBuilder.buildIntrinsicTrunc(Dst: Ty, Src0: SrcReg, Flags);
9325 auto Zero = MIRBuilder.buildFConstant(Res: Ty, Val: 0.0);
9326
9327 auto Lt0 = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OLT, Res: CondTy,
9328 Op0: SrcReg, Op1: Zero, Flags);
9329 auto NeTrunc = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_ONE, Res: CondTy,
9330 Op0: SrcReg, Op1: Trunc, Flags);
9331 auto And = MIRBuilder.buildAnd(Dst: CondTy, Src0: Lt0, Src1: NeTrunc);
9332 auto AddVal = MIRBuilder.buildSITOFP(Dst: Ty, Src0: And);
9333
9334 MIRBuilder.buildFAdd(Dst: DstReg, Src0: Trunc, Src1: AddVal, Flags);
9335 MI.eraseFromParent();
9336 return Legalized;
9337}
9338
9339LegalizerHelper::LegalizeResult
9340LegalizerHelper::lowerMergeValues(MachineInstr &MI) {
9341 const unsigned NumOps = MI.getNumOperands();
9342 auto [DstReg, DstTy, Src0Reg, Src0Ty] = MI.getFirst2RegLLTs();
9343 unsigned PartSize = Src0Ty.getSizeInBits();
9344
9345 LLT WideTy = LLT::integer(SizeInBits: DstTy.getSizeInBits());
9346 Register ResultReg = MIRBuilder.buildZExt(Res: WideTy, Op: Src0Reg).getReg(Idx: 0);
9347
9348 for (unsigned I = 2; I != NumOps; ++I) {
9349 const unsigned Offset = (I - 1) * PartSize;
9350
9351 Register SrcReg = MI.getOperand(i: I).getReg();
9352 auto ZextInput = MIRBuilder.buildZExt(Res: WideTy, Op: SrcReg);
9353
9354 Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg :
9355 MRI.createGenericVirtualRegister(Ty: WideTy);
9356
9357 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: Offset);
9358 auto Shl = MIRBuilder.buildShl(Dst: WideTy, Src0: ZextInput, Src1: ShiftAmt);
9359 MIRBuilder.buildOr(Dst: NextResult, Src0: ResultReg, Src1: Shl);
9360 ResultReg = NextResult;
9361 }
9362
9363 if (DstTy.isPointer()) {
9364 if (MIRBuilder.getDataLayout().isNonIntegralAddressSpace(
9365 AddrSpace: DstTy.getAddressSpace())) {
9366 LLVM_DEBUG(dbgs() << "Not casting nonintegral address space\n");
9367 return UnableToLegalize;
9368 }
9369
9370 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: ResultReg);
9371 }
9372
9373 MI.eraseFromParent();
9374 return Legalized;
9375}
9376
9377LegalizerHelper::LegalizeResult
9378LegalizerHelper::lowerUnmergeValues(MachineInstr &MI) {
9379 const unsigned NumDst = MI.getNumOperands() - 1;
9380 Register SrcReg = MI.getOperand(i: NumDst).getReg();
9381 Register Dst0Reg = MI.getOperand(i: 0).getReg();
9382 LLT DstTy = MRI.getType(Reg: Dst0Reg);
9383 if (DstTy.isPointer())
9384 return UnableToLegalize; // TODO
9385
9386 SrcReg = coerceToInteger(Val: SrcReg);
9387 if (!SrcReg)
9388 return UnableToLegalize;
9389
9390 // Expand scalarizing unmerge as bitcast to integer and shift.
9391 LLT IntTy = MRI.getType(Reg: SrcReg);
9392
9393 MIRBuilder.buildTrunc(Res: Dst0Reg, Op: SrcReg);
9394
9395 const unsigned DstSize = DstTy.getSizeInBits();
9396 unsigned Offset = DstSize;
9397 for (unsigned I = 1; I != NumDst; ++I, Offset += DstSize) {
9398 auto ShiftAmt = MIRBuilder.buildConstant(Res: IntTy, Val: Offset);
9399 auto Shift = MIRBuilder.buildLShr(Dst: IntTy, Src0: SrcReg, Src1: ShiftAmt);
9400 MIRBuilder.buildTrunc(Res: MI.getOperand(i: I), Op: Shift);
9401 }
9402
9403 MI.eraseFromParent();
9404 return Legalized;
9405}
9406
9407/// Lower a vector extract or insert by writing the vector to a stack temporary
9408/// and reloading the element or vector.
9409///
9410/// %dst = G_EXTRACT_VECTOR_ELT %vec, %idx
9411/// =>
9412/// %stack_temp = G_FRAME_INDEX
9413/// G_STORE %vec, %stack_temp
9414/// %idx = clamp(%idx, %vec.getNumElements())
9415/// %element_ptr = G_PTR_ADD %stack_temp, %idx
9416/// %dst = G_LOAD %element_ptr
9417LegalizerHelper::LegalizeResult
9418LegalizerHelper::lowerExtractInsertVectorElt(MachineInstr &MI) {
9419 Register DstReg = MI.getOperand(i: 0).getReg();
9420 Register SrcVec = MI.getOperand(i: 1).getReg();
9421 Register InsertVal;
9422 if (MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)
9423 InsertVal = MI.getOperand(i: 2).getReg();
9424
9425 Register Idx = MI.getOperand(i: MI.getNumOperands() - 1).getReg();
9426
9427 LLT VecTy = MRI.getType(Reg: SrcVec);
9428 LLT EltTy = VecTy.getElementType();
9429 unsigned NumElts = VecTy.getNumElements();
9430
9431 int64_t IdxVal;
9432 if (mi_match(R: Idx, MRI, P: m_ICst(Cst&: IdxVal)) && IdxVal <= NumElts) {
9433 SmallVector<Register, 8> SrcRegs;
9434 extractParts(Reg: SrcVec, Ty: EltTy, NumParts: NumElts, VRegs&: SrcRegs, MIRBuilder, MRI);
9435
9436 if (InsertVal) {
9437 SrcRegs[IdxVal] = MI.getOperand(i: 2).getReg();
9438 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: SrcRegs);
9439 } else {
9440 MIRBuilder.buildCopy(Res: DstReg, Op: SrcRegs[IdxVal]);
9441 }
9442
9443 MI.eraseFromParent();
9444 return Legalized;
9445 }
9446
9447 if (!EltTy.isByteSized()) { // Not implemented.
9448 LLVM_DEBUG(dbgs() << "Can't handle non-byte element vectors yet\n");
9449 return UnableToLegalize;
9450 }
9451
9452 unsigned EltBytes = EltTy.getSizeInBytes();
9453 Align VecAlign = getStackTemporaryAlignment(Ty: VecTy);
9454 Align EltAlign;
9455
9456 MachinePointerInfo PtrInfo;
9457 auto StackTemp = createStackTemporary(
9458 Bytes: TypeSize::getFixed(ExactSize: VecTy.getSizeInBytes()), Alignment: VecAlign, PtrInfo);
9459 MIRBuilder.buildStore(Val: SrcVec, Addr: StackTemp, PtrInfo, Alignment: VecAlign);
9460
9461 // Get the pointer to the element, and be sure not to hit undefined behavior
9462 // if the index is out of bounds.
9463 Register EltPtr = getVectorElementPointer(VecPtr: StackTemp.getReg(Idx: 0), VecTy, Index: Idx);
9464
9465 if (mi_match(R: Idx, MRI, P: m_ICst(Cst&: IdxVal))) {
9466 int64_t Offset = IdxVal * EltBytes;
9467 PtrInfo = PtrInfo.getWithOffset(O: Offset);
9468 EltAlign = commonAlignment(A: VecAlign, Offset);
9469 } else {
9470 // We lose information with a variable offset.
9471 EltAlign = getStackTemporaryAlignment(Ty: EltTy);
9472 PtrInfo = MachinePointerInfo(MRI.getType(Reg: EltPtr).getAddressSpace());
9473 }
9474
9475 if (InsertVal) {
9476 // Write the inserted element
9477 MIRBuilder.buildStore(Val: InsertVal, Addr: EltPtr, PtrInfo, Alignment: EltAlign);
9478
9479 // Reload the whole vector.
9480 MIRBuilder.buildLoad(Res: DstReg, Addr: StackTemp, PtrInfo, Alignment: VecAlign);
9481 } else {
9482 MIRBuilder.buildLoad(Res: DstReg, Addr: EltPtr, PtrInfo, Alignment: EltAlign);
9483 }
9484
9485 MI.eraseFromParent();
9486 return Legalized;
9487}
9488
9489LegalizerHelper::LegalizeResult
9490LegalizerHelper::lowerShuffleVector(MachineInstr &MI) {
9491 auto [DstReg, DstTy, Src0Reg, Src0Ty, Src1Reg, Src1Ty] =
9492 MI.getFirst3RegLLTs();
9493 LLT IdxTy = LLT::scalar(SizeInBits: 32);
9494
9495 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
9496 Register Undef;
9497 SmallVector<Register, 32> BuildVec;
9498 LLT EltTy = DstTy.getScalarType();
9499
9500 DenseMap<unsigned, Register> CachedExtract;
9501
9502 for (int Idx : Mask) {
9503 if (Idx < 0) {
9504 if (!Undef.isValid())
9505 Undef = MIRBuilder.buildUndef(Res: EltTy).getReg(Idx: 0);
9506 BuildVec.push_back(Elt: Undef);
9507 continue;
9508 }
9509
9510 assert(!Src0Ty.isScalar() && "Unexpected scalar G_SHUFFLE_VECTOR");
9511
9512 int NumElts = Src0Ty.getNumElements();
9513 Register SrcVec = Idx < NumElts ? Src0Reg : Src1Reg;
9514 int ExtractIdx = Idx < NumElts ? Idx : Idx - NumElts;
9515 auto [It, Inserted] = CachedExtract.try_emplace(Key: Idx);
9516 if (Inserted) {
9517 auto IdxK = MIRBuilder.buildConstant(Res: IdxTy, Val: ExtractIdx);
9518 It->second =
9519 MIRBuilder.buildExtractVectorElement(Res: EltTy, Val: SrcVec, Idx: IdxK).getReg(Idx: 0);
9520 }
9521 BuildVec.push_back(Elt: It->second);
9522 }
9523
9524 assert(DstTy.isVector() && "Unexpected scalar G_SHUFFLE_VECTOR");
9525 MIRBuilder.buildBuildVector(Res: DstReg, Ops: BuildVec);
9526 MI.eraseFromParent();
9527 return Legalized;
9528}
9529
9530LegalizerHelper::LegalizeResult
9531LegalizerHelper::lowerVECTOR_COMPRESS(llvm::MachineInstr &MI) {
9532 auto [Dst, DstTy, Vec, VecTy, Mask, MaskTy, Passthru, PassthruTy] =
9533 MI.getFirst4RegLLTs();
9534
9535 if (VecTy.isScalableVector())
9536 report_fatal_error(reason: "Cannot expand masked_compress for scalable vectors.");
9537
9538 Align VecAlign = getStackTemporaryAlignment(Ty: VecTy);
9539 MachinePointerInfo PtrInfo;
9540 Register StackPtr =
9541 createStackTemporary(Bytes: TypeSize::getFixed(ExactSize: VecTy.getSizeInBytes()), Alignment: VecAlign,
9542 PtrInfo)
9543 .getReg(Idx: 0);
9544 MachinePointerInfo ValPtrInfo =
9545 MachinePointerInfo::getUnknownStack(MF&: *MI.getMF());
9546
9547 LLT IdxTy = LLT::scalar(SizeInBits: 32);
9548 LLT ValTy = VecTy.getElementType();
9549 Align ValAlign = getStackTemporaryAlignment(Ty: ValTy);
9550
9551 auto OutPos = MIRBuilder.buildConstant(Res: IdxTy, Val: 0);
9552
9553 bool HasPassthru =
9554 MRI.getVRegDef(Reg: Passthru)->getOpcode() != TargetOpcode::G_IMPLICIT_DEF;
9555
9556 if (HasPassthru)
9557 MIRBuilder.buildStore(Val: Passthru, Addr: StackPtr, PtrInfo, Alignment: VecAlign);
9558
9559 Register LastWriteVal;
9560 std::optional<APInt> PassthruSplatVal =
9561 isConstantOrConstantSplatVector(Def: Passthru, MRI);
9562
9563 if (PassthruSplatVal.has_value()) {
9564 LastWriteVal =
9565 MIRBuilder.buildConstant(Res: ValTy, Val: PassthruSplatVal.value()).getReg(Idx: 0);
9566 } else if (HasPassthru) {
9567 auto Popcount = MIRBuilder.buildZExt(Res: MaskTy.changeElementSize(NewEltSize: 32), Op: Mask);
9568 Popcount = MIRBuilder.buildInstr(Opc: TargetOpcode::G_VECREDUCE_ADD,
9569 DstOps: {LLT::scalar(SizeInBits: 32)}, SrcOps: {Popcount});
9570
9571 Register LastElmtPtr =
9572 getVectorElementPointer(VecPtr: StackPtr, VecTy, Index: Popcount.getReg(Idx: 0));
9573 LastWriteVal =
9574 MIRBuilder.buildLoad(Res: ValTy, Addr: LastElmtPtr, PtrInfo: ValPtrInfo, Alignment: ValAlign)
9575 .getReg(Idx: 0);
9576 }
9577
9578 unsigned NumElmts = VecTy.getNumElements();
9579 for (unsigned I = 0; I < NumElmts; ++I) {
9580 auto Idx = MIRBuilder.buildConstant(Res: IdxTy, Val: I);
9581 auto Val = MIRBuilder.buildExtractVectorElement(Res: ValTy, Val: Vec, Idx);
9582 Register ElmtPtr =
9583 getVectorElementPointer(VecPtr: StackPtr, VecTy, Index: OutPos.getReg(Idx: 0));
9584 MIRBuilder.buildStore(Val, Addr: ElmtPtr, PtrInfo: ValPtrInfo, Alignment: ValAlign);
9585
9586 LLT MaskITy = MaskTy.getElementType();
9587 auto MaskI = MIRBuilder.buildExtractVectorElement(Res: MaskITy, Val: Mask, Idx);
9588 if (MaskITy.getSizeInBits() > 1)
9589 MaskI = MIRBuilder.buildTrunc(Res: LLT::scalar(SizeInBits: 1), Op: MaskI);
9590
9591 MaskI = MIRBuilder.buildZExt(Res: IdxTy, Op: MaskI);
9592 OutPos = MIRBuilder.buildAdd(Dst: IdxTy, Src0: OutPos, Src1: MaskI);
9593
9594 if (HasPassthru && I == NumElmts - 1) {
9595 auto EndOfVector =
9596 MIRBuilder.buildConstant(Res: IdxTy, Val: VecTy.getNumElements() - 1);
9597 auto AllLanesSelected = MIRBuilder.buildICmp(
9598 Pred: CmpInst::ICMP_UGT, Res: LLT::scalar(SizeInBits: 1), Op0: OutPos, Op1: EndOfVector);
9599 OutPos = MIRBuilder.buildInstr(Opc: TargetOpcode::G_UMIN, DstOps: {IdxTy},
9600 SrcOps: {OutPos, EndOfVector});
9601 ElmtPtr = getVectorElementPointer(VecPtr: StackPtr, VecTy, Index: OutPos.getReg(Idx: 0));
9602
9603 LastWriteVal =
9604 MIRBuilder.buildSelect(Res: ValTy, Tst: AllLanesSelected, Op0: Val, Op1: LastWriteVal)
9605 .getReg(Idx: 0);
9606 MIRBuilder.buildStore(Val: LastWriteVal, Addr: ElmtPtr, PtrInfo: ValPtrInfo, Alignment: ValAlign);
9607 }
9608 }
9609
9610 // TODO: Use StackPtr's FrameIndex alignment.
9611 MIRBuilder.buildLoad(Res: Dst, Addr: StackPtr, PtrInfo, Alignment: VecAlign);
9612
9613 MI.eraseFromParent();
9614 return Legalized;
9615}
9616
9617Register LegalizerHelper::getDynStackAllocTargetPtr(Register SPReg,
9618 Register AllocSize,
9619 Align Alignment,
9620 LLT PtrTy) {
9621 LLT IntPtrTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
9622
9623 auto SPTmp = MIRBuilder.buildCopy(Res: PtrTy, Op: SPReg);
9624 SPTmp = MIRBuilder.buildCast(Dst: IntPtrTy, Src: SPTmp);
9625
9626 // Subtract the final alloc from the SP. We use G_PTRTOINT here so we don't
9627 // have to generate an extra instruction to negate the alloc and then use
9628 // G_PTR_ADD to add the negative offset.
9629 auto Alloc = MIRBuilder.buildSub(Dst: IntPtrTy, Src0: SPTmp, Src1: AllocSize);
9630 if (Alignment > Align(1)) {
9631 APInt AlignMask(IntPtrTy.getSizeInBits(), Alignment.value(), true);
9632 AlignMask.negate();
9633 auto AlignCst = MIRBuilder.buildConstant(Res: IntPtrTy, Val: AlignMask);
9634 Alloc = MIRBuilder.buildAnd(Dst: IntPtrTy, Src0: Alloc, Src1: AlignCst);
9635 }
9636
9637 return MIRBuilder.buildCast(Dst: PtrTy, Src: Alloc).getReg(Idx: 0);
9638}
9639
9640LegalizerHelper::LegalizeResult
9641LegalizerHelper::lowerDynStackAlloc(MachineInstr &MI) {
9642 const auto &MF = *MI.getMF();
9643 const auto &TFI = *MF.getSubtarget().getFrameLowering();
9644 if (TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp)
9645 return UnableToLegalize;
9646
9647 Register Dst = MI.getOperand(i: 0).getReg();
9648 Register AllocSize = MI.getOperand(i: 1).getReg();
9649 Align Alignment = assumeAligned(Value: MI.getOperand(i: 2).getImm());
9650
9651 LLT PtrTy = MRI.getType(Reg: Dst);
9652 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
9653 Register SPTmp =
9654 getDynStackAllocTargetPtr(SPReg, AllocSize, Alignment, PtrTy);
9655
9656 MIRBuilder.buildCopy(Res: SPReg, Op: SPTmp);
9657 MIRBuilder.buildCopy(Res: Dst, Op: SPTmp);
9658
9659 MI.eraseFromParent();
9660 return Legalized;
9661}
9662
9663LegalizerHelper::LegalizeResult
9664LegalizerHelper::lowerStackSave(MachineInstr &MI) {
9665 Register StackPtr = TLI.getStackPointerRegisterToSaveRestore();
9666 if (!StackPtr)
9667 return UnableToLegalize;
9668
9669 MIRBuilder.buildCopy(Res: MI.getOperand(i: 0), Op: StackPtr);
9670 MI.eraseFromParent();
9671 return Legalized;
9672}
9673
9674LegalizerHelper::LegalizeResult
9675LegalizerHelper::lowerStackRestore(MachineInstr &MI) {
9676 Register StackPtr = TLI.getStackPointerRegisterToSaveRestore();
9677 if (!StackPtr)
9678 return UnableToLegalize;
9679
9680 MIRBuilder.buildCopy(Res: StackPtr, Op: MI.getOperand(i: 0));
9681 MI.eraseFromParent();
9682 return Legalized;
9683}
9684
9685LegalizerHelper::LegalizeResult
9686LegalizerHelper::lowerExtract(MachineInstr &MI) {
9687 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
9688 unsigned Offset = MI.getOperand(i: 2).getImm();
9689
9690 // Extract sub-vector or one element
9691 if (SrcTy.isVector()) {
9692 unsigned SrcEltSize = SrcTy.getElementType().getSizeInBits();
9693 unsigned DstSize = DstTy.getSizeInBits();
9694
9695 if ((Offset % SrcEltSize == 0) && (DstSize % SrcEltSize == 0) &&
9696 (Offset + DstSize <= SrcTy.getSizeInBits())) {
9697 // Unmerge and allow access to each Src element for the artifact combiner.
9698 auto Unmerge = MIRBuilder.buildUnmerge(Res: SrcTy.getElementType(), Op: SrcReg);
9699
9700 // Take element(s) we need to extract and copy it (merge them).
9701 SmallVector<Register, 8> SubVectorElts;
9702 for (unsigned Idx = Offset / SrcEltSize;
9703 Idx < (Offset + DstSize) / SrcEltSize; ++Idx) {
9704 SubVectorElts.push_back(Elt: Unmerge.getReg(Idx));
9705 }
9706 if (SubVectorElts.size() == 1)
9707 MIRBuilder.buildCopy(Res: DstReg, Op: SubVectorElts[0]);
9708 else
9709 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: SubVectorElts);
9710
9711 MI.eraseFromParent();
9712 return Legalized;
9713 }
9714 }
9715
9716 const DataLayout &DL = MIRBuilder.getDataLayout();
9717 if ((SrcTy.isPointer() &&
9718 DL.isNonIntegralAddressSpace(AddrSpace: SrcTy.getAddressSpace())) ||
9719 (DstTy.isPointer() &&
9720 DL.isNonIntegralAddressSpace(AddrSpace: DstTy.getAddressSpace()))) {
9721 LLVM_DEBUG(dbgs() << "Not casting non-integral address space integer\n");
9722 return UnableToLegalize;
9723 }
9724
9725 if ((DstTy.isScalar() || DstTy.isPointer()) &&
9726 (SrcTy.isScalar() || SrcTy.isPointer() ||
9727 (SrcTy.isVector() && DstTy == SrcTy.getElementType()))) {
9728 LLT SrcIntTy = SrcTy;
9729 if (!SrcTy.isScalar()) {
9730 SrcIntTy = LLT::scalar(SizeInBits: SrcTy.getSizeInBits());
9731 SrcReg = MIRBuilder.buildCast(Dst: SrcIntTy, Src: SrcReg).getReg(Idx: 0);
9732 }
9733
9734 Register ResultReg = DstReg;
9735 if (DstTy.isPointer())
9736 ResultReg =
9737 MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: DstTy.getSizeInBits()));
9738
9739 if (Offset == 0)
9740 MIRBuilder.buildTrunc(Res: ResultReg, Op: SrcReg);
9741 else {
9742 auto ShiftAmt = MIRBuilder.buildConstant(Res: SrcIntTy, Val: Offset);
9743 auto Shr = MIRBuilder.buildLShr(Dst: SrcIntTy, Src0: SrcReg, Src1: ShiftAmt);
9744 MIRBuilder.buildTrunc(Res: ResultReg, Op: Shr);
9745 }
9746
9747 if (DstTy.isPointer())
9748 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: ResultReg);
9749
9750 MI.eraseFromParent();
9751 return Legalized;
9752 }
9753
9754 return UnableToLegalize;
9755}
9756
9757LegalizerHelper::LegalizeResult LegalizerHelper::lowerInsert(MachineInstr &MI) {
9758 auto [Dst, Src, InsertSrc] = MI.getFirst3Regs();
9759 uint64_t Offset = MI.getOperand(i: 3).getImm();
9760
9761 LLT DstTy = MRI.getType(Reg: Src);
9762 LLT InsertTy = MRI.getType(Reg: InsertSrc);
9763
9764 const DataLayout &DL = MIRBuilder.getDataLayout();
9765 bool IsNonIntegralInsert =
9766 InsertTy.isPointerOrPointerVector() &&
9767 DL.isNonIntegralAddressSpace(AddrSpace: InsertTy.getAddressSpace());
9768 bool IsNonIntegralDst = DstTy.isPointerOrPointerVector() &&
9769 DL.isNonIntegralAddressSpace(AddrSpace: DstTy.getAddressSpace());
9770
9771 // Insert sub-vector or one element
9772 if (DstTy.isVector()) {
9773 LLT EltTy = DstTy.getElementType();
9774
9775 if ((IsNonIntegralInsert || IsNonIntegralDst) && InsertTy != EltTy) {
9776 LLVM_DEBUG(dbgs() << "Not casting non-integral address space integer\n");
9777 return UnableToLegalize;
9778 }
9779
9780 unsigned EltSize = EltTy.getSizeInBits();
9781 unsigned InsertSize = InsertTy.getSizeInBits();
9782
9783 if ((Offset % EltSize == 0) && (InsertSize % EltSize == 0) &&
9784 (Offset + InsertSize <= DstTy.getSizeInBits())) {
9785 auto UnmergeSrc = MIRBuilder.buildUnmerge(Res: EltTy, Op: Src);
9786 SmallVector<Register, 8> DstElts;
9787 unsigned Idx = 0;
9788 // Elements from Src before insert start Offset
9789 for (; Idx < Offset / EltSize; ++Idx) {
9790 DstElts.push_back(Elt: UnmergeSrc.getReg(Idx));
9791 }
9792
9793 // Replace elements in Src with elements from InsertSrc
9794 if (InsertTy.getSizeInBits() > EltSize) {
9795 auto UnmergeInsertSrc = MIRBuilder.buildUnmerge(Res: EltTy, Op: InsertSrc);
9796 for (unsigned i = 0; Idx < (Offset + InsertSize) / EltSize;
9797 ++Idx, ++i) {
9798 DstElts.push_back(Elt: UnmergeInsertSrc.getReg(Idx: i));
9799 }
9800 } else {
9801 if (InsertTy.isPointer() && !EltTy.isPointer())
9802 InsertSrc = MIRBuilder.buildPtrToInt(Dst: EltTy, Src: InsertSrc).getReg(Idx: 0);
9803 else if (!InsertTy.isPointer() && EltTy.isPointer())
9804 InsertSrc = MIRBuilder.buildIntToPtr(Dst: EltTy, Src: InsertSrc).getReg(Idx: 0);
9805 DstElts.push_back(Elt: InsertSrc);
9806 ++Idx;
9807 }
9808
9809 // Remaining elements from Src after insert
9810 for (; Idx < DstTy.getNumElements(); ++Idx) {
9811 DstElts.push_back(Elt: UnmergeSrc.getReg(Idx));
9812 }
9813
9814 MIRBuilder.buildMergeLikeInstr(Res: Dst, Ops: DstElts);
9815 MI.eraseFromParent();
9816 return Legalized;
9817 }
9818 }
9819
9820 if (InsertTy.isVector() ||
9821 (DstTy.isVector() && DstTy.getElementType() != InsertTy))
9822 return UnableToLegalize;
9823
9824 if (IsNonIntegralDst || IsNonIntegralInsert) {
9825 LLVM_DEBUG(dbgs() << "Not casting non-integral address space integer\n");
9826 return UnableToLegalize;
9827 }
9828
9829 LLT IntDstTy = DstTy;
9830
9831 if (!DstTy.isScalar()) {
9832 IntDstTy = LLT::scalar(SizeInBits: DstTy.getSizeInBits());
9833 Src = MIRBuilder.buildCast(Dst: IntDstTy, Src).getReg(Idx: 0);
9834 }
9835
9836 if (!InsertTy.isScalar()) {
9837 const LLT IntInsertTy = LLT::scalar(SizeInBits: InsertTy.getSizeInBits());
9838 InsertSrc = MIRBuilder.buildPtrToInt(Dst: IntInsertTy, Src: InsertSrc).getReg(Idx: 0);
9839 }
9840
9841 Register ExtInsSrc = MIRBuilder.buildZExt(Res: IntDstTy, Op: InsertSrc).getReg(Idx: 0);
9842 if (Offset != 0) {
9843 auto ShiftAmt = MIRBuilder.buildConstant(Res: IntDstTy, Val: Offset);
9844 ExtInsSrc = MIRBuilder.buildShl(Dst: IntDstTy, Src0: ExtInsSrc, Src1: ShiftAmt).getReg(Idx: 0);
9845 }
9846
9847 APInt MaskVal = APInt::getBitsSetWithWrap(
9848 numBits: DstTy.getSizeInBits(), loBit: Offset + InsertTy.getSizeInBits(), hiBit: Offset);
9849
9850 auto Mask = MIRBuilder.buildConstant(Res: IntDstTy, Val: MaskVal);
9851 auto MaskedSrc = MIRBuilder.buildAnd(Dst: IntDstTy, Src0: Src, Src1: Mask);
9852 auto Or = MIRBuilder.buildOr(Dst: IntDstTy, Src0: MaskedSrc, Src1: ExtInsSrc);
9853
9854 MIRBuilder.buildCast(Dst, Src: Or);
9855 MI.eraseFromParent();
9856 return Legalized;
9857}
9858
9859LegalizerHelper::LegalizeResult
9860LegalizerHelper::lowerSADDO_SSUBO(MachineInstr &MI) {
9861 auto [Dst0, Dst0Ty, Dst1, Dst1Ty, LHS, LHSTy, RHS, RHSTy] =
9862 MI.getFirst4RegLLTs();
9863 const bool IsAdd = MI.getOpcode() == TargetOpcode::G_SADDO;
9864
9865 LLT Ty = Dst0Ty;
9866 LLT BoolTy = Dst1Ty;
9867
9868 Register NewDst0 = MRI.cloneVirtualRegister(VReg: Dst0);
9869
9870 if (IsAdd)
9871 MIRBuilder.buildAdd(Dst: NewDst0, Src0: LHS, Src1: RHS);
9872 else
9873 MIRBuilder.buildSub(Dst: NewDst0, Src0: LHS, Src1: RHS);
9874
9875 // TODO: If SADDSAT/SSUBSAT is legal, compare results to detect overflow.
9876
9877 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
9878
9879 if (IsAdd) {
9880 // For an addition, the result should be less than one of the operands (LHS)
9881 // if and only if the other operand (RHS) is negative, otherwise there will
9882 // be overflow.
9883 auto ResultLowerThanLHS =
9884 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: NewDst0, Op1: LHS);
9885 auto RHSNegative =
9886 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: RHS, Op1: Zero);
9887 MIRBuilder.buildXor(Dst: Dst1, Src0: RHSNegative, Src1: ResultLowerThanLHS);
9888 } else {
9889 // For subtraction, overflow occurs when the signed comparison of operands
9890 // doesn't match the sign of the result.
9891 auto LHSLessThanRHS =
9892 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: LHS, Op1: RHS);
9893 auto ResultNegative =
9894 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: NewDst0, Op1: Zero);
9895 MIRBuilder.buildXor(Dst: Dst1, Src0: LHSLessThanRHS, Src1: ResultNegative);
9896 }
9897
9898 MIRBuilder.buildCopy(Res: Dst0, Op: NewDst0);
9899 MI.eraseFromParent();
9900
9901 return Legalized;
9902}
9903
9904LegalizerHelper::LegalizeResult LegalizerHelper::lowerSADDE(MachineInstr &MI) {
9905 auto [Res, OvOut, LHS, RHS, CarryIn] = MI.getFirst5Regs();
9906 const LLT Ty = MRI.getType(Reg: Res);
9907
9908 // sum = LHS + RHS + zext(CarryIn)
9909 auto Tmp = MIRBuilder.buildAdd(Dst: Ty, Src0: LHS, Src1: RHS);
9910 auto CarryZ = MIRBuilder.buildZExt(Res: Ty, Op: CarryIn);
9911 auto Sum = MIRBuilder.buildAdd(Dst: Ty, Src0: Tmp, Src1: CarryZ);
9912 MIRBuilder.buildCopy(Res, Op: Sum);
9913
9914 // OvOut = icmp slt ((sum ^ lhs) & (sum ^ rhs)), 0
9915 auto AX = MIRBuilder.buildXor(Dst: Ty, Src0: Sum, Src1: LHS);
9916 auto BX = MIRBuilder.buildXor(Dst: Ty, Src0: Sum, Src1: RHS);
9917 auto T = MIRBuilder.buildAnd(Dst: Ty, Src0: AX, Src1: BX);
9918
9919 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
9920 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: OvOut, Op0: T, Op1: Zero);
9921
9922 MI.eraseFromParent();
9923 return Legalized;
9924}
9925
9926LegalizerHelper::LegalizeResult LegalizerHelper::lowerSSUBE(MachineInstr &MI) {
9927 auto [Res, OvOut, LHS, RHS, CarryIn] = MI.getFirst5Regs();
9928 const LLT Ty = MRI.getType(Reg: Res);
9929
9930 // Diff = LHS - (RHS + zext(CarryIn))
9931 auto CarryZ = MIRBuilder.buildZExt(Res: Ty, Op: CarryIn);
9932 auto RHSPlusCI = MIRBuilder.buildAdd(Dst: Ty, Src0: RHS, Src1: CarryZ);
9933 auto Diff = MIRBuilder.buildSub(Dst: Ty, Src0: LHS, Src1: RHSPlusCI);
9934 MIRBuilder.buildCopy(Res, Op: Diff);
9935
9936 // ov = msb((LHS ^ RHS) & (LHS ^ Diff))
9937 auto X1 = MIRBuilder.buildXor(Dst: Ty, Src0: LHS, Src1: RHS);
9938 auto X2 = MIRBuilder.buildXor(Dst: Ty, Src0: LHS, Src1: Diff);
9939 auto T = MIRBuilder.buildAnd(Dst: Ty, Src0: X1, Src1: X2);
9940 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
9941 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: OvOut, Op0: T, Op1: Zero);
9942
9943 MI.eraseFromParent();
9944 return Legalized;
9945}
9946
9947LegalizerHelper::LegalizeResult
9948LegalizerHelper::lowerAddSubSatToMinMax(MachineInstr &MI) {
9949 auto [Res, LHS, RHS] = MI.getFirst3Regs();
9950 LLT Ty = MRI.getType(Reg: Res);
9951 bool IsSigned;
9952 bool IsAdd;
9953 unsigned BaseOp;
9954 switch (MI.getOpcode()) {
9955 default:
9956 llvm_unreachable("unexpected addsat/subsat opcode");
9957 case TargetOpcode::G_UADDSAT:
9958 IsSigned = false;
9959 IsAdd = true;
9960 BaseOp = TargetOpcode::G_ADD;
9961 break;
9962 case TargetOpcode::G_SADDSAT:
9963 IsSigned = true;
9964 IsAdd = true;
9965 BaseOp = TargetOpcode::G_ADD;
9966 break;
9967 case TargetOpcode::G_USUBSAT:
9968 IsSigned = false;
9969 IsAdd = false;
9970 BaseOp = TargetOpcode::G_SUB;
9971 break;
9972 case TargetOpcode::G_SSUBSAT:
9973 IsSigned = true;
9974 IsAdd = false;
9975 BaseOp = TargetOpcode::G_SUB;
9976 break;
9977 }
9978
9979 if (IsSigned) {
9980 // sadd.sat(a, b) ->
9981 // hi = 0x7fffffff - smax(a, 0)
9982 // lo = 0x80000000 - smin(a, 0)
9983 // a + smin(smax(lo, b), hi)
9984 // ssub.sat(a, b) ->
9985 // lo = smax(a, -1) - 0x7fffffff
9986 // hi = smin(a, -1) - 0x80000000
9987 // a - smin(smax(lo, b), hi)
9988 // TODO: AMDGPU can use a "median of 3" instruction here:
9989 // a +/- med3(lo, b, hi)
9990 uint64_t NumBits = Ty.getScalarSizeInBits();
9991 auto MaxVal =
9992 MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMaxValue(numBits: NumBits));
9993 auto MinVal =
9994 MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMinValue(numBits: NumBits));
9995 MachineInstrBuilder Hi, Lo;
9996 if (IsAdd) {
9997 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
9998 Hi = MIRBuilder.buildSub(Dst: Ty, Src0: MaxVal, Src1: MIRBuilder.buildSMax(Dst: Ty, Src0: LHS, Src1: Zero));
9999 Lo = MIRBuilder.buildSub(Dst: Ty, Src0: MinVal, Src1: MIRBuilder.buildSMin(Dst: Ty, Src0: LHS, Src1: Zero));
10000 } else {
10001 auto NegOne = MIRBuilder.buildConstant(Res: Ty, Val: -1);
10002 Lo = MIRBuilder.buildSub(Dst: Ty, Src0: MIRBuilder.buildSMax(Dst: Ty, Src0: LHS, Src1: NegOne),
10003 Src1: MaxVal);
10004 Hi = MIRBuilder.buildSub(Dst: Ty, Src0: MIRBuilder.buildSMin(Dst: Ty, Src0: LHS, Src1: NegOne),
10005 Src1: MinVal);
10006 }
10007 auto RHSClamped =
10008 MIRBuilder.buildSMin(Dst: Ty, Src0: MIRBuilder.buildSMax(Dst: Ty, Src0: Lo, Src1: RHS), Src1: Hi);
10009 MIRBuilder.buildInstr(Opc: BaseOp, DstOps: {Res}, SrcOps: {LHS, RHSClamped});
10010 } else {
10011 // uadd.sat(a, b) -> a + umin(~a, b)
10012 // usub.sat(a, b) -> a - umin(a, b)
10013 Register Not = IsAdd ? MIRBuilder.buildNot(Dst: Ty, Src0: LHS).getReg(Idx: 0) : LHS;
10014 auto Min = MIRBuilder.buildUMin(Dst: Ty, Src0: Not, Src1: RHS);
10015 MIRBuilder.buildInstr(Opc: BaseOp, DstOps: {Res}, SrcOps: {LHS, Min});
10016 }
10017
10018 MI.eraseFromParent();
10019 return Legalized;
10020}
10021
10022LegalizerHelper::LegalizeResult
10023LegalizerHelper::lowerAddSubSatToAddoSubo(MachineInstr &MI) {
10024 auto [Res, LHS, RHS] = MI.getFirst3Regs();
10025 LLT Ty = MRI.getType(Reg: Res);
10026 LLT BoolTy = Ty.changeElementSize(NewEltSize: 1);
10027 bool IsSigned;
10028 bool IsAdd;
10029 unsigned OverflowOp;
10030 switch (MI.getOpcode()) {
10031 default:
10032 llvm_unreachable("unexpected addsat/subsat opcode");
10033 case TargetOpcode::G_UADDSAT:
10034 IsSigned = false;
10035 IsAdd = true;
10036 OverflowOp = TargetOpcode::G_UADDO;
10037 break;
10038 case TargetOpcode::G_SADDSAT:
10039 IsSigned = true;
10040 IsAdd = true;
10041 OverflowOp = TargetOpcode::G_SADDO;
10042 break;
10043 case TargetOpcode::G_USUBSAT:
10044 IsSigned = false;
10045 IsAdd = false;
10046 OverflowOp = TargetOpcode::G_USUBO;
10047 break;
10048 case TargetOpcode::G_SSUBSAT:
10049 IsSigned = true;
10050 IsAdd = false;
10051 OverflowOp = TargetOpcode::G_SSUBO;
10052 break;
10053 }
10054
10055 auto OverflowRes =
10056 MIRBuilder.buildInstr(Opc: OverflowOp, DstOps: {Ty, BoolTy}, SrcOps: {LHS, RHS});
10057 Register Tmp = OverflowRes.getReg(Idx: 0);
10058 Register Ov = OverflowRes.getReg(Idx: 1);
10059 MachineInstrBuilder Clamp;
10060 if (IsSigned) {
10061 // sadd.sat(a, b) ->
10062 // {tmp, ov} = saddo(a, b)
10063 // ov ? (tmp >>s 31) + 0x80000000 : r
10064 // ssub.sat(a, b) ->
10065 // {tmp, ov} = ssubo(a, b)
10066 // ov ? (tmp >>s 31) + 0x80000000 : r
10067 uint64_t NumBits = Ty.getScalarSizeInBits();
10068 auto ShiftAmount = MIRBuilder.buildConstant(Res: Ty, Val: NumBits - 1);
10069 auto Sign = MIRBuilder.buildAShr(Dst: Ty, Src0: Tmp, Src1: ShiftAmount);
10070 auto MinVal =
10071 MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMinValue(numBits: NumBits));
10072 Clamp = MIRBuilder.buildAdd(Dst: Ty, Src0: Sign, Src1: MinVal);
10073 } else {
10074 // uadd.sat(a, b) ->
10075 // {tmp, ov} = uaddo(a, b)
10076 // ov ? 0xffffffff : tmp
10077 // usub.sat(a, b) ->
10078 // {tmp, ov} = usubo(a, b)
10079 // ov ? 0 : tmp
10080 Clamp = MIRBuilder.buildConstant(Res: Ty, Val: IsAdd ? -1 : 0);
10081 }
10082 MIRBuilder.buildSelect(Res, Tst: Ov, Op0: Clamp, Op1: Tmp);
10083
10084 MI.eraseFromParent();
10085 return Legalized;
10086}
10087
10088LegalizerHelper::LegalizeResult
10089LegalizerHelper::lowerShlSat(MachineInstr &MI) {
10090 assert((MI.getOpcode() == TargetOpcode::G_SSHLSAT ||
10091 MI.getOpcode() == TargetOpcode::G_USHLSAT) &&
10092 "Expected shlsat opcode!");
10093 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SSHLSAT;
10094 auto [Res, LHS, RHS] = MI.getFirst3Regs();
10095 LLT Ty = MRI.getType(Reg: Res);
10096 LLT BoolTy = Ty.changeElementSize(NewEltSize: 1);
10097
10098 unsigned BW = Ty.getScalarSizeInBits();
10099 auto Result = MIRBuilder.buildShl(Dst: Ty, Src0: LHS, Src1: RHS);
10100 auto Orig = IsSigned ? MIRBuilder.buildAShr(Dst: Ty, Src0: Result, Src1: RHS)
10101 : MIRBuilder.buildLShr(Dst: Ty, Src0: Result, Src1: RHS);
10102
10103 MachineInstrBuilder SatVal;
10104 if (IsSigned) {
10105 auto SatMin = MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMinValue(numBits: BW));
10106 auto SatMax = MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMaxValue(numBits: BW));
10107 auto Cmp = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: LHS,
10108 Op1: MIRBuilder.buildConstant(Res: Ty, Val: 0));
10109 SatVal = MIRBuilder.buildSelect(Res: Ty, Tst: Cmp, Op0: SatMin, Op1: SatMax);
10110 } else {
10111 SatVal = MIRBuilder.buildConstant(Res: Ty, Val: APInt::getMaxValue(numBits: BW));
10112 }
10113 auto Ov = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: BoolTy, Op0: LHS, Op1: Orig);
10114 MIRBuilder.buildSelect(Res, Tst: Ov, Op0: SatVal, Op1: Result);
10115
10116 MI.eraseFromParent();
10117 return Legalized;
10118}
10119
10120LegalizerHelper::LegalizeResult LegalizerHelper::lowerBswap(MachineInstr &MI) {
10121 auto [Dst, Src] = MI.getFirst2Regs();
10122 const LLT Ty = MRI.getType(Reg: Src);
10123 unsigned SizeInBytes = (Ty.getScalarSizeInBits() + 7) / 8;
10124 unsigned BaseShiftAmt = (SizeInBytes - 1) * 8;
10125
10126 // Swap most and least significant byte, set remaining bytes in Res to zero.
10127 auto ShiftAmt = MIRBuilder.buildConstant(Res: Ty, Val: BaseShiftAmt);
10128 auto LSByteShiftedLeft = MIRBuilder.buildShl(Dst: Ty, Src0: Src, Src1: ShiftAmt);
10129 auto MSByteShiftedRight = MIRBuilder.buildLShr(Dst: Ty, Src0: Src, Src1: ShiftAmt);
10130 auto Res = MIRBuilder.buildOr(Dst: Ty, Src0: MSByteShiftedRight, Src1: LSByteShiftedLeft);
10131
10132 // Set i-th high/low byte in Res to i-th low/high byte from Src.
10133 for (unsigned i = 1; i < SizeInBytes / 2; ++i) {
10134 // AND with Mask leaves byte i unchanged and sets remaining bytes to 0.
10135 APInt APMask = APInt::getBitsSet(numBits: SizeInBytes * 8, loBit: i * 8, hiBit: i * 8 + 8);
10136 auto Mask = MIRBuilder.buildConstant(Res: Ty, Val: APMask);
10137 auto ShiftAmt = MIRBuilder.buildConstant(Res: Ty, Val: BaseShiftAmt - 16 * i);
10138 // Low byte shifted left to place of high byte: (Src & Mask) << ShiftAmt.
10139 auto LoByte = MIRBuilder.buildAnd(Dst: Ty, Src0: Src, Src1: Mask);
10140 auto LoShiftedLeft = MIRBuilder.buildShl(Dst: Ty, Src0: LoByte, Src1: ShiftAmt);
10141 Res = MIRBuilder.buildOr(Dst: Ty, Src0: Res, Src1: LoShiftedLeft);
10142 // High byte shifted right to place of low byte: (Src >> ShiftAmt) & Mask.
10143 auto SrcShiftedRight = MIRBuilder.buildLShr(Dst: Ty, Src0: Src, Src1: ShiftAmt);
10144 auto HiShiftedRight = MIRBuilder.buildAnd(Dst: Ty, Src0: SrcShiftedRight, Src1: Mask);
10145 Res = MIRBuilder.buildOr(Dst: Ty, Src0: Res, Src1: HiShiftedRight);
10146 }
10147 Res.getInstr()->getOperand(i: 0).setReg(Dst);
10148
10149 MI.eraseFromParent();
10150 return Legalized;
10151}
10152
10153//{ (Src & Mask) >> N } | { (Src << N) & Mask }
10154static MachineInstrBuilder SwapN(unsigned N, DstOp Dst, MachineIRBuilder &B,
10155 MachineInstrBuilder Src, const APInt &Mask) {
10156 const LLT Ty = Dst.getLLTTy(MRI: *B.getMRI());
10157 MachineInstrBuilder C_N = B.buildConstant(Res: Ty, Val: N);
10158 MachineInstrBuilder MaskLoNTo0 = B.buildConstant(Res: Ty, Val: Mask);
10159 auto LHS = B.buildLShr(Dst: Ty, Src0: B.buildAnd(Dst: Ty, Src0: Src, Src1: MaskLoNTo0), Src1: C_N);
10160 auto RHS = B.buildAnd(Dst: Ty, Src0: B.buildShl(Dst: Ty, Src0: Src, Src1: C_N), Src1: MaskLoNTo0);
10161 return B.buildOr(Dst, Src0: LHS, Src1: RHS);
10162}
10163
10164LegalizerHelper::LegalizeResult
10165LegalizerHelper::lowerBitreverse(MachineInstr &MI) {
10166 auto [Dst, Src] = MI.getFirst2Regs();
10167 const LLT SrcTy = MRI.getType(Reg: Src);
10168 unsigned Size = SrcTy.getScalarSizeInBits();
10169 unsigned VSize = SrcTy.getSizeInBits();
10170
10171 if (Size >= 8) {
10172 if (SrcTy.isVector() && (VSize % 8 == 0) &&
10173 (LI.isLegal(Query: {TargetOpcode::G_BITREVERSE,
10174 {LLT::fixed_vector(NumElements: VSize / 8, ScalarTy: LLT::integer(SizeInBits: 8)),
10175 LLT::fixed_vector(NumElements: VSize / 8, ScalarTy: LLT::integer(SizeInBits: 8))}}))) {
10176 // If bitreverse is legal for i8 vector of the same size, then cast
10177 // to i8 vector type.
10178 // e.g. v4s32 -> v16s8
10179 LLT VTy = LLT::fixed_vector(NumElements: VSize / 8, ScalarTy: LLT::integer(SizeInBits: 8));
10180 auto BSWAP = MIRBuilder.buildBSwap(Dst: SrcTy, Src0: Src);
10181 auto Cast = MIRBuilder.buildBitcast(Dst: VTy, Src: BSWAP);
10182 auto RBIT = MIRBuilder.buildBitReverse(Dst: VTy, Src: Cast);
10183 MIRBuilder.buildBitcast(Dst, Src: RBIT);
10184 } else {
10185 MachineInstrBuilder BSWAP =
10186 MIRBuilder.buildInstr(Opc: TargetOpcode::G_BSWAP, DstOps: {SrcTy}, SrcOps: {Src});
10187
10188 // swap high and low 4 bits in 8 bit blocks 7654|3210 -> 3210|7654
10189 // [(val & 0xF0F0F0F0) >> 4] | [(val & 0x0F0F0F0F) << 4]
10190 // -> [(val & 0xF0F0F0F0) >> 4] | [(val << 4) & 0xF0F0F0F0]
10191 MachineInstrBuilder Swap4 = SwapN(N: 4, Dst: SrcTy, B&: MIRBuilder, Src: BSWAP,
10192 Mask: APInt::getSplat(NewLen: Size, V: APInt(8, 0xF0)));
10193
10194 // swap high and low 2 bits in 4 bit blocks 32|10 76|54 -> 10|32 54|76
10195 // [(val & 0xCCCCCCCC) >> 2] & [(val & 0x33333333) << 2]
10196 // -> [(val & 0xCCCCCCCC) >> 2] & [(val << 2) & 0xCCCCCCCC]
10197 MachineInstrBuilder Swap2 = SwapN(N: 2, Dst: SrcTy, B&: MIRBuilder, Src: Swap4,
10198 Mask: APInt::getSplat(NewLen: Size, V: APInt(8, 0xCC)));
10199
10200 // swap high and low 1 bit in 2 bit blocks 1|0 3|2 5|4 7|6 -> 0|1 2|3 4|5
10201 // 6|7
10202 // [(val & 0xAAAAAAAA) >> 1] & [(val & 0x55555555) << 1]
10203 // -> [(val & 0xAAAAAAAA) >> 1] & [(val << 1) & 0xAAAAAAAA]
10204 SwapN(N: 1, Dst, B&: MIRBuilder, Src: Swap2, Mask: APInt::getSplat(NewLen: Size, V: APInt(8, 0xAA)));
10205 }
10206 } else {
10207 // Expand bitreverse for types smaller than 8 bits.
10208 MachineInstrBuilder Tmp;
10209 for (unsigned I = 0, J = Size - 1; I < Size; ++I, --J) {
10210 MachineInstrBuilder Tmp2;
10211 if (I < J) {
10212 auto ShAmt = MIRBuilder.buildConstant(Res: SrcTy, Val: J - I);
10213 Tmp2 = MIRBuilder.buildShl(Dst: SrcTy, Src0: Src, Src1: ShAmt);
10214 } else {
10215 auto ShAmt = MIRBuilder.buildConstant(Res: SrcTy, Val: I - J);
10216 Tmp2 = MIRBuilder.buildLShr(Dst: SrcTy, Src0: Src, Src1: ShAmt);
10217 }
10218
10219 auto Mask = MIRBuilder.buildConstant(Res: SrcTy, Val: 1ULL << J);
10220 Tmp2 = MIRBuilder.buildAnd(Dst: SrcTy, Src0: Tmp2, Src1: Mask);
10221 if (I == 0)
10222 Tmp = Tmp2;
10223 else
10224 Tmp = MIRBuilder.buildOr(Dst: SrcTy, Src0: Tmp, Src1: Tmp2);
10225 }
10226 MIRBuilder.buildCopy(Res: Dst, Op: Tmp);
10227 }
10228
10229 MI.eraseFromParent();
10230 return Legalized;
10231}
10232
10233LegalizerHelper::LegalizeResult
10234LegalizerHelper::lowerReadWriteRegister(MachineInstr &MI) {
10235 MachineFunction &MF = MIRBuilder.getMF();
10236
10237 bool IsRead = MI.getOpcode() == TargetOpcode::G_READ_REGISTER;
10238 int NameOpIdx = IsRead ? 1 : 0;
10239 int ValRegIndex = IsRead ? 0 : 1;
10240
10241 Register ValReg = MI.getOperand(i: ValRegIndex).getReg();
10242 const LLT Ty = MRI.getType(Reg: ValReg);
10243 const MDString *RegStr = cast<MDString>(
10244 Val: cast<MDNode>(Val: MI.getOperand(i: NameOpIdx).getMetadata())->getOperand(I: 0));
10245
10246 Register PhysReg = TLI.getRegisterByName(RegName: RegStr->getString().data(), Ty, MF);
10247 if (!PhysReg) {
10248 const Function &Fn = MF.getFunction();
10249 Fn.getContext().diagnose(DI: DiagnosticInfoGenericWithLoc(
10250 "invalid register \"" + Twine(RegStr->getString().data()) + "\" for " +
10251 (IsRead ? "llvm.read_register" : "llvm.write_register"),
10252 Fn, MI.getDebugLoc()));
10253 if (IsRead)
10254 MIRBuilder.buildUndef(Res: ValReg);
10255
10256 MI.eraseFromParent();
10257 return Legalized;
10258 }
10259
10260 if (IsRead)
10261 MIRBuilder.buildCopy(Res: ValReg, Op: PhysReg);
10262 else
10263 MIRBuilder.buildCopy(Res: PhysReg, Op: ValReg);
10264
10265 MI.eraseFromParent();
10266 return Legalized;
10267}
10268
10269LegalizerHelper::LegalizeResult
10270LegalizerHelper::lowerSMULH_UMULH(MachineInstr &MI) {
10271 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SMULH;
10272 unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
10273 Register Result = MI.getOperand(i: 0).getReg();
10274 LLT OrigTy = MRI.getType(Reg: Result);
10275 auto SizeInBits = OrigTy.getScalarSizeInBits();
10276 LLT WideTy = OrigTy.changeElementSize(NewEltSize: SizeInBits * 2);
10277
10278 auto LHS = MIRBuilder.buildInstr(Opc: ExtOp, DstOps: {WideTy}, SrcOps: {MI.getOperand(i: 1)});
10279 auto RHS = MIRBuilder.buildInstr(Opc: ExtOp, DstOps: {WideTy}, SrcOps: {MI.getOperand(i: 2)});
10280 auto Mul = MIRBuilder.buildMul(Dst: WideTy, Src0: LHS, Src1: RHS);
10281 unsigned ShiftOp = IsSigned ? TargetOpcode::G_ASHR : TargetOpcode::G_LSHR;
10282
10283 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: SizeInBits);
10284 auto Shifted = MIRBuilder.buildInstr(Opc: ShiftOp, DstOps: {WideTy}, SrcOps: {Mul, ShiftAmt});
10285 MIRBuilder.buildTrunc(Res: Result, Op: Shifted);
10286
10287 MI.eraseFromParent();
10288 return Legalized;
10289}
10290
10291LegalizerHelper::LegalizeResult
10292LegalizerHelper::lowerISFPCLASS(MachineInstr &MI) {
10293 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
10294 FPClassTest Mask = static_cast<FPClassTest>(MI.getOperand(i: 2).getImm());
10295
10296 if (Mask == fcNone) {
10297 MIRBuilder.buildConstant(Res: DstReg, Val: 0);
10298 MI.eraseFromParent();
10299 return Legalized;
10300 }
10301 if (Mask == fcAllFlags) {
10302 MIRBuilder.buildConstant(Res: DstReg, Val: 1);
10303 MI.eraseFromParent();
10304 return Legalized;
10305 }
10306
10307 // TODO: Try inverting the test with getInvertedFPClassTest like the DAG
10308 // version
10309
10310 unsigned BitSize = SrcTy.getScalarSizeInBits();
10311 const fltSemantics &Semantics = getFltSemanticForLLT(Ty: SrcTy.getScalarType());
10312
10313 LLT IntTy = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: BitSize));
10314 auto AsInt = SrcTy == IntTy ? MIRBuilder.buildCopy(Res: IntTy, Op: SrcReg)
10315 : MIRBuilder.buildBitcast(Dst: IntTy, Src: SrcReg);
10316
10317 // Various masks.
10318 APInt SignBit = APInt::getSignMask(BitWidth: BitSize);
10319 APInt ValueMask = APInt::getSignedMaxValue(numBits: BitSize); // All bits but sign.
10320 APInt Inf = APFloat::getInf(Sem: Semantics).bitcastToAPInt(); // Exp and int bit.
10321 APInt ExpMask = Inf;
10322 APInt AllOneMantissa = APFloat::getLargest(Sem: Semantics).bitcastToAPInt() & ~Inf;
10323 APInt QNaNBitMask =
10324 APInt::getOneBitSet(numBits: BitSize, BitNo: AllOneMantissa.getActiveBits() - 1);
10325 APInt InversionMask = APInt::getAllOnes(numBits: DstTy.getScalarSizeInBits());
10326
10327 auto SignBitC = MIRBuilder.buildConstant(Res: IntTy, Val: SignBit);
10328 auto ValueMaskC = MIRBuilder.buildConstant(Res: IntTy, Val: ValueMask);
10329 auto InfC = MIRBuilder.buildConstant(Res: IntTy, Val: Inf);
10330 auto ExpMaskC = MIRBuilder.buildConstant(Res: IntTy, Val: ExpMask);
10331 auto ZeroC = MIRBuilder.buildConstant(Res: IntTy, Val: 0);
10332
10333 auto Abs = MIRBuilder.buildAnd(Dst: IntTy, Src0: AsInt, Src1: ValueMaskC);
10334 auto Sign =
10335 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_NE, Res: DstTy, Op0: AsInt, Op1: Abs);
10336
10337 auto Res = MIRBuilder.buildConstant(Res: DstTy, Val: 0);
10338 // Clang doesn't support capture of structured bindings:
10339 LLT DstTyCopy = DstTy;
10340 const auto appendToRes = [&](MachineInstrBuilder ToAppend) {
10341 Res = MIRBuilder.buildOr(Dst: DstTyCopy, Src0: Res, Src1: ToAppend);
10342 };
10343
10344 // Tests that involve more than one class should be processed first.
10345 if ((Mask & fcFinite) == fcFinite) {
10346 // finite(V) ==> abs(V) u< exp_mask
10347 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: Abs,
10348 Op1: ExpMaskC));
10349 Mask &= ~fcFinite;
10350 } else if ((Mask & fcFinite) == fcPosFinite) {
10351 // finite(V) && V > 0 ==> V u< exp_mask
10352 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: AsInt,
10353 Op1: ExpMaskC));
10354 Mask &= ~fcPosFinite;
10355 } else if ((Mask & fcFinite) == fcNegFinite) {
10356 // finite(V) && V < 0 ==> abs(V) u< exp_mask && signbit == 1
10357 auto Cmp = MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: Abs,
10358 Op1: ExpMaskC);
10359 auto And = MIRBuilder.buildAnd(Dst: DstTy, Src0: Cmp, Src1: Sign);
10360 appendToRes(And);
10361 Mask &= ~fcNegFinite;
10362 }
10363
10364 if (FPClassTest PartialCheck = Mask & (fcZero | fcSubnormal)) {
10365 // fcZero | fcSubnormal => test all exponent bits are 0
10366 // TODO: Handle sign bit specific cases
10367 // TODO: Handle inverted case
10368 if (PartialCheck == (fcZero | fcSubnormal)) {
10369 auto ExpBits = MIRBuilder.buildAnd(Dst: IntTy, Src0: AsInt, Src1: ExpMaskC);
10370 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10371 Op0: ExpBits, Op1: ZeroC));
10372 Mask &= ~PartialCheck;
10373 }
10374 }
10375
10376 // Check for individual classes.
10377 if (FPClassTest PartialCheck = Mask & fcZero) {
10378 if (PartialCheck == fcPosZero)
10379 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10380 Op0: AsInt, Op1: ZeroC));
10381 else if (PartialCheck == fcZero)
10382 appendToRes(
10383 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy, Op0: Abs, Op1: ZeroC));
10384 else // fcNegZero
10385 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10386 Op0: AsInt, Op1: SignBitC));
10387 }
10388
10389 if (FPClassTest PartialCheck = Mask & fcSubnormal) {
10390 // issubnormal(V) ==> unsigned(abs(V) - 1) u< (all mantissa bits set)
10391 // issubnormal(V) && V>0 ==> unsigned(V - 1) u< (all mantissa bits set)
10392 auto V = (PartialCheck == fcPosSubnormal) ? AsInt : Abs;
10393 auto OneC = MIRBuilder.buildConstant(Res: IntTy, Val: 1);
10394 auto VMinusOne = MIRBuilder.buildSub(Dst: IntTy, Src0: V, Src1: OneC);
10395 auto SubnormalRes =
10396 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: VMinusOne,
10397 Op1: MIRBuilder.buildConstant(Res: IntTy, Val: AllOneMantissa));
10398 if (PartialCheck == fcNegSubnormal)
10399 SubnormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: SubnormalRes, Src1: Sign);
10400 appendToRes(SubnormalRes);
10401 }
10402
10403 if (FPClassTest PartialCheck = Mask & fcInf) {
10404 if (PartialCheck == fcPosInf)
10405 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10406 Op0: AsInt, Op1: InfC));
10407 else if (PartialCheck == fcInf)
10408 appendToRes(
10409 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy, Op0: Abs, Op1: InfC));
10410 else { // fcNegInf
10411 APInt NegInf = APFloat::getInf(Sem: Semantics, Negative: true).bitcastToAPInt();
10412 auto NegInfC = MIRBuilder.buildConstant(Res: IntTy, Val: NegInf);
10413 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10414 Op0: AsInt, Op1: NegInfC));
10415 }
10416 }
10417
10418 if (FPClassTest PartialCheck = Mask & fcNan) {
10419 auto InfWithQnanBitC = MIRBuilder.buildConstant(Res: IntTy, Val: Inf | QNaNBitMask);
10420 if (PartialCheck == fcNan) {
10421 // isnan(V) ==> abs(V) u> int(inf)
10422 appendToRes(
10423 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: DstTy, Op0: Abs, Op1: InfC));
10424 } else if (PartialCheck == fcQNan) {
10425 // isquiet(V) ==> abs(V) u>= (unsigned(Inf) | quiet_bit)
10426 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGE, Res: DstTy, Op0: Abs,
10427 Op1: InfWithQnanBitC));
10428 } else { // fcSNan
10429 // issignaling(V) ==> abs(V) u> unsigned(Inf) &&
10430 // abs(V) u< (unsigned(Inf) | quiet_bit)
10431 auto IsNan =
10432 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: DstTy, Op0: Abs, Op1: InfC);
10433 auto IsNotQnan = MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy,
10434 Op0: Abs, Op1: InfWithQnanBitC);
10435 appendToRes(MIRBuilder.buildAnd(Dst: DstTy, Src0: IsNan, Src1: IsNotQnan));
10436 }
10437 }
10438
10439 if (FPClassTest PartialCheck = Mask & fcNormal) {
10440 // isnormal(V) ==> (0 u< exp u< max_exp) ==> (unsigned(exp-1) u<
10441 // (max_exp-1))
10442 APInt ExpLSB = ExpMask & ~(ExpMask.shl(shiftAmt: 1));
10443 auto ExpMinusOne = MIRBuilder.buildSub(
10444 Dst: IntTy, Src0: Abs, Src1: MIRBuilder.buildConstant(Res: IntTy, Val: ExpLSB));
10445 APInt MaxExpMinusOne = ExpMask - ExpLSB;
10446 auto NormalRes =
10447 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: ExpMinusOne,
10448 Op1: MIRBuilder.buildConstant(Res: IntTy, Val: MaxExpMinusOne));
10449 if (PartialCheck == fcNegNormal)
10450 NormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: NormalRes, Src1: Sign);
10451 else if (PartialCheck == fcPosNormal) {
10452 auto PosSign = MIRBuilder.buildXor(
10453 Dst: DstTy, Src0: Sign, Src1: MIRBuilder.buildConstant(Res: DstTy, Val: InversionMask));
10454 NormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: NormalRes, Src1: PosSign);
10455 }
10456 appendToRes(NormalRes);
10457 }
10458
10459 MIRBuilder.buildCopy(Res: DstReg, Op: Res);
10460 MI.eraseFromParent();
10461 return Legalized;
10462}
10463
10464LegalizerHelper::LegalizeResult LegalizerHelper::lowerSelect(MachineInstr &MI) {
10465 // Implement G_SELECT in terms of XOR, AND, OR.
10466 auto [DstReg, DstTy, MaskReg, MaskTy, Op1Reg, Op1Ty, Op2Reg, Op2Ty] =
10467 MI.getFirst4RegLLTs();
10468
10469 LLT Op1TyInt =
10470 Op1Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: Op1Ty.getScalarSizeInBits()));
10471
10472 bool IsEltPtr = DstTy.isPointerOrPointerVector();
10473 if (IsEltPtr) {
10474 LLT ScalarPtrTy = LLT::integer(SizeInBits: DstTy.getScalarSizeInBits());
10475 LLT NewTy = DstTy.changeElementType(NewEltTy: ScalarPtrTy);
10476 Op1Reg = MIRBuilder.buildPtrToInt(Dst: NewTy, Src: Op1Reg).getReg(Idx: 0);
10477 Op1Ty = MRI.getType(Reg: Op1Reg);
10478 Op2Reg = MIRBuilder.buildPtrToInt(Dst: NewTy, Src: Op2Reg).getReg(Idx: 0);
10479 Op2Ty = MRI.getType(Reg: Op2Reg);
10480 DstTy = NewTy;
10481 }
10482
10483 if (MaskTy.isScalar()) {
10484 // Turn the scalar condition into a vector condition mask if needed.
10485
10486 Register MaskElt = MaskReg;
10487
10488 // The condition was potentially zero extended before, but we want a sign
10489 // extended boolean.
10490 if (MaskTy != LLT::scalar(SizeInBits: 1))
10491 MaskElt = MIRBuilder.buildSExtInReg(Res: MaskTy, Op: MaskElt, ImmOp: 1).getReg(Idx: 0);
10492
10493 // Continue the sign extension (or truncate) to match the data type.
10494 MaskTy = DstTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: DstTy.getScalarSizeInBits()));
10495 MaskElt =
10496 MIRBuilder.buildSExtOrTrunc(Res: MaskTy.getScalarType(), Op: MaskElt).getReg(Idx: 0);
10497
10498 if (DstTy.isVector()) {
10499 // Generate a vector splat idiom.
10500 auto ShufSplat = MIRBuilder.buildShuffleSplat(Res: MaskTy, Src: MaskElt);
10501 MaskReg = ShufSplat.getReg(Idx: 0);
10502 } else {
10503 MaskReg = MaskElt;
10504 }
10505 } else if (!DstTy.isVector()) {
10506 // Cannot handle the case that mask is a vector and dst is a scalar.
10507 return UnableToLegalize;
10508 }
10509
10510 if (MaskTy.getSizeInBits() != DstTy.getSizeInBits()) {
10511 return UnableToLegalize;
10512 }
10513
10514 if (!Op1Ty.getScalarType().isAnyScalar() &&
10515 !Op1Ty.getScalarType().isInteger())
10516 Op1Reg = MIRBuilder.buildBitcast(Dst: Op1TyInt, Src: Op1Reg).getReg(Idx: 0);
10517
10518 if (!Op2Ty.getScalarType().isAnyScalar() &&
10519 !Op2Ty.getScalarType().isInteger()) {
10520 auto Op2TyInt =
10521 Op2Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: Op2Ty.getScalarSizeInBits()));
10522 Op2Reg = MIRBuilder.buildBitcast(Dst: Op2TyInt, Src: Op2Reg).getReg(Idx: 0);
10523 }
10524
10525 auto NotMask = MIRBuilder.buildNot(Dst: MaskTy, Src0: MaskReg);
10526 auto NewOp1 = MIRBuilder.buildAnd(Dst: MaskTy, Src0: Op1Reg, Src1: MaskReg);
10527 auto NewOp2 = MIRBuilder.buildAnd(Dst: MaskTy, Src0: Op2Reg, Src1: NotMask);
10528 if (IsEltPtr) {
10529 auto Or = MIRBuilder.buildOr(Dst: DstTy, Src0: NewOp1, Src1: NewOp2);
10530 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: Or);
10531 } else {
10532 if (DstTy == Op1TyInt)
10533 MIRBuilder.buildOr(Dst: DstReg, Src0: NewOp1, Src1: NewOp2);
10534 else {
10535 auto Or = MIRBuilder.buildOr(Dst: Op1TyInt, Src0: NewOp1, Src1: NewOp2);
10536 MIRBuilder.buildBitcast(Dst: DstReg, Src: Or.getReg(Idx: 0));
10537 }
10538 }
10539 MI.eraseFromParent();
10540 return Legalized;
10541}
10542
10543LegalizerHelper::LegalizeResult LegalizerHelper::lowerDIVREM(MachineInstr &MI) {
10544 // Split DIVREM into individual instructions.
10545 unsigned Opcode = MI.getOpcode();
10546
10547 MIRBuilder.buildInstr(
10548 Opc: Opcode == TargetOpcode::G_SDIVREM ? TargetOpcode::G_SDIV
10549 : TargetOpcode::G_UDIV,
10550 DstOps: {MI.getOperand(i: 0).getReg()}, SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3)});
10551 MIRBuilder.buildInstr(
10552 Opc: Opcode == TargetOpcode::G_SDIVREM ? TargetOpcode::G_SREM
10553 : TargetOpcode::G_UREM,
10554 DstOps: {MI.getOperand(i: 1).getReg()}, SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3)});
10555 MI.eraseFromParent();
10556 return Legalized;
10557}
10558
10559LegalizerHelper::LegalizeResult
10560LegalizerHelper::lowerAbsToAddXor(MachineInstr &MI) {
10561 // Expand %res = G_ABS %a into:
10562 // %v1 = G_ASHR %a, scalar_size-1
10563 // %v2 = G_ADD %a, %v1
10564 // %res = G_XOR %v2, %v1
10565 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
10566 Register OpReg = MI.getOperand(i: 1).getReg();
10567 auto ShiftAmt =
10568 MIRBuilder.buildConstant(Res: DstTy, Val: DstTy.getScalarSizeInBits() - 1);
10569 auto Shift = MIRBuilder.buildAShr(Dst: DstTy, Src0: OpReg, Src1: ShiftAmt);
10570 auto Add = MIRBuilder.buildAdd(Dst: DstTy, Src0: OpReg, Src1: Shift);
10571 MIRBuilder.buildXor(Dst: MI.getOperand(i: 0).getReg(), Src0: Add, Src1: Shift);
10572 MI.eraseFromParent();
10573 return Legalized;
10574}
10575
10576LegalizerHelper::LegalizeResult
10577LegalizerHelper::lowerAbsToMaxNeg(MachineInstr &MI) {
10578 // Expand %res = G_ABS %a into:
10579 // %v1 = G_CONSTANT 0
10580 // %v2 = G_SUB %v1, %a
10581 // %res = G_SMAX %a, %v2
10582 Register SrcReg = MI.getOperand(i: 1).getReg();
10583 LLT Ty = MRI.getType(Reg: SrcReg);
10584 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
10585 auto Sub = MIRBuilder.buildSub(Dst: Ty, Src0: Zero, Src1: SrcReg);
10586 MIRBuilder.buildSMax(Dst: MI.getOperand(i: 0), Src0: SrcReg, Src1: Sub);
10587 MI.eraseFromParent();
10588 return Legalized;
10589}
10590
10591LegalizerHelper::LegalizeResult
10592LegalizerHelper::lowerAbsToCNeg(MachineInstr &MI) {
10593 Register SrcReg = MI.getOperand(i: 1).getReg();
10594 Register DestReg = MI.getOperand(i: 0).getReg();
10595 LLT Ty = MRI.getType(Reg: SrcReg), IType = LLT::scalar(SizeInBits: 1);
10596 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0).getReg(Idx: 0);
10597 auto Sub = MIRBuilder.buildSub(Dst: Ty, Src0: Zero, Src1: SrcReg).getReg(Idx: 0);
10598 auto ICmp = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SGT, Res: IType, Op0: SrcReg, Op1: Zero);
10599 MIRBuilder.buildSelect(Res: DestReg, Tst: ICmp, Op0: SrcReg, Op1: Sub);
10600 MI.eraseFromParent();
10601 return Legalized;
10602}
10603
10604LegalizerHelper::LegalizeResult
10605LegalizerHelper::lowerAbsDiffToSelect(MachineInstr &MI) {
10606 assert((MI.getOpcode() == TargetOpcode::G_ABDS ||
10607 MI.getOpcode() == TargetOpcode::G_ABDU) &&
10608 "Expected G_ABDS or G_ABDU instruction");
10609
10610 auto [DstReg, LHS, RHS] = MI.getFirst3Regs();
10611 LLT Ty = MRI.getType(Reg: LHS);
10612
10613 // abds(lhs, rhs) -> select(sgt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
10614 // abdu(lhs, rhs) -> select(ugt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
10615 Register LHSSub = MIRBuilder.buildSub(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10616 Register RHSSub = MIRBuilder.buildSub(Dst: Ty, Src0: RHS, Src1: LHS).getReg(Idx: 0);
10617 CmpInst::Predicate Pred = (MI.getOpcode() == TargetOpcode::G_ABDS)
10618 ? CmpInst::ICMP_SGT
10619 : CmpInst::ICMP_UGT;
10620 auto ICmp = MIRBuilder.buildICmp(Pred, Res: LLT::scalar(SizeInBits: 1), Op0: LHS, Op1: RHS);
10621 MIRBuilder.buildSelect(Res: DstReg, Tst: ICmp, Op0: LHSSub, Op1: RHSSub);
10622
10623 MI.eraseFromParent();
10624 return Legalized;
10625}
10626
10627LegalizerHelper::LegalizeResult
10628LegalizerHelper::lowerAbsDiffToMinMax(MachineInstr &MI) {
10629 assert((MI.getOpcode() == TargetOpcode::G_ABDS ||
10630 MI.getOpcode() == TargetOpcode::G_ABDU) &&
10631 "Expected G_ABDS or G_ABDU instruction");
10632
10633 auto [DstReg, LHS, RHS] = MI.getFirst3Regs();
10634 LLT Ty = MRI.getType(Reg: LHS);
10635
10636 // abds(lhs, rhs) -→ sub(smax(lhs, rhs), smin(lhs, rhs))
10637 // abdu(lhs, rhs) -→ sub(umax(lhs, rhs), umin(lhs, rhs))
10638 Register MaxReg, MinReg;
10639 if (MI.getOpcode() == TargetOpcode::G_ABDS) {
10640 MaxReg = MIRBuilder.buildSMax(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10641 MinReg = MIRBuilder.buildSMin(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10642 } else {
10643 MaxReg = MIRBuilder.buildUMax(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10644 MinReg = MIRBuilder.buildUMin(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10645 }
10646 MIRBuilder.buildSub(Dst: DstReg, Src0: MaxReg, Src1: MinReg);
10647
10648 MI.eraseFromParent();
10649 return Legalized;
10650}
10651
10652LegalizerHelper::LegalizeResult LegalizerHelper::lowerFAbs(MachineInstr &MI) {
10653 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
10654 LLT TyInt =
10655 DstTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: DstTy.getScalarSizeInBits()));
10656 Register CastedSrc = SrcReg;
10657
10658 if (!(SrcTy.getScalarType().isAnyScalar() ||
10659 SrcTy.getScalarType().isInteger())) {
10660 auto SrcTyInt =
10661 SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: SrcTy.getScalarSizeInBits()));
10662 CastedSrc = MIRBuilder.buildBitcast(Dst: SrcTyInt, Src: SrcReg).getReg(Idx: 0);
10663 }
10664
10665 if (MRI.getType(Reg: DstReg) != TyInt) {
10666 // Reset sign bit
10667 Register NewDst =
10668 MIRBuilder
10669 .buildAnd(Dst: TyInt, Src0: CastedSrc,
10670 Src1: MIRBuilder.buildConstant(
10671 Res: TyInt, Val: APInt::getSignedMaxValue(
10672 numBits: DstTy.getScalarSizeInBits())))
10673 .getReg(Idx: 0);
10674
10675 MIRBuilder.buildBitcast(Dst: DstReg, Src: NewDst);
10676 } else
10677 MIRBuilder
10678 .buildAnd(
10679 Dst: DstReg, Src0: CastedSrc,
10680 Src1: MIRBuilder.buildConstant(
10681 Res: TyInt, Val: APInt::getSignedMaxValue(numBits: DstTy.getScalarSizeInBits())))
10682 .getReg(Idx: 0);
10683
10684 MI.eraseFromParent();
10685 return Legalized;
10686}
10687
10688LegalizerHelper::LegalizeResult
10689LegalizerHelper::lowerVectorReduction(MachineInstr &MI) {
10690 Register SrcReg = MI.getOperand(i: 1).getReg();
10691 LLT SrcTy = MRI.getType(Reg: SrcReg);
10692 LLT DstTy = MRI.getType(Reg: SrcReg);
10693
10694 // The source could be a scalar if the IR type was <1 x sN>.
10695 if (SrcTy.isScalar()) {
10696 if (DstTy.getSizeInBits() > SrcTy.getSizeInBits())
10697 return UnableToLegalize; // FIXME: handle extension.
10698 // This can be just a plain copy.
10699 Observer.changingInstr(MI);
10700 MI.setDesc(MIRBuilder.getTII().get(Opcode: TargetOpcode::COPY));
10701 Observer.changedInstr(MI);
10702 return Legalized;
10703 }
10704 return UnableToLegalize;
10705}
10706
10707LegalizerHelper::LegalizeResult LegalizerHelper::lowerVAArg(MachineInstr &MI) {
10708 MachineFunction &MF = *MI.getMF();
10709 const DataLayout &DL = MIRBuilder.getDataLayout();
10710 LLVMContext &Ctx = MF.getFunction().getContext();
10711 Register ListPtr = MI.getOperand(i: 1).getReg();
10712 LLT PtrTy = MRI.getType(Reg: ListPtr);
10713
10714 // LstPtr is a pointer to the head of the list. Get the address
10715 // of the head of the list.
10716 Align PtrAlignment = DL.getABITypeAlign(Ty: getTypeForLLT(Ty: PtrTy, C&: Ctx));
10717 MachineMemOperand *PtrLoadMMO = MF.getMachineMemOperand(
10718 PtrInfo: MachinePointerInfo(), f: MachineMemOperand::MOLoad, MemTy: PtrTy, base_alignment: PtrAlignment);
10719 auto VAList = MIRBuilder.buildLoad(Res: PtrTy, Addr: ListPtr, MMO&: *PtrLoadMMO).getReg(Idx: 0);
10720
10721 const Align A(MI.getOperand(i: 2).getImm());
10722 LLT PtrTyAsScalarTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
10723 if (A > TLI.getMinStackArgumentAlignment()) {
10724 Register AlignAmt =
10725 MIRBuilder.buildConstant(Res: PtrTyAsScalarTy, Val: A.value() - 1).getReg(Idx: 0);
10726 auto AddDst = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: VAList, Op1: AlignAmt);
10727 auto AndDst = MIRBuilder.buildMaskLowPtrBits(Res: PtrTy, Op0: AddDst, NumBits: Log2(A));
10728 VAList = AndDst.getReg(Idx: 0);
10729 }
10730
10731 // Increment the pointer, VAList, to the next vaarg
10732 // The list should be bumped by the size of element in the current head of
10733 // list.
10734 Register Dst = MI.getOperand(i: 0).getReg();
10735 LLT LLTTy = MRI.getType(Reg: Dst);
10736 Type *Ty = getTypeForLLT(Ty: LLTTy, C&: Ctx);
10737 auto IncAmt =
10738 MIRBuilder.buildConstant(Res: PtrTyAsScalarTy, Val: DL.getTypeAllocSize(Ty));
10739 auto Succ = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: VAList, Op1: IncAmt);
10740
10741 // Store the increment VAList to the legalized pointer
10742 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
10743 PtrInfo: MachinePointerInfo(), f: MachineMemOperand::MOStore, MemTy: PtrTy, base_alignment: PtrAlignment);
10744 MIRBuilder.buildStore(Val: Succ, Addr: ListPtr, MMO&: *StoreMMO);
10745 // Load the actual argument out of the pointer VAList
10746 Align EltAlignment = DL.getABITypeAlign(Ty);
10747 MachineMemOperand *EltLoadMMO = MF.getMachineMemOperand(
10748 PtrInfo: MachinePointerInfo(), f: MachineMemOperand::MOLoad, MemTy: LLTTy, base_alignment: EltAlignment);
10749 MIRBuilder.buildLoad(Res: Dst, Addr: VAList, MMO&: *EltLoadMMO);
10750
10751 MI.eraseFromParent();
10752 return Legalized;
10753}
10754
10755LegalizerHelper::LegalizeResult LegalizerHelper::lowerMulfix(MachineInstr &MI) {
10756 [[maybe_unused]] unsigned OpCode = MI.getOpcode();
10757 assert((OpCode == TargetOpcode::G_SMULFIX ||
10758 OpCode == TargetOpcode::G_UMULFIX) &&
10759 "Operator must be either G_SMULFIX or G_UMULFIX!");
10760 auto [Dst, LHS, RHS] = MI.getFirst3Regs();
10761 LLT Ty = MRI.getType(Reg: Dst);
10762 unsigned Scale = MI.getOperand(i: 3).getImm();
10763
10764 if (Scale == 0) {
10765 MIRBuilder.buildMul(Dst, Src0: LHS, Src1: RHS);
10766 MI.eraseFromParent();
10767 return Legalized;
10768 }
10769
10770 // TODO: Port other lowerng paths from SelectionDAG.
10771 LLT WideTy = Ty.changeElementSize(NewEltSize: Ty.getScalarSizeInBits() * 2);
10772 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: Scale);
10773 MachineInstrBuilder ExtLHS{}, ExtRHS{}, Shift{};
10774 if (MI.getOpcode() == TargetOpcode::G_SMULFIX) {
10775 ExtLHS = MIRBuilder.buildSExt(Res: WideTy, Op: LHS);
10776 ExtRHS = MIRBuilder.buildSExt(Res: WideTy, Op: RHS);
10777 } else {
10778 ExtLHS = MIRBuilder.buildZExt(Res: WideTy, Op: LHS);
10779 ExtRHS = MIRBuilder.buildZExt(Res: WideTy, Op: RHS);
10780 }
10781
10782 auto Mul = MIRBuilder.buildMul(Dst: WideTy, Src0: ExtLHS, Src1: ExtRHS);
10783 if (MI.getOpcode() == TargetOpcode::G_SMULFIX)
10784 Shift = MIRBuilder.buildAShr(Dst: WideTy, Src0: Mul, Src1: ShiftAmt);
10785 else
10786 Shift = MIRBuilder.buildLShr(Dst: WideTy, Src0: Mul, Src1: ShiftAmt);
10787
10788 MIRBuilder.buildTrunc(Res: Dst, Op: Shift);
10789
10790 MI.eraseFromParent();
10791 return Legalized;
10792}
10793
10794// Get a vectorized representation of the memset value operand, GISel edition.
10795static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB) {
10796 MachineRegisterInfo &MRI = *MIB.getMRI();
10797 unsigned NumBits = Ty.getScalarSizeInBits();
10798 auto ValVRegAndVal = getIConstantVRegValWithLookThrough(VReg: Val, MRI);
10799 if (!Ty.isVector() && ValVRegAndVal) {
10800 APInt Scalar = ValVRegAndVal->Value.trunc(width: 8);
10801 APInt SplatVal = APInt::getSplat(NewLen: NumBits, V: Scalar);
10802 return MIB.buildConstant(Res: Ty, Val: SplatVal).getReg(Idx: 0);
10803 }
10804
10805 // Extend the byte value to the larger type, and then multiply by a magic
10806 // value 0x010101... in order to replicate it across every byte.
10807 // Unless it's zero, in which case just emit a larger G_CONSTANT 0.
10808 if (ValVRegAndVal && ValVRegAndVal->Value == 0) {
10809 return MIB.buildConstant(Res: Ty, Val: 0).getReg(Idx: 0);
10810 }
10811
10812 LLT ExtType = Ty.getScalarType();
10813 auto ZExt = MIB.buildZExtOrTrunc(Res: ExtType, Op: Val);
10814 if (NumBits > 8) {
10815 APInt Magic = APInt::getSplat(NewLen: NumBits, V: APInt(8, 0x01));
10816 auto MagicMI = MIB.buildConstant(Res: ExtType, Val: Magic);
10817 Val = MIB.buildMul(Dst: ExtType, Src0: ZExt, Src1: MagicMI).getReg(Idx: 0);
10818 }
10819
10820 // For vector types create a G_BUILD_VECTOR.
10821 if (Ty.isVector())
10822 Val = MIB.buildSplatBuildVector(Res: Ty, Src: Val).getReg(Idx: 0);
10823
10824 return Val;
10825}
10826
10827LegalizerHelper::LegalizeResult
10828LegalizerHelper::lowerMemset(MachineInstr &MI, Register Dst, Register Val,
10829 uint64_t KnownLen, Align Alignment,
10830 bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
10831 auto &MF = *MI.getParent()->getParent();
10832 const auto &TLI = *MF.getSubtarget().getTargetLowering();
10833 auto &DL = MF.getDataLayout();
10834 LLVMContext &C = MF.getFunction().getContext();
10835
10836 assert(KnownLen != 0 && "Have a zero length memset length!");
10837 assert(!MemOps.empty() && "Expected at least one memory op");
10838
10839 MachineFrameInfo &MFI = MF.getFrameInfo();
10840 MachineInstr *FIDef = getOpcodeDef(Opcode: TargetOpcode::G_FRAME_INDEX, Reg: Dst, MRI);
10841 const auto &DstMMO = **MI.memoperands_begin();
10842
10843 if (DstAlignCanChange) {
10844 // Get an estimate of the type from the LLT.
10845 Type *IRTy = getTypeForLLT(Ty: MemOps[0], C);
10846 Align NewAlign = DL.getABITypeAlign(Ty: IRTy);
10847 if (NewAlign > Alignment) {
10848 Alignment = NewAlign;
10849 unsigned FI = FIDef->getOperand(i: 1).getIndex();
10850 // Give the stack frame object a larger alignment if needed.
10851 if (MFI.getObjectAlign(ObjectIdx: FI) < Alignment)
10852 MFI.setObjectAlignment(ObjectIdx: FI, Alignment);
10853 }
10854 }
10855
10856 MachineIRBuilder MIB(MI);
10857 // Find the largest store and generate the bit pattern for it.
10858 LLT LargestTy = MemOps[0];
10859 for (unsigned i = 1; i < MemOps.size(); i++)
10860 if (MemOps[i].getSizeInBits() > LargestTy.getSizeInBits())
10861 LargestTy = MemOps[i];
10862
10863 // The memset stored value is always defined as an s8, so in order to make it
10864 // work with larger store types we need to repeat the bit pattern across the
10865 // wider type.
10866 Register MemSetValue = getMemsetValue(Val, Ty: LargestTy, MIB);
10867
10868 if (!MemSetValue)
10869 return UnableToLegalize;
10870
10871 // Generate the stores. For each store type in the list, we generate the
10872 // matching store of that type to the destination address.
10873 LLT PtrTy = MRI.getType(Reg: Dst);
10874 unsigned DstOff = 0;
10875 unsigned Size = KnownLen;
10876 for (unsigned I = 0; I < MemOps.size(); I++) {
10877 LLT Ty = MemOps[I];
10878 unsigned TySize = Ty.getSizeInBytes();
10879 if (TySize > Size) {
10880 // Issuing an unaligned load / store pair that overlaps with the previous
10881 // pair. Adjust the offset accordingly.
10882 assert(I == MemOps.size() - 1 && I != 0);
10883 DstOff -= TySize - Size;
10884 }
10885
10886 // If this store is smaller than the largest store see whether we can get
10887 // the smaller value for free with a truncate.
10888 Register Value = MemSetValue;
10889 if (Ty.getSizeInBits() < LargestTy.getSizeInBits()) {
10890 MVT VT = getMVTForLLT(Ty);
10891 MVT LargestVT = getMVTForLLT(Ty: LargestTy);
10892 if (!LargestTy.isVector() && !Ty.isVector() &&
10893 TLI.isTruncateFree(FromVT: LargestVT, ToVT: VT))
10894 Value = MIB.buildTrunc(Res: Ty, Op: MemSetValue).getReg(Idx: 0);
10895 else
10896 Value = getMemsetValue(Val, Ty, MIB);
10897 if (!Value)
10898 return UnableToLegalize;
10899 }
10900
10901 auto *StoreMMO = MF.getMachineMemOperand(MMO: &DstMMO, Offset: DstOff, Ty);
10902
10903 Register Ptr = Dst;
10904 if (DstOff != 0) {
10905 auto Offset =
10906 MIB.buildConstant(Res: LLT::scalar(SizeInBits: PtrTy.getSizeInBits()), Val: DstOff);
10907 Ptr = MIB.buildObjectPtrOffset(Res: PtrTy, Op0: Dst, Op1: Offset).getReg(Idx: 0);
10908 }
10909
10910 MIB.buildStore(Val: Value, Addr: Ptr, MMO&: *StoreMMO);
10911 DstOff += Ty.getSizeInBytes();
10912 Size -= TySize;
10913 }
10914
10915 MI.eraseFromParent();
10916 return Legalized;
10917}
10918
10919LegalizerHelper::LegalizeResult
10920LegalizerHelper::lowerMemcpy(MachineInstr &MI, Register Dst, Register Src,
10921 uint64_t KnownLen, Align Alignment,
10922 bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
10923 auto &MF = *MI.getParent()->getParent();
10924 auto &DL = MF.getDataLayout();
10925 LLVMContext &C = MF.getFunction().getContext();
10926
10927 assert(KnownLen != 0 && "Have a zero length memcpy length!");
10928 assert(!MemOps.empty() && "Expected at least one memory op");
10929
10930 MachineFrameInfo &MFI = MF.getFrameInfo();
10931 MachineInstr *FIDef = getOpcodeDef(Opcode: TargetOpcode::G_FRAME_INDEX, Reg: Dst, MRI);
10932
10933 // FIXME: infer better src pointer alignment like SelectionDAG does here.
10934 // FIXME: also use the equivalent of isMemSrcFromConstant and alwaysinlining
10935 // if the memcpy is in a tail call position.
10936
10937 const auto &DstMMO = **MI.memoperands_begin();
10938 const auto &SrcMMO = **std::next(x: MI.memoperands_begin());
10939
10940 if (DstAlignCanChange) {
10941 // Get an estimate of the type from the LLT.
10942 Type *IRTy = getTypeForLLT(Ty: MemOps[0], C);
10943 Align NewAlign = DL.getABITypeAlign(Ty: IRTy);
10944
10945 // Don't promote to an alignment that would require dynamic stack
10946 // realignment.
10947 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
10948 if (!TRI->hasStackRealignment(MF))
10949 if (MaybeAlign StackAlign = DL.getStackAlignment())
10950 NewAlign = std::min(a: NewAlign, b: *StackAlign);
10951
10952 if (NewAlign > Alignment) {
10953 Alignment = NewAlign;
10954 unsigned FI = FIDef->getOperand(i: 1).getIndex();
10955 // Give the stack frame object a larger alignment if needed.
10956 if (MFI.getObjectAlign(ObjectIdx: FI) < Alignment)
10957 MFI.setObjectAlignment(ObjectIdx: FI, Alignment);
10958 }
10959 }
10960
10961 LLVM_DEBUG(dbgs() << "Inlining memcpy: " << MI << " into loads & stores\n");
10962
10963 MachineIRBuilder MIB(MI);
10964 // Now we need to emit a pair of load and stores for each of the types we've
10965 // collected. I.e. for each type, generate a load from the source pointer of
10966 // that type width, and then generate a corresponding store to the dest buffer
10967 // of that value loaded. This can result in a sequence of loads and stores
10968 // mixed types, depending on what the target specifies as good types to use.
10969 unsigned CurrOffset = 0;
10970 unsigned Size = KnownLen;
10971 for (auto CopyTy : MemOps) {
10972 // Issuing an unaligned load / store pair that overlaps with the previous
10973 // pair. Adjust the offset accordingly.
10974 if (CopyTy.getSizeInBytes() > Size)
10975 CurrOffset -= CopyTy.getSizeInBytes() - Size;
10976
10977 // Construct MMOs for the accesses.
10978 auto *LoadMMO =
10979 MF.getMachineMemOperand(MMO: &SrcMMO, Offset: CurrOffset, Size: CopyTy.getSizeInBytes());
10980 auto *StoreMMO =
10981 MF.getMachineMemOperand(MMO: &DstMMO, Offset: CurrOffset, Size: CopyTy.getSizeInBytes());
10982
10983 // Create the load.
10984 Register LoadPtr = Src;
10985 Register Offset;
10986 if (CurrOffset != 0) {
10987 LLT SrcTy = MRI.getType(Reg: Src);
10988 Offset =
10989 MIB.buildConstant(Res: LLT::integer(SizeInBits: SrcTy.getSizeInBits()), Val: CurrOffset)
10990 .getReg(Idx: 0);
10991 LoadPtr = MIB.buildObjectPtrOffset(Res: SrcTy, Op0: Src, Op1: Offset).getReg(Idx: 0);
10992 }
10993 auto LdVal = MIB.buildLoad(Res: CopyTy, Addr: LoadPtr, MMO&: *LoadMMO);
10994
10995 // Create the store.
10996 Register StorePtr = Dst;
10997 if (CurrOffset != 0) {
10998 LLT DstTy = MRI.getType(Reg: Dst);
10999 StorePtr = MIB.buildObjectPtrOffset(Res: DstTy, Op0: Dst, Op1: Offset).getReg(Idx: 0);
11000 }
11001 MIB.buildStore(Val: LdVal, Addr: StorePtr, MMO&: *StoreMMO);
11002 CurrOffset += CopyTy.getSizeInBytes();
11003 Size -= CopyTy.getSizeInBytes();
11004 }
11005
11006 MI.eraseFromParent();
11007 return Legalized;
11008}
11009
11010LegalizerHelper::LegalizeResult
11011LegalizerHelper::lowerMemmove(MachineInstr &MI, Register Dst, Register Src,
11012 uint64_t KnownLen, Align Alignment,
11013 bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
11014 auto &MF = *MI.getParent()->getParent();
11015 auto &DL = MF.getDataLayout();
11016 LLVMContext &C = MF.getFunction().getContext();
11017
11018 assert(KnownLen != 0 && "Have a zero length memmove length!");
11019 assert(!MemOps.empty() && "Expected at least one memory op");
11020
11021 MachineFrameInfo &MFI = MF.getFrameInfo();
11022 MachineInstr *FIDef = getOpcodeDef(Opcode: TargetOpcode::G_FRAME_INDEX, Reg: Dst, MRI);
11023 const auto &DstMMO = **MI.memoperands_begin();
11024 const auto &SrcMMO = **std::next(x: MI.memoperands_begin());
11025
11026 if (DstAlignCanChange) {
11027 // Get an estimate of the type from the LLT.
11028 Type *IRTy = getTypeForLLT(Ty: MemOps[0], C);
11029 Align NewAlign = DL.getABITypeAlign(Ty: IRTy);
11030
11031 // Don't promote to an alignment that would require dynamic stack
11032 // realignment.
11033 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
11034 if (!TRI->hasStackRealignment(MF))
11035 if (MaybeAlign StackAlign = DL.getStackAlignment())
11036 NewAlign = std::min(a: NewAlign, b: *StackAlign);
11037
11038 if (NewAlign > Alignment) {
11039 Alignment = NewAlign;
11040 unsigned FI = FIDef->getOperand(i: 1).getIndex();
11041 // Give the stack frame object a larger alignment if needed.
11042 if (MFI.getObjectAlign(ObjectIdx: FI) < Alignment)
11043 MFI.setObjectAlignment(ObjectIdx: FI, Alignment);
11044 }
11045 }
11046
11047 LLVM_DEBUG(dbgs() << "Inlining memmove: " << MI << " into loads & stores\n");
11048
11049 MachineIRBuilder MIB(MI);
11050 // Memmove requires that we perform the loads first before issuing the stores.
11051 // Apart from that, this loop is pretty much doing the same thing as the
11052 // memcpy codegen function.
11053 unsigned CurrOffset = 0;
11054 SmallVector<Register, 16> LoadVals;
11055 for (auto CopyTy : MemOps) {
11056 // Construct MMO for the load.
11057 auto *LoadMMO =
11058 MF.getMachineMemOperand(MMO: &SrcMMO, Offset: CurrOffset, Size: CopyTy.getSizeInBytes());
11059
11060 // Create the load.
11061 Register LoadPtr = Src;
11062 if (CurrOffset != 0) {
11063 LLT SrcTy = MRI.getType(Reg: Src);
11064 auto Offset =
11065 MIB.buildConstant(Res: LLT::scalar(SizeInBits: SrcTy.getSizeInBits()), Val: CurrOffset);
11066 LoadPtr = MIB.buildObjectPtrOffset(Res: SrcTy, Op0: Src, Op1: Offset).getReg(Idx: 0);
11067 }
11068 LoadVals.push_back(Elt: MIB.buildLoad(Res: CopyTy, Addr: LoadPtr, MMO&: *LoadMMO).getReg(Idx: 0));
11069 CurrOffset += CopyTy.getSizeInBytes();
11070 }
11071
11072 CurrOffset = 0;
11073 for (unsigned I = 0; I < MemOps.size(); ++I) {
11074 LLT CopyTy = MemOps[I];
11075 // Now store the values loaded.
11076 auto *StoreMMO =
11077 MF.getMachineMemOperand(MMO: &DstMMO, Offset: CurrOffset, Size: CopyTy.getSizeInBytes());
11078
11079 Register StorePtr = Dst;
11080 if (CurrOffset != 0) {
11081 LLT DstTy = MRI.getType(Reg: Dst);
11082 auto Offset =
11083 MIB.buildConstant(Res: LLT::scalar(SizeInBits: DstTy.getSizeInBits()), Val: CurrOffset);
11084 StorePtr = MIB.buildObjectPtrOffset(Res: DstTy, Op0: Dst, Op1: Offset).getReg(Idx: 0);
11085 }
11086 MIB.buildStore(Val: LoadVals[I], Addr: StorePtr, MMO&: *StoreMMO);
11087 CurrOffset += CopyTy.getSizeInBytes();
11088 }
11089 MI.eraseFromParent();
11090 return Legalized;
11091}
11092
11093LegalizerHelper::LegalizeResult LegalizerHelper::lowerMemCpyFamily(
11094 MachineInstr &MI, Register Dst, Register Src, uint64_t KnownLen,
11095 Align Alignment, bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
11096 const unsigned Opc = MI.getOpcode();
11097 assert((Opc == TargetOpcode::G_MEMCPY ||
11098 Opc == TargetOpcode::G_MEMCPY_INLINE ||
11099 Opc == TargetOpcode::G_MEMMOVE || Opc == TargetOpcode::G_MEMSET ||
11100 Opc == TargetOpcode::G_MEMSET_INLINE) &&
11101 "Expected memcpy like instruction");
11102
11103 if (KnownLen == 0) {
11104 MI.eraseFromParent();
11105 return Legalized;
11106 }
11107
11108 if (Opc == TargetOpcode::G_MEMCPY || Opc == TargetOpcode::G_MEMCPY_INLINE) {
11109 return lowerMemcpy(MI, Dst, Src, KnownLen, Alignment, DstAlignCanChange,
11110 MemOps);
11111 }
11112 if (Opc == TargetOpcode::G_MEMMOVE)
11113 return lowerMemmove(MI, Dst, Src, KnownLen, Alignment, DstAlignCanChange,
11114 MemOps);
11115 if (Opc == TargetOpcode::G_MEMSET || Opc == TargetOpcode::G_MEMSET_INLINE)
11116 return lowerMemset(MI, Dst, Val: Src, KnownLen, Alignment, DstAlignCanChange,
11117 MemOps);
11118 return UnableToLegalize;
11119}
11120
11121LegalizerHelper::LegalizeResult
11122LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen) {
11123 Register Dst, Src;
11124 uint64_t KnownLen;
11125 Align Alignment;
11126 bool DstAlignCanChange;
11127 std::vector<LLT> MemOps;
11128 if (!canLowerMemCpyFamily(MI, MRI, MaxLen, Dst, Src, KnownLen, Alignment,
11129 DstAlignCanChange, MemOps))
11130 return UnableToLegalize;
11131 return lowerMemCpyFamily(MI, Dst, Src, KnownLen, Alignment, DstAlignCanChange,
11132 MemOps);
11133}
11134