1//===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file implements the LegalizerHelper class to legalize
10/// individual instructions and the LegalizeMachineIR wrapper pass for the
11/// primary legalization.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
16#include "llvm/CodeGen/GlobalISel/CallLowering.h"
17#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
18#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
19#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
20#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
21#include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h"
22#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
23#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24#include "llvm/CodeGen/GlobalISel/Utils.h"
25#include "llvm/CodeGen/LowLevelTypeUtils.h"
26#include "llvm/CodeGen/MachineConstantPool.h"
27#include "llvm/CodeGen/MachineFrameInfo.h"
28#include "llvm/CodeGen/MachineRegisterInfo.h"
29#include "llvm/CodeGen/RuntimeLibcallUtil.h"
30#include "llvm/CodeGen/TargetFrameLowering.h"
31#include "llvm/CodeGen/TargetInstrInfo.h"
32#include "llvm/CodeGen/TargetLowering.h"
33#include "llvm/CodeGen/TargetOpcodes.h"
34#include "llvm/CodeGen/TargetSubtargetInfo.h"
35#include "llvm/IR/Instructions.h"
36#include "llvm/Support/Debug.h"
37#include "llvm/Support/MathExtras.h"
38#include "llvm/Support/raw_ostream.h"
39#include "llvm/Target/TargetMachine.h"
40#include <cassert>
41#include <numeric>
42#include <optional>
43
44#define DEBUG_TYPE "legalizer"
45
46using namespace llvm;
47using namespace LegalizeActions;
48using namespace MIPatternMatch;
49
50/// Try to break down \p OrigTy into \p NarrowTy sized pieces.
51///
52/// Returns the number of \p NarrowTy elements needed to reconstruct \p OrigTy,
53/// with any leftover piece as type \p LeftoverTy
54///
55/// Returns -1 in the first element of the pair if the breakdown is not
56/// satisfiable.
57static std::pair<int, int>
58getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) {
59 assert(!LeftoverTy.isValid() && "this is an out argument");
60
61 unsigned Size = OrigTy.getSizeInBits();
62 unsigned NarrowSize = NarrowTy.getSizeInBits();
63 unsigned NumParts = Size / NarrowSize;
64 unsigned LeftoverSize = Size - NumParts * NarrowSize;
65 assert(Size > NarrowSize);
66
67 if (LeftoverSize == 0)
68 return {NumParts, 0};
69
70 if (NarrowTy.isVector()) {
71 unsigned EltSize = OrigTy.getScalarSizeInBits();
72 if (LeftoverSize % EltSize != 0)
73 return {-1, -1};
74 LeftoverTy = OrigTy.changeElementCount(
75 EC: ElementCount::getFixed(MinVal: LeftoverSize / EltSize));
76 } else {
77 LeftoverTy = LLT::integer(SizeInBits: LeftoverSize);
78 }
79
80 int NumLeftover = LeftoverSize / LeftoverTy.getSizeInBits();
81 return std::make_pair(x&: NumParts, y&: NumLeftover);
82}
83
84static Type *getFloatTypeForLLT(LLVMContext &Ctx, LLT Ty) {
85
86 if (!Ty.isScalar())
87 return nullptr;
88
89 switch (Ty.getSizeInBits()) {
90 case 16:
91 return Type::getHalfTy(C&: Ctx);
92 case 32:
93 return Type::getFloatTy(C&: Ctx);
94 case 64:
95 return Type::getDoubleTy(C&: Ctx);
96 case 80:
97 return Type::getX86_FP80Ty(C&: Ctx);
98 case 128:
99 return Type::getFP128Ty(C&: Ctx);
100 default:
101 return nullptr;
102 }
103}
104
105LegalizerHelper::LegalizerHelper(MachineFunction &MF,
106 GISelChangeObserver &Observer,
107 MachineIRBuilder &Builder,
108 const LibcallLoweringInfo *Libcalls)
109 : MIRBuilder(Builder), Observer(Observer), MRI(MF.getRegInfo()),
110 LI(*MF.getSubtarget().getLegalizerInfo()),
111 TLI(*MF.getSubtarget().getTargetLowering()), Libcalls(Libcalls) {}
112
113LegalizerHelper::LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI,
114 GISelChangeObserver &Observer,
115 MachineIRBuilder &B,
116 const LibcallLoweringInfo *Libcalls,
117 GISelValueTracking *VT)
118 : MIRBuilder(B), Observer(Observer), MRI(MF.getRegInfo()), LI(LI),
119 TLI(*MF.getSubtarget().getTargetLowering()), Libcalls(Libcalls), VT(VT) {}
120
121LegalizerHelper::LegalizeResult
122LegalizerHelper::legalizeInstrStep(MachineInstr &MI,
123 LostDebugLocObserver &LocObserver) {
124 LLVM_DEBUG(dbgs() << "\nLegalizing: " << MI);
125
126 MIRBuilder.setInstrAndDebugLoc(MI);
127
128 if (isa<GIntrinsic>(Val: MI))
129 return LI.legalizeIntrinsic(Helper&: *this, MI) ? Legalized : UnableToLegalize;
130 auto Step = LI.getAction(MI, MRI);
131 switch (Step.Action) {
132 case Legal:
133 LLVM_DEBUG(dbgs() << ".. Already legal\n");
134 return AlreadyLegal;
135 case Libcall:
136 LLVM_DEBUG(dbgs() << ".. Convert to libcall\n");
137 return libcall(MI, LocObserver);
138 case NarrowScalar:
139 LLVM_DEBUG(dbgs() << ".. Narrow scalar\n");
140 return narrowScalar(MI, TypeIdx: Step.TypeIdx, NarrowTy: Step.NewType);
141 case WidenScalar:
142 LLVM_DEBUG(dbgs() << ".. Widen scalar\n");
143 return widenScalar(MI, TypeIdx: Step.TypeIdx, WideTy: Step.NewType);
144 case Bitcast:
145 LLVM_DEBUG(dbgs() << ".. Bitcast type\n");
146 return bitcast(MI, TypeIdx: Step.TypeIdx, Ty: Step.NewType);
147 case Lower:
148 LLVM_DEBUG(dbgs() << ".. Lower\n");
149 return lower(MI, TypeIdx: Step.TypeIdx, Ty: Step.NewType);
150 case FewerElements:
151 LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n");
152 return fewerElementsVector(MI, TypeIdx: Step.TypeIdx, NarrowTy: Step.NewType);
153 case MoreElements:
154 LLVM_DEBUG(dbgs() << ".. Increase number of elements\n");
155 return moreElementsVector(MI, TypeIdx: Step.TypeIdx, MoreTy: Step.NewType);
156 case Custom:
157 LLVM_DEBUG(dbgs() << ".. Custom legalization\n");
158 return LI.legalizeCustom(Helper&: *this, MI, LocObserver) ? Legalized
159 : UnableToLegalize;
160 default:
161 LLVM_DEBUG(dbgs() << ".. Unable to legalize\n");
162 return UnableToLegalize;
163 }
164}
165
166void LegalizerHelper::insertParts(Register DstReg,
167 LLT ResultTy, LLT PartTy,
168 ArrayRef<Register> PartRegs,
169 LLT LeftoverTy,
170 ArrayRef<Register> LeftoverRegs) {
171 if (!LeftoverTy.isValid()) {
172 assert(LeftoverRegs.empty());
173
174 if (!ResultTy.isVector()) {
175 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: PartRegs);
176 return;
177 }
178
179 if (PartTy.isVector())
180 MIRBuilder.buildConcatVectors(Res: DstReg, Ops: PartRegs);
181 else
182 MIRBuilder.buildBuildVector(Res: DstReg, Ops: PartRegs);
183 return;
184 }
185
186 // Merge sub-vectors with different number of elements and insert into DstReg.
187 if (ResultTy.isVector()) {
188 assert(LeftoverRegs.size() == 1 && "Expected one leftover register");
189 SmallVector<Register, 8> AllRegs(PartRegs);
190 AllRegs.append(in_start: LeftoverRegs.begin(), in_end: LeftoverRegs.end());
191 return mergeMixedSubvectors(DstReg, PartRegs: AllRegs);
192 }
193
194 SmallVector<Register> GCDRegs;
195 LLT GCDTy = getGCDType(OrigTy: getGCDType(OrigTy: ResultTy, TargetTy: LeftoverTy), TargetTy: PartTy);
196 for (auto PartReg : concat<const Register>(Ranges&: PartRegs, Ranges&: LeftoverRegs))
197 extractGCDType(Parts&: GCDRegs, GCDTy, SrcReg: PartReg);
198 LLT ResultLCMTy = buildLCMMergePieces(DstTy: ResultTy, NarrowTy: LeftoverTy, GCDTy, VRegs&: GCDRegs);
199 buildWidenedRemergeToDst(DstReg, LCMTy: ResultLCMTy, RemergeRegs: GCDRegs);
200}
201
202void LegalizerHelper::appendVectorElts(SmallVectorImpl<Register> &Elts,
203 Register Reg) {
204 LLT Ty = MRI.getType(Reg);
205 SmallVector<Register, 8> RegElts;
206 extractParts(Reg, Ty: Ty.getScalarType(), NumParts: Ty.getNumElements(), VRegs&: RegElts,
207 MIRBuilder, MRI);
208 Elts.append(RHS: RegElts);
209}
210
211/// Merge \p PartRegs with different types into \p DstReg.
212void LegalizerHelper::mergeMixedSubvectors(Register DstReg,
213 ArrayRef<Register> PartRegs) {
214 SmallVector<Register, 8> AllElts;
215 for (unsigned i = 0; i < PartRegs.size() - 1; ++i)
216 appendVectorElts(Elts&: AllElts, Reg: PartRegs[i]);
217
218 Register Leftover = PartRegs[PartRegs.size() - 1];
219 if (!MRI.getType(Reg: Leftover).isVector())
220 AllElts.push_back(Elt: Leftover);
221 else
222 appendVectorElts(Elts&: AllElts, Reg: Leftover);
223
224 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: AllElts);
225}
226
227/// Append the result registers of G_UNMERGE_VALUES \p MI to \p Regs.
228static void getUnmergeResults(SmallVectorImpl<Register> &Regs,
229 const MachineInstr &MI) {
230 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
231
232 const int StartIdx = Regs.size();
233 const int NumResults = MI.getNumOperands() - 1;
234 Regs.resize(N: Regs.size() + NumResults);
235 for (int I = 0; I != NumResults; ++I)
236 Regs[StartIdx + I] = MI.getOperand(i: I).getReg();
237}
238
239void LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts,
240 LLT GCDTy, Register SrcReg) {
241 LLT SrcTy = MRI.getType(Reg: SrcReg);
242 if (SrcTy == GCDTy) {
243 // If the source already evenly divides the result type, we don't need to do
244 // anything.
245 Parts.push_back(Elt: SrcReg);
246 } else {
247 // Need to split into common type sized pieces.
248 auto Unmerge = MIRBuilder.buildUnmerge(Res: GCDTy, Op: SrcReg);
249 getUnmergeResults(Regs&: Parts, MI: *Unmerge);
250 }
251}
252
253LLT LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts, LLT DstTy,
254 LLT NarrowTy, Register SrcReg) {
255 LLT SrcTy = MRI.getType(Reg: SrcReg);
256 LLT GCDTy = getGCDType(OrigTy: getGCDType(OrigTy: SrcTy, TargetTy: NarrowTy), TargetTy: DstTy);
257 extractGCDType(Parts, GCDTy, SrcReg);
258 return GCDTy;
259}
260
261LLT LegalizerHelper::buildLCMMergePieces(LLT DstTy, LLT NarrowTy, LLT GCDTy,
262 SmallVectorImpl<Register> &VRegs,
263 unsigned PadStrategy) {
264 LLT LCMTy = getLCMType(OrigTy: DstTy, TargetTy: NarrowTy);
265
266 int NumParts = LCMTy.getSizeInBits() / NarrowTy.getSizeInBits();
267 int NumSubParts = NarrowTy.getSizeInBits() / GCDTy.getSizeInBits();
268 int NumOrigSrc = VRegs.size();
269
270 Register PadReg;
271
272 // Get a value we can use to pad the source value if the sources won't evenly
273 // cover the result type.
274 if (NumOrigSrc < NumParts * NumSubParts) {
275 if (PadStrategy == TargetOpcode::G_ZEXT)
276 PadReg = MIRBuilder.buildConstant(Res: GCDTy, Val: 0).getReg(Idx: 0);
277 else if (PadStrategy == TargetOpcode::G_ANYEXT)
278 PadReg = MIRBuilder.buildUndef(Res: GCDTy).getReg(Idx: 0);
279 else {
280 assert(PadStrategy == TargetOpcode::G_SEXT);
281
282 // Shift the sign bit of the low register through the high register.
283 auto ShiftAmt =
284 MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: GCDTy.getSizeInBits() - 1);
285 PadReg = MIRBuilder.buildAShr(Dst: GCDTy, Src0: VRegs.back(), Src1: ShiftAmt).getReg(Idx: 0);
286 }
287 }
288
289 // Registers for the final merge to be produced.
290 SmallVector<Register, 4> Remerge(NumParts);
291
292 // Registers needed for intermediate merges, which will be merged into a
293 // source for Remerge.
294 SmallVector<Register, 4> SubMerge(NumSubParts);
295
296 // Once we've fully read off the end of the original source bits, we can reuse
297 // the same high bits for remaining padding elements.
298 Register AllPadReg;
299
300 // Build merges to the LCM type to cover the original result type.
301 for (int I = 0; I != NumParts; ++I) {
302 bool AllMergePartsArePadding = true;
303
304 // Build the requested merges to the requested type.
305 for (int J = 0; J != NumSubParts; ++J) {
306 int Idx = I * NumSubParts + J;
307 if (Idx >= NumOrigSrc) {
308 SubMerge[J] = PadReg;
309 continue;
310 }
311
312 SubMerge[J] = VRegs[Idx];
313
314 // There are meaningful bits here we can't reuse later.
315 AllMergePartsArePadding = false;
316 }
317
318 // If we've filled up a complete piece with padding bits, we can directly
319 // emit the natural sized constant if applicable, rather than a merge of
320 // smaller constants.
321 if (AllMergePartsArePadding && !AllPadReg) {
322 if (PadStrategy == TargetOpcode::G_ANYEXT)
323 AllPadReg = MIRBuilder.buildUndef(Res: NarrowTy).getReg(Idx: 0);
324 else if (PadStrategy == TargetOpcode::G_ZEXT)
325 AllPadReg = MIRBuilder.buildConstant(Res: NarrowTy, Val: 0).getReg(Idx: 0);
326
327 // If this is a sign extension, we can't materialize a trivial constant
328 // with the right type and have to produce a merge.
329 }
330
331 if (AllPadReg) {
332 // Avoid creating additional instructions if we're just adding additional
333 // copies of padding bits.
334 Remerge[I] = AllPadReg;
335 continue;
336 }
337
338 if (NumSubParts == 1)
339 Remerge[I] = SubMerge[0];
340 else
341 Remerge[I] = MIRBuilder.buildMergeLikeInstr(Res: NarrowTy, Ops: SubMerge).getReg(Idx: 0);
342
343 // In the sign extend padding case, re-use the first all-signbit merge.
344 if (AllMergePartsArePadding && !AllPadReg)
345 AllPadReg = Remerge[I];
346 }
347
348 VRegs = std::move(Remerge);
349 return LCMTy;
350}
351
352void LegalizerHelper::buildWidenedRemergeToDst(Register DstReg, LLT LCMTy,
353 ArrayRef<Register> RemergeRegs) {
354 LLT DstTy = MRI.getType(Reg: DstReg);
355
356 // Create the merge to the widened source, and extract the relevant bits into
357 // the result.
358
359 if (DstTy == LCMTy) {
360 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: RemergeRegs);
361 return;
362 }
363
364 auto Remerge = MIRBuilder.buildMergeLikeInstr(Res: LCMTy, Ops: RemergeRegs);
365 if (DstTy.isScalar() && LCMTy.isScalar()) {
366 MIRBuilder.buildTrunc(Res: DstReg, Op: Remerge);
367 return;
368 }
369
370 if (LCMTy.isVector()) {
371 unsigned NumDefs = LCMTy.getSizeInBits() / DstTy.getSizeInBits();
372 SmallVector<Register, 8> UnmergeDefs(NumDefs);
373 UnmergeDefs[0] = DstReg;
374 for (unsigned I = 1; I != NumDefs; ++I)
375 UnmergeDefs[I] = MRI.createGenericVirtualRegister(Ty: DstTy);
376
377 MIRBuilder.buildUnmerge(Res: UnmergeDefs,
378 Op: MIRBuilder.buildMergeLikeInstr(Res: LCMTy, Ops: RemergeRegs));
379 return;
380 }
381
382 llvm_unreachable("unhandled case");
383}
384
385static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
386#define RTLIBCASE_INT(LibcallPrefix) \
387 do { \
388 switch (Size) { \
389 case 32: \
390 return RTLIB::LibcallPrefix##32; \
391 case 64: \
392 return RTLIB::LibcallPrefix##64; \
393 case 128: \
394 return RTLIB::LibcallPrefix##128; \
395 default: \
396 llvm_unreachable("unexpected size"); \
397 } \
398 } while (0)
399
400#define RTLIBCASE(LibcallPrefix) \
401 do { \
402 switch (Size) { \
403 case 32: \
404 return RTLIB::LibcallPrefix##32; \
405 case 64: \
406 return RTLIB::LibcallPrefix##64; \
407 case 80: \
408 return RTLIB::LibcallPrefix##80; \
409 case 128: \
410 return RTLIB::LibcallPrefix##128; \
411 default: \
412 llvm_unreachable("unexpected size"); \
413 } \
414 } while (0)
415
416 switch (Opcode) {
417 case TargetOpcode::G_LROUND:
418 RTLIBCASE(LROUND_F);
419 case TargetOpcode::G_LLROUND:
420 RTLIBCASE(LLROUND_F);
421 case TargetOpcode::G_MUL:
422 RTLIBCASE_INT(MUL_I);
423 case TargetOpcode::G_SDIV:
424 RTLIBCASE_INT(SDIV_I);
425 case TargetOpcode::G_UDIV:
426 RTLIBCASE_INT(UDIV_I);
427 case TargetOpcode::G_SREM:
428 RTLIBCASE_INT(SREM_I);
429 case TargetOpcode::G_UREM:
430 RTLIBCASE_INT(UREM_I);
431 case TargetOpcode::G_CTLZ_ZERO_POISON:
432 RTLIBCASE_INT(CTLZ_I);
433 case TargetOpcode::G_FADD:
434 RTLIBCASE(ADD_F);
435 case TargetOpcode::G_FSUB:
436 RTLIBCASE(SUB_F);
437 case TargetOpcode::G_FMUL:
438 RTLIBCASE(MUL_F);
439 case TargetOpcode::G_FDIV:
440 RTLIBCASE(DIV_F);
441 case TargetOpcode::G_FEXP:
442 RTLIBCASE(EXP_F);
443 case TargetOpcode::G_FEXP2:
444 RTLIBCASE(EXP2_F);
445 case TargetOpcode::G_FEXP10:
446 RTLIBCASE(EXP10_F);
447 case TargetOpcode::G_FREM:
448 RTLIBCASE(REM_F);
449 case TargetOpcode::G_FPOW:
450 RTLIBCASE(POW_F);
451 case TargetOpcode::G_FPOWI:
452 RTLIBCASE(POWI_F);
453 case TargetOpcode::G_FMA:
454 RTLIBCASE(FMA_F);
455 case TargetOpcode::G_FSIN:
456 RTLIBCASE(SIN_F);
457 case TargetOpcode::G_FCOS:
458 RTLIBCASE(COS_F);
459 case TargetOpcode::G_FTAN:
460 RTLIBCASE(TAN_F);
461 case TargetOpcode::G_FASIN:
462 RTLIBCASE(ASIN_F);
463 case TargetOpcode::G_FACOS:
464 RTLIBCASE(ACOS_F);
465 case TargetOpcode::G_FATAN:
466 RTLIBCASE(ATAN_F);
467 case TargetOpcode::G_FATAN2:
468 RTLIBCASE(ATAN2_F);
469 case TargetOpcode::G_FSINH:
470 RTLIBCASE(SINH_F);
471 case TargetOpcode::G_FCOSH:
472 RTLIBCASE(COSH_F);
473 case TargetOpcode::G_FTANH:
474 RTLIBCASE(TANH_F);
475 case TargetOpcode::G_FSINCOS:
476 RTLIBCASE(SINCOS_F);
477 case TargetOpcode::G_FMODF:
478 RTLIBCASE(MODF_F);
479 case TargetOpcode::G_FLOG10:
480 RTLIBCASE(LOG10_F);
481 case TargetOpcode::G_FLOG:
482 RTLIBCASE(LOG_F);
483 case TargetOpcode::G_FLOG2:
484 RTLIBCASE(LOG2_F);
485 case TargetOpcode::G_FLDEXP:
486 RTLIBCASE(LDEXP_F);
487 case TargetOpcode::G_FCEIL:
488 RTLIBCASE(CEIL_F);
489 case TargetOpcode::G_FFLOOR:
490 RTLIBCASE(FLOOR_F);
491 case TargetOpcode::G_FMINNUM:
492 RTLIBCASE(FMIN_F);
493 case TargetOpcode::G_FMAXNUM:
494 RTLIBCASE(FMAX_F);
495 case TargetOpcode::G_FMINIMUMNUM:
496 RTLIBCASE(FMINIMUM_NUM_F);
497 case TargetOpcode::G_FMAXIMUMNUM:
498 RTLIBCASE(FMAXIMUM_NUM_F);
499 case TargetOpcode::G_FSQRT:
500 RTLIBCASE(SQRT_F);
501 case TargetOpcode::G_FRINT:
502 RTLIBCASE(RINT_F);
503 case TargetOpcode::G_FNEARBYINT:
504 RTLIBCASE(NEARBYINT_F);
505 case TargetOpcode::G_INTRINSIC_TRUNC:
506 RTLIBCASE(TRUNC_F);
507 case TargetOpcode::G_INTRINSIC_ROUND:
508 RTLIBCASE(ROUND_F);
509 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
510 RTLIBCASE(ROUNDEVEN_F);
511 case TargetOpcode::G_INTRINSIC_LRINT:
512 RTLIBCASE(LRINT_F);
513 case TargetOpcode::G_INTRINSIC_LLRINT:
514 RTLIBCASE(LLRINT_F);
515 }
516 llvm_unreachable("Unknown libcall function");
517#undef RTLIBCASE_INT
518#undef RTLIBCASE
519}
520
521static bool hasSwiftErrorArg(MachineFunction &MF) {
522 const TargetLowering &TLI = *MF.getSubtarget().getTargetLowering();
523 return TLI.supportSwiftError() &&
524 MF.getFunction().getAttributes().hasAttrSomewhere(
525 Kind: Attribute::SwiftError);
526}
527
528/// True if an instruction is in tail position in its caller. Intended for
529/// legalizing libcalls as tail calls when possible.
530static bool isLibCallInTailPosition(const CallLowering::ArgInfo &Result,
531 MachineInstr &MI,
532 const TargetInstrInfo &TII,
533 MachineRegisterInfo &MRI) {
534 MachineBasicBlock &MBB = *MI.getParent();
535 const Function &F = MBB.getParent()->getFunction();
536
537 // Conservatively require the attributes of the call to match those of
538 // the return. Ignore NoAlias and NonNull because they don't affect the
539 // call sequence.
540 AttributeList CallerAttrs = F.getAttributes();
541 if (AttrBuilder(F.getContext(), CallerAttrs.getRetAttrs())
542 .removeAttribute(Val: Attribute::NoAlias)
543 .removeAttribute(Val: Attribute::NonNull)
544 .hasAttributes())
545 return false;
546
547 // It's not safe to eliminate the sign / zero extension of the return value.
548 if (CallerAttrs.hasRetAttr(Kind: Attribute::ZExt) ||
549 CallerAttrs.hasRetAttr(Kind: Attribute::SExt))
550 return false;
551
552 // Only tail call if the following instruction is a standard return or if we
553 // have a `thisreturn` callee, and a sequence like:
554 //
555 // G_MEMCPY %0, %1, %2
556 // $x0 = COPY %0
557 // RET_ReallyLR implicit $x0
558 auto Next = next_nodbg(It: MI.getIterator(), End: MBB.instr_end());
559 if (Next != MBB.instr_end() && Next->isCopy()) {
560 if (MI.getOpcode() == TargetOpcode::G_BZERO)
561 return false;
562
563 // For MEMCPY/MOMMOVE/MEMSET these will be the first use (the dst), as the
564 // mempy/etc routines return the same parameter. For other it will be the
565 // returned value.
566 Register VReg = MI.getOperand(i: 0).getReg();
567 if (!VReg.isVirtual() || VReg != Next->getOperand(i: 1).getReg())
568 return false;
569
570 Register PReg = Next->getOperand(i: 0).getReg();
571 if (!PReg.isPhysical())
572 return false;
573
574 auto Ret = next_nodbg(It: Next, End: MBB.instr_end());
575 if (Ret == MBB.instr_end() || !Ret->isReturn())
576 return false;
577
578 if (Ret->getNumImplicitOperands() != 1)
579 return false;
580
581 if (!Ret->getOperand(i: 0).isReg() || PReg != Ret->getOperand(i: 0).getReg())
582 return false;
583
584 // Skip over the COPY that we just validated.
585 Next = Ret;
586 }
587
588 if (Next == MBB.instr_end() || TII.isTailCall(Inst: *Next) || !Next->isReturn())
589 return false;
590
591 return true;
592}
593
594LegalizerHelper::LegalizeResult LegalizerHelper::createLibcall(
595 const char *Name, const CallLowering::ArgInfo &Result,
596 ArrayRef<CallLowering::ArgInfo> Args, const CallingConv::ID CC,
597 LostDebugLocObserver &LocObserver, MachineInstr *MI) const {
598 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
599
600 CallLowering::CallLoweringInfo Info;
601 Info.CallConv = CC;
602 Info.Callee = MachineOperand::CreateES(SymName: Name);
603 Info.OrigRet = Result;
604 if (MI)
605 Info.IsTailCall =
606 (Result.Ty->isVoidTy() ||
607 Result.Ty == MIRBuilder.getMF().getFunction().getReturnType()) &&
608 isLibCallInTailPosition(Result, MI&: *MI, TII: MIRBuilder.getTII(),
609 MRI&: *MIRBuilder.getMRI()) &&
610 // Lowering doesn't support tail calling inside a function with
611 // a swifterror argument yet.
612 !hasSwiftErrorArg(MF&: MIRBuilder.getMF());
613
614 llvm::append_range(C&: Info.OrigArgs, R&: Args);
615 if (!CLI.lowerCall(MIRBuilder, Info))
616 return LegalizerHelper::UnableToLegalize;
617
618 if (MI && Info.LoweredTailCall) {
619 assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?");
620
621 // Check debug locations before removing the return.
622 LocObserver.checkpoint(CheckDebugLocs: true);
623
624 // We must have a return following the call (or debug insts) to get past
625 // isLibCallInTailPosition.
626 do {
627 MachineInstr *Next = MI->getNextNode();
628 assert(Next &&
629 (Next->isCopy() || Next->isReturn() || Next->isDebugInstr()) &&
630 "Expected instr following MI to be return or debug inst?");
631 // We lowered a tail call, so the call is now the return from the block.
632 // Delete the old return.
633 Next->eraseFromParent();
634 } while (MI->getNextNode());
635
636 // We expect to lose the debug location from the return.
637 LocObserver.checkpoint(CheckDebugLocs: false);
638 }
639 return LegalizerHelper::Legalized;
640}
641
642LegalizerHelper::LegalizeResult LegalizerHelper::createLibcall(
643 RTLIB::Libcall Libcall, const CallLowering::ArgInfo &Result,
644 ArrayRef<CallLowering::ArgInfo> Args, LostDebugLocObserver &LocObserver,
645 MachineInstr *MI) const {
646 if (!Libcalls)
647 return LegalizerHelper::UnableToLegalize;
648
649 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: Libcall);
650 if (LibcallImpl == RTLIB::Unsupported)
651 return LegalizerHelper::UnableToLegalize;
652
653 StringRef Name = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: LibcallImpl);
654 const CallingConv::ID CC = Libcalls->getLibcallImplCallingConv(Call: LibcallImpl);
655 return createLibcall(Name: Name.data(), Result, Args, CC, LocObserver, MI);
656}
657
658// Useful for libcalls where all operands have the same type.
659LegalizerHelper::LegalizeResult
660LegalizerHelper::simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder,
661 unsigned Size, Type *OpType,
662 LostDebugLocObserver &LocObserver) const {
663 auto Libcall = getRTLibDesc(Opcode: MI.getOpcode(), Size);
664
665 // FIXME: What does the original arg index mean here?
666 SmallVector<CallLowering::ArgInfo, 3> Args;
667 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands()))
668 Args.push_back(Elt: {MO.getReg(), OpType, 0});
669 return createLibcall(Libcall, Result: {MI.getOperand(i: 0).getReg(), OpType, 0}, Args,
670 LocObserver, MI: &MI);
671}
672
673LegalizerHelper::LegalizeResult LegalizerHelper::emitSincosLibcall(
674 MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size, Type *OpType,
675 LostDebugLocObserver &LocObserver) {
676 MachineFunction &MF = *MI.getMF();
677 MachineRegisterInfo &MRI = MF.getRegInfo();
678
679 Register DstSin = MI.getOperand(i: 0).getReg();
680 Register DstCos = MI.getOperand(i: 1).getReg();
681 Register Src = MI.getOperand(i: 2).getReg();
682 LLT DstTy = MRI.getType(Reg: DstSin);
683
684 int MemSize = DstTy.getSizeInBytes();
685 Align Alignment = getStackTemporaryAlignment(Type: DstTy);
686 const DataLayout &DL = MIRBuilder.getDataLayout();
687 unsigned AddrSpace = DL.getAllocaAddrSpace();
688 MachinePointerInfo PtrInfo;
689
690 Register StackPtrSin =
691 createStackTemporary(Bytes: TypeSize::getFixed(ExactSize: MemSize), Alignment, PtrInfo)
692 .getReg(Idx: 0);
693 Register StackPtrCos =
694 createStackTemporary(Bytes: TypeSize::getFixed(ExactSize: MemSize), Alignment, PtrInfo)
695 .getReg(Idx: 0);
696
697 auto &Ctx = MF.getFunction().getContext();
698 auto LibcallResult = createLibcall(
699 Libcall: getRTLibDesc(Opcode: MI.getOpcode(), Size), Result: {{0}, Type::getVoidTy(C&: Ctx), 0},
700 Args: {{Src, OpType, 0},
701 {StackPtrSin, PointerType::get(C&: Ctx, AddressSpace: AddrSpace), 1},
702 {StackPtrCos, PointerType::get(C&: Ctx, AddressSpace: AddrSpace), 2}},
703 LocObserver, MI: &MI);
704
705 if (LibcallResult != LegalizeResult::Legalized)
706 return LegalizerHelper::UnableToLegalize;
707
708 MachineMemOperand *LoadMMOSin = MF.getMachineMemOperand(
709 PtrInfo, F: MachineMemOperand::MOLoad, Size: MemSize, BaseAlignment: Alignment);
710 MachineMemOperand *LoadMMOCos = MF.getMachineMemOperand(
711 PtrInfo, F: MachineMemOperand::MOLoad, Size: MemSize, BaseAlignment: Alignment);
712
713 MIRBuilder.buildLoad(Res: DstSin, Addr: StackPtrSin, MMO&: *LoadMMOSin);
714 MIRBuilder.buildLoad(Res: DstCos, Addr: StackPtrCos, MMO&: *LoadMMOCos);
715 MI.eraseFromParent();
716
717 return LegalizerHelper::Legalized;
718}
719
720LegalizerHelper::LegalizeResult
721LegalizerHelper::emitModfLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder,
722 unsigned Size, Type *OpType,
723 LostDebugLocObserver &LocObserver) {
724 MachineFunction &MF = MIRBuilder.getMF();
725 MachineRegisterInfo &MRI = MF.getRegInfo();
726
727 Register DstFrac = MI.getOperand(i: 0).getReg();
728 Register DstInt = MI.getOperand(i: 1).getReg();
729 Register Src = MI.getOperand(i: 2).getReg();
730 LLT DstTy = MRI.getType(Reg: DstFrac);
731
732 int MemSize = DstTy.getSizeInBytes();
733 Align Alignment = getStackTemporaryAlignment(Type: DstTy);
734 const DataLayout &DL = MIRBuilder.getDataLayout();
735 unsigned AddrSpace = DL.getAllocaAddrSpace();
736 MachinePointerInfo PtrInfo;
737
738 Register StackPtrInt =
739 createStackTemporary(Bytes: TypeSize::getFixed(ExactSize: MemSize), Alignment, PtrInfo)
740 .getReg(Idx: 0);
741
742 auto &Ctx = MF.getFunction().getContext();
743 auto LibcallResult = createLibcall(
744 Libcall: getRTLibDesc(Opcode: MI.getOpcode(), Size), Result: {DstFrac, OpType, 0},
745 Args: {{Src, OpType, 0}, {StackPtrInt, PointerType::get(C&: Ctx, AddressSpace: AddrSpace), 1}},
746 LocObserver, MI: &MI);
747
748 if (LibcallResult != LegalizeResult::Legalized)
749 return LegalizerHelper::UnableToLegalize;
750
751 MachineMemOperand *LoadMMOInt = MF.getMachineMemOperand(
752 PtrInfo, F: MachineMemOperand::MOLoad, Size: MemSize, BaseAlignment: Alignment);
753
754 MIRBuilder.buildLoad(Res: DstInt, Addr: StackPtrInt, MMO&: *LoadMMOInt);
755 MI.eraseFromParent();
756
757 return LegalizerHelper::Legalized;
758}
759
760static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
761 Type *FromType) {
762 auto ToMVT = MVT::getVT(Ty: ToType);
763 auto FromMVT = MVT::getVT(Ty: FromType);
764
765 switch (Opcode) {
766 case TargetOpcode::G_FPEXT:
767 return RTLIB::getFPEXT(OpVT: FromMVT, RetVT: ToMVT);
768 case TargetOpcode::G_FPTRUNC:
769 return RTLIB::getFPROUND(OpVT: FromMVT, RetVT: ToMVT);
770 case TargetOpcode::G_FPTOSI:
771 return RTLIB::getFPTOSINT(OpVT: FromMVT, RetVT: ToMVT);
772 case TargetOpcode::G_FPTOUI:
773 return RTLIB::getFPTOUINT(OpVT: FromMVT, RetVT: ToMVT);
774 case TargetOpcode::G_SITOFP:
775 return RTLIB::getSINTTOFP(OpVT: FromMVT, RetVT: ToMVT);
776 case TargetOpcode::G_UITOFP:
777 return RTLIB::getUINTTOFP(OpVT: FromMVT, RetVT: ToMVT);
778 }
779 llvm_unreachable("Unsupported libcall function");
780}
781
782LegalizerHelper::LegalizeResult LegalizerHelper::conversionLibcall(
783 MachineInstr &MI, Type *ToType, Type *FromType,
784 LostDebugLocObserver &LocObserver, bool IsSigned) const {
785 CallLowering::ArgInfo Arg = {MI.getOperand(i: 1).getReg(), FromType, 0};
786 if (FromType->isIntegerTy()) {
787 if (TLI.shouldSignExtendTypeInLibCall(Ty: FromType, IsSigned))
788 Arg.Flags[0].setSExt();
789 else
790 Arg.Flags[0].setZExt();
791 }
792
793 RTLIB::Libcall Libcall = getConvRTLibDesc(Opcode: MI.getOpcode(), ToType, FromType);
794 return createLibcall(Libcall, Result: {MI.getOperand(i: 0).getReg(), ToType, 0}, Args: Arg,
795 LocObserver, MI: &MI);
796}
797
798LegalizerHelper::LegalizeResult
799LegalizerHelper::createMemLibcall(MachineRegisterInfo &MRI, MachineInstr &MI,
800 LostDebugLocObserver &LocObserver) const {
801 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
802
803 SmallVector<CallLowering::ArgInfo, 3> Args;
804 // Add all the args, except for the last which is an imm denoting 'tail'.
805 for (unsigned i = 0; i < MI.getNumOperands() - 1; ++i) {
806 Register Reg = MI.getOperand(i).getReg();
807
808 // Need derive an IR type for call lowering.
809 LLT OpLLT = MRI.getType(Reg);
810 Type *OpTy = nullptr;
811 if (OpLLT.isPointer())
812 OpTy = PointerType::get(C&: Ctx, AddressSpace: OpLLT.getAddressSpace());
813 else
814 OpTy = IntegerType::get(C&: Ctx, NumBits: OpLLT.getSizeInBits());
815 Args.push_back(Elt: {Reg, OpTy, 0});
816 }
817
818 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
819 RTLIB::Libcall RTLibcall;
820 unsigned Opc = MI.getOpcode();
821 switch (Opc) {
822 case TargetOpcode::G_BZERO:
823 RTLibcall = RTLIB::BZERO;
824 break;
825 case TargetOpcode::G_MEMCPY:
826 RTLibcall = RTLIB::MEMCPY;
827 Args[0].Flags[0].setReturned();
828 break;
829 case TargetOpcode::G_MEMMOVE:
830 RTLibcall = RTLIB::MEMMOVE;
831 Args[0].Flags[0].setReturned();
832 break;
833 case TargetOpcode::G_MEMSET:
834 RTLibcall = RTLIB::MEMSET;
835 Args[0].Flags[0].setReturned();
836 break;
837 default:
838 llvm_unreachable("unsupported opcode");
839 }
840
841 if (!Libcalls) // FIXME: Should be mandatory
842 return LegalizerHelper::UnableToLegalize;
843
844 RTLIB::LibcallImpl RTLibcallImpl = Libcalls->getLibcallImpl(Call: RTLibcall);
845
846 // Unsupported libcall on the target.
847 if (RTLibcallImpl == RTLIB::Unsupported) {
848 LLVM_DEBUG(dbgs() << ".. .. Could not find libcall name for "
849 << MIRBuilder.getTII().getName(Opc) << "\n");
850 return LegalizerHelper::UnableToLegalize;
851 }
852
853 CallLowering::CallLoweringInfo Info;
854 Info.CallConv = Libcalls->getLibcallImplCallingConv(Call: RTLibcallImpl);
855
856 StringRef LibcallName =
857 RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: RTLibcallImpl);
858 Info.Callee = MachineOperand::CreateES(SymName: LibcallName.data());
859 Info.OrigRet = CallLowering::ArgInfo({0}, Type::getVoidTy(C&: Ctx), 0);
860 Info.IsTailCall =
861 MI.getOperand(i: MI.getNumOperands() - 1).getImm() &&
862 isLibCallInTailPosition(Result: Info.OrigRet, MI, TII: MIRBuilder.getTII(), MRI) &&
863 // Lowering doesn't support tail calling inside a function with
864 // a swifterror argument yet.
865 !hasSwiftErrorArg(MF&: MIRBuilder.getMF());
866
867 llvm::append_range(C&: Info.OrigArgs, R&: Args);
868 if (!CLI.lowerCall(MIRBuilder, Info))
869 return LegalizerHelper::UnableToLegalize;
870
871 if (Info.LoweredTailCall) {
872 assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?");
873
874 // Check debug locations before removing the return.
875 LocObserver.checkpoint(CheckDebugLocs: true);
876
877 // We must have a return following the call (or debug insts) to get past
878 // isLibCallInTailPosition.
879 do {
880 MachineInstr *Next = MI.getNextNode();
881 assert(Next &&
882 (Next->isCopy() || Next->isReturn() || Next->isDebugInstr()) &&
883 "Expected instr following MI to be return or debug inst?");
884 // We lowered a tail call, so the call is now the return from the block.
885 // Delete the old return.
886 Next->eraseFromParent();
887 } while (MI.getNextNode());
888
889 // We expect to lose the debug location from the return.
890 LocObserver.checkpoint(CheckDebugLocs: false);
891 }
892
893 return LegalizerHelper::Legalized;
894}
895
896static RTLIB::Libcall getOutlineAtomicLibcall(MachineInstr &MI) {
897 unsigned Opc = MI.getOpcode();
898 auto &AtomicMI = cast<GMemOperation>(Val&: MI);
899 auto &MMO = AtomicMI.getMMO();
900 auto Ordering = MMO.getMergedOrdering();
901 LLT MemType = MMO.getMemoryType();
902 uint64_t MemSize = MemType.getSizeInBytes();
903 if (MemType.isVector())
904 return RTLIB::UNKNOWN_LIBCALL;
905
906#define LCALLS(A, B) {A##B##_RELAX, A##B##_ACQ, A##B##_REL, A##B##_ACQ_REL}
907#define LCALL5(A) \
908 LCALLS(A, 1), LCALLS(A, 2), LCALLS(A, 4), LCALLS(A, 8), LCALLS(A, 16)
909 switch (Opc) {
910 case TargetOpcode::G_ATOMIC_CMPXCHG:
911 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
912 const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_CAS)};
913 return getOutlineAtomicHelper(LC, Order: Ordering, MemSize);
914 }
915 case TargetOpcode::G_ATOMICRMW_XCHG: {
916 const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_SWP)};
917 return getOutlineAtomicHelper(LC, Order: Ordering, MemSize);
918 }
919 case TargetOpcode::G_ATOMICRMW_ADD:
920 case TargetOpcode::G_ATOMICRMW_SUB: {
921 const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDADD)};
922 return getOutlineAtomicHelper(LC, Order: Ordering, MemSize);
923 }
924 case TargetOpcode::G_ATOMICRMW_AND: {
925 const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDCLR)};
926 return getOutlineAtomicHelper(LC, Order: Ordering, MemSize);
927 }
928 case TargetOpcode::G_ATOMICRMW_OR: {
929 const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDSET)};
930 return getOutlineAtomicHelper(LC, Order: Ordering, MemSize);
931 }
932 case TargetOpcode::G_ATOMICRMW_XOR: {
933 const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDEOR)};
934 return getOutlineAtomicHelper(LC, Order: Ordering, MemSize);
935 }
936 default:
937 return RTLIB::UNKNOWN_LIBCALL;
938 }
939#undef LCALLS
940#undef LCALL5
941}
942
943LegalizerHelper::LegalizeResult
944LegalizerHelper::createAtomicLibcall(MachineInstr &MI) const {
945 auto &Ctx = MIRBuilder.getContext();
946
947 Type *RetTy;
948 SmallVector<Register> RetRegs;
949 SmallVector<CallLowering::ArgInfo, 3> Args;
950 unsigned Opc = MI.getOpcode();
951 switch (Opc) {
952 case TargetOpcode::G_ATOMIC_CMPXCHG:
953 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
954 Register Success;
955 LLT SuccessLLT;
956 auto [Ret, RetLLT, Mem, MemLLT, Cmp, CmpLLT, New, NewLLT] =
957 MI.getFirst4RegLLTs();
958 RetRegs.push_back(Elt: Ret);
959 RetTy = IntegerType::get(C&: Ctx, NumBits: RetLLT.getSizeInBits());
960 if (Opc == TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS) {
961 std::tie(args&: Ret, args&: RetLLT, args&: Success, args&: SuccessLLT, args&: Mem, args&: MemLLT, args&: Cmp, args&: CmpLLT, args&: New,
962 args&: NewLLT) = MI.getFirst5RegLLTs();
963 RetRegs.push_back(Elt: Success);
964 RetTy = StructType::get(
965 Context&: Ctx, Elements: {RetTy, IntegerType::get(C&: Ctx, NumBits: SuccessLLT.getSizeInBits())});
966 }
967 Args.push_back(Elt: {Cmp, IntegerType::get(C&: Ctx, NumBits: CmpLLT.getSizeInBits()), 0});
968 Args.push_back(Elt: {New, IntegerType::get(C&: Ctx, NumBits: NewLLT.getSizeInBits()), 0});
969 Args.push_back(Elt: {Mem, PointerType::get(C&: Ctx, AddressSpace: MemLLT.getAddressSpace()), 0});
970 break;
971 }
972 case TargetOpcode::G_ATOMICRMW_XCHG:
973 case TargetOpcode::G_ATOMICRMW_ADD:
974 case TargetOpcode::G_ATOMICRMW_SUB:
975 case TargetOpcode::G_ATOMICRMW_AND:
976 case TargetOpcode::G_ATOMICRMW_OR:
977 case TargetOpcode::G_ATOMICRMW_XOR: {
978 auto [Ret, RetLLT, Mem, MemLLT, Val, ValLLT] = MI.getFirst3RegLLTs();
979 RetRegs.push_back(Elt: Ret);
980 RetTy = IntegerType::get(C&: Ctx, NumBits: RetLLT.getSizeInBits());
981 if (Opc == TargetOpcode::G_ATOMICRMW_AND)
982 Val =
983 MIRBuilder.buildXor(Dst: ValLLT, Src0: MIRBuilder.buildConstant(Res: ValLLT, Val: -1), Src1: Val)
984 .getReg(Idx: 0);
985 else if (Opc == TargetOpcode::G_ATOMICRMW_SUB)
986 Val =
987 MIRBuilder.buildSub(Dst: ValLLT, Src0: MIRBuilder.buildConstant(Res: ValLLT, Val: 0), Src1: Val)
988 .getReg(Idx: 0);
989 Args.push_back(Elt: {Val, IntegerType::get(C&: Ctx, NumBits: ValLLT.getSizeInBits()), 0});
990 Args.push_back(Elt: {Mem, PointerType::get(C&: Ctx, AddressSpace: MemLLT.getAddressSpace()), 0});
991 break;
992 }
993 default:
994 llvm_unreachable("unsupported opcode");
995 }
996
997 if (!Libcalls) // FIXME: Should be mandatory
998 return LegalizerHelper::UnableToLegalize;
999
1000 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
1001 RTLIB::Libcall RTLibcall = getOutlineAtomicLibcall(MI);
1002 RTLIB::LibcallImpl RTLibcallImpl = Libcalls->getLibcallImpl(Call: RTLibcall);
1003
1004 // Unsupported libcall on the target.
1005 if (RTLibcallImpl == RTLIB::Unsupported) {
1006 LLVM_DEBUG(dbgs() << ".. .. Could not find libcall name for "
1007 << MIRBuilder.getTII().getName(Opc) << "\n");
1008 return LegalizerHelper::UnableToLegalize;
1009 }
1010
1011 CallLowering::CallLoweringInfo Info;
1012 Info.CallConv = Libcalls->getLibcallImplCallingConv(Call: RTLibcallImpl);
1013
1014 StringRef LibcallName =
1015 RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: RTLibcallImpl);
1016 Info.Callee = MachineOperand::CreateES(SymName: LibcallName.data());
1017 Info.OrigRet = CallLowering::ArgInfo(RetRegs, RetTy, 0);
1018
1019 llvm::append_range(C&: Info.OrigArgs, R&: Args);
1020 if (!CLI.lowerCall(MIRBuilder, Info))
1021 return LegalizerHelper::UnableToLegalize;
1022
1023 return LegalizerHelper::Legalized;
1024}
1025
1026static RTLIB::Libcall
1027getStateLibraryFunctionFor(MachineInstr &MI, const TargetLowering &TLI) {
1028 RTLIB::Libcall RTLibcall;
1029 switch (MI.getOpcode()) {
1030 case TargetOpcode::G_GET_FPENV:
1031 RTLibcall = RTLIB::FEGETENV;
1032 break;
1033 case TargetOpcode::G_SET_FPENV:
1034 case TargetOpcode::G_RESET_FPENV:
1035 RTLibcall = RTLIB::FESETENV;
1036 break;
1037 case TargetOpcode::G_GET_FPMODE:
1038 RTLibcall = RTLIB::FEGETMODE;
1039 break;
1040 case TargetOpcode::G_SET_FPMODE:
1041 case TargetOpcode::G_RESET_FPMODE:
1042 RTLibcall = RTLIB::FESETMODE;
1043 break;
1044 default:
1045 llvm_unreachable("Unexpected opcode");
1046 }
1047 return RTLibcall;
1048}
1049
1050// Some library functions that read FP state (fegetmode, fegetenv) write the
1051// state into a region in memory. IR intrinsics that do the same operations
1052// (get_fpmode, get_fpenv) return the state as integer value. To implement these
1053// intrinsics via the library functions, we need to use temporary variable,
1054// for example:
1055//
1056// %0:_(s32) = G_GET_FPMODE
1057//
1058// is transformed to:
1059//
1060// %1:_(p0) = G_FRAME_INDEX %stack.0
1061// BL &fegetmode
1062// %0:_(s32) = G_LOAD % 1
1063//
1064LegalizerHelper::LegalizeResult
1065LegalizerHelper::createGetStateLibcall(MachineInstr &MI,
1066 LostDebugLocObserver &LocObserver) {
1067 const DataLayout &DL = MIRBuilder.getDataLayout();
1068 auto &MF = MIRBuilder.getMF();
1069 auto &MRI = *MIRBuilder.getMRI();
1070 auto &Ctx = MF.getFunction().getContext();
1071
1072 // Create temporary, where library function will put the read state.
1073 Register Dst = MI.getOperand(i: 0).getReg();
1074 LLT StateTy = MRI.getType(Reg: Dst);
1075 TypeSize StateSize = StateTy.getSizeInBytes();
1076 Align TempAlign = getStackTemporaryAlignment(Type: StateTy);
1077 MachinePointerInfo TempPtrInfo;
1078 auto Temp = createStackTemporary(Bytes: StateSize, Alignment: TempAlign, PtrInfo&: TempPtrInfo);
1079
1080 // Create a call to library function, with the temporary as an argument.
1081 unsigned TempAddrSpace = DL.getAllocaAddrSpace();
1082 Type *StatePtrTy = PointerType::get(C&: Ctx, AddressSpace: TempAddrSpace);
1083 RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI);
1084 auto Res = createLibcall(
1085 Libcall: RTLibcall, Result: CallLowering::ArgInfo({0}, Type::getVoidTy(C&: Ctx), 0),
1086 Args: CallLowering::ArgInfo({Temp.getReg(Idx: 0), StatePtrTy, 0}), LocObserver,
1087 MI: nullptr);
1088 if (Res != LegalizerHelper::Legalized)
1089 return Res;
1090
1091 // Create a load from the temporary.
1092 MachineMemOperand *MMO = MF.getMachineMemOperand(
1093 PtrInfo: TempPtrInfo, F: MachineMemOperand::MOLoad, MemTy: StateTy, BaseAlignment: TempAlign);
1094 MIRBuilder.buildLoadInstr(Opcode: TargetOpcode::G_LOAD, Res: Dst, Addr: Temp, MMO&: *MMO);
1095
1096 return LegalizerHelper::Legalized;
1097}
1098
1099// Similar to `createGetStateLibcall` the function calls a library function
1100// using transient space in stack. In this case the library function reads
1101// content of memory region.
1102LegalizerHelper::LegalizeResult
1103LegalizerHelper::createSetStateLibcall(MachineInstr &MI,
1104 LostDebugLocObserver &LocObserver) {
1105 const DataLayout &DL = MIRBuilder.getDataLayout();
1106 auto &MF = MIRBuilder.getMF();
1107 auto &MRI = *MIRBuilder.getMRI();
1108 auto &Ctx = MF.getFunction().getContext();
1109
1110 // Create temporary, where library function will get the new state.
1111 Register Src = MI.getOperand(i: 0).getReg();
1112 LLT StateTy = MRI.getType(Reg: Src);
1113 TypeSize StateSize = StateTy.getSizeInBytes();
1114 Align TempAlign = getStackTemporaryAlignment(Type: StateTy);
1115 MachinePointerInfo TempPtrInfo;
1116 auto Temp = createStackTemporary(Bytes: StateSize, Alignment: TempAlign, PtrInfo&: TempPtrInfo);
1117
1118 // Put the new state into the temporary.
1119 MachineMemOperand *MMO = MF.getMachineMemOperand(
1120 PtrInfo: TempPtrInfo, F: MachineMemOperand::MOStore, MemTy: StateTy, BaseAlignment: TempAlign);
1121 MIRBuilder.buildStore(Val: Src, Addr: Temp, MMO&: *MMO);
1122
1123 // Create a call to library function, with the temporary as an argument.
1124 unsigned TempAddrSpace = DL.getAllocaAddrSpace();
1125 Type *StatePtrTy = PointerType::get(C&: Ctx, AddressSpace: TempAddrSpace);
1126 RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI);
1127 return createLibcall(Libcall: RTLibcall,
1128 Result: CallLowering::ArgInfo({0}, Type::getVoidTy(C&: Ctx), 0),
1129 Args: CallLowering::ArgInfo({Temp.getReg(Idx: 0), StatePtrTy, 0}),
1130 LocObserver, MI: nullptr);
1131}
1132
1133/// Returns the corresponding libcall for the given Pred and
1134/// the ICMP predicate that should be generated to compare with #0
1135/// after the libcall.
1136static std::pair<RTLIB::Libcall, CmpInst::Predicate>
1137getFCMPLibcallDesc(const CmpInst::Predicate Pred, unsigned Size) {
1138#define RTLIBCASE_CMP(LibcallPrefix, ICmpPred) \
1139 do { \
1140 switch (Size) { \
1141 case 32: \
1142 return {RTLIB::LibcallPrefix##32, ICmpPred}; \
1143 case 64: \
1144 return {RTLIB::LibcallPrefix##64, ICmpPred}; \
1145 case 128: \
1146 return {RTLIB::LibcallPrefix##128, ICmpPred}; \
1147 default: \
1148 llvm_unreachable("unexpected size"); \
1149 } \
1150 } while (0)
1151
1152 // These use the three-way (-1/0/1) compare libcalls, whose result is tested
1153 // against 0 with a signed integer predicate. Unordered (UO) is a boolean.
1154 switch (Pred) {
1155 case CmpInst::FCMP_OEQ:
1156 RTLIBCASE_CMP(FCMP3_PRED_OEQ_F, CmpInst::ICMP_EQ);
1157 case CmpInst::FCMP_UNE:
1158 RTLIBCASE_CMP(FCMP3_PRED_UNE_F, CmpInst::ICMP_NE);
1159 case CmpInst::FCMP_OGE:
1160 RTLIBCASE_CMP(FCMP3_PRED_OGE_F, CmpInst::ICMP_SGE);
1161 case CmpInst::FCMP_OLT:
1162 RTLIBCASE_CMP(FCMP3_PRED_OLT_F, CmpInst::ICMP_SLT);
1163 case CmpInst::FCMP_OLE:
1164 RTLIBCASE_CMP(FCMP3_PRED_OLE_F, CmpInst::ICMP_SLE);
1165 case CmpInst::FCMP_OGT:
1166 RTLIBCASE_CMP(FCMP3_PRED_OGT_F, CmpInst::ICMP_SGT);
1167 case CmpInst::FCMP_UNO:
1168 RTLIBCASE_CMP(UO_F, CmpInst::ICMP_NE);
1169 default:
1170 return {RTLIB::UNKNOWN_LIBCALL, CmpInst::BAD_ICMP_PREDICATE};
1171 }
1172}
1173
1174LegalizerHelper::LegalizeResult
1175LegalizerHelper::createFCMPLibcall(MachineInstr &MI,
1176 LostDebugLocObserver &LocObserver) {
1177 auto &MF = MIRBuilder.getMF();
1178 auto &Ctx = MF.getFunction().getContext();
1179 const GFCmp *Cmp = cast<GFCmp>(Val: &MI);
1180
1181 LLT OpLLT = MRI.getType(Reg: Cmp->getLHSReg());
1182 unsigned Size = OpLLT.getSizeInBits();
1183 if ((Size != 32 && Size != 64 && Size != 128) ||
1184 OpLLT != MRI.getType(Reg: Cmp->getRHSReg()))
1185 return UnableToLegalize;
1186
1187 Type *OpType = getFloatTypeForLLT(Ctx, Ty: OpLLT);
1188
1189 // DstReg type is s32
1190 const Register DstReg = Cmp->getReg(Idx: 0);
1191 LLT DstTy = MRI.getType(Reg: DstReg);
1192 const auto Cond = Cmp->getCond();
1193 Type *RetTy = EVT(TLI.getCmpLibcallReturnType()).getTypeForEVT(Context&: Ctx);
1194
1195 // Reference:
1196 // https://gcc.gnu.org/onlinedocs/gccint/Soft-float-library-routines.html#Comparison-functions-1
1197 // Generates a libcall followed by ICMP.
1198 const auto BuildLibcall = [&](const RTLIB::Libcall Libcall,
1199 const CmpInst::Predicate ICmpPred,
1200 const DstOp &Res) -> Register {
1201 // FCMP libcall returns an integer, and needs an ICMP with #0.
1202 LLT TempLLT = LLT::integer(SizeInBits: RetTy->getIntegerBitWidth());
1203 Register Temp = MRI.createGenericVirtualRegister(Ty: TempLLT);
1204 // Generate libcall, holding result in Temp
1205 const auto Status = createLibcall(
1206 Libcall, Result: {Temp, RetTy, 0},
1207 Args: {{Cmp->getLHSReg(), OpType, 0}, {Cmp->getRHSReg(), OpType, 1}},
1208 LocObserver, MI: &MI);
1209 if (Status != Legalized)
1210 return {};
1211
1212 // Compare temp with #0 to get the final result.
1213 return MIRBuilder
1214 .buildICmp(Pred: ICmpPred, Res, Op0: Temp, Op1: MIRBuilder.buildConstant(Res: TempLLT, Val: 0))
1215 .getReg(Idx: 0);
1216 };
1217
1218 // Simple case if we have a direct mapping from predicate to libcall
1219 if (const auto [Libcall, ICmpPred] = getFCMPLibcallDesc(Pred: Cond, Size);
1220 Libcall != RTLIB::UNKNOWN_LIBCALL &&
1221 ICmpPred != CmpInst::BAD_ICMP_PREDICATE) {
1222 if (BuildLibcall(Libcall, ICmpPred, DstReg)) {
1223 return Legalized;
1224 }
1225 return UnableToLegalize;
1226 }
1227
1228 // No direct mapping found, should be generated as combination of libcalls.
1229
1230 switch (Cond) {
1231 case CmpInst::FCMP_UEQ: {
1232 // FCMP_UEQ: unordered or equal
1233 // Convert into (FCMP_OEQ || FCMP_UNO).
1234
1235 const auto [OeqLibcall, OeqPred] =
1236 getFCMPLibcallDesc(Pred: CmpInst::FCMP_OEQ, Size);
1237 const auto Oeq = BuildLibcall(OeqLibcall, OeqPred, DstTy);
1238
1239 const auto [UnoLibcall, UnoPred] =
1240 getFCMPLibcallDesc(Pred: CmpInst::FCMP_UNO, Size);
1241 const auto Uno = BuildLibcall(UnoLibcall, UnoPred, DstTy);
1242 if (Oeq && Uno)
1243 MIRBuilder.buildOr(Dst: DstReg, Src0: Oeq, Src1: Uno);
1244 else
1245 return UnableToLegalize;
1246
1247 break;
1248 }
1249 case CmpInst::FCMP_ONE: {
1250 // FCMP_ONE: ordered and operands are unequal
1251 // Convert into (!FCMP_OEQ && !FCMP_UNO).
1252
1253 // We inverse the predicate instead of generating a NOT
1254 // to save one instruction.
1255 // On AArch64 isel can even select two cmp into a single ccmp.
1256 const auto [OeqLibcall, OeqPred] =
1257 getFCMPLibcallDesc(Pred: CmpInst::FCMP_OEQ, Size);
1258 const auto NotOeq =
1259 BuildLibcall(OeqLibcall, CmpInst::getInversePredicate(pred: OeqPred), DstTy);
1260
1261 const auto [UnoLibcall, UnoPred] =
1262 getFCMPLibcallDesc(Pred: CmpInst::FCMP_UNO, Size);
1263 const auto NotUno =
1264 BuildLibcall(UnoLibcall, CmpInst::getInversePredicate(pred: UnoPred), DstTy);
1265
1266 if (NotOeq && NotUno)
1267 MIRBuilder.buildAnd(Dst: DstReg, Src0: NotOeq, Src1: NotUno);
1268 else
1269 return UnableToLegalize;
1270
1271 break;
1272 }
1273 case CmpInst::FCMP_ULT:
1274 case CmpInst::FCMP_UGE:
1275 case CmpInst::FCMP_UGT:
1276 case CmpInst::FCMP_ULE:
1277 case CmpInst::FCMP_ORD: {
1278 // Convert into: !(inverse(Pred))
1279 // E.g. FCMP_ULT becomes !FCMP_OGE
1280 // This is equivalent to the following, but saves some instructions.
1281 // MIRBuilder.buildNot(
1282 // PredTy,
1283 // MIRBuilder.buildFCmp(CmpInst::getInversePredicate(Pred), PredTy,
1284 // Op1, Op2));
1285 const auto [InversedLibcall, InversedPred] =
1286 getFCMPLibcallDesc(Pred: CmpInst::getInversePredicate(pred: Cond), Size);
1287 if (!BuildLibcall(InversedLibcall,
1288 CmpInst::getInversePredicate(pred: InversedPred), DstReg))
1289 return UnableToLegalize;
1290 break;
1291 }
1292 default:
1293 return UnableToLegalize;
1294 }
1295
1296 return Legalized;
1297}
1298
1299// The function is used to legalize operations that set default environment
1300// state. In C library a call like `fesetmode(FE_DFL_MODE)` is used for that.
1301// On most targets supported in glibc FE_DFL_MODE is defined as
1302// `((const femode_t *) -1)`. Such assumption is used here. If for some target
1303// it is not true, the target must provide custom lowering.
1304LegalizerHelper::LegalizeResult
1305LegalizerHelper::createResetStateLibcall(MachineInstr &MI,
1306 LostDebugLocObserver &LocObserver) {
1307 const DataLayout &DL = MIRBuilder.getDataLayout();
1308 auto &MF = MIRBuilder.getMF();
1309 auto &Ctx = MF.getFunction().getContext();
1310
1311 // Create an argument for the library function.
1312 unsigned AddrSpace = DL.getDefaultGlobalsAddressSpace();
1313 Type *StatePtrTy = PointerType::get(C&: Ctx, AddressSpace: AddrSpace);
1314 unsigned PtrSize = DL.getPointerSizeInBits(AS: AddrSpace);
1315 LLT MemTy = LLT::pointer(AddressSpace: AddrSpace, SizeInBits: PtrSize);
1316 auto DefValue = MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: PtrSize), Val: -1LL);
1317 DstOp Dest(MRI.createGenericVirtualRegister(Ty: MemTy));
1318 MIRBuilder.buildIntToPtr(Dst: Dest, Src: DefValue);
1319
1320 RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI);
1321 return createLibcall(
1322 Libcall: RTLibcall, Result: CallLowering::ArgInfo({0}, Type::getVoidTy(C&: Ctx), 0),
1323 Args: CallLowering::ArgInfo({Dest.getReg(), StatePtrTy, 0}), LocObserver, MI: &MI);
1324}
1325
1326LegalizerHelper::LegalizeResult
1327LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
1328 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
1329
1330 switch (MI.getOpcode()) {
1331 default:
1332 return UnableToLegalize;
1333 case TargetOpcode::G_MUL:
1334 case TargetOpcode::G_SDIV:
1335 case TargetOpcode::G_UDIV:
1336 case TargetOpcode::G_SREM:
1337 case TargetOpcode::G_UREM:
1338 case TargetOpcode::G_CTLZ_ZERO_POISON: {
1339 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1340 unsigned Size = LLTy.getSizeInBits();
1341 Type *HLTy = IntegerType::get(C&: Ctx, NumBits: Size);
1342 auto Status = simpleLibcall(MI, MIRBuilder, Size, OpType: HLTy, LocObserver);
1343 if (Status != Legalized)
1344 return Status;
1345 break;
1346 }
1347 case TargetOpcode::G_FADD:
1348 case TargetOpcode::G_FSUB:
1349 case TargetOpcode::G_FMUL:
1350 case TargetOpcode::G_FDIV:
1351 case TargetOpcode::G_FMA:
1352 case TargetOpcode::G_FPOW:
1353 case TargetOpcode::G_FREM:
1354 case TargetOpcode::G_FCOS:
1355 case TargetOpcode::G_FSIN:
1356 case TargetOpcode::G_FTAN:
1357 case TargetOpcode::G_FACOS:
1358 case TargetOpcode::G_FASIN:
1359 case TargetOpcode::G_FATAN:
1360 case TargetOpcode::G_FATAN2:
1361 case TargetOpcode::G_FCOSH:
1362 case TargetOpcode::G_FSINH:
1363 case TargetOpcode::G_FTANH:
1364 case TargetOpcode::G_FLOG10:
1365 case TargetOpcode::G_FLOG:
1366 case TargetOpcode::G_FLOG2:
1367 case TargetOpcode::G_FEXP:
1368 case TargetOpcode::G_FEXP2:
1369 case TargetOpcode::G_FEXP10:
1370 case TargetOpcode::G_FCEIL:
1371 case TargetOpcode::G_FFLOOR:
1372 case TargetOpcode::G_FMINNUM:
1373 case TargetOpcode::G_FMAXNUM:
1374 case TargetOpcode::G_FMINIMUMNUM:
1375 case TargetOpcode::G_FMAXIMUMNUM:
1376 case TargetOpcode::G_FSQRT:
1377 case TargetOpcode::G_FRINT:
1378 case TargetOpcode::G_FNEARBYINT:
1379 case TargetOpcode::G_INTRINSIC_TRUNC:
1380 case TargetOpcode::G_INTRINSIC_ROUND:
1381 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: {
1382 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1383 unsigned Size = LLTy.getSizeInBits();
1384 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1385 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1386 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1387 return UnableToLegalize;
1388 }
1389 auto Status = simpleLibcall(MI, MIRBuilder, Size, OpType: HLTy, LocObserver);
1390 if (Status != Legalized)
1391 return Status;
1392 break;
1393 }
1394 case TargetOpcode::G_FSINCOS: {
1395 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1396 unsigned Size = LLTy.getSizeInBits();
1397 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1398 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1399 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1400 return UnableToLegalize;
1401 }
1402 return emitSincosLibcall(MI, MIRBuilder, Size, OpType: HLTy, LocObserver);
1403 }
1404 case TargetOpcode::G_FMODF: {
1405 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1406 unsigned Size = LLTy.getSizeInBits();
1407 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1408 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1409 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1410 return UnableToLegalize;
1411 }
1412 return emitModfLibcall(MI, MIRBuilder, Size, OpType: HLTy, LocObserver);
1413 }
1414 case TargetOpcode::G_LROUND:
1415 case TargetOpcode::G_LLROUND:
1416 case TargetOpcode::G_INTRINSIC_LRINT:
1417 case TargetOpcode::G_INTRINSIC_LLRINT: {
1418 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
1419 unsigned Size = LLTy.getSizeInBits();
1420 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1421 Type *ITy = IntegerType::get(
1422 C&: Ctx, NumBits: MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits());
1423 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1424 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1425 return UnableToLegalize;
1426 }
1427 auto Libcall = getRTLibDesc(Opcode: MI.getOpcode(), Size);
1428 LegalizeResult Status =
1429 createLibcall(Libcall, Result: {MI.getOperand(i: 0).getReg(), ITy, 0},
1430 Args: {{MI.getOperand(i: 1).getReg(), HLTy, 0}}, LocObserver, MI: &MI);
1431 if (Status != Legalized)
1432 return Status;
1433 MI.eraseFromParent();
1434 return Legalized;
1435 }
1436 case TargetOpcode::G_FPOWI:
1437 case TargetOpcode::G_FLDEXP: {
1438 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1439 unsigned Size = LLTy.getSizeInBits();
1440 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1441 Type *ITy = IntegerType::get(
1442 C&: Ctx, NumBits: MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits());
1443 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1444 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1445 return UnableToLegalize;
1446 }
1447 auto Libcall = getRTLibDesc(Opcode: MI.getOpcode(), Size);
1448 SmallVector<CallLowering::ArgInfo, 2> Args = {
1449 {MI.getOperand(i: 1).getReg(), HLTy, 0},
1450 {MI.getOperand(i: 2).getReg(), ITy, 1}};
1451 Args[1].Flags[0].setSExt();
1452 LegalizeResult Status = createLibcall(
1453 Libcall, Result: {MI.getOperand(i: 0).getReg(), HLTy, 0}, Args, LocObserver, MI: &MI);
1454 if (Status != Legalized)
1455 return Status;
1456 break;
1457 }
1458 case TargetOpcode::G_FPEXT:
1459 case TargetOpcode::G_FPTRUNC: {
1460 Type *FromTy = getFloatTypeForLLT(Ctx, Ty: MRI.getType(Reg: MI.getOperand(i: 1).getReg()));
1461 Type *ToTy = getFloatTypeForLLT(Ctx, Ty: MRI.getType(Reg: MI.getOperand(i: 0).getReg()));
1462 if (!FromTy || !ToTy)
1463 return UnableToLegalize;
1464 LegalizeResult Status = conversionLibcall(MI, ToType: ToTy, FromType: FromTy, LocObserver);
1465 if (Status != Legalized)
1466 return Status;
1467 break;
1468 }
1469 case TargetOpcode::G_FCMP: {
1470 LegalizeResult Status = createFCMPLibcall(MI, LocObserver);
1471 if (Status != Legalized)
1472 return Status;
1473 MI.eraseFromParent();
1474 return Status;
1475 }
1476 case TargetOpcode::G_FPTOSI:
1477 case TargetOpcode::G_FPTOUI: {
1478 // FIXME: Support other types
1479 Type *FromTy =
1480 getFloatTypeForLLT(Ctx, Ty: MRI.getType(Reg: MI.getOperand(i: 1).getReg()));
1481 unsigned ToSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
1482 if ((ToSize != 32 && ToSize != 64 && ToSize != 128) || !FromTy)
1483 return UnableToLegalize;
1484 LegalizeResult Status = conversionLibcall(MI, ToType: Type::getIntNTy(C&: Ctx, N: ToSize),
1485 FromType: FromTy, LocObserver);
1486 if (Status != Legalized)
1487 return Status;
1488 break;
1489 }
1490 case TargetOpcode::G_SITOFP:
1491 case TargetOpcode::G_UITOFP: {
1492 unsigned FromSize = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
1493 Type *ToTy =
1494 getFloatTypeForLLT(Ctx, Ty: MRI.getType(Reg: MI.getOperand(i: 0).getReg()));
1495 if ((FromSize != 32 && FromSize != 64 && FromSize != 128) || !ToTy)
1496 return UnableToLegalize;
1497 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SITOFP;
1498 LegalizeResult Status = conversionLibcall(
1499 MI, ToType: ToTy, FromType: Type::getIntNTy(C&: Ctx, N: FromSize), LocObserver, IsSigned);
1500 if (Status != Legalized)
1501 return Status;
1502 break;
1503 }
1504 case TargetOpcode::G_ATOMICRMW_XCHG:
1505 case TargetOpcode::G_ATOMICRMW_ADD:
1506 case TargetOpcode::G_ATOMICRMW_SUB:
1507 case TargetOpcode::G_ATOMICRMW_AND:
1508 case TargetOpcode::G_ATOMICRMW_OR:
1509 case TargetOpcode::G_ATOMICRMW_XOR:
1510 case TargetOpcode::G_ATOMIC_CMPXCHG:
1511 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
1512 auto Status = createAtomicLibcall(MI);
1513 if (Status != Legalized)
1514 return Status;
1515 break;
1516 }
1517 case TargetOpcode::G_BZERO:
1518 case TargetOpcode::G_MEMCPY:
1519 case TargetOpcode::G_MEMMOVE:
1520 case TargetOpcode::G_MEMSET: {
1521 LegalizeResult Result =
1522 createMemLibcall(MRI&: *MIRBuilder.getMRI(), MI, LocObserver);
1523 if (Result != Legalized)
1524 return Result;
1525 MI.eraseFromParent();
1526 return Result;
1527 }
1528 case TargetOpcode::G_GET_FPENV:
1529 case TargetOpcode::G_GET_FPMODE: {
1530 LegalizeResult Result = createGetStateLibcall(MI, LocObserver);
1531 if (Result != Legalized)
1532 return Result;
1533 break;
1534 }
1535 case TargetOpcode::G_SET_FPENV:
1536 case TargetOpcode::G_SET_FPMODE: {
1537 LegalizeResult Result = createSetStateLibcall(MI, LocObserver);
1538 if (Result != Legalized)
1539 return Result;
1540 break;
1541 }
1542 case TargetOpcode::G_RESET_FPENV:
1543 case TargetOpcode::G_RESET_FPMODE: {
1544 LegalizeResult Result = createResetStateLibcall(MI, LocObserver);
1545 if (Result != Legalized)
1546 return Result;
1547 break;
1548 }
1549 }
1550
1551 MI.eraseFromParent();
1552 return Legalized;
1553}
1554
1555LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
1556 unsigned TypeIdx,
1557 LLT NarrowTy) {
1558 uint64_t SizeOp0 = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
1559 uint64_t NarrowSize = NarrowTy.getSizeInBits();
1560
1561 switch (MI.getOpcode()) {
1562 default:
1563 return UnableToLegalize;
1564 case TargetOpcode::G_IMPLICIT_DEF: {
1565 Register DstReg = MI.getOperand(i: 0).getReg();
1566 LLT DstTy = MRI.getType(Reg: DstReg);
1567
1568 // If SizeOp0 is not an exact multiple of NarrowSize, emit
1569 // G_ANYEXT(G_IMPLICIT_DEF). Cast result to vector if needed.
1570 // FIXME: Although this would also be legal for the general case, it causes
1571 // a lot of regressions in the emitted code (superfluous COPYs, artifact
1572 // combines not being hit). This seems to be a problem related to the
1573 // artifact combiner.
1574 if (SizeOp0 % NarrowSize != 0) {
1575 LLT ImplicitTy = DstTy.changeElementType(NewEltTy: NarrowTy);
1576 Register ImplicitReg = MIRBuilder.buildUndef(Res: ImplicitTy).getReg(Idx: 0);
1577 MIRBuilder.buildAnyExt(Res: DstReg, Op: ImplicitReg);
1578
1579 MI.eraseFromParent();
1580 return Legalized;
1581 }
1582
1583 int NumParts = SizeOp0 / NarrowSize;
1584
1585 SmallVector<Register, 2> DstRegs;
1586 for (int i = 0; i < NumParts; ++i)
1587 DstRegs.push_back(Elt: MIRBuilder.buildUndef(Res: NarrowTy).getReg(Idx: 0));
1588
1589 if (DstTy.isVector())
1590 MIRBuilder.buildBuildVector(Res: DstReg, Ops: DstRegs);
1591 else
1592 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
1593 MI.eraseFromParent();
1594 return Legalized;
1595 }
1596 case TargetOpcode::G_CONSTANT: {
1597 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1598 const APInt &Val = MI.getOperand(i: 1).getCImm()->getValue();
1599 unsigned TotalSize = Ty.getSizeInBits();
1600 unsigned NarrowSize = NarrowTy.getSizeInBits();
1601 int NumParts = TotalSize / NarrowSize;
1602
1603 SmallVector<Register, 4> PartRegs;
1604 for (int I = 0; I != NumParts; ++I) {
1605 unsigned Offset = I * NarrowSize;
1606 auto K = MIRBuilder.buildConstant(Res: NarrowTy,
1607 Val: Val.lshr(shiftAmt: Offset).trunc(width: NarrowSize));
1608 PartRegs.push_back(Elt: K.getReg(Idx: 0));
1609 }
1610
1611 LLT LeftoverTy;
1612 unsigned LeftoverBits = TotalSize - NumParts * NarrowSize;
1613 SmallVector<Register, 1> LeftoverRegs;
1614 if (LeftoverBits != 0) {
1615 LeftoverTy = LLT::scalar(SizeInBits: LeftoverBits);
1616 auto K = MIRBuilder.buildConstant(
1617 Res: LeftoverTy,
1618 Val: Val.lshr(shiftAmt: NumParts * NarrowSize).trunc(width: LeftoverBits));
1619 LeftoverRegs.push_back(Elt: K.getReg(Idx: 0));
1620 }
1621
1622 insertParts(DstReg: MI.getOperand(i: 0).getReg(),
1623 ResultTy: Ty, PartTy: NarrowTy, PartRegs, LeftoverTy, LeftoverRegs);
1624
1625 MI.eraseFromParent();
1626 return Legalized;
1627 }
1628 case TargetOpcode::G_SEXT:
1629 case TargetOpcode::G_ZEXT:
1630 case TargetOpcode::G_ANYEXT:
1631 return narrowScalarExt(MI, TypeIdx, Ty: NarrowTy);
1632 case TargetOpcode::G_TRUNC: {
1633 if (TypeIdx != 1)
1634 return UnableToLegalize;
1635
1636 uint64_t SizeOp1 = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
1637 if (NarrowTy.getSizeInBits() * 2 != SizeOp1) {
1638 LLVM_DEBUG(dbgs() << "Can't narrow trunc to type " << NarrowTy << "\n");
1639 return UnableToLegalize;
1640 }
1641
1642 auto Unmerge = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: MI.getOperand(i: 1));
1643 MIRBuilder.buildCopy(Res: MI.getOperand(i: 0), Op: Unmerge.getReg(Idx: 0));
1644 MI.eraseFromParent();
1645 return Legalized;
1646 }
1647 case TargetOpcode::G_CONSTANT_FOLD_BARRIER:
1648 case TargetOpcode::G_FREEZE: {
1649 if (TypeIdx != 0)
1650 return UnableToLegalize;
1651
1652 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1653 // Should widen scalar first
1654 if (Ty.getSizeInBits() % NarrowTy.getSizeInBits() != 0)
1655 return UnableToLegalize;
1656
1657 auto Unmerge = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: MI.getOperand(i: 1).getReg());
1658 SmallVector<Register, 8> Parts;
1659 for (unsigned i = 0; i < Unmerge->getNumDefs(); ++i) {
1660 Parts.push_back(
1661 Elt: MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {NarrowTy}, SrcOps: {Unmerge.getReg(Idx: i)})
1662 .getReg(Idx: 0));
1663 }
1664
1665 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0).getReg(), Ops: Parts);
1666 MI.eraseFromParent();
1667 return Legalized;
1668 }
1669 case TargetOpcode::G_ADD:
1670 case TargetOpcode::G_SUB:
1671 case TargetOpcode::G_SADDO:
1672 case TargetOpcode::G_SSUBO:
1673 case TargetOpcode::G_SADDE:
1674 case TargetOpcode::G_SSUBE:
1675 case TargetOpcode::G_UADDO:
1676 case TargetOpcode::G_USUBO:
1677 case TargetOpcode::G_UADDE:
1678 case TargetOpcode::G_USUBE:
1679 return narrowScalarAddSub(MI, TypeIdx, NarrowTy);
1680 case TargetOpcode::G_MUL:
1681 case TargetOpcode::G_UMULH:
1682 return narrowScalarMul(MI, Ty: NarrowTy);
1683 case TargetOpcode::G_EXTRACT:
1684 return narrowScalarExtract(MI, TypeIdx, Ty: NarrowTy);
1685 case TargetOpcode::G_INSERT:
1686 return narrowScalarInsert(MI, TypeIdx, Ty: NarrowTy);
1687 case TargetOpcode::G_LOAD: {
1688 auto &LoadMI = cast<GLoad>(Val&: MI);
1689 Register DstReg = LoadMI.getDstReg();
1690 LLT DstTy = MRI.getType(Reg: DstReg);
1691 if (DstTy.isVector())
1692 return UnableToLegalize;
1693
1694 if (8 * LoadMI.getMemSize().getValue() != DstTy.getSizeInBits()) {
1695 Register TmpReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
1696 MIRBuilder.buildLoad(Res: TmpReg, Addr: LoadMI.getPointerReg(), MMO&: LoadMI.getMMO());
1697 MIRBuilder.buildAnyExt(Res: DstReg, Op: TmpReg);
1698 LoadMI.eraseFromParent();
1699 return Legalized;
1700 }
1701
1702 return reduceLoadStoreWidth(MI&: LoadMI, TypeIdx, NarrowTy);
1703 }
1704 case TargetOpcode::G_ZEXTLOAD:
1705 case TargetOpcode::G_SEXTLOAD:
1706 case TargetOpcode::G_FPEXTLOAD: {
1707 auto &LoadMI = cast<GExtLoad>(Val&: MI);
1708 Register DstReg = LoadMI.getDstReg();
1709 Register PtrReg = LoadMI.getPointerReg();
1710
1711 Register TmpReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
1712 auto &MMO = LoadMI.getMMO();
1713 unsigned MemSize = MMO.getSizeInBits().getValue();
1714
1715 if (MemSize == NarrowSize) {
1716 MIRBuilder.buildLoad(Res: TmpReg, Addr: PtrReg, MMO);
1717 } else if (MemSize < NarrowSize) {
1718 MIRBuilder.buildLoadInstr(Opcode: LoadMI.getOpcode(), Res: TmpReg, Addr: PtrReg, MMO);
1719 } else if (MemSize > NarrowSize) {
1720 // FIXME: Need to split the load.
1721 return UnableToLegalize;
1722 }
1723
1724 if (isa<GZExtLoad>(Val: LoadMI))
1725 MIRBuilder.buildZExt(Res: DstReg, Op: TmpReg);
1726 else if (isa<GSExtLoad>(Val: LoadMI))
1727 MIRBuilder.buildSExt(Res: DstReg, Op: TmpReg);
1728 else
1729 MIRBuilder.buildFPExt(Res: DstReg, Op: TmpReg);
1730
1731 LoadMI.eraseFromParent();
1732 return Legalized;
1733 }
1734 case TargetOpcode::G_STORE: {
1735 auto &StoreMI = cast<GStore>(Val&: MI);
1736
1737 Register SrcReg = StoreMI.getValueReg();
1738 LLT SrcTy = MRI.getType(Reg: SrcReg);
1739 if (SrcTy.isVector())
1740 return UnableToLegalize;
1741
1742 int NumParts = SizeOp0 / NarrowSize;
1743 unsigned HandledSize = NumParts * NarrowTy.getSizeInBits();
1744 unsigned LeftoverBits = SrcTy.getSizeInBits() - HandledSize;
1745 if (SrcTy.isVector() && LeftoverBits != 0)
1746 return UnableToLegalize;
1747
1748 if (8 * StoreMI.getMemSize().getValue() != SrcTy.getSizeInBits()) {
1749 Register TmpReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
1750 MIRBuilder.buildTrunc(Res: TmpReg, Op: SrcReg);
1751 MIRBuilder.buildStore(Val: TmpReg, Addr: StoreMI.getPointerReg(), MMO&: StoreMI.getMMO());
1752 StoreMI.eraseFromParent();
1753 return Legalized;
1754 }
1755
1756 return reduceLoadStoreWidth(MI&: StoreMI, TypeIdx: 0, NarrowTy);
1757 }
1758 case TargetOpcode::G_FPTRUNCSTORE: {
1759 auto &StoreMI = cast<GFPTruncStore>(Val&: MI);
1760 Register SrcReg = StoreMI.getValueReg();
1761 Register PtrReg = StoreMI.getPointerReg();
1762
1763 auto &MMO = StoreMI.getMMO();
1764 unsigned MemSize = MMO.getSizeInBits().getValue();
1765 if (MemSize > NarrowSize) {
1766 return UnableToLegalize;
1767 }
1768
1769 auto TmpReg = MIRBuilder.buildFPTrunc(Res: NarrowTy, Op: SrcReg);
1770 if (MemSize == NarrowSize) {
1771 MIRBuilder.buildStore(Val: TmpReg, Addr: PtrReg, MMO);
1772 } else if (MemSize < NarrowSize) {
1773 MIRBuilder.buildStoreInstr(Opcode: TargetOpcode::G_FPTRUNCSTORE, Val: TmpReg, Addr: PtrReg,
1774 MMO);
1775 }
1776
1777 StoreMI.eraseFromParent();
1778 return Legalized;
1779 }
1780 case TargetOpcode::G_SELECT:
1781 return narrowScalarSelect(MI, TypeIdx, Ty: NarrowTy);
1782 case TargetOpcode::G_AND:
1783 case TargetOpcode::G_OR:
1784 case TargetOpcode::G_XOR: {
1785 // Legalize bitwise operation:
1786 // A = BinOp<Ty> B, C
1787 // into:
1788 // B1, ..., BN = G_UNMERGE_VALUES B
1789 // C1, ..., CN = G_UNMERGE_VALUES C
1790 // A1 = BinOp<Ty/N> B1, C2
1791 // ...
1792 // AN = BinOp<Ty/N> BN, CN
1793 // A = G_MERGE_VALUES A1, ..., AN
1794 return narrowScalarBasic(MI, TypeIdx, Ty: NarrowTy);
1795 }
1796 case TargetOpcode::G_SHL:
1797 case TargetOpcode::G_LSHR:
1798 case TargetOpcode::G_ASHR:
1799 return narrowScalarShift(MI, TypeIdx, Ty: NarrowTy);
1800 case TargetOpcode::G_CTLZ:
1801 case TargetOpcode::G_CTLZ_ZERO_POISON:
1802 case TargetOpcode::G_CTTZ:
1803 case TargetOpcode::G_CTTZ_ZERO_POISON:
1804 case TargetOpcode::G_CTLS:
1805 case TargetOpcode::G_CTPOP:
1806 if (TypeIdx == 1)
1807 switch (MI.getOpcode()) {
1808 case TargetOpcode::G_CTLZ:
1809 case TargetOpcode::G_CTLZ_ZERO_POISON:
1810 return narrowScalarCTLZ(MI, TypeIdx, Ty: NarrowTy);
1811 case TargetOpcode::G_CTTZ:
1812 case TargetOpcode::G_CTTZ_ZERO_POISON:
1813 return narrowScalarCTTZ(MI, TypeIdx, Ty: NarrowTy);
1814 case TargetOpcode::G_CTPOP:
1815 return narrowScalarCTPOP(MI, TypeIdx, Ty: NarrowTy);
1816 case TargetOpcode::G_CTLS:
1817 return narrowScalarCTLS(MI, TypeIdx, Ty: NarrowTy);
1818 default:
1819 return UnableToLegalize;
1820 }
1821
1822 Observer.changingInstr(MI);
1823 narrowScalarDst(MI, NarrowTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_ZEXT);
1824 Observer.changedInstr(MI);
1825 return Legalized;
1826 case TargetOpcode::G_INTTOPTR:
1827 if (TypeIdx != 1)
1828 return UnableToLegalize;
1829
1830 Observer.changingInstr(MI);
1831 narrowScalarSrc(MI, NarrowTy, OpIdx: 1);
1832 Observer.changedInstr(MI);
1833 return Legalized;
1834 case TargetOpcode::G_PTRTOINT:
1835 if (TypeIdx != 0)
1836 return UnableToLegalize;
1837
1838 Observer.changingInstr(MI);
1839 narrowScalarDst(MI, NarrowTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_ZEXT);
1840 Observer.changedInstr(MI);
1841 return Legalized;
1842 case TargetOpcode::G_PHI: {
1843 // FIXME: add support for when SizeOp0 isn't an exact multiple of
1844 // NarrowSize.
1845 if (SizeOp0 % NarrowSize != 0)
1846 return UnableToLegalize;
1847
1848 unsigned NumParts = SizeOp0 / NarrowSize;
1849 SmallVector<Register, 2> DstRegs(NumParts);
1850 SmallVector<SmallVector<Register, 2>, 2> SrcRegs(MI.getNumOperands() / 2);
1851 Observer.changingInstr(MI);
1852 for (unsigned i = 1; i < MI.getNumOperands(); i += 2) {
1853 MachineBasicBlock &OpMBB = *MI.getOperand(i: i + 1).getMBB();
1854 MIRBuilder.setInsertPt(MBB&: OpMBB, II: OpMBB.getFirstTerminatorForward());
1855 extractParts(Reg: MI.getOperand(i).getReg(), Ty: NarrowTy, NumParts,
1856 VRegs&: SrcRegs[i / 2], MIRBuilder, MRI);
1857 }
1858 MachineBasicBlock &MBB = *MI.getParent();
1859 MIRBuilder.setInsertPt(MBB, II: MI);
1860 for (unsigned i = 0; i < NumParts; ++i) {
1861 DstRegs[i] = MRI.createGenericVirtualRegister(Ty: NarrowTy);
1862 MachineInstrBuilder MIB =
1863 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_PHI).addDef(RegNo: DstRegs[i]);
1864 for (unsigned j = 1; j < MI.getNumOperands(); j += 2)
1865 MIB.addUse(RegNo: SrcRegs[j / 2][i]).add(MO: MI.getOperand(i: j + 1));
1866 }
1867 MIRBuilder.setInsertPt(MBB, II: MBB.getFirstNonPHI());
1868 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0), Ops: DstRegs);
1869 Observer.changedInstr(MI);
1870 MI.eraseFromParent();
1871 return Legalized;
1872 }
1873 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1874 case TargetOpcode::G_INSERT_VECTOR_ELT: {
1875 if (TypeIdx != 2)
1876 return UnableToLegalize;
1877
1878 int OpIdx = MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
1879 Observer.changingInstr(MI);
1880 narrowScalarSrc(MI, NarrowTy, OpIdx);
1881 Observer.changedInstr(MI);
1882 return Legalized;
1883 }
1884 case TargetOpcode::G_ICMP: {
1885 Register LHS = MI.getOperand(i: 2).getReg();
1886 LLT SrcTy = MRI.getType(Reg: LHS);
1887 CmpInst::Predicate Pred =
1888 static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
1889
1890 LLT LeftoverTy; // Example: s88 -> s64 (NarrowTy) + s24 (leftover)
1891 SmallVector<Register, 4> LHSPartRegs, LHSLeftoverRegs;
1892 if (!extractParts(Reg: LHS, RegTy: SrcTy, MainTy: NarrowTy, LeftoverTy, VRegs&: LHSPartRegs,
1893 LeftoverVRegs&: LHSLeftoverRegs, MIRBuilder, MRI))
1894 return UnableToLegalize;
1895
1896 LLT Unused; // Matches LeftoverTy; G_ICMP LHS and RHS are the same type.
1897 SmallVector<Register, 4> RHSPartRegs, RHSLeftoverRegs;
1898 if (!extractParts(Reg: MI.getOperand(i: 3).getReg(), RegTy: SrcTy, MainTy: NarrowTy, LeftoverTy&: Unused,
1899 VRegs&: RHSPartRegs, LeftoverVRegs&: RHSLeftoverRegs, MIRBuilder, MRI))
1900 return UnableToLegalize;
1901
1902 // We now have the LHS and RHS of the compare split into narrow-type
1903 // registers, plus potentially some leftover type.
1904 Register Dst = MI.getOperand(i: 0).getReg();
1905 LLT ResTy = MRI.getType(Reg: Dst);
1906 if (ICmpInst::isEquality(P: Pred)) {
1907 // For each part on the LHS and RHS, keep track of the result of XOR-ing
1908 // them together. For each equal part, the result should be all 0s. For
1909 // each non-equal part, we'll get at least one 1.
1910 auto Zero = MIRBuilder.buildConstant(Res: NarrowTy, Val: 0);
1911 SmallVector<Register, 4> Xors;
1912 for (auto LHSAndRHS : zip(t&: LHSPartRegs, u&: RHSPartRegs)) {
1913 auto LHS = std::get<0>(t&: LHSAndRHS);
1914 auto RHS = std::get<1>(t&: LHSAndRHS);
1915 auto Xor = MIRBuilder.buildXor(Dst: NarrowTy, Src0: LHS, Src1: RHS).getReg(Idx: 0);
1916 Xors.push_back(Elt: Xor);
1917 }
1918
1919 // Build a G_XOR for each leftover register. Each G_XOR must be widened
1920 // to the desired narrow type so that we can OR them together later.
1921 SmallVector<Register, 4> WidenedXors;
1922 for (auto LHSAndRHS : zip(t&: LHSLeftoverRegs, u&: RHSLeftoverRegs)) {
1923 auto LHS = std::get<0>(t&: LHSAndRHS);
1924 auto RHS = std::get<1>(t&: LHSAndRHS);
1925 auto Xor = MIRBuilder.buildXor(Dst: LeftoverTy, Src0: LHS, Src1: RHS).getReg(Idx: 0);
1926 LLT GCDTy = extractGCDType(Parts&: WidenedXors, DstTy: NarrowTy, NarrowTy: LeftoverTy, SrcReg: Xor);
1927 buildLCMMergePieces(DstTy: LeftoverTy, NarrowTy, GCDTy, VRegs&: WidenedXors,
1928 /* PadStrategy = */ TargetOpcode::G_ZEXT);
1929 llvm::append_range(C&: Xors, R&: WidenedXors);
1930 }
1931
1932 // Now, for each part we broke up, we know if they are equal/not equal
1933 // based off the G_XOR. We can OR these all together and compare against
1934 // 0 to get the result.
1935 assert(Xors.size() >= 2 && "Should have gotten at least two Xors?");
1936 auto Or = MIRBuilder.buildOr(Dst: NarrowTy, Src0: Xors[0], Src1: Xors[1]);
1937 for (unsigned I = 2, E = Xors.size(); I < E; ++I)
1938 Or = MIRBuilder.buildOr(Dst: NarrowTy, Src0: Or, Src1: Xors[I]);
1939 MIRBuilder.buildICmp(Pred, Res: Dst, Op0: Or, Op1: Zero);
1940 } else {
1941 Register CmpIn;
1942 for (unsigned I = 0, E = LHSPartRegs.size(); I != E; ++I) {
1943 Register CmpOut;
1944 CmpInst::Predicate PartPred;
1945
1946 if (I == E - 1 && LHSLeftoverRegs.empty()) {
1947 PartPred = Pred;
1948 CmpOut = Dst;
1949 } else {
1950 PartPred = ICmpInst::getUnsignedPredicate(Pred);
1951 CmpOut = MRI.createGenericVirtualRegister(Ty: ResTy);
1952 }
1953
1954 if (!CmpIn) {
1955 MIRBuilder.buildICmp(Pred: PartPred, Res: CmpOut, Op0: LHSPartRegs[I],
1956 Op1: RHSPartRegs[I]);
1957 } else {
1958 auto Cmp = MIRBuilder.buildICmp(Pred: PartPred, Res: ResTy, Op0: LHSPartRegs[I],
1959 Op1: RHSPartRegs[I]);
1960 auto CmpEq = MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: ResTy,
1961 Op0: LHSPartRegs[I], Op1: RHSPartRegs[I]);
1962 MIRBuilder.buildSelect(Res: CmpOut, Tst: CmpEq, Op0: CmpIn, Op1: Cmp);
1963 }
1964
1965 CmpIn = CmpOut;
1966 }
1967
1968 for (unsigned I = 0, E = LHSLeftoverRegs.size(); I != E; ++I) {
1969 Register CmpOut;
1970 CmpInst::Predicate PartPred;
1971
1972 if (I == E - 1) {
1973 PartPred = Pred;
1974 CmpOut = Dst;
1975 } else {
1976 PartPred = ICmpInst::getUnsignedPredicate(Pred);
1977 CmpOut = MRI.createGenericVirtualRegister(Ty: ResTy);
1978 }
1979
1980 if (!CmpIn) {
1981 MIRBuilder.buildICmp(Pred: PartPred, Res: CmpOut, Op0: LHSLeftoverRegs[I],
1982 Op1: RHSLeftoverRegs[I]);
1983 } else {
1984 auto Cmp = MIRBuilder.buildICmp(Pred: PartPred, Res: ResTy, Op0: LHSLeftoverRegs[I],
1985 Op1: RHSLeftoverRegs[I]);
1986 auto CmpEq =
1987 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: ResTy,
1988 Op0: LHSLeftoverRegs[I], Op1: RHSLeftoverRegs[I]);
1989 MIRBuilder.buildSelect(Res: CmpOut, Tst: CmpEq, Op0: CmpIn, Op1: Cmp);
1990 }
1991
1992 CmpIn = CmpOut;
1993 }
1994 }
1995 MI.eraseFromParent();
1996 return Legalized;
1997 }
1998 case TargetOpcode::G_FCMP:
1999 if (TypeIdx != 0)
2000 return UnableToLegalize;
2001
2002 Observer.changingInstr(MI);
2003 narrowScalarDst(MI, NarrowTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_ZEXT);
2004 Observer.changedInstr(MI);
2005 return Legalized;
2006
2007 case TargetOpcode::G_SEXT_INREG: {
2008 if (TypeIdx != 0)
2009 return UnableToLegalize;
2010
2011 int64_t SizeInBits = MI.getOperand(i: 2).getImm();
2012
2013 // So long as the new type has more bits than the bits we're extending we
2014 // don't need to break it apart.
2015 if (NarrowTy.getScalarSizeInBits() > SizeInBits) {
2016 Observer.changingInstr(MI);
2017 // We don't lose any non-extension bits by truncating the src and
2018 // sign-extending the dst.
2019 MachineOperand &MO1 = MI.getOperand(i: 1);
2020 auto TruncMIB = MIRBuilder.buildTrunc(Res: NarrowTy, Op: MO1);
2021 MO1.setReg(TruncMIB.getReg(Idx: 0));
2022
2023 MachineOperand &MO2 = MI.getOperand(i: 0);
2024 Register DstExt = MRI.createGenericVirtualRegister(Ty: NarrowTy);
2025 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2026 MIRBuilder.buildSExt(Res: MO2, Op: DstExt);
2027 MO2.setReg(DstExt);
2028 Observer.changedInstr(MI);
2029 return Legalized;
2030 }
2031
2032 // Break it apart. Components below the extension point are unmodified. The
2033 // component containing the extension point becomes a narrower SEXT_INREG.
2034 // Components above it are ashr'd from the component containing the
2035 // extension point.
2036 if (SizeOp0 % NarrowSize != 0)
2037 return UnableToLegalize;
2038 int NumParts = SizeOp0 / NarrowSize;
2039
2040 // List the registers where the destination will be scattered.
2041 SmallVector<Register, 2> DstRegs;
2042 // List the registers where the source will be split.
2043 SmallVector<Register, 2> SrcRegs;
2044
2045 // Create all the temporary registers.
2046 for (int i = 0; i < NumParts; ++i) {
2047 Register SrcReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
2048
2049 SrcRegs.push_back(Elt: SrcReg);
2050 }
2051
2052 // Explode the big arguments into smaller chunks.
2053 MIRBuilder.buildUnmerge(Res: SrcRegs, Op: MI.getOperand(i: 1));
2054
2055 Register AshrCstReg =
2056 MIRBuilder.buildConstant(Res: NarrowTy, Val: NarrowTy.getScalarSizeInBits() - 1)
2057 .getReg(Idx: 0);
2058 Register FullExtensionReg;
2059 Register PartialExtensionReg;
2060
2061 // Do the operation on each small part.
2062 for (int i = 0; i < NumParts; ++i) {
2063 if ((i + 1) * NarrowTy.getScalarSizeInBits() <= SizeInBits) {
2064 DstRegs.push_back(Elt: SrcRegs[i]);
2065 PartialExtensionReg = DstRegs.back();
2066 } else if (i * NarrowTy.getScalarSizeInBits() >= SizeInBits) {
2067 assert(PartialExtensionReg &&
2068 "Expected to visit partial extension before full");
2069 if (FullExtensionReg) {
2070 DstRegs.push_back(Elt: FullExtensionReg);
2071 continue;
2072 }
2073 DstRegs.push_back(
2074 Elt: MIRBuilder.buildAShr(Dst: NarrowTy, Src0: PartialExtensionReg, Src1: AshrCstReg)
2075 .getReg(Idx: 0));
2076 FullExtensionReg = DstRegs.back();
2077 } else {
2078 DstRegs.push_back(
2079 Elt: MIRBuilder
2080 .buildInstr(
2081 Opc: TargetOpcode::G_SEXT_INREG, DstOps: {NarrowTy},
2082 SrcOps: {SrcRegs[i], SizeInBits % NarrowTy.getScalarSizeInBits()})
2083 .getReg(Idx: 0));
2084 PartialExtensionReg = DstRegs.back();
2085 }
2086 }
2087
2088 // Gather the destination registers into the final destination.
2089 Register DstReg = MI.getOperand(i: 0).getReg();
2090 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
2091 MI.eraseFromParent();
2092 return Legalized;
2093 }
2094 case TargetOpcode::G_BSWAP:
2095 case TargetOpcode::G_BITREVERSE: {
2096 if (SizeOp0 % NarrowSize != 0)
2097 return UnableToLegalize;
2098
2099 Observer.changingInstr(MI);
2100 SmallVector<Register, 2> SrcRegs, DstRegs;
2101 unsigned NumParts = SizeOp0 / NarrowSize;
2102 extractParts(Reg: MI.getOperand(i: 1).getReg(), Ty: NarrowTy, NumParts, VRegs&: SrcRegs,
2103 MIRBuilder, MRI);
2104
2105 for (unsigned i = 0; i < NumParts; ++i) {
2106 auto DstPart = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {NarrowTy},
2107 SrcOps: {SrcRegs[NumParts - 1 - i]});
2108 DstRegs.push_back(Elt: DstPart.getReg(Idx: 0));
2109 }
2110
2111 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0), Ops: DstRegs);
2112
2113 Observer.changedInstr(MI);
2114 MI.eraseFromParent();
2115 return Legalized;
2116 }
2117 case TargetOpcode::G_PTR_ADD:
2118 case TargetOpcode::G_PTRMASK: {
2119 if (TypeIdx != 1)
2120 return UnableToLegalize;
2121 Observer.changingInstr(MI);
2122 narrowScalarSrc(MI, NarrowTy, OpIdx: 2);
2123 Observer.changedInstr(MI);
2124 return Legalized;
2125 }
2126 case TargetOpcode::G_FPTOUI:
2127 case TargetOpcode::G_FPTOSI:
2128 case TargetOpcode::G_FPTOUI_SAT:
2129 case TargetOpcode::G_FPTOSI_SAT:
2130 return narrowScalarFPTOI(MI, TypeIdx, Ty: NarrowTy);
2131 case TargetOpcode::G_FPEXT:
2132 if (TypeIdx != 0)
2133 return UnableToLegalize;
2134 Observer.changingInstr(MI);
2135 narrowScalarDst(MI, NarrowTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_FPEXT);
2136 Observer.changedInstr(MI);
2137 return Legalized;
2138 case TargetOpcode::G_FLDEXP:
2139 case TargetOpcode::G_STRICT_FLDEXP:
2140 return narrowScalarFLDEXP(MI, TypeIdx, Ty: NarrowTy);
2141 case TargetOpcode::G_VSCALE: {
2142 Register Dst = MI.getOperand(i: 0).getReg();
2143 LLT Ty = MRI.getType(Reg: Dst);
2144
2145 // Assume VSCALE(1) fits into a legal integer
2146 const APInt One(NarrowTy.getSizeInBits(), 1);
2147 auto VScaleBase = MIRBuilder.buildVScale(Res: NarrowTy, MinElts: One);
2148 auto ZExt = MIRBuilder.buildZExt(Res: Ty, Op: VScaleBase);
2149 auto C = MIRBuilder.buildConstant(Res: Ty, Val: *MI.getOperand(i: 1).getCImm());
2150 MIRBuilder.buildMul(Dst, Src0: ZExt, Src1: C);
2151
2152 MI.eraseFromParent();
2153 return Legalized;
2154 }
2155 }
2156}
2157
2158Register LegalizerHelper::coerceToInteger(Register Val) {
2159 LLT Ty = MRI.getType(Reg: Val);
2160 if (Ty.isScalar() && !Ty.isFloat())
2161 return Val;
2162
2163 const DataLayout &DL = MIRBuilder.getDataLayout();
2164 LLT NewTy = LLT::integer(SizeInBits: Ty.getSizeInBits());
2165
2166 if (Ty.isFloat())
2167 return MIRBuilder.buildBitcast(Dst: NewTy, Src: Val).getReg(Idx: 0);
2168
2169 if (Ty.isPointer()) {
2170 if (DL.isNonIntegralAddressSpace(AddrSpace: Ty.getAddressSpace()))
2171 return Register();
2172 return MIRBuilder.buildPtrToInt(Dst: NewTy, Src: Val).getReg(Idx: 0);
2173 }
2174
2175 Register NewVal = Val;
2176
2177 assert(Ty.isVector());
2178 if (Ty.isPointerVector())
2179 NewVal = MIRBuilder.buildPtrToInt(Dst: NewTy, Src: NewVal).getReg(Idx: 0);
2180 return MIRBuilder.buildBitcast(Dst: NewTy, Src: NewVal).getReg(Idx: 0);
2181}
2182
2183void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy,
2184 unsigned OpIdx, unsigned ExtOpcode) {
2185 MachineOperand &MO = MI.getOperand(i: OpIdx);
2186 auto ExtB = MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {WideTy}, SrcOps: {MO});
2187 MO.setReg(ExtB.getReg(Idx: 0));
2188}
2189
2190void LegalizerHelper::widenScalarSrcUsingFPExt(MachineInstr &MI, LLT WideTy,
2191 unsigned OpIdx) {
2192 MachineOperand &MO = MI.getOperand(i: OpIdx);
2193 auto ExtB = MIRBuilder.buildInstr(Opc: TargetOpcode::G_FPEXT, DstOps: {WideTy}, SrcOps: {MO},
2194 Flags: MI.getFlags());
2195 MO.setReg(ExtB.getReg(Idx: 0));
2196}
2197
2198void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy,
2199 unsigned OpIdx) {
2200 MachineOperand &MO = MI.getOperand(i: OpIdx);
2201 auto ExtB = MIRBuilder.buildTrunc(Res: NarrowTy, Op: MO);
2202 MO.setReg(ExtB.getReg(Idx: 0));
2203}
2204
2205void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy,
2206 unsigned OpIdx, unsigned TruncOpcode) {
2207 MachineOperand &MO = MI.getOperand(i: OpIdx);
2208 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2209 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2210 MIRBuilder.buildInstr(Opc: TruncOpcode, DstOps: {MO}, SrcOps: {DstExt});
2211 MO.setReg(DstExt);
2212}
2213
2214void LegalizerHelper::widenScalarDstUsingFPTrunc(MachineInstr &MI, LLT WideTy,
2215 unsigned OpIdx) {
2216 MachineOperand &MO = MI.getOperand(i: OpIdx);
2217 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2218 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2219 MIRBuilder.buildInstr(Opc: TargetOpcode::G_FPTRUNC, DstOps: {MO}, SrcOps: {DstExt}, Flags: MI.getFlags());
2220 MO.setReg(DstExt);
2221}
2222
2223void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy,
2224 unsigned OpIdx, unsigned ExtOpcode) {
2225 MachineOperand &MO = MI.getOperand(i: OpIdx);
2226 Register DstTrunc = MRI.createGenericVirtualRegister(Ty: NarrowTy);
2227 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2228 MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {MO}, SrcOps: {DstTrunc});
2229 MO.setReg(DstTrunc);
2230}
2231
2232void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy,
2233 unsigned OpIdx) {
2234 MachineOperand &MO = MI.getOperand(i: OpIdx);
2235 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2236 Register Dst = MO.getReg();
2237 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2238 MO.setReg(DstExt);
2239 MIRBuilder.buildDeleteTrailingVectorElements(Res: Dst, Op0: DstExt);
2240}
2241
2242void LegalizerHelper::moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy,
2243 unsigned OpIdx) {
2244 MachineOperand &MO = MI.getOperand(i: OpIdx);
2245 MO.setReg(MIRBuilder.buildPadVectorWithUndefElements(Res: MoreTy, Op0: MO).getReg(Idx: 0));
2246}
2247
2248void LegalizerHelper::bitcastSrc(MachineInstr &MI, LLT CastTy, unsigned OpIdx) {
2249 MachineOperand &Op = MI.getOperand(i: OpIdx);
2250 Op.setReg(MIRBuilder.buildBitcast(Dst: CastTy, Src: Op).getReg(Idx: 0));
2251}
2252
2253void LegalizerHelper::bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx) {
2254 MachineOperand &MO = MI.getOperand(i: OpIdx);
2255 Register CastDst = MRI.createGenericVirtualRegister(Ty: CastTy);
2256 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2257 MIRBuilder.buildBitcast(Dst: MO, Src: CastDst);
2258 MO.setReg(CastDst);
2259}
2260
2261LegalizerHelper::LegalizeResult
2262LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx,
2263 LLT WideTy) {
2264 if (TypeIdx != 1)
2265 return UnableToLegalize;
2266
2267 auto [DstReg, DstTy, Src1Reg, Src1Ty] = MI.getFirst2RegLLTs();
2268 if (DstTy.isVector())
2269 return UnableToLegalize;
2270
2271 LLT SrcTy = MRI.getType(Reg: Src1Reg);
2272 const int DstSize = DstTy.getSizeInBits();
2273 const int SrcSize = SrcTy.getSizeInBits();
2274 const int WideSize = WideTy.getSizeInBits();
2275 const int NumMerge = (DstSize + WideSize - 1) / WideSize;
2276
2277 unsigned NumOps = MI.getNumOperands();
2278 unsigned NumSrc = MI.getNumOperands() - 1;
2279 unsigned PartSize = DstTy.getSizeInBits() / NumSrc;
2280
2281 if (WideSize >= DstSize) {
2282 // Directly pack the bits in the target type.
2283 Register ResultReg = MIRBuilder.buildZExt(Res: WideTy, Op: Src1Reg).getReg(Idx: 0);
2284
2285 for (unsigned I = 2; I != NumOps; ++I) {
2286 const unsigned Offset = (I - 1) * PartSize;
2287
2288 Register SrcReg = MI.getOperand(i: I).getReg();
2289 assert(MRI.getType(SrcReg) == LLT::scalar(PartSize));
2290
2291 auto ZextInput = MIRBuilder.buildZExt(Res: WideTy, Op: SrcReg);
2292
2293 Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg :
2294 MRI.createGenericVirtualRegister(Ty: WideTy);
2295
2296 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: Offset);
2297 auto Shl = MIRBuilder.buildShl(Dst: WideTy, Src0: ZextInput, Src1: ShiftAmt);
2298 MIRBuilder.buildOr(Dst: NextResult, Src0: ResultReg, Src1: Shl);
2299 ResultReg = NextResult;
2300 }
2301
2302 if (WideSize > DstSize)
2303 MIRBuilder.buildTrunc(Res: DstReg, Op: ResultReg);
2304 else if (DstTy.isPointer())
2305 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: ResultReg);
2306 else if (DstTy != WideTy)
2307 MIRBuilder.buildBitcast(Dst: DstReg, Src: ResultReg);
2308
2309 MI.eraseFromParent();
2310 return Legalized;
2311 }
2312
2313 // Unmerge the original values to the GCD type, and recombine to the next
2314 // multiple greater than the original type.
2315 //
2316 // %3:_(s12) = G_MERGE_VALUES %0:_(s4), %1:_(s4), %2:_(s4) -> s6
2317 // %4:_(s2), %5:_(s2) = G_UNMERGE_VALUES %0
2318 // %6:_(s2), %7:_(s2) = G_UNMERGE_VALUES %1
2319 // %8:_(s2), %9:_(s2) = G_UNMERGE_VALUES %2
2320 // %10:_(s6) = G_MERGE_VALUES %4, %5, %6
2321 // %11:_(s6) = G_MERGE_VALUES %7, %8, %9
2322 // %12:_(s12) = G_MERGE_VALUES %10, %11
2323 //
2324 // Padding with undef if necessary:
2325 //
2326 // %2:_(s8) = G_MERGE_VALUES %0:_(s4), %1:_(s4) -> s6
2327 // %3:_(s2), %4:_(s2) = G_UNMERGE_VALUES %0
2328 // %5:_(s2), %6:_(s2) = G_UNMERGE_VALUES %1
2329 // %7:_(s2) = G_IMPLICIT_DEF
2330 // %8:_(s6) = G_MERGE_VALUES %3, %4, %5
2331 // %9:_(s6) = G_MERGE_VALUES %6, %7, %7
2332 // %10:_(s12) = G_MERGE_VALUES %8, %9
2333
2334 const int GCD = std::gcd(m: SrcSize, n: WideSize);
2335 LLT GCDTy = WideTy.changeElementSize(NewEltSize: GCD);
2336
2337 SmallVector<Register, 8> NewMergeRegs;
2338 SmallVector<Register, 8> Unmerges;
2339 LLT WideDstTy = WideTy.changeElementSize(NewEltSize: NumMerge * WideSize);
2340
2341 // Decompose the original operands if they don't evenly divide.
2342 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands())) {
2343 Register SrcReg = MO.getReg();
2344 if (GCD == SrcSize) {
2345 Unmerges.push_back(Elt: SrcReg);
2346 } else {
2347 auto Unmerge = MIRBuilder.buildUnmerge(Res: GCDTy, Op: SrcReg);
2348 for (int J = 0, JE = Unmerge->getNumOperands() - 1; J != JE; ++J)
2349 Unmerges.push_back(Elt: Unmerge.getReg(Idx: J));
2350 }
2351 }
2352
2353 // Pad with undef to the next size that is a multiple of the requested size.
2354 if (static_cast<int>(Unmerges.size()) != NumMerge * WideSize) {
2355 Register UndefReg = MIRBuilder.buildUndef(Res: GCDTy).getReg(Idx: 0);
2356 for (int I = Unmerges.size(); I != NumMerge * WideSize; ++I)
2357 Unmerges.push_back(Elt: UndefReg);
2358 }
2359
2360 const int PartsPerGCD = WideSize / GCD;
2361
2362 // Build merges of each piece.
2363 ArrayRef<Register> Slicer(Unmerges);
2364 for (int I = 0; I != NumMerge; ++I, Slicer = Slicer.drop_front(N: PartsPerGCD)) {
2365 auto Merge =
2366 MIRBuilder.buildMergeLikeInstr(Res: WideTy, Ops: Slicer.take_front(N: PartsPerGCD));
2367 NewMergeRegs.push_back(Elt: Merge.getReg(Idx: 0));
2368 }
2369
2370 // A truncate may be necessary if the requested type doesn't evenly divide the
2371 // original result type.
2372 if (DstTy.getSizeInBits() == WideDstTy.getSizeInBits()) {
2373 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: NewMergeRegs);
2374 } else {
2375 auto FinalMerge = MIRBuilder.buildMergeLikeInstr(Res: WideDstTy, Ops: NewMergeRegs);
2376 MIRBuilder.buildTrunc(Res: DstReg, Op: FinalMerge.getReg(Idx: 0));
2377 }
2378
2379 MI.eraseFromParent();
2380 return Legalized;
2381}
2382
2383LegalizerHelper::LegalizeResult
2384LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx,
2385 LLT WideTy) {
2386 if (TypeIdx != 0)
2387 return UnableToLegalize;
2388
2389 int NumDst = MI.getNumOperands() - 1;
2390 Register SrcReg = MI.getOperand(i: NumDst).getReg();
2391 LLT SrcTy = MRI.getType(Reg: SrcReg);
2392 if (SrcTy.isVector())
2393 return UnableToLegalize;
2394
2395 Register Dst0Reg = MI.getOperand(i: 0).getReg();
2396 LLT DstTy = MRI.getType(Reg: Dst0Reg);
2397 if (!DstTy.isScalar())
2398 return UnableToLegalize;
2399
2400 if (WideTy.getSizeInBits() >= SrcTy.getSizeInBits()) {
2401 if (SrcTy.isPointer()) {
2402 const DataLayout &DL = MIRBuilder.getDataLayout();
2403 if (DL.isNonIntegralAddressSpace(AddrSpace: SrcTy.getAddressSpace())) {
2404 LLVM_DEBUG(
2405 dbgs() << "Not casting non-integral address space integer\n");
2406 return UnableToLegalize;
2407 }
2408
2409 SrcTy = LLT::scalar(SizeInBits: SrcTy.getSizeInBits());
2410 SrcReg = MIRBuilder.buildPtrToInt(Dst: SrcTy, Src: SrcReg).getReg(Idx: 0);
2411 }
2412
2413 // Widen SrcTy to WideTy. This does not affect the result, but since the
2414 // user requested this size, it is probably better handled than SrcTy and
2415 // should reduce the total number of legalization artifacts.
2416 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
2417 SrcTy = WideTy;
2418 SrcReg = MIRBuilder.buildAnyExt(Res: WideTy, Op: SrcReg).getReg(Idx: 0);
2419 }
2420
2421 // Theres no unmerge type to target. Directly extract the bits from the
2422 // source type
2423 unsigned DstSize = DstTy.getSizeInBits();
2424
2425 if (SrcTy.isFloat()) {
2426 SrcReg = coerceToInteger(Val: SrcReg);
2427 SrcTy = MRI.getType(Reg: SrcReg);
2428 }
2429
2430 MIRBuilder.buildTrunc(Res: Dst0Reg, Op: SrcReg);
2431 for (int I = 1; I != NumDst; ++I) {
2432 auto ShiftAmt = MIRBuilder.buildConstant(Res: SrcTy, Val: DstSize * I);
2433 auto Shr = MIRBuilder.buildLShr(Dst: SrcTy, Src0: SrcReg, Src1: ShiftAmt);
2434 MIRBuilder.buildTrunc(Res: MI.getOperand(i: I), Op: Shr);
2435 }
2436
2437 MI.eraseFromParent();
2438 return Legalized;
2439 }
2440
2441 // Extend the source to a wider type.
2442 LLT LCMTy = getLCMType(OrigTy: SrcTy, TargetTy: WideTy);
2443
2444 Register WideSrc = SrcReg;
2445 if (LCMTy.getSizeInBits() != SrcTy.getSizeInBits()) {
2446 // TODO: If this is an integral address space, cast to integer and anyext.
2447 if (SrcTy.isPointer()) {
2448 LLVM_DEBUG(dbgs() << "Widening pointer source types not implemented\n");
2449 return UnableToLegalize;
2450 }
2451
2452 WideSrc = MIRBuilder.buildAnyExt(Res: LCMTy, Op: WideSrc).getReg(Idx: 0);
2453 }
2454
2455 auto Unmerge = MIRBuilder.buildUnmerge(Res: WideTy, Op: WideSrc);
2456
2457 // Create a sequence of unmerges and merges to the original results. Since we
2458 // may have widened the source, we will need to pad the results with dead defs
2459 // to cover the source register.
2460 // e.g. widen s48 to s64:
2461 // %1:_(s48), %2:_(s48) = G_UNMERGE_VALUES %0:_(s96)
2462 //
2463 // =>
2464 // %4:_(s192) = G_ANYEXT %0:_(s96)
2465 // %5:_(s64), %6, %7 = G_UNMERGE_VALUES %4 ; Requested unmerge
2466 // ; unpack to GCD type, with extra dead defs
2467 // %8:_(s16), %9, %10, %11 = G_UNMERGE_VALUES %5:_(s64)
2468 // %12:_(s16), %13, dead %14, dead %15 = G_UNMERGE_VALUES %6:_(s64)
2469 // dead %16:_(s16), dead %17, dead %18, dead %18 = G_UNMERGE_VALUES %7:_(s64)
2470 // %1:_(s48) = G_MERGE_VALUES %8:_(s16), %9, %10 ; Remerge to destination
2471 // %2:_(s48) = G_MERGE_VALUES %11:_(s16), %12, %13 ; Remerge to destination
2472 const LLT GCDTy = getGCDType(OrigTy: WideTy, TargetTy: DstTy);
2473 const int NumUnmerge = Unmerge->getNumOperands() - 1;
2474 const int PartsPerRemerge = DstTy.getSizeInBits() / GCDTy.getSizeInBits();
2475
2476 // Directly unmerge to the destination without going through a GCD type
2477 // if possible
2478 if (PartsPerRemerge == 1) {
2479 const int PartsPerUnmerge = WideTy.getSizeInBits() / DstTy.getSizeInBits();
2480
2481 for (int I = 0; I != NumUnmerge; ++I) {
2482 auto MIB = MIRBuilder.buildInstr(Opcode: TargetOpcode::G_UNMERGE_VALUES);
2483
2484 for (int J = 0; J != PartsPerUnmerge; ++J) {
2485 int Idx = I * PartsPerUnmerge + J;
2486 if (Idx < NumDst)
2487 MIB.addDef(RegNo: MI.getOperand(i: Idx).getReg());
2488 else {
2489 // Create dead def for excess components.
2490 MIB.addDef(RegNo: MRI.createGenericVirtualRegister(Ty: DstTy));
2491 }
2492 }
2493
2494 MIB.addUse(RegNo: Unmerge.getReg(Idx: I));
2495 }
2496 } else {
2497 SmallVector<Register, 16> Parts;
2498 for (int J = 0; J != NumUnmerge; ++J)
2499 extractGCDType(Parts, GCDTy, SrcReg: Unmerge.getReg(Idx: J));
2500
2501 SmallVector<Register, 8> RemergeParts;
2502 for (int I = 0; I != NumDst; ++I) {
2503 for (int J = 0; J < PartsPerRemerge; ++J) {
2504 const int Idx = I * PartsPerRemerge + J;
2505 RemergeParts.emplace_back(Args&: Parts[Idx]);
2506 }
2507
2508 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: I).getReg(), Ops: RemergeParts);
2509 RemergeParts.clear();
2510 }
2511 }
2512
2513 MI.eraseFromParent();
2514 return Legalized;
2515}
2516
2517LegalizerHelper::LegalizeResult
2518LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx,
2519 LLT WideTy) {
2520 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
2521 unsigned Offset = MI.getOperand(i: 2).getImm();
2522
2523 if (TypeIdx == 0) {
2524 if (SrcTy.isVector() || DstTy.isVector())
2525 return UnableToLegalize;
2526
2527 SrcOp Src(SrcReg);
2528 if (SrcTy.isPointer()) {
2529 // Extracts from pointers can be handled only if they are really just
2530 // simple integers.
2531 const DataLayout &DL = MIRBuilder.getDataLayout();
2532 if (DL.isNonIntegralAddressSpace(AddrSpace: SrcTy.getAddressSpace()))
2533 return UnableToLegalize;
2534
2535 LLT SrcAsIntTy = LLT::scalar(SizeInBits: SrcTy.getSizeInBits());
2536 Src = MIRBuilder.buildPtrToInt(Dst: SrcAsIntTy, Src);
2537 SrcTy = SrcAsIntTy;
2538 }
2539
2540 if (DstTy.isPointer())
2541 return UnableToLegalize;
2542
2543 if (Offset == 0) {
2544 // Avoid a shift in the degenerate case.
2545 MIRBuilder.buildTrunc(Res: DstReg,
2546 Op: MIRBuilder.buildAnyExtOrTrunc(Res: WideTy, Op: Src));
2547 MI.eraseFromParent();
2548 return Legalized;
2549 }
2550
2551 // Do a shift in the source type.
2552 LLT ShiftTy = SrcTy;
2553 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
2554 Src = MIRBuilder.buildAnyExt(Res: WideTy, Op: Src);
2555 ShiftTy = WideTy;
2556 }
2557
2558 auto LShr = MIRBuilder.buildLShr(
2559 Dst: ShiftTy, Src0: Src, Src1: MIRBuilder.buildConstant(Res: ShiftTy, Val: Offset));
2560 MIRBuilder.buildTrunc(Res: DstReg, Op: LShr);
2561 MI.eraseFromParent();
2562 return Legalized;
2563 }
2564
2565 if (SrcTy.isScalar()) {
2566 Observer.changingInstr(MI);
2567 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2568 Observer.changedInstr(MI);
2569 return Legalized;
2570 }
2571
2572 if (!SrcTy.isVector())
2573 return UnableToLegalize;
2574
2575 if (DstTy != SrcTy.getElementType())
2576 return UnableToLegalize;
2577
2578 if (Offset % SrcTy.getScalarSizeInBits() != 0)
2579 return UnableToLegalize;
2580
2581 Observer.changingInstr(MI);
2582 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2583
2584 MI.getOperand(i: 2).setImm((WideTy.getSizeInBits() / SrcTy.getSizeInBits()) *
2585 Offset);
2586 widenScalarDst(MI, WideTy: WideTy.getScalarType(), OpIdx: 0);
2587 Observer.changedInstr(MI);
2588 return Legalized;
2589}
2590
2591LegalizerHelper::LegalizeResult
2592LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx,
2593 LLT WideTy) {
2594 if (TypeIdx != 0 || WideTy.isVector())
2595 return UnableToLegalize;
2596 Observer.changingInstr(MI);
2597 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2598 widenScalarDst(MI, WideTy);
2599 Observer.changedInstr(MI);
2600 return Legalized;
2601}
2602
2603LegalizerHelper::LegalizeResult
2604LegalizerHelper::widenScalarAddSubOverflow(MachineInstr &MI, unsigned TypeIdx,
2605 LLT WideTy) {
2606 unsigned Opcode;
2607 unsigned ExtOpcode;
2608 std::optional<Register> CarryIn;
2609 switch (MI.getOpcode()) {
2610 default:
2611 llvm_unreachable("Unexpected opcode!");
2612 case TargetOpcode::G_SADDO:
2613 Opcode = TargetOpcode::G_ADD;
2614 ExtOpcode = TargetOpcode::G_SEXT;
2615 break;
2616 case TargetOpcode::G_SSUBO:
2617 Opcode = TargetOpcode::G_SUB;
2618 ExtOpcode = TargetOpcode::G_SEXT;
2619 break;
2620 case TargetOpcode::G_UADDO:
2621 Opcode = TargetOpcode::G_ADD;
2622 ExtOpcode = TargetOpcode::G_ZEXT;
2623 break;
2624 case TargetOpcode::G_USUBO:
2625 Opcode = TargetOpcode::G_SUB;
2626 ExtOpcode = TargetOpcode::G_ZEXT;
2627 break;
2628 case TargetOpcode::G_SADDE:
2629 Opcode = TargetOpcode::G_UADDE;
2630 ExtOpcode = TargetOpcode::G_SEXT;
2631 CarryIn = MI.getOperand(i: 4).getReg();
2632 break;
2633 case TargetOpcode::G_SSUBE:
2634 Opcode = TargetOpcode::G_USUBE;
2635 ExtOpcode = TargetOpcode::G_SEXT;
2636 CarryIn = MI.getOperand(i: 4).getReg();
2637 break;
2638 case TargetOpcode::G_UADDE:
2639 Opcode = TargetOpcode::G_UADDE;
2640 ExtOpcode = TargetOpcode::G_ZEXT;
2641 CarryIn = MI.getOperand(i: 4).getReg();
2642 break;
2643 case TargetOpcode::G_USUBE:
2644 Opcode = TargetOpcode::G_USUBE;
2645 ExtOpcode = TargetOpcode::G_ZEXT;
2646 CarryIn = MI.getOperand(i: 4).getReg();
2647 break;
2648 }
2649
2650 if (TypeIdx == 1) {
2651 unsigned BoolExtOp = MIRBuilder.getBoolExtOp(IsVec: WideTy.isVector(), IsFP: false);
2652
2653 Observer.changingInstr(MI);
2654 if (CarryIn)
2655 widenScalarSrc(MI, WideTy, OpIdx: 4, ExtOpcode: BoolExtOp);
2656 widenScalarDst(MI, WideTy, OpIdx: 1);
2657
2658 Observer.changedInstr(MI);
2659 return Legalized;
2660 }
2661
2662 auto LHSExt = MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {WideTy}, SrcOps: {MI.getOperand(i: 2)});
2663 auto RHSExt = MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {WideTy}, SrcOps: {MI.getOperand(i: 3)});
2664 // Do the arithmetic in the larger type.
2665 Register NewOp;
2666 if (CarryIn) {
2667 LLT CarryOutTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
2668 NewOp = MIRBuilder
2669 .buildInstr(Opc: Opcode, DstOps: {WideTy, CarryOutTy},
2670 SrcOps: {LHSExt, RHSExt, *CarryIn})
2671 .getReg(Idx: 0);
2672 } else {
2673 NewOp = MIRBuilder.buildInstr(Opc: Opcode, DstOps: {WideTy}, SrcOps: {LHSExt, RHSExt}).getReg(Idx: 0);
2674 }
2675 LLT OrigTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2676 auto TruncOp = MIRBuilder.buildTrunc(Res: OrigTy, Op: NewOp);
2677 auto ExtOp = MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {WideTy}, SrcOps: {TruncOp});
2678 // There is no overflow if the ExtOp is the same as NewOp.
2679 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: MI.getOperand(i: 1), Op0: NewOp, Op1: ExtOp);
2680 // Now trunc the NewOp to the original result.
2681 MIRBuilder.buildTrunc(Res: MI.getOperand(i: 0), Op: NewOp);
2682 MI.eraseFromParent();
2683 return Legalized;
2684}
2685
2686LegalizerHelper::LegalizeResult
2687LegalizerHelper::widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx,
2688 LLT WideTy) {
2689 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SADDSAT ||
2690 MI.getOpcode() == TargetOpcode::G_SSUBSAT ||
2691 MI.getOpcode() == TargetOpcode::G_SSHLSAT;
2692 bool IsShift = MI.getOpcode() == TargetOpcode::G_SSHLSAT ||
2693 MI.getOpcode() == TargetOpcode::G_USHLSAT;
2694 // We can convert this to:
2695 // 1. Any extend iN to iM
2696 // 2. SHL by M-N
2697 // 3. [US][ADD|SUB|SHL]SAT
2698 // 4. L/ASHR by M-N
2699 //
2700 // It may be more efficient to lower this to a min and a max operation in
2701 // the higher precision arithmetic if the promoted operation isn't legal,
2702 // but this decision is up to the target's lowering request.
2703 Register DstReg = MI.getOperand(i: 0).getReg();
2704
2705 unsigned NewBits = WideTy.getScalarSizeInBits();
2706 unsigned SHLAmount = NewBits - MRI.getType(Reg: DstReg).getScalarSizeInBits();
2707
2708 // Shifts must zero-extend the RHS to preserve the unsigned quantity, and
2709 // must not left shift the RHS to preserve the shift amount.
2710 auto LHS = MIRBuilder.buildAnyExt(Res: WideTy, Op: MI.getOperand(i: 1));
2711 auto RHS = IsShift ? MIRBuilder.buildZExt(Res: WideTy, Op: MI.getOperand(i: 2))
2712 : MIRBuilder.buildAnyExt(Res: WideTy, Op: MI.getOperand(i: 2));
2713 auto ShiftK = MIRBuilder.buildConstant(Res: WideTy, Val: SHLAmount);
2714 auto ShiftL = MIRBuilder.buildShl(Dst: WideTy, Src0: LHS, Src1: ShiftK);
2715 auto ShiftR = IsShift ? RHS : MIRBuilder.buildShl(Dst: WideTy, Src0: RHS, Src1: ShiftK);
2716
2717 auto WideInst = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {WideTy},
2718 SrcOps: {ShiftL, ShiftR}, Flags: MI.getFlags());
2719
2720 // Use a shift that will preserve the number of sign bits when the trunc is
2721 // folded away.
2722 auto Result = IsSigned ? MIRBuilder.buildAShr(Dst: WideTy, Src0: WideInst, Src1: ShiftK)
2723 : MIRBuilder.buildLShr(Dst: WideTy, Src0: WideInst, Src1: ShiftK);
2724
2725 MIRBuilder.buildTrunc(Res: DstReg, Op: Result);
2726 MI.eraseFromParent();
2727 return Legalized;
2728}
2729
2730LegalizerHelper::LegalizeResult
2731LegalizerHelper::widenScalarMulo(MachineInstr &MI, unsigned TypeIdx,
2732 LLT WideTy) {
2733 if (TypeIdx == 1) {
2734 Observer.changingInstr(MI);
2735 widenScalarDst(MI, WideTy, OpIdx: 1);
2736 Observer.changedInstr(MI);
2737 return Legalized;
2738 }
2739
2740 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SMULO;
2741 auto [Result, OriginalOverflow, LHS, RHS] = MI.getFirst4Regs();
2742 LLT SrcTy = MRI.getType(Reg: LHS);
2743 LLT OverflowTy = MRI.getType(Reg: OriginalOverflow);
2744 unsigned SrcBitWidth = SrcTy.getScalarSizeInBits();
2745
2746 // To determine if the result overflowed in the larger type, we extend the
2747 // input to the larger type, do the multiply (checking if it overflows),
2748 // then also check the high bits of the result to see if overflow happened
2749 // there.
2750 unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
2751 auto LeftOperand = MIRBuilder.buildInstr(Opc: ExtOp, DstOps: {WideTy}, SrcOps: {LHS});
2752 auto RightOperand = MIRBuilder.buildInstr(Opc: ExtOp, DstOps: {WideTy}, SrcOps: {RHS});
2753
2754 // Multiplication cannot overflow if the WideTy is >= 2 * original width,
2755 // so we don't need to check the overflow result of larger type Mulo.
2756 bool WideMulCanOverflow = WideTy.getScalarSizeInBits() < 2 * SrcBitWidth;
2757
2758 unsigned MulOpc =
2759 WideMulCanOverflow ? MI.getOpcode() : (unsigned)TargetOpcode::G_MUL;
2760
2761 MachineInstrBuilder Mulo;
2762 if (WideMulCanOverflow)
2763 Mulo = MIRBuilder.buildInstr(Opc: MulOpc, DstOps: {WideTy, OverflowTy},
2764 SrcOps: {LeftOperand, RightOperand});
2765 else
2766 Mulo = MIRBuilder.buildInstr(Opc: MulOpc, DstOps: {WideTy}, SrcOps: {LeftOperand, RightOperand});
2767
2768 auto Mul = Mulo->getOperand(i: 0);
2769 MIRBuilder.buildTrunc(Res: Result, Op: Mul);
2770
2771 MachineInstrBuilder ExtResult;
2772 // Overflow occurred if it occurred in the larger type, or if the high part
2773 // of the result does not zero/sign-extend the low part. Check this second
2774 // possibility first.
2775 if (IsSigned) {
2776 // For signed, overflow occurred when the high part does not sign-extend
2777 // the low part.
2778 ExtResult = MIRBuilder.buildSExtInReg(Res: WideTy, Op: Mul, ImmOp: SrcBitWidth);
2779 } else {
2780 // Unsigned overflow occurred when the high part does not zero-extend the
2781 // low part.
2782 ExtResult = MIRBuilder.buildZExtInReg(Res: WideTy, Op: Mul, ImmOp: SrcBitWidth);
2783 }
2784
2785 if (WideMulCanOverflow) {
2786 auto Overflow =
2787 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: OverflowTy, Op0: Mul, Op1: ExtResult);
2788 // Finally check if the multiplication in the larger type itself overflowed.
2789 MIRBuilder.buildOr(Dst: OriginalOverflow, Src0: Mulo->getOperand(i: 1), Src1: Overflow);
2790 } else {
2791 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: OriginalOverflow, Op0: Mul, Op1: ExtResult);
2792 }
2793 MI.eraseFromParent();
2794 return Legalized;
2795}
2796
2797LegalizerHelper::LegalizeResult
2798LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
2799 unsigned Opcode = MI.getOpcode();
2800 switch (Opcode) {
2801 default:
2802 return UnableToLegalize;
2803 case TargetOpcode::G_ATOMICRMW_XCHG:
2804 case TargetOpcode::G_ATOMICRMW_ADD:
2805 case TargetOpcode::G_ATOMICRMW_SUB:
2806 case TargetOpcode::G_ATOMICRMW_AND:
2807 case TargetOpcode::G_ATOMICRMW_OR:
2808 case TargetOpcode::G_ATOMICRMW_XOR:
2809 case TargetOpcode::G_ATOMICRMW_MIN:
2810 case TargetOpcode::G_ATOMICRMW_MAX:
2811 case TargetOpcode::G_ATOMICRMW_UMIN:
2812 case TargetOpcode::G_ATOMICRMW_UMAX:
2813 assert(TypeIdx == 0 && "atomicrmw with second scalar type");
2814 Observer.changingInstr(MI);
2815 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
2816 widenScalarDst(MI, WideTy, OpIdx: 0);
2817 Observer.changedInstr(MI);
2818 return Legalized;
2819 case TargetOpcode::G_ATOMIC_CMPXCHG:
2820 assert(TypeIdx == 0 && "G_ATOMIC_CMPXCHG with second scalar type");
2821 Observer.changingInstr(MI);
2822 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
2823 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ANYEXT);
2824 widenScalarDst(MI, WideTy, OpIdx: 0);
2825 Observer.changedInstr(MI);
2826 return Legalized;
2827 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS:
2828 if (TypeIdx == 0) {
2829 Observer.changingInstr(MI);
2830 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ANYEXT);
2831 widenScalarSrc(MI, WideTy, OpIdx: 4, ExtOpcode: TargetOpcode::G_ANYEXT);
2832 widenScalarDst(MI, WideTy, OpIdx: 0);
2833 Observer.changedInstr(MI);
2834 return Legalized;
2835 }
2836 assert(TypeIdx == 1 &&
2837 "G_ATOMIC_CMPXCHG_WITH_SUCCESS with third scalar type");
2838 Observer.changingInstr(MI);
2839 widenScalarDst(MI, WideTy, OpIdx: 1);
2840 Observer.changedInstr(MI);
2841 return Legalized;
2842 case TargetOpcode::G_EXTRACT:
2843 return widenScalarExtract(MI, TypeIdx, WideTy);
2844 case TargetOpcode::G_INSERT:
2845 return widenScalarInsert(MI, TypeIdx, WideTy);
2846 case TargetOpcode::G_MERGE_VALUES:
2847 return widenScalarMergeValues(MI, TypeIdx, WideTy);
2848 case TargetOpcode::G_UNMERGE_VALUES:
2849 return widenScalarUnmergeValues(MI, TypeIdx, WideTy);
2850 case TargetOpcode::G_SADDO:
2851 case TargetOpcode::G_SSUBO:
2852 case TargetOpcode::G_UADDO:
2853 case TargetOpcode::G_USUBO:
2854 case TargetOpcode::G_SADDE:
2855 case TargetOpcode::G_SSUBE:
2856 case TargetOpcode::G_UADDE:
2857 case TargetOpcode::G_USUBE:
2858 return widenScalarAddSubOverflow(MI, TypeIdx, WideTy);
2859 case TargetOpcode::G_UMULO:
2860 case TargetOpcode::G_SMULO:
2861 return widenScalarMulo(MI, TypeIdx, WideTy);
2862 case TargetOpcode::G_SADDSAT:
2863 case TargetOpcode::G_SSUBSAT:
2864 case TargetOpcode::G_SSHLSAT:
2865 case TargetOpcode::G_UADDSAT:
2866 case TargetOpcode::G_USUBSAT:
2867 case TargetOpcode::G_USHLSAT:
2868 return widenScalarAddSubShlSat(MI, TypeIdx, WideTy);
2869 case TargetOpcode::G_CTTZ:
2870 case TargetOpcode::G_CTTZ_ZERO_POISON:
2871 case TargetOpcode::G_CTLZ:
2872 case TargetOpcode::G_CTLZ_ZERO_POISON:
2873 case TargetOpcode::G_CTLS:
2874 case TargetOpcode::G_CTPOP: {
2875 if (TypeIdx == 0) {
2876 Observer.changingInstr(MI);
2877 widenScalarDst(MI, WideTy, OpIdx: 0);
2878 Observer.changedInstr(MI);
2879 return Legalized;
2880 }
2881
2882 Register SrcReg = MI.getOperand(i: 1).getReg();
2883
2884 // First extend the input.
2885 unsigned ExtOpc;
2886 switch (Opcode) {
2887 case TargetOpcode::G_CTTZ:
2888 case TargetOpcode::G_CTTZ_ZERO_POISON:
2889 case TargetOpcode::G_CTLZ_ZERO_POISON: // poison shifted out below
2890 ExtOpc = TargetOpcode::G_ANYEXT;
2891 break;
2892 case TargetOpcode::G_CTLS:
2893 ExtOpc = TargetOpcode::G_SEXT;
2894 break;
2895 default:
2896 ExtOpc = TargetOpcode::G_ZEXT;
2897 }
2898
2899 auto MIBSrc = MIRBuilder.buildInstr(Opc: ExtOpc, DstOps: {WideTy}, SrcOps: {SrcReg});
2900 LLT CurTy = MRI.getType(Reg: SrcReg);
2901 unsigned NewOpc = Opcode;
2902 if (NewOpc == TargetOpcode::G_CTTZ) {
2903 // The count is the same in the larger type except if the original
2904 // value was zero. This can be handled by setting the bit just off
2905 // the top of the original type.
2906 auto TopBit = APInt::getOneBitSet(numBits: WideTy.getScalarSizeInBits(),
2907 BitNo: CurTy.getScalarSizeInBits());
2908 MIBSrc = MIRBuilder.buildOr(
2909 Dst: WideTy, Src0: MIBSrc, Src1: MIRBuilder.buildConstant(Res: WideTy, Val: TopBit));
2910 // Now we know the operand is non-zero, use the more relaxed opcode.
2911 NewOpc = TargetOpcode::G_CTTZ_ZERO_POISON;
2912 }
2913
2914 unsigned SizeDiff =
2915 WideTy.getScalarSizeInBits() - CurTy.getScalarSizeInBits();
2916
2917 if (Opcode == TargetOpcode::G_CTLZ_ZERO_POISON) {
2918 // An optimization where the result is the CTLZ after the left shift by
2919 // (Difference in widety and current ty), that is,
2920 // MIBSrc = MIBSrc << (sizeinbits(WideTy) - sizeinbits(CurTy))
2921 // Result = ctlz MIBSrc
2922 MIBSrc = MIRBuilder.buildShl(Dst: WideTy, Src0: MIBSrc,
2923 Src1: MIRBuilder.buildConstant(Res: WideTy, Val: SizeDiff));
2924 }
2925
2926 // Perform the operation at the larger size.
2927 auto MIBNewOp = MIRBuilder.buildInstr(Opc: NewOpc, DstOps: {WideTy}, SrcOps: {MIBSrc});
2928 // This is already the correct result for CTPOP and CTTZs
2929 if (Opcode == TargetOpcode::G_CTLZ || Opcode == TargetOpcode::G_CTLS) {
2930 // The correct result is NewOp - (Difference in widety and current ty).
2931 // At this stage SUB is guaranteed to be positive no-wrap,
2932 // that to be used in further KnownBits optimizations for CTLZ.
2933 MIBNewOp = MIRBuilder.buildSub(
2934 Dst: WideTy, Src0: MIBNewOp, Src1: MIRBuilder.buildConstant(Res: WideTy, Val: SizeDiff),
2935 Flags: Opcode == TargetOpcode::G_CTLZ
2936 ? std::optional<unsigned>(MachineInstr::NoUWrap)
2937 : std::nullopt);
2938 }
2939
2940 MIRBuilder.buildZExtOrTrunc(Res: MI.getOperand(i: 0), Op: MIBNewOp);
2941 MI.eraseFromParent();
2942 return Legalized;
2943 }
2944 case TargetOpcode::G_BSWAP: {
2945 Observer.changingInstr(MI);
2946 Register DstReg = MI.getOperand(i: 0).getReg();
2947
2948 Register ShrReg = MRI.createGenericVirtualRegister(Ty: WideTy);
2949 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2950 Register ShiftAmtReg = MRI.createGenericVirtualRegister(Ty: WideTy);
2951 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2952
2953 MI.getOperand(i: 0).setReg(DstExt);
2954
2955 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2956
2957 LLT Ty = MRI.getType(Reg: DstReg);
2958 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
2959 MIRBuilder.buildConstant(Res: ShiftAmtReg, Val: DiffBits);
2960 MIRBuilder.buildLShr(Dst: ShrReg, Src0: DstExt, Src1: ShiftAmtReg);
2961
2962 MIRBuilder.buildTrunc(Res: DstReg, Op: ShrReg);
2963 Observer.changedInstr(MI);
2964 return Legalized;
2965 }
2966 case TargetOpcode::G_BITREVERSE: {
2967 Observer.changingInstr(MI);
2968
2969 Register DstReg = MI.getOperand(i: 0).getReg();
2970 LLT Ty = MRI.getType(Reg: DstReg);
2971 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
2972
2973 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2974 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2975 MI.getOperand(i: 0).setReg(DstExt);
2976 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2977
2978 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: DiffBits);
2979 auto Shift = MIRBuilder.buildLShr(Dst: WideTy, Src0: DstExt, Src1: ShiftAmt);
2980 MIRBuilder.buildTrunc(Res: DstReg, Op: Shift);
2981 Observer.changedInstr(MI);
2982 return Legalized;
2983 }
2984 case TargetOpcode::G_FREEZE:
2985 case TargetOpcode::G_CONSTANT_FOLD_BARRIER:
2986 Observer.changingInstr(MI);
2987 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2988 widenScalarDst(MI, WideTy);
2989 Observer.changedInstr(MI);
2990 return Legalized;
2991
2992 case TargetOpcode::G_ABS:
2993 Observer.changingInstr(MI);
2994 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_SEXT);
2995 widenScalarDst(MI, WideTy);
2996 Observer.changedInstr(MI);
2997 return Legalized;
2998
2999 case TargetOpcode::G_ADD:
3000 case TargetOpcode::G_AND:
3001 case TargetOpcode::G_MUL:
3002 case TargetOpcode::G_OR:
3003 case TargetOpcode::G_XOR:
3004 case TargetOpcode::G_SUB:
3005 case TargetOpcode::G_SHUFFLE_VECTOR:
3006 // Perform operation at larger width (any extension is fines here, high bits
3007 // don't affect the result) and then truncate the result back to the
3008 // original type.
3009 Observer.changingInstr(MI);
3010 // The G_ANYEXTs below leave the new high bits unconstrained, so no-wrap and
3011 // disjoint claims proved at the narrow width no longer hold. Paths that
3012 // widen with value-preserving G_ZEXT/G_SEXT keep their flags.
3013 MI.clearFlags(flags: MachineInstr::NoUWrap | MachineInstr::NoSWrap |
3014 MachineInstr::Disjoint);
3015 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3016 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
3017 widenScalarDst(MI, WideTy);
3018 Observer.changedInstr(MI);
3019 return Legalized;
3020
3021 case TargetOpcode::G_SBFX:
3022 case TargetOpcode::G_UBFX:
3023 Observer.changingInstr(MI);
3024
3025 if (TypeIdx == 0) {
3026 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3027 widenScalarDst(MI, WideTy);
3028 } else {
3029 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3030 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ZEXT);
3031 }
3032
3033 Observer.changedInstr(MI);
3034 return Legalized;
3035
3036 case TargetOpcode::G_SHL:
3037 Observer.changingInstr(MI);
3038
3039 if (TypeIdx == 0) {
3040 // Widening the result with G_ANYEXT invalidates the no-wrap flags, as in
3041 // the G_ADD/G_SUB/G_MUL case above. TypeIdx 1 widens only the shift
3042 // amount, which is value-preserving, so it keeps them.
3043 MI.clearFlags(flags: MachineInstr::NoUWrap | MachineInstr::NoSWrap);
3044 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3045 widenScalarDst(MI, WideTy);
3046 } else {
3047 assert(TypeIdx == 1);
3048 // The "number of bits to shift" operand must preserve its value as an
3049 // unsigned integer:
3050 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3051 }
3052
3053 Observer.changedInstr(MI);
3054 return Legalized;
3055
3056 case TargetOpcode::G_ROTR:
3057 case TargetOpcode::G_ROTL:
3058 if (TypeIdx != 1)
3059 return UnableToLegalize;
3060
3061 Observer.changingInstr(MI);
3062 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3063 Observer.changedInstr(MI);
3064 return Legalized;
3065
3066 case TargetOpcode::G_SDIV:
3067 case TargetOpcode::G_SREM:
3068 case TargetOpcode::G_SMIN:
3069 case TargetOpcode::G_SMAX:
3070 case TargetOpcode::G_ABDS:
3071 Observer.changingInstr(MI);
3072 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_SEXT);
3073 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_SEXT);
3074 widenScalarDst(MI, WideTy);
3075 Observer.changedInstr(MI);
3076 return Legalized;
3077
3078 case TargetOpcode::G_SDIVREM:
3079 Observer.changingInstr(MI);
3080 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_SEXT);
3081 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_SEXT);
3082 widenScalarDst(MI, WideTy);
3083 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: --MIRBuilder.getInsertPt());
3084 widenScalarDst(MI, WideTy, OpIdx: 1);
3085 Observer.changedInstr(MI);
3086 return Legalized;
3087
3088 case TargetOpcode::G_ASHR:
3089 case TargetOpcode::G_LSHR:
3090 Observer.changingInstr(MI);
3091
3092 if (TypeIdx == 0) {
3093 unsigned CvtOp = Opcode == TargetOpcode::G_ASHR ? TargetOpcode::G_SEXT
3094 : TargetOpcode::G_ZEXT;
3095
3096 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: CvtOp);
3097 widenScalarDst(MI, WideTy);
3098 } else {
3099 assert(TypeIdx == 1);
3100 // The "number of bits to shift" operand must preserve its value as an
3101 // unsigned integer:
3102 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3103 }
3104
3105 Observer.changedInstr(MI);
3106 return Legalized;
3107 case TargetOpcode::G_UDIV:
3108 case TargetOpcode::G_UREM:
3109 case TargetOpcode::G_ABDU:
3110 Observer.changingInstr(MI);
3111 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ZEXT);
3112 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3113 widenScalarDst(MI, WideTy);
3114 Observer.changedInstr(MI);
3115 return Legalized;
3116 case TargetOpcode::G_UDIVREM:
3117 Observer.changingInstr(MI);
3118 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3119 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ZEXT);
3120 widenScalarDst(MI, WideTy);
3121 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: --MIRBuilder.getInsertPt());
3122 widenScalarDst(MI, WideTy, OpIdx: 1);
3123 Observer.changedInstr(MI);
3124 return Legalized;
3125 case TargetOpcode::G_UMIN:
3126 case TargetOpcode::G_UMAX: {
3127 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3128
3129 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
3130 unsigned ExtOpc =
3131 TLI.isSExtCheaperThanZExt(FromTy: getApproximateEVTForLLT(Ty, Ctx),
3132 ToTy: getApproximateEVTForLLT(Ty: WideTy, Ctx))
3133 ? TargetOpcode::G_SEXT
3134 : TargetOpcode::G_ZEXT;
3135
3136 Observer.changingInstr(MI);
3137 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: ExtOpc);
3138 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: ExtOpc);
3139 widenScalarDst(MI, WideTy);
3140 Observer.changedInstr(MI);
3141 return Legalized;
3142 }
3143
3144 case TargetOpcode::G_SELECT:
3145 Observer.changingInstr(MI);
3146 if (TypeIdx == 0) {
3147 // Perform operation at larger width (any extension is fine here, high
3148 // bits don't affect the result) and then truncate the result back to the
3149 // original type.
3150 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
3151 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ANYEXT);
3152 widenScalarDst(MI, WideTy);
3153 } else {
3154 bool IsVec = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).isVector();
3155 // Explicit extension is required here since high bits affect the result.
3156 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: MIRBuilder.getBoolExtOp(IsVec, IsFP: false));
3157 }
3158 Observer.changedInstr(MI);
3159 return Legalized;
3160
3161 case TargetOpcode::G_FPEXT:
3162 if (TypeIdx != 1)
3163 return UnableToLegalize;
3164
3165 Observer.changingInstr(MI);
3166 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3167 Observer.changedInstr(MI);
3168 return Legalized;
3169 case TargetOpcode::G_FPTOSI:
3170 case TargetOpcode::G_FPTOUI:
3171 case TargetOpcode::G_INTRINSIC_LRINT:
3172 case TargetOpcode::G_INTRINSIC_LLRINT:
3173 case TargetOpcode::G_IS_FPCLASS:
3174 Observer.changingInstr(MI);
3175
3176 if (TypeIdx == 0)
3177 widenScalarDst(MI, WideTy);
3178 else
3179 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3180
3181 Observer.changedInstr(MI);
3182 return Legalized;
3183 case TargetOpcode::G_SITOFP:
3184 Observer.changingInstr(MI);
3185
3186 if (TypeIdx == 0)
3187 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3188 else
3189 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_SEXT);
3190
3191 Observer.changedInstr(MI);
3192 return Legalized;
3193 case TargetOpcode::G_UITOFP:
3194 Observer.changingInstr(MI);
3195
3196 if (TypeIdx == 0)
3197 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3198 else
3199 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ZEXT);
3200
3201 Observer.changedInstr(MI);
3202 return Legalized;
3203 case TargetOpcode::G_FPTOSI_SAT:
3204 case TargetOpcode::G_FPTOUI_SAT:
3205 Observer.changingInstr(MI);
3206
3207 if (TypeIdx == 0) {
3208 Register OldDst = MI.getOperand(i: 0).getReg();
3209 LLT Ty = MRI.getType(Reg: OldDst);
3210 Register ExtReg = MRI.createGenericVirtualRegister(Ty: WideTy);
3211 Register NewDst;
3212 MI.getOperand(i: 0).setReg(ExtReg);
3213 uint64_t ShortBits = Ty.getScalarSizeInBits();
3214 uint64_t WideBits = WideTy.getScalarSizeInBits();
3215 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
3216 if (Opcode == TargetOpcode::G_FPTOSI_SAT) {
3217 // z = i16 fptosi_sat(a)
3218 // ->
3219 // x = i32 fptosi_sat(a)
3220 // y = smin(x, 32767)
3221 // z = smax(y, -32768)
3222 auto MaxVal = MIRBuilder.buildConstant(
3223 Res: WideTy, Val: APInt::getSignedMaxValue(numBits: ShortBits).sext(width: WideBits));
3224 auto MinVal = MIRBuilder.buildConstant(
3225 Res: WideTy, Val: APInt::getSignedMinValue(numBits: ShortBits).sext(width: WideBits));
3226 Register MidReg =
3227 MIRBuilder.buildSMin(Dst: WideTy, Src0: ExtReg, Src1: MaxVal).getReg(Idx: 0);
3228 NewDst = MIRBuilder.buildSMax(Dst: WideTy, Src0: MidReg, Src1: MinVal).getReg(Idx: 0);
3229 } else {
3230 // z = i16 fptoui_sat(a)
3231 // ->
3232 // x = i32 fptoui_sat(a)
3233 // y = smin(x, 65535)
3234 auto MaxVal = MIRBuilder.buildConstant(
3235 Res: WideTy, Val: APInt::getAllOnes(numBits: ShortBits).zext(width: WideBits));
3236 NewDst = MIRBuilder.buildUMin(Dst: WideTy, Src0: ExtReg, Src1: MaxVal).getReg(Idx: 0);
3237 }
3238 MIRBuilder.buildTrunc(Res: OldDst, Op: NewDst);
3239 } else
3240 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3241
3242 Observer.changedInstr(MI);
3243 return Legalized;
3244 case TargetOpcode::G_LOAD:
3245 case TargetOpcode::G_SEXTLOAD:
3246 case TargetOpcode::G_ZEXTLOAD:
3247 case TargetOpcode::G_FPEXTLOAD:
3248 Observer.changingInstr(MI);
3249 widenScalarDst(MI, WideTy);
3250 Observer.changedInstr(MI);
3251 return Legalized;
3252
3253 case TargetOpcode::G_STORE: {
3254 if (TypeIdx != 0)
3255 return UnableToLegalize;
3256
3257 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3258 assert(!Ty.isPointerOrPointerVector() && "Can't widen type");
3259 if (!Ty.isScalar()) {
3260 // We need to widen the vector element type.
3261 Observer.changingInstr(MI);
3262 widenScalarSrc(MI, WideTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_ANYEXT);
3263 // We also need to adjust the MMO to turn this into a truncating store.
3264 MachineMemOperand &MMO = **MI.memoperands_begin();
3265 MachineFunction &MF = MIRBuilder.getMF();
3266 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo: MMO.getPointerInfo(), Ty);
3267 MI.setMemRefs(MF, MemRefs: {NewMMO});
3268 Observer.changedInstr(MI);
3269 return Legalized;
3270 }
3271
3272 Observer.changingInstr(MI);
3273
3274 unsigned ExtType = Ty.getScalarSizeInBits() == 1 ?
3275 TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT;
3276 widenScalarSrc(MI, WideTy, OpIdx: 0, ExtOpcode: ExtType);
3277
3278 Observer.changedInstr(MI);
3279 return Legalized;
3280 }
3281 case TargetOpcode::G_FPTRUNCSTORE:
3282 if (TypeIdx != 0)
3283 return UnableToLegalize;
3284 Observer.changingInstr(MI);
3285 widenScalarSrc(MI, WideTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_FPEXT);
3286 Observer.changedInstr(MI);
3287 return Legalized;
3288 case TargetOpcode::G_CONSTANT: {
3289 MachineOperand &SrcMO = MI.getOperand(i: 1);
3290 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
3291 unsigned ExtOpc = LI.getExtOpcodeForWideningConstant(
3292 SmallTy: MRI.getType(Reg: MI.getOperand(i: 0).getReg()));
3293 assert((ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::G_SEXT ||
3294 ExtOpc == TargetOpcode::G_ANYEXT) &&
3295 "Illegal Extend");
3296 const APInt &SrcVal = SrcMO.getCImm()->getValue();
3297 const APInt &Val = (ExtOpc == TargetOpcode::G_SEXT)
3298 ? SrcVal.sext(width: WideTy.getSizeInBits())
3299 : SrcVal.zext(width: WideTy.getSizeInBits());
3300 Observer.changingInstr(MI);
3301 SrcMO.setCImm(ConstantInt::get(Context&: Ctx, V: Val));
3302
3303 widenScalarDst(MI, WideTy);
3304 Observer.changedInstr(MI);
3305 return Legalized;
3306 }
3307 case TargetOpcode::G_FCONSTANT: {
3308 // To avoid changing the bits of the constant due to extension to a larger
3309 // type and then using G_FPTRUNC, we simply convert to a G_CONSTANT.
3310 MachineOperand &SrcMO = MI.getOperand(i: 1);
3311 APInt Val = SrcMO.getFPImm()->getValueAPF().bitcastToAPInt();
3312 MIRBuilder.setInstrAndDebugLoc(MI);
3313 auto IntCst = MIRBuilder.buildConstant(Res: MI.getOperand(i: 0).getReg(), Val);
3314 widenScalarDst(MI&: *IntCst, WideTy, OpIdx: 0, TruncOpcode: TargetOpcode::G_TRUNC);
3315 MI.eraseFromParent();
3316 return Legalized;
3317 }
3318 case TargetOpcode::G_IMPLICIT_DEF: {
3319 Observer.changingInstr(MI);
3320 widenScalarDst(MI, WideTy);
3321 Observer.changedInstr(MI);
3322 return Legalized;
3323 }
3324 case TargetOpcode::G_BRCOND:
3325 Observer.changingInstr(MI);
3326 widenScalarSrc(MI, WideTy, OpIdx: 0, ExtOpcode: MIRBuilder.getBoolExtOp(IsVec: false, IsFP: false));
3327 Observer.changedInstr(MI);
3328 return Legalized;
3329
3330 case TargetOpcode::G_FCMP:
3331 Observer.changingInstr(MI);
3332 if (TypeIdx == 0)
3333 widenScalarDst(MI, WideTy);
3334 else {
3335 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 2);
3336 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 3);
3337 }
3338 Observer.changedInstr(MI);
3339 return Legalized;
3340
3341 case TargetOpcode::G_ICMP:
3342 Observer.changingInstr(MI);
3343 if (TypeIdx == 0)
3344 widenScalarDst(MI, WideTy);
3345 else {
3346 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
3347 CmpInst::Predicate Pred =
3348 static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
3349
3350 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
3351 unsigned ExtOpcode =
3352 (CmpInst::isSigned(Pred) ||
3353 TLI.isSExtCheaperThanZExt(FromTy: getApproximateEVTForLLT(Ty: SrcTy, Ctx),
3354 ToTy: getApproximateEVTForLLT(Ty: WideTy, Ctx)))
3355 ? TargetOpcode::G_SEXT
3356 : TargetOpcode::G_ZEXT;
3357 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode);
3358 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode);
3359 }
3360 Observer.changedInstr(MI);
3361 return Legalized;
3362
3363 case TargetOpcode::G_PTR_ADD:
3364 assert(TypeIdx == 1 && "unable to legalize pointer of G_PTR_ADD");
3365 Observer.changingInstr(MI);
3366 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_SEXT);
3367 Observer.changedInstr(MI);
3368 return Legalized;
3369
3370 case TargetOpcode::G_PHI: {
3371 assert(TypeIdx == 0 && "Expecting only Idx 0");
3372
3373 Observer.changingInstr(MI);
3374 for (unsigned I = 1; I < MI.getNumOperands(); I += 2) {
3375 MachineBasicBlock &OpMBB = *MI.getOperand(i: I + 1).getMBB();
3376 MIRBuilder.setInsertPt(MBB&: OpMBB, II: OpMBB.getFirstTerminatorForward());
3377 widenScalarSrc(MI, WideTy, OpIdx: I, ExtOpcode: TargetOpcode::G_ANYEXT);
3378 }
3379
3380 MachineBasicBlock &MBB = *MI.getParent();
3381 MIRBuilder.setInsertPt(MBB, II: --MBB.getFirstNonPHI());
3382 widenScalarDst(MI, WideTy);
3383 Observer.changedInstr(MI);
3384 return Legalized;
3385 }
3386 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
3387 if (TypeIdx == 0) {
3388 Register VecReg = MI.getOperand(i: 1).getReg();
3389 LLT VecTy = MRI.getType(Reg: VecReg);
3390 Observer.changingInstr(MI);
3391
3392 widenScalarSrc(MI, WideTy: LLT::vector(EC: VecTy.getElementCount(), ScalarTy: WideTy), OpIdx: 1,
3393 ExtOpcode: TargetOpcode::G_ANYEXT);
3394
3395 widenScalarDst(MI, WideTy, OpIdx: 0);
3396 Observer.changedInstr(MI);
3397 return Legalized;
3398 }
3399
3400 if (TypeIdx != 2)
3401 return UnableToLegalize;
3402 Observer.changingInstr(MI);
3403 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3404 Observer.changedInstr(MI);
3405 return Legalized;
3406 }
3407 case TargetOpcode::G_INSERT_VECTOR_ELT: {
3408 if (TypeIdx == 0) {
3409 Observer.changingInstr(MI);
3410 const LLT WideEltTy = WideTy.getElementType();
3411
3412 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3413 widenScalarSrc(MI, WideTy: WideEltTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
3414 widenScalarDst(MI, WideTy, OpIdx: 0);
3415 Observer.changedInstr(MI);
3416 return Legalized;
3417 }
3418
3419 if (TypeIdx == 1) {
3420 Observer.changingInstr(MI);
3421
3422 Register VecReg = MI.getOperand(i: 1).getReg();
3423 LLT VecTy = MRI.getType(Reg: VecReg);
3424 LLT WideVecTy = VecTy.changeVectorElementType(NewEltTy: WideTy);
3425
3426 widenScalarSrc(MI, WideTy: WideVecTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3427 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
3428 widenScalarDst(MI, WideTy: WideVecTy, OpIdx: 0);
3429 Observer.changedInstr(MI);
3430 return Legalized;
3431 }
3432
3433 if (TypeIdx == 2) {
3434 Observer.changingInstr(MI);
3435 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ZEXT);
3436 Observer.changedInstr(MI);
3437 return Legalized;
3438 }
3439
3440 return UnableToLegalize;
3441 }
3442 case TargetOpcode::G_FADD:
3443 case TargetOpcode::G_FMUL:
3444 case TargetOpcode::G_FSUB:
3445 case TargetOpcode::G_FMA:
3446 case TargetOpcode::G_FMAD:
3447 case TargetOpcode::G_FNEG:
3448 case TargetOpcode::G_FABS:
3449 case TargetOpcode::G_FCANONICALIZE:
3450 case TargetOpcode::G_FMINNUM:
3451 case TargetOpcode::G_FMAXNUM:
3452 case TargetOpcode::G_FMINNUM_IEEE:
3453 case TargetOpcode::G_FMAXNUM_IEEE:
3454 case TargetOpcode::G_FMINIMUM:
3455 case TargetOpcode::G_FMAXIMUM:
3456 case TargetOpcode::G_FMINIMUMNUM:
3457 case TargetOpcode::G_FMAXIMUMNUM:
3458 case TargetOpcode::G_FDIV:
3459 case TargetOpcode::G_FREM:
3460 case TargetOpcode::G_FCEIL:
3461 case TargetOpcode::G_FFLOOR:
3462 case TargetOpcode::G_FCOS:
3463 case TargetOpcode::G_FSIN:
3464 case TargetOpcode::G_FTAN:
3465 case TargetOpcode::G_FACOS:
3466 case TargetOpcode::G_FASIN:
3467 case TargetOpcode::G_FATAN:
3468 case TargetOpcode::G_FATAN2:
3469 case TargetOpcode::G_FCOSH:
3470 case TargetOpcode::G_FSINH:
3471 case TargetOpcode::G_FTANH:
3472 case TargetOpcode::G_FLOG10:
3473 case TargetOpcode::G_FLOG:
3474 case TargetOpcode::G_FLOG2:
3475 case TargetOpcode::G_FRINT:
3476 case TargetOpcode::G_FNEARBYINT:
3477 case TargetOpcode::G_FSQRT:
3478 case TargetOpcode::G_FEXP:
3479 case TargetOpcode::G_FEXP2:
3480 case TargetOpcode::G_FEXP10:
3481 case TargetOpcode::G_FPOW:
3482 case TargetOpcode::G_INTRINSIC_TRUNC:
3483 case TargetOpcode::G_INTRINSIC_ROUND:
3484 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
3485 assert(TypeIdx == 0);
3486 Observer.changingInstr(MI);
3487
3488 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I)
3489 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: I);
3490
3491 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3492 Observer.changedInstr(MI);
3493 return Legalized;
3494 case TargetOpcode::G_FMODF: {
3495 Observer.changingInstr(MI);
3496 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 2);
3497
3498 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 1);
3499 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: --MIRBuilder.getInsertPt());
3500 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3501 Observer.changedInstr(MI);
3502 return Legalized;
3503 }
3504 case TargetOpcode::G_FPOWI:
3505 case TargetOpcode::G_FLDEXP:
3506 case TargetOpcode::G_STRICT_FLDEXP: {
3507 if (TypeIdx == 0) {
3508 if (Opcode == TargetOpcode::G_STRICT_FLDEXP)
3509 return UnableToLegalize;
3510
3511 Observer.changingInstr(MI);
3512 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3513 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3514 Observer.changedInstr(MI);
3515 return Legalized;
3516 }
3517
3518 if (TypeIdx == 1) {
3519 // For some reason SelectionDAG tries to promote to a libcall without
3520 // actually changing the integer type for promotion.
3521 Observer.changingInstr(MI);
3522 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_SEXT);
3523 Observer.changedInstr(MI);
3524 return Legalized;
3525 }
3526
3527 return UnableToLegalize;
3528 }
3529 case TargetOpcode::G_FFREXP: {
3530 Observer.changingInstr(MI);
3531
3532 if (TypeIdx == 0) {
3533 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 2);
3534 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3535 } else {
3536 widenScalarDst(MI, WideTy, OpIdx: 1);
3537 }
3538
3539 Observer.changedInstr(MI);
3540 return Legalized;
3541 }
3542 case TargetOpcode::G_LROUND:
3543 case TargetOpcode::G_LLROUND:
3544 Observer.changingInstr(MI);
3545
3546 if (TypeIdx == 0)
3547 widenScalarDst(MI, WideTy);
3548 else
3549 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3550
3551 Observer.changedInstr(MI);
3552 return Legalized;
3553
3554 case TargetOpcode::G_INTTOPTR:
3555 if (TypeIdx != 1)
3556 return UnableToLegalize;
3557
3558 Observer.changingInstr(MI);
3559 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ZEXT);
3560 Observer.changedInstr(MI);
3561 return Legalized;
3562 case TargetOpcode::G_PTRTOINT:
3563 if (TypeIdx != 0)
3564 return UnableToLegalize;
3565
3566 Observer.changingInstr(MI);
3567 widenScalarDst(MI, WideTy, OpIdx: 0);
3568 Observer.changedInstr(MI);
3569 return Legalized;
3570 case TargetOpcode::G_BUILD_VECTOR: {
3571 Observer.changingInstr(MI);
3572
3573 const LLT WideEltTy = TypeIdx == 1 ? WideTy : WideTy.getElementType();
3574 for (int I = 1, E = MI.getNumOperands(); I != E; ++I)
3575 widenScalarSrc(MI, WideTy: WideEltTy, OpIdx: I, ExtOpcode: TargetOpcode::G_ANYEXT);
3576
3577 // Avoid changing the result vector type if the source element type was
3578 // requested.
3579 if (TypeIdx == 1) {
3580 MI.setDesc(MIRBuilder.getTII().get(Opcode: TargetOpcode::G_BUILD_VECTOR_TRUNC));
3581 } else {
3582 widenScalarDst(MI, WideTy, OpIdx: 0);
3583 }
3584
3585 Observer.changedInstr(MI);
3586 return Legalized;
3587 }
3588 case TargetOpcode::G_SEXT_INREG:
3589 if (TypeIdx != 0)
3590 return UnableToLegalize;
3591
3592 Observer.changingInstr(MI);
3593 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3594 widenScalarDst(MI, WideTy, OpIdx: 0, TruncOpcode: TargetOpcode::G_TRUNC);
3595 Observer.changedInstr(MI);
3596 return Legalized;
3597 case TargetOpcode::G_PTRMASK: {
3598 if (TypeIdx != 1)
3599 return UnableToLegalize;
3600 Observer.changingInstr(MI);
3601 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3602 Observer.changedInstr(MI);
3603 return Legalized;
3604 }
3605 case TargetOpcode::G_VECREDUCE_ADD: {
3606 if (TypeIdx != 1)
3607 return UnableToLegalize;
3608 Observer.changingInstr(MI);
3609 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3610 widenScalarDst(MI, WideTy: WideTy.getScalarType(), OpIdx: 0, TruncOpcode: TargetOpcode::G_TRUNC);
3611 Observer.changedInstr(MI);
3612 return Legalized;
3613 }
3614 case TargetOpcode::G_VECREDUCE_FADD:
3615 case TargetOpcode::G_VECREDUCE_FMUL:
3616 case TargetOpcode::G_VECREDUCE_FMIN:
3617 case TargetOpcode::G_VECREDUCE_FMAX:
3618 case TargetOpcode::G_VECREDUCE_FMINIMUM:
3619 case TargetOpcode::G_VECREDUCE_FMAXIMUM: {
3620 if (TypeIdx != 0)
3621 return UnableToLegalize;
3622 Observer.changingInstr(MI);
3623 Register VecReg = MI.getOperand(i: 1).getReg();
3624 LLT VecTy = MRI.getType(Reg: VecReg);
3625 LLT WideVecTy = VecTy.changeElementType(NewEltTy: WideTy);
3626 widenScalarSrcUsingFPExt(MI, WideTy: WideVecTy, OpIdx: 1);
3627 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3628 Observer.changedInstr(MI);
3629 return Legalized;
3630 }
3631 case TargetOpcode::G_VSCALE: {
3632 MachineOperand &SrcMO = MI.getOperand(i: 1);
3633 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
3634 const APInt &SrcVal = SrcMO.getCImm()->getValue();
3635 // The CImm is always a signed value
3636 const APInt Val = SrcVal.sext(width: WideTy.getSizeInBits());
3637 Observer.changingInstr(MI);
3638 SrcMO.setCImm(ConstantInt::get(Context&: Ctx, V: Val));
3639 widenScalarDst(MI, WideTy);
3640 Observer.changedInstr(MI);
3641 return Legalized;
3642 }
3643 case TargetOpcode::G_SPLAT_VECTOR: {
3644 if (TypeIdx != 1)
3645 return UnableToLegalize;
3646
3647 Observer.changingInstr(MI);
3648 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3649 Observer.changedInstr(MI);
3650 return Legalized;
3651 }
3652 case TargetOpcode::G_INSERT_SUBVECTOR: {
3653 if (TypeIdx != 0)
3654 return UnableToLegalize;
3655
3656 GInsertSubvector &IS = cast<GInsertSubvector>(Val&: MI);
3657 Register BigVec = IS.getBigVec();
3658 Register SubVec = IS.getSubVec();
3659
3660 LLT SubVecTy = MRI.getType(Reg: SubVec);
3661 LLT SubVecWideTy = SubVecTy.changeElementType(NewEltTy: WideTy.getElementType());
3662
3663 // Widen the G_INSERT_SUBVECTOR
3664 auto BigZExt = MIRBuilder.buildZExt(Res: WideTy, Op: BigVec);
3665 auto SubZExt = MIRBuilder.buildZExt(Res: SubVecWideTy, Op: SubVec);
3666 auto WideInsert = MIRBuilder.buildInsertSubvector(Res: WideTy, Src0: BigZExt, Src1: SubZExt,
3667 Index: IS.getIndexImm());
3668
3669 // Truncate back down
3670 auto SplatZero = MIRBuilder.buildSplatVector(
3671 Res: WideTy, Val: MIRBuilder.buildConstant(Res: WideTy.getElementType(), Val: 0));
3672 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_NE, Res: IS.getReg(Idx: 0), Op0: WideInsert,
3673 Op1: SplatZero);
3674
3675 MI.eraseFromParent();
3676
3677 return Legalized;
3678 }
3679 case TargetOpcode::G_BITCAST:
3680 if (WideTy.isVector())
3681 return UnableToLegalize;
3682 Observer.changingInstr(MI);
3683 if (TypeIdx == 0)
3684 widenScalarDst(MI, WideTy, OpIdx: 0, TruncOpcode: TargetOpcode::G_TRUNC);
3685 else
3686 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3687 Observer.changedInstr(MI);
3688
3689 Register Dst = MI.getOperand(i: 0).getReg();
3690 Register Src = MI.getOperand(i: 1).getReg();
3691 if (MRI.getType(Reg: Dst) == MRI.getType(Reg: Src)) {
3692 Observer.changingAllUsesOfReg(MRI, Reg: Dst);
3693 MRI.replaceRegWith(FromReg: Dst, ToReg: Src);
3694 Observer.finishedChangingAllUsesOfReg();
3695 MI.eraseFromParent();
3696 }
3697
3698 return Legalized;
3699 }
3700}
3701
3702static void getUnmergePieces(SmallVectorImpl<Register> &Pieces,
3703 MachineIRBuilder &B, Register Src, LLT Ty) {
3704 auto Unmerge = B.buildUnmerge(Res: Ty, Op: Src);
3705 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
3706 Pieces.push_back(Elt: Unmerge.getReg(Idx: I));
3707}
3708
3709static void emitLoadFromConstantPool(Register DstReg, const Constant *ConstVal,
3710 MachineIRBuilder &MIRBuilder) {
3711 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
3712 MachineFunction &MF = MIRBuilder.getMF();
3713 const DataLayout &DL = MIRBuilder.getDataLayout();
3714 unsigned AddrSpace = DL.getDefaultGlobalsAddressSpace();
3715 LLT AddrPtrTy = LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL.getPointerSizeInBits(AS: AddrSpace));
3716 LLT DstLLT = MRI.getType(Reg: DstReg);
3717
3718 Align Alignment(DL.getABITypeAlign(Ty: ConstVal->getType()));
3719
3720 auto Addr = MIRBuilder.buildConstantPool(
3721 Res: AddrPtrTy,
3722 Idx: MF.getConstantPool()->getConstantPoolIndex(C: ConstVal, Alignment));
3723
3724 MachineMemOperand *MMO =
3725 MF.getMachineMemOperand(PtrInfo: MachinePointerInfo::getConstantPool(MF),
3726 F: MachineMemOperand::MOLoad, MemTy: DstLLT, BaseAlignment: Alignment);
3727
3728 MIRBuilder.buildLoadInstr(Opcode: TargetOpcode::G_LOAD, Res: DstReg, Addr, MMO&: *MMO);
3729}
3730
3731LegalizerHelper::LegalizeResult
3732LegalizerHelper::lowerConstant(MachineInstr &MI) {
3733 const MachineOperand &ConstOperand = MI.getOperand(i: 1);
3734 const Constant *ConstantVal = ConstOperand.getCImm();
3735
3736 emitLoadFromConstantPool(DstReg: MI.getOperand(i: 0).getReg(), ConstVal: ConstantVal, MIRBuilder);
3737 MI.eraseFromParent();
3738
3739 return Legalized;
3740}
3741
3742LegalizerHelper::LegalizeResult
3743LegalizerHelper::lowerFConstant(MachineInstr &MI) {
3744 const MachineOperand &ConstOperand = MI.getOperand(i: 1);
3745 const Constant *ConstantVal = ConstOperand.getFPImm();
3746
3747 emitLoadFromConstantPool(DstReg: MI.getOperand(i: 0).getReg(), ConstVal: ConstantVal, MIRBuilder);
3748 MI.eraseFromParent();
3749
3750 return Legalized;
3751}
3752
3753LegalizerHelper::LegalizeResult
3754LegalizerHelper::lowerBitcast(MachineInstr &MI) {
3755 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
3756 if (SrcTy.isVector()) {
3757 LLT SrcEltTy = SrcTy.getElementType();
3758 SmallVector<Register, 8> SrcRegs;
3759
3760 if (DstTy.isVector()) {
3761 LLT DstEltTy = DstTy.getElementType();
3762 ElementCount DstEC = DstTy.getElementCount();
3763 ElementCount SrcEC = SrcTy.getElementCount();
3764
3765 if (!SrcEC.isKnownMultipleOf(RHS: DstEC) && !DstEC.isKnownMultipleOf(RHS: SrcEC)) {
3766 // Split non-integer element ratio bitcast
3767 //
3768 // %1:_(<3 x s16>) = G_BITCAST %0:_(<2 x s24>)
3769 //
3770 // =>
3771 //
3772 // %2:_(<6 x s8>) = G_BITCAST %0:_(<2 x s24>)
3773 // %1:_(<3 x s16>) = G_BITCAST %2:_(<6 x s8>)
3774 unsigned SrcEltSize = SrcEltTy.getScalarSizeInBits();
3775 unsigned PieceSize =
3776 std::gcd(m: SrcEltSize, n: DstEltTy.getScalarSizeInBits());
3777 LLT PieceTy = LLT::integer(SizeInBits: PieceSize);
3778
3779 if (!PieceTy.isByteSized()) {
3780 // Split bitcast whose pieces are not whole bytes through a scalar
3781 //
3782 // %1:_(<3 x s8>) = G_BITCAST %0:_(<2 x s12>)
3783 //
3784 // =>
3785 //
3786 // %2:_(s24) = G_BITCAST %0:_(<2 x s12>)
3787 // %1:_(<3 x s8>) = G_BITCAST %2:_(s24)
3788 LLT ScalarTy = LLT::integer(SizeInBits: SrcTy.getSizeInBits());
3789 Register ScalarReg = MIRBuilder.buildBitcast(Dst: ScalarTy, Src).getReg(Idx: 0);
3790 MIRBuilder.buildBitcast(Dst, Src: ScalarReg);
3791 MI.eraseFromParent();
3792 return Legalized;
3793 }
3794
3795 LLT PiecesVecTy =
3796 LLT::vector(EC: SrcEC * (SrcEltSize / PieceSize), ScalarTy: PieceTy);
3797 Register PiecesReg =
3798 MIRBuilder.buildBitcast(Dst: PiecesVecTy, Src).getReg(Idx: 0);
3799 MIRBuilder.buildBitcast(Dst, Src: PiecesReg);
3800 MI.eraseFromParent();
3801 return Legalized;
3802 }
3803
3804 unsigned NumDstElt = DstEC.getKnownMinValue();
3805 unsigned NumSrcElt = SrcEC.getKnownMinValue();
3806
3807 LLT DstCastTy = DstEltTy; // Intermediate bitcast result type
3808 LLT SrcPartTy = SrcEltTy; // Original unmerge result type.
3809
3810 // If there's an element size mismatch, insert intermediate casts to match
3811 // the result element type.
3812 if (NumSrcElt < NumDstElt) { // Source element type is larger.
3813 // %1:_(<4 x s8>) = G_BITCAST %0:_(<2 x s16>)
3814 //
3815 // =>
3816 //
3817 // %2:_(s16), %3:_(s16) = G_UNMERGE_VALUES %0
3818 // %3:_(<2 x s8>) = G_BITCAST %2
3819 // %4:_(<2 x s8>) = G_BITCAST %3
3820 // %1:_(<4 x s8>) = G_CONCAT_VECTORS %3, %4
3821 DstCastTy = DstTy.changeVectorElementCount(
3822 EC: ElementCount::getFixed(MinVal: NumDstElt / NumSrcElt));
3823 SrcPartTy = SrcEltTy;
3824 } else if (NumSrcElt > NumDstElt) { // Source element type is smaller.
3825 //
3826 // %1:_(<2 x s16>) = G_BITCAST %0:_(<4 x s8>)
3827 //
3828 // =>
3829 //
3830 // %2:_(<2 x s8>), %3:_(<2 x s8>) = G_UNMERGE_VALUES %0
3831 // %3:_(s16) = G_BITCAST %2
3832 // %4:_(s16) = G_BITCAST %3
3833 // %1:_(<2 x s16>) = G_BUILD_VECTOR %3, %4
3834 SrcPartTy = SrcTy.changeVectorElementCount(
3835 EC: ElementCount::getFixed(MinVal: NumSrcElt / NumDstElt));
3836 DstCastTy = DstEltTy;
3837 }
3838
3839 getUnmergePieces(Pieces&: SrcRegs, B&: MIRBuilder, Src, Ty: SrcPartTy);
3840 for (Register &SrcReg : SrcRegs)
3841 SrcReg = MIRBuilder.buildBitcast(Dst: DstCastTy, Src: SrcReg).getReg(Idx: 0);
3842 } else
3843 getUnmergePieces(Pieces&: SrcRegs, B&: MIRBuilder, Src, Ty: SrcEltTy);
3844
3845 MIRBuilder.buildMergeLikeInstr(Res: Dst, Ops: SrcRegs);
3846 MI.eraseFromParent();
3847 return Legalized;
3848 }
3849
3850 if (DstTy.isVector()) {
3851 SmallVector<Register, 8> SrcRegs;
3852 getUnmergePieces(Pieces&: SrcRegs, B&: MIRBuilder, Src, Ty: DstTy.getElementType());
3853 MIRBuilder.buildMergeLikeInstr(Res: Dst, Ops: SrcRegs);
3854 MI.eraseFromParent();
3855 return Legalized;
3856 }
3857
3858 return UnableToLegalize;
3859}
3860
3861/// Figure out the bit offset into a register when coercing a vector index for
3862/// the wide element type. This is only for the case when promoting vector to
3863/// one with larger elements.
3864//
3865///
3866/// %offset_idx = G_AND %idx, ~(-1 << Log2(DstEltSize / SrcEltSize))
3867/// %offset_bits = G_SHL %offset_idx, Log2(SrcEltSize)
3868static Register getBitcastWiderVectorElementOffset(MachineIRBuilder &B,
3869 Register Idx,
3870 unsigned NewEltSize,
3871 unsigned OldEltSize) {
3872 const unsigned Log2EltRatio = Log2_32(Value: NewEltSize / OldEltSize);
3873 LLT IdxTy = B.getMRI()->getType(Reg: Idx);
3874
3875 // Now figure out the amount we need to shift to get the target bits.
3876 auto OffsetMask = B.buildConstant(
3877 Res: IdxTy, Val: ~(APInt::getAllOnes(numBits: IdxTy.getSizeInBits()) << Log2EltRatio));
3878 auto OffsetIdx = B.buildAnd(Dst: IdxTy, Src0: Idx, Src1: OffsetMask);
3879 return B.buildShl(Dst: IdxTy, Src0: OffsetIdx,
3880 Src1: B.buildConstant(Res: IdxTy, Val: Log2_32(Value: OldEltSize))).getReg(Idx: 0);
3881}
3882
3883/// Perform a G_EXTRACT_VECTOR_ELT in a different sized vector element. If this
3884/// is casting to a vector with a smaller element size, perform multiple element
3885/// extracts and merge the results. If this is coercing to a vector with larger
3886/// elements, index the bitcasted vector and extract the target element with bit
3887/// operations. This is intended to force the indexing in the native register
3888/// size for architectures that can dynamically index the register file.
3889LegalizerHelper::LegalizeResult
3890LegalizerHelper::bitcastExtractVectorElt(MachineInstr &MI, unsigned TypeIdx,
3891 LLT CastTy) {
3892 if (TypeIdx != 1)
3893 return UnableToLegalize;
3894
3895 auto [Dst, DstTy, SrcVec, SrcVecTy, Idx, IdxTy] = MI.getFirst3RegLLTs();
3896
3897 LLT SrcEltTy = SrcVecTy.getElementType();
3898 unsigned NewNumElts = CastTy.isVector() ? CastTy.getNumElements() : 1;
3899 unsigned OldNumElts = SrcVecTy.getNumElements();
3900
3901 LLT NewEltTy = CastTy.getScalarType();
3902 Register CastVec = MIRBuilder.buildBitcast(Dst: CastTy, Src: SrcVec).getReg(Idx: 0);
3903
3904 const unsigned NewEltSize = NewEltTy.getSizeInBits();
3905 const unsigned OldEltSize = SrcEltTy.getSizeInBits();
3906 if (NewNumElts > OldNumElts) {
3907 // Decreasing the vector element size
3908 //
3909 // e.g. i64 = extract_vector_elt x:v2i64, y:i32
3910 // =>
3911 // v4i32:castx = bitcast x:v2i64
3912 //
3913 // i64 = bitcast
3914 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
3915 // (i32 (extract_vector_elt castx, (2 * y + 1)))
3916 //
3917 if (NewNumElts % OldNumElts != 0)
3918 return UnableToLegalize;
3919
3920 // Type of the intermediate result vector.
3921 const unsigned NewEltsPerOldElt = NewNumElts / OldNumElts;
3922 LLT MidTy =
3923 CastTy.changeElementCount(EC: ElementCount::getFixed(MinVal: NewEltsPerOldElt));
3924
3925 auto NewEltsPerOldEltK = MIRBuilder.buildConstant(Res: IdxTy, Val: NewEltsPerOldElt);
3926
3927 SmallVector<Register, 8> NewOps(NewEltsPerOldElt);
3928 auto NewBaseIdx = MIRBuilder.buildMul(Dst: IdxTy, Src0: Idx, Src1: NewEltsPerOldEltK);
3929
3930 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
3931 auto IdxOffset = MIRBuilder.buildConstant(Res: IdxTy, Val: I);
3932 auto TmpIdx = MIRBuilder.buildAdd(Dst: IdxTy, Src0: NewBaseIdx, Src1: IdxOffset);
3933 auto Elt = MIRBuilder.buildExtractVectorElement(Res: NewEltTy, Val: CastVec, Idx: TmpIdx);
3934 NewOps[I] = Elt.getReg(Idx: 0);
3935 }
3936
3937 auto NewVec = MIRBuilder.buildBuildVector(Res: MidTy, Ops: NewOps);
3938 MIRBuilder.buildBitcast(Dst, Src: NewVec);
3939 MI.eraseFromParent();
3940 return Legalized;
3941 }
3942
3943 if (NewNumElts < OldNumElts) {
3944 if (NewEltSize % OldEltSize != 0)
3945 return UnableToLegalize;
3946
3947 // This only depends on powers of 2 because we use bit tricks to figure out
3948 // the bit offset we need to shift to get the target element. A general
3949 // expansion could emit division/multiply.
3950 if (!isPowerOf2_32(Value: NewEltSize / OldEltSize))
3951 return UnableToLegalize;
3952
3953 // Increasing the vector element size.
3954 // %elt:_(small_elt) = G_EXTRACT_VECTOR_ELT %vec:_(<N x small_elt>), %idx
3955 //
3956 // =>
3957 //
3958 // %cast = G_BITCAST %vec
3959 // %scaled_idx = G_LSHR %idx, Log2(DstEltSize / SrcEltSize)
3960 // %wide_elt = G_EXTRACT_VECTOR_ELT %cast, %scaled_idx
3961 // %offset_idx = G_AND %idx, ~(-1 << Log2(DstEltSize / SrcEltSize))
3962 // %offset_bits = G_SHL %offset_idx, Log2(SrcEltSize)
3963 // %elt_bits = G_LSHR %wide_elt, %offset_bits
3964 // %elt = G_TRUNC %elt_bits
3965
3966 const unsigned Log2EltRatio = Log2_32(Value: NewEltSize / OldEltSize);
3967 auto Log2Ratio = MIRBuilder.buildConstant(Res: IdxTy, Val: Log2EltRatio);
3968
3969 // Divide to get the index in the wider element type.
3970 auto ScaledIdx = MIRBuilder.buildLShr(Dst: IdxTy, Src0: Idx, Src1: Log2Ratio);
3971
3972 Register WideElt = CastVec;
3973 if (CastTy.isVector()) {
3974 WideElt = MIRBuilder.buildExtractVectorElement(Res: NewEltTy, Val: CastVec,
3975 Idx: ScaledIdx).getReg(Idx: 0);
3976 }
3977
3978 // Compute the bit offset into the register of the target element.
3979 Register OffsetBits = getBitcastWiderVectorElementOffset(
3980 B&: MIRBuilder, Idx, NewEltSize, OldEltSize);
3981
3982 // Shift the wide element to get the target element.
3983 auto ExtractedBits = MIRBuilder.buildLShr(Dst: NewEltTy, Src0: WideElt, Src1: OffsetBits);
3984 MIRBuilder.buildTrunc(Res: Dst, Op: ExtractedBits);
3985 MI.eraseFromParent();
3986 return Legalized;
3987 }
3988
3989 return UnableToLegalize;
3990}
3991
3992/// Emit code to insert \p InsertReg into \p TargetRet at \p OffsetBits in \p
3993/// TargetReg, while preserving other bits in \p TargetReg.
3994///
3995/// (InsertReg << Offset) | (TargetReg & ~(-1 >> InsertReg.size()) << Offset)
3996static Register buildBitFieldInsert(MachineIRBuilder &B,
3997 Register TargetReg, Register InsertReg,
3998 Register OffsetBits) {
3999 LLT TargetTy = B.getMRI()->getType(Reg: TargetReg);
4000 LLT InsertTy = B.getMRI()->getType(Reg: InsertReg);
4001 auto ZextVal = B.buildZExt(Res: TargetTy, Op: InsertReg);
4002 auto ShiftedInsertVal = B.buildShl(Dst: TargetTy, Src0: ZextVal, Src1: OffsetBits);
4003
4004 // Produce a bitmask of the value to insert
4005 auto EltMask = B.buildConstant(
4006 Res: TargetTy, Val: APInt::getLowBitsSet(numBits: TargetTy.getSizeInBits(),
4007 loBitsSet: InsertTy.getSizeInBits()));
4008 // Shift it into position
4009 auto ShiftedMask = B.buildShl(Dst: TargetTy, Src0: EltMask, Src1: OffsetBits);
4010 auto InvShiftedMask = B.buildNot(Dst: TargetTy, Src0: ShiftedMask);
4011
4012 // Clear out the bits in the wide element
4013 auto MaskedOldElt = B.buildAnd(Dst: TargetTy, Src0: TargetReg, Src1: InvShiftedMask);
4014
4015 // The value to insert has all zeros already, so stick it into the masked
4016 // wide element.
4017 return B.buildOr(Dst: TargetTy, Src0: MaskedOldElt, Src1: ShiftedInsertVal).getReg(Idx: 0);
4018}
4019
4020/// Perform a G_INSERT_VECTOR_ELT in a different sized vector element. If this
4021/// is increasing the element size, perform the indexing in the target element
4022/// type, and use bit operations to insert at the element position. This is
4023/// intended for architectures that can dynamically index the register file and
4024/// want to force indexing in the native register size.
4025LegalizerHelper::LegalizeResult
4026LegalizerHelper::bitcastInsertVectorElt(MachineInstr &MI, unsigned TypeIdx,
4027 LLT CastTy) {
4028 if (TypeIdx != 0)
4029 return UnableToLegalize;
4030
4031 auto [Dst, DstTy, SrcVec, SrcVecTy, Val, ValTy, Idx, IdxTy] =
4032 MI.getFirst4RegLLTs();
4033 LLT VecTy = DstTy;
4034
4035 LLT VecEltTy = VecTy.getElementType();
4036 LLT NewEltTy = CastTy.isVector() ? CastTy.getElementType() : CastTy;
4037 const unsigned NewEltSize = NewEltTy.getSizeInBits();
4038 const unsigned OldEltSize = VecEltTy.getSizeInBits();
4039
4040 unsigned NewNumElts = CastTy.isVector() ? CastTy.getNumElements() : 1;
4041 unsigned OldNumElts = VecTy.getNumElements();
4042
4043 Register CastVec = MIRBuilder.buildBitcast(Dst: CastTy, Src: SrcVec).getReg(Idx: 0);
4044 if (NewNumElts < OldNumElts) {
4045 if (NewEltSize % OldEltSize != 0)
4046 return UnableToLegalize;
4047
4048 // This only depends on powers of 2 because we use bit tricks to figure out
4049 // the bit offset we need to shift to get the target element. A general
4050 // expansion could emit division/multiply.
4051 if (!isPowerOf2_32(Value: NewEltSize / OldEltSize))
4052 return UnableToLegalize;
4053
4054 const unsigned Log2EltRatio = Log2_32(Value: NewEltSize / OldEltSize);
4055 auto Log2Ratio = MIRBuilder.buildConstant(Res: IdxTy, Val: Log2EltRatio);
4056
4057 // Divide to get the index in the wider element type.
4058 auto ScaledIdx = MIRBuilder.buildLShr(Dst: IdxTy, Src0: Idx, Src1: Log2Ratio);
4059
4060 Register ExtractedElt = CastVec;
4061 if (CastTy.isVector()) {
4062 ExtractedElt = MIRBuilder.buildExtractVectorElement(Res: NewEltTy, Val: CastVec,
4063 Idx: ScaledIdx).getReg(Idx: 0);
4064 }
4065
4066 // Compute the bit offset into the register of the target element.
4067 Register OffsetBits = getBitcastWiderVectorElementOffset(
4068 B&: MIRBuilder, Idx, NewEltSize, OldEltSize);
4069
4070 Register InsertedElt = buildBitFieldInsert(B&: MIRBuilder, TargetReg: ExtractedElt,
4071 InsertReg: Val, OffsetBits);
4072 if (CastTy.isVector()) {
4073 InsertedElt = MIRBuilder.buildInsertVectorElement(
4074 Res: CastTy, Val: CastVec, Elt: InsertedElt, Idx: ScaledIdx).getReg(Idx: 0);
4075 }
4076
4077 MIRBuilder.buildBitcast(Dst, Src: InsertedElt);
4078 MI.eraseFromParent();
4079 return Legalized;
4080 }
4081
4082 return UnableToLegalize;
4083}
4084
4085// This attempts to handle G_CONCAT_VECTORS with illegal operands, particularly
4086// those that have smaller than legal operands.
4087//
4088// <16 x s8> = G_CONCAT_VECTORS <4 x s8>, <4 x s8>, <4 x s8>, <4 x s8>
4089//
4090// ===>
4091//
4092// s32 = G_BITCAST <4 x s8>
4093// s32 = G_BITCAST <4 x s8>
4094// s32 = G_BITCAST <4 x s8>
4095// s32 = G_BITCAST <4 x s8>
4096// <4 x s32> = G_BUILD_VECTOR s32, s32, s32, s32
4097// <16 x s8> = G_BITCAST <4 x s32>
4098LegalizerHelper::LegalizeResult
4099LegalizerHelper::bitcastConcatVector(MachineInstr &MI, unsigned TypeIdx,
4100 LLT CastTy) {
4101 // Convert it to CONCAT instruction
4102 auto ConcatMI = dyn_cast<GConcatVectors>(Val: &MI);
4103 if (!ConcatMI) {
4104 return UnableToLegalize;
4105 }
4106
4107 // Check if bitcast is Legal
4108 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
4109 LLT SrcScalTy = CastTy.getScalarType();
4110
4111 // Check if the build vector is Legal
4112 if (!LI.isLegal(Query: {TargetOpcode::G_BUILD_VECTOR, {CastTy, SrcScalTy}})) {
4113 return UnableToLegalize;
4114 }
4115
4116 // Bitcast the sources
4117 SmallVector<Register> BitcastRegs;
4118 for (unsigned i = 0; i < ConcatMI->getNumSources(); i++) {
4119 BitcastRegs.push_back(
4120 Elt: MIRBuilder.buildBitcast(Dst: SrcScalTy, Src: ConcatMI->getSourceReg(I: i))
4121 .getReg(Idx: 0));
4122 }
4123
4124 // Build the scalar values into a vector
4125 Register BuildReg =
4126 MIRBuilder.buildBuildVector(Res: CastTy, Ops: BitcastRegs).getReg(Idx: 0);
4127 MIRBuilder.buildBitcast(Dst: DstReg, Src: BuildReg);
4128
4129 MI.eraseFromParent();
4130 return Legalized;
4131}
4132
4133// This bitcasts a shuffle vector to a different type currently of the same
4134// element size. Mostly used to legalize ptr vectors, where ptrtoint/inttoptr
4135// will be used instead.
4136//
4137// <16 x p0> = G_CONCAT_VECTORS <4 x p0>, <4 x p0>, mask
4138// ===>
4139// <4 x s64> = G_PTRTOINT <4 x p0>
4140// <4 x s64> = G_PTRTOINT <4 x p0>
4141// <16 x s64> = G_CONCAT_VECTORS <4 x s64>, <4 x s64>, mask
4142// <16 x p0> = G_INTTOPTR <16 x s64>
4143LegalizerHelper::LegalizeResult
4144LegalizerHelper::bitcastShuffleVector(MachineInstr &MI, unsigned TypeIdx,
4145 LLT CastTy) {
4146 auto ShuffleMI = cast<GShuffleVector>(Val: &MI);
4147 LLT DstTy = MRI.getType(Reg: ShuffleMI->getReg(Idx: 0));
4148 LLT SrcTy = MRI.getType(Reg: ShuffleMI->getReg(Idx: 1));
4149
4150 // We currently only handle vectors of the same size.
4151 if (TypeIdx != 0 ||
4152 CastTy.getScalarSizeInBits() != DstTy.getScalarSizeInBits() ||
4153 CastTy.getElementCount() != DstTy.getElementCount())
4154 return UnableToLegalize;
4155
4156 LLT NewSrcTy = SrcTy.changeElementType(NewEltTy: CastTy.getScalarType());
4157
4158 auto Inp1 = MIRBuilder.buildCast(Dst: NewSrcTy, Src: ShuffleMI->getReg(Idx: 1));
4159 auto Inp2 = MIRBuilder.buildCast(Dst: NewSrcTy, Src: ShuffleMI->getReg(Idx: 2));
4160 auto Shuf =
4161 MIRBuilder.buildShuffleVector(Res: CastTy, Src1: Inp1, Src2: Inp2, Mask: ShuffleMI->getMask());
4162 MIRBuilder.buildCast(Dst: ShuffleMI->getReg(Idx: 0), Src: Shuf);
4163
4164 MI.eraseFromParent();
4165 return Legalized;
4166}
4167
4168/// This attempts to bitcast G_EXTRACT_SUBVECTOR to CastTy.
4169///
4170/// <vscale x 8 x i1> = G_EXTRACT_SUBVECTOR <vscale x 16 x i1>, N
4171///
4172/// ===>
4173///
4174/// <vscale x 2 x i1> = G_BITCAST <vscale x 16 x i1>
4175/// <vscale x 1 x i8> = G_EXTRACT_SUBVECTOR <vscale x 2 x i1>, N / 8
4176/// <vscale x 8 x i1> = G_BITCAST <vscale x 1 x i8>
4177LegalizerHelper::LegalizeResult
4178LegalizerHelper::bitcastExtractSubvector(MachineInstr &MI, unsigned TypeIdx,
4179 LLT CastTy) {
4180 auto ES = cast<GExtractSubvector>(Val: &MI);
4181
4182 if (!CastTy.isVector())
4183 return UnableToLegalize;
4184
4185 if (TypeIdx != 0)
4186 return UnableToLegalize;
4187
4188 Register Dst = ES->getReg(Idx: 0);
4189 Register Src = ES->getSrcVec();
4190 uint64_t Idx = ES->getIndexImm();
4191
4192 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4193
4194 LLT DstTy = MRI.getType(Reg: Dst);
4195 LLT SrcTy = MRI.getType(Reg: Src);
4196 ElementCount DstTyEC = DstTy.getElementCount();
4197 ElementCount SrcTyEC = SrcTy.getElementCount();
4198 auto DstTyMinElts = DstTyEC.getKnownMinValue();
4199 auto SrcTyMinElts = SrcTyEC.getKnownMinValue();
4200
4201 if (DstTy == CastTy)
4202 return Legalized;
4203
4204 if (DstTy.getSizeInBits() != CastTy.getSizeInBits())
4205 return UnableToLegalize;
4206
4207 unsigned CastEltSize = CastTy.getElementType().getSizeInBits();
4208 unsigned DstEltSize = DstTy.getElementType().getSizeInBits();
4209 if (CastEltSize < DstEltSize)
4210 return UnableToLegalize;
4211
4212 auto AdjustAmt = CastEltSize / DstEltSize;
4213 if (Idx % AdjustAmt != 0 || DstTyMinElts % AdjustAmt != 0 ||
4214 SrcTyMinElts % AdjustAmt != 0)
4215 return UnableToLegalize;
4216
4217 Idx /= AdjustAmt;
4218 SrcTy = LLT::vector(EC: SrcTyEC.divideCoefficientBy(RHS: AdjustAmt), ScalarSizeInBits: AdjustAmt);
4219 auto CastVec = MIRBuilder.buildBitcast(Dst: SrcTy, Src);
4220 auto PromotedES = MIRBuilder.buildExtractSubvector(Res: CastTy, Src: CastVec, Index: Idx);
4221 MIRBuilder.buildBitcast(Dst, Src: PromotedES);
4222
4223 ES->eraseFromParent();
4224 return Legalized;
4225}
4226
4227/// This attempts to bitcast G_INSERT_SUBVECTOR to CastTy.
4228///
4229/// <vscale x 16 x i1> = G_INSERT_SUBVECTOR <vscale x 16 x i1>,
4230/// <vscale x 8 x i1>,
4231/// N
4232///
4233/// ===>
4234///
4235/// <vscale x 2 x i8> = G_BITCAST <vscale x 16 x i1>
4236/// <vscale x 1 x i8> = G_BITCAST <vscale x 8 x i1>
4237/// <vscale x 2 x i8> = G_INSERT_SUBVECTOR <vscale x 2 x i8>,
4238/// <vscale x 1 x i8>, N / 8
4239/// <vscale x 16 x i1> = G_BITCAST <vscale x 2 x i8>
4240LegalizerHelper::LegalizeResult
4241LegalizerHelper::bitcastInsertSubvector(MachineInstr &MI, unsigned TypeIdx,
4242 LLT CastTy) {
4243 auto ES = cast<GInsertSubvector>(Val: &MI);
4244
4245 if (!CastTy.isVector())
4246 return UnableToLegalize;
4247
4248 if (TypeIdx != 0)
4249 return UnableToLegalize;
4250
4251 Register Dst = ES->getReg(Idx: 0);
4252 Register BigVec = ES->getBigVec();
4253 Register SubVec = ES->getSubVec();
4254 uint64_t Idx = ES->getIndexImm();
4255
4256 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4257
4258 LLT DstTy = MRI.getType(Reg: Dst);
4259 LLT BigVecTy = MRI.getType(Reg: BigVec);
4260 LLT SubVecTy = MRI.getType(Reg: SubVec);
4261
4262 if (DstTy == CastTy)
4263 return Legalized;
4264
4265 if (DstTy.getSizeInBits() != CastTy.getSizeInBits())
4266 return UnableToLegalize;
4267
4268 ElementCount DstTyEC = DstTy.getElementCount();
4269 ElementCount BigVecTyEC = BigVecTy.getElementCount();
4270 ElementCount SubVecTyEC = SubVecTy.getElementCount();
4271 auto DstTyMinElts = DstTyEC.getKnownMinValue();
4272 auto BigVecTyMinElts = BigVecTyEC.getKnownMinValue();
4273 auto SubVecTyMinElts = SubVecTyEC.getKnownMinValue();
4274
4275 unsigned CastEltSize = CastTy.getElementType().getSizeInBits();
4276 unsigned DstEltSize = DstTy.getElementType().getSizeInBits();
4277 if (CastEltSize < DstEltSize)
4278 return UnableToLegalize;
4279
4280 auto AdjustAmt = CastEltSize / DstEltSize;
4281 if (Idx % AdjustAmt != 0 || DstTyMinElts % AdjustAmt != 0 ||
4282 BigVecTyMinElts % AdjustAmt != 0 || SubVecTyMinElts % AdjustAmt != 0)
4283 return UnableToLegalize;
4284
4285 Idx /= AdjustAmt;
4286 BigVecTy = LLT::vector(EC: BigVecTyEC.divideCoefficientBy(RHS: AdjustAmt), ScalarSizeInBits: AdjustAmt);
4287 SubVecTy = LLT::vector(EC: SubVecTyEC.divideCoefficientBy(RHS: AdjustAmt), ScalarSizeInBits: AdjustAmt);
4288 auto CastBigVec = MIRBuilder.buildBitcast(Dst: BigVecTy, Src: BigVec);
4289 auto CastSubVec = MIRBuilder.buildBitcast(Dst: SubVecTy, Src: SubVec);
4290 auto PromotedIS =
4291 MIRBuilder.buildInsertSubvector(Res: CastTy, Src0: CastBigVec, Src1: CastSubVec, Index: Idx);
4292 MIRBuilder.buildBitcast(Dst, Src: PromotedIS);
4293
4294 ES->eraseFromParent();
4295 return Legalized;
4296}
4297
4298LegalizerHelper::LegalizeResult LegalizerHelper::lowerLoad(GAnyLoad &LoadMI) {
4299 // Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT
4300 Register DstReg = LoadMI.getDstReg();
4301 Register PtrReg = LoadMI.getPointerReg();
4302 LLT DstTy = MRI.getType(Reg: DstReg);
4303 MachineMemOperand &MMO = LoadMI.getMMO();
4304 LLT MemTy = MMO.getMemoryType();
4305 MachineFunction &MF = MIRBuilder.getMF();
4306
4307 LLT EltTy = MemTy.getScalarType();
4308
4309 unsigned MemSizeInBits = MemTy.getSizeInBits();
4310 unsigned MemStoreSizeInBits = 8 * MemTy.getSizeInBytes();
4311
4312 if (MemSizeInBits != MemStoreSizeInBits) {
4313 if (MemTy.isVector())
4314 return UnableToLegalize;
4315
4316 // Promote to a byte-sized load if not loading an integral number of
4317 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
4318 LLT WideMemTy = EltTy.changeElementSize(NewEltSize: MemStoreSizeInBits);
4319 MachineMemOperand *NewMMO =
4320 MF.getMachineMemOperand(MMO: &MMO, PtrInfo: MMO.getPointerInfo(), Ty: WideMemTy);
4321
4322 Register LoadReg = DstReg;
4323 LLT LoadTy = DstTy;
4324
4325 // If this wasn't already an extending load, we need to widen the result
4326 // register to avoid creating a load with a narrower result than the source.
4327 if (MemStoreSizeInBits > DstTy.getSizeInBits()) {
4328 LoadTy = WideMemTy;
4329 LoadReg = MRI.createGenericVirtualRegister(Ty: WideMemTy);
4330 }
4331
4332 if (isa<GSExtLoad>(Val: LoadMI)) {
4333 auto NewLoad = MIRBuilder.buildLoad(Res: LoadTy, Addr: PtrReg, MMO&: *NewMMO);
4334 MIRBuilder.buildSExtInReg(Res: LoadReg, Op: NewLoad, ImmOp: MemSizeInBits);
4335 } else if (isa<GZExtLoad>(Val: LoadMI) || WideMemTy == LoadTy) {
4336 auto NewLoad = MIRBuilder.buildLoad(Res: LoadTy, Addr: PtrReg, MMO&: *NewMMO);
4337 // The extra bits are guaranteed to be zero, since we stored them that
4338 // way. A zext load from Wide thus automatically gives zext from MemVT.
4339 MIRBuilder.buildAssertZExt(Res: LoadReg, Op: NewLoad, Size: MemSizeInBits);
4340 } else {
4341 MIRBuilder.buildLoad(Res: LoadReg, Addr: PtrReg, MMO&: *NewMMO);
4342 }
4343
4344 if (DstTy != LoadTy)
4345 MIRBuilder.buildTrunc(Res: DstReg, Op: LoadReg);
4346
4347 LoadMI.eraseFromParent();
4348 return Legalized;
4349 }
4350
4351 // Big endian lowering not implemented.
4352 if (MIRBuilder.getDataLayout().isBigEndian())
4353 return UnableToLegalize;
4354
4355 // This load needs splitting into power of 2 sized loads.
4356 //
4357 // Our strategy here is to generate anyextending loads for the smaller
4358 // types up to next power-2 result type, and then combine the two larger
4359 // result values together, before truncating back down to the non-pow-2
4360 // type.
4361 // E.g. v1 = i24 load =>
4362 // v2 = i32 zextload (2 byte)
4363 // v3 = i32 load (1 byte)
4364 // v4 = i32 shl v3, 16
4365 // v5 = i32 or v4, v2
4366 // v1 = i24 trunc v5
4367 // By doing this we generate the correct truncate which should get
4368 // combined away as an artifact with a matching extend.
4369
4370 uint64_t LargeSplitSize, SmallSplitSize;
4371
4372 if (!isPowerOf2_32(Value: MemSizeInBits)) {
4373 // This load needs splitting into power of 2 sized loads.
4374 LargeSplitSize = llvm::bit_floor(Value: MemSizeInBits);
4375 SmallSplitSize = MemSizeInBits - LargeSplitSize;
4376 } else {
4377 // This is already a power of 2, but we still need to split this in half.
4378 //
4379 // Assume we're being asked to decompose an unaligned load.
4380 // TODO: If this requires multiple splits, handle them all at once.
4381 auto &Ctx = MF.getFunction().getContext();
4382 if (TLI.allowsMemoryAccess(Context&: Ctx, DL: MIRBuilder.getDataLayout(), Ty: MemTy, MMO))
4383 return UnableToLegalize;
4384
4385 SmallSplitSize = LargeSplitSize = MemSizeInBits / 2;
4386 }
4387
4388 if (MemTy.isVector()) {
4389 // TODO: Handle vector extloads
4390 if (MemTy != DstTy)
4391 return UnableToLegalize;
4392
4393 Align Alignment = LoadMI.getAlign();
4394 // Given an alignment larger than the size of the memory, we can increase
4395 // the size of the load without needing to scalarize it.
4396 if (Alignment.value() * 8 > MemSizeInBits &&
4397 isPowerOf2_64(Value: DstTy.getScalarSizeInBits())) {
4398 LLT MoreTy = DstTy.changeVectorElementCount(
4399 EC: ElementCount::getFixed(MinVal: NextPowerOf2(A: DstTy.getNumElements())));
4400 MachineMemOperand *NewMMO = MF.getMachineMemOperand(MMO: &MMO, Offset: 0, Ty: MoreTy);
4401 auto NewLoad = MIRBuilder.buildLoad(Res: MoreTy, Addr: PtrReg, MMO&: *NewMMO);
4402 MIRBuilder.buildDeleteTrailingVectorElements(Res: LoadMI.getReg(Idx: 0),
4403 Op0: NewLoad.getReg(Idx: 0));
4404 LoadMI.eraseFromParent();
4405 return Legalized;
4406 }
4407
4408 // TODO: We can do better than scalarizing the vector and at least split it
4409 // in half.
4410 return reduceLoadStoreWidth(MI&: LoadMI, TypeIdx: 0, NarrowTy: DstTy.getElementType());
4411 }
4412
4413 MachineMemOperand *LargeMMO =
4414 MF.getMachineMemOperand(MMO: &MMO, Offset: 0, Size: LargeSplitSize / 8);
4415 MachineMemOperand *SmallMMO =
4416 MF.getMachineMemOperand(MMO: &MMO, Offset: LargeSplitSize / 8, Size: SmallSplitSize / 8);
4417
4418 LLT PtrTy = MRI.getType(Reg: PtrReg);
4419 unsigned AnyExtSize = PowerOf2Ceil(A: DstTy.getSizeInBits());
4420
4421 LLT AnyExtTy = LLT::integer(SizeInBits: AnyExtSize);
4422 LLT OffsetCstRes = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
4423
4424 auto LargeLoad = MIRBuilder.buildLoadInstr(Opcode: TargetOpcode::G_ZEXTLOAD, Res: AnyExtTy,
4425 Addr: PtrReg, MMO&: *LargeMMO);
4426
4427 auto OffsetCst = MIRBuilder.buildConstant(Res: OffsetCstRes, Val: LargeSplitSize / 8);
4428 Register PtrAddReg = MRI.createGenericVirtualRegister(Ty: PtrTy);
4429 auto SmallPtr = MIRBuilder.buildObjectPtrOffset(Res: PtrAddReg, Op0: PtrReg, Op1: OffsetCst);
4430 auto SmallLoad = MIRBuilder.buildLoadInstr(Opcode: LoadMI.getOpcode(), Res: AnyExtTy,
4431 Addr: SmallPtr, MMO&: *SmallMMO);
4432
4433 auto ShiftAmt = MIRBuilder.buildConstant(Res: AnyExtTy, Val: LargeSplitSize);
4434 auto Shift = MIRBuilder.buildShl(Dst: AnyExtTy, Src0: SmallLoad, Src1: ShiftAmt);
4435
4436 if (AnyExtTy == DstTy)
4437 MIRBuilder.buildOr(Dst: DstReg, Src0: Shift, Src1: LargeLoad);
4438 else if (AnyExtTy.getSizeInBits() != DstTy.getSizeInBits()) {
4439 auto Or = MIRBuilder.buildOr(Dst: AnyExtTy, Src0: Shift, Src1: LargeLoad);
4440 LLT IntDstTy = DstTy.changeToInteger();
4441 if (IntDstTy == DstTy) {
4442 MIRBuilder.buildTrunc(Res: DstReg, Op: {Or});
4443 } else {
4444 auto Trunc = MIRBuilder.buildTrunc(Res: IntDstTy, Op: Or);
4445 MIRBuilder.buildBitcast(Dst: DstReg, Src: Trunc);
4446 }
4447 } else if (DstTy.isPointer()) {
4448 auto Or = MIRBuilder.buildOr(Dst: AnyExtTy, Src0: Shift, Src1: LargeLoad);
4449
4450 // FIXME: We currently consider this to be illegal for non-integral address
4451 // spaces, but we need still need a way to reinterpret the bits.
4452 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: Or);
4453 } else {
4454 auto Or = MIRBuilder.buildOr(Dst: AnyExtTy, Src0: Shift, Src1: LargeLoad);
4455 MIRBuilder.buildBitcast(Dst: DstReg, Src: Or);
4456 }
4457
4458 LoadMI.eraseFromParent();
4459 return Legalized;
4460}
4461
4462LegalizerHelper::LegalizeResult LegalizerHelper::lowerStore(GStore &StoreMI) {
4463 // Lower a non-power of 2 store into multiple pow-2 stores.
4464 // E.g. split an i24 store into an i16 store + i8 store.
4465 // We do this by first extending the stored value to the next largest power
4466 // of 2 type, and then using truncating stores to store the components.
4467 // By doing this, likewise with G_LOAD, generate an extend that can be
4468 // artifact-combined away instead of leaving behind extracts.
4469 Register SrcReg = StoreMI.getValueReg();
4470 Register PtrReg = StoreMI.getPointerReg();
4471 LLT SrcTy = MRI.getType(Reg: SrcReg);
4472 MachineFunction &MF = MIRBuilder.getMF();
4473 MachineMemOperand &MMO = **StoreMI.memoperands_begin();
4474 LLT MemTy = MMO.getMemoryType();
4475
4476 unsigned StoreWidth = MemTy.getSizeInBits();
4477 unsigned StoreSizeInBits = 8 * MemTy.getSizeInBytes();
4478
4479 if (StoreWidth != StoreSizeInBits && !SrcTy.isVector()) {
4480 // Promote to a byte-sized store with upper bits zero if not
4481 // storing an integral number of bytes. For example, promote
4482 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
4483 LLT WideTy = LLT::integer(SizeInBits: StoreSizeInBits);
4484
4485 if (StoreSizeInBits > SrcTy.getSizeInBits()) {
4486 // Avoid creating a store with a narrower source than result.
4487 SrcReg = MIRBuilder.buildAnyExt(Res: WideTy, Op: SrcReg).getReg(Idx: 0);
4488 SrcTy = WideTy;
4489 }
4490
4491 auto ZextInReg = MIRBuilder.buildZExtInReg(Res: SrcTy, Op: SrcReg, ImmOp: StoreWidth);
4492
4493 MachineMemOperand *NewMMO =
4494 MF.getMachineMemOperand(MMO: &MMO, PtrInfo: MMO.getPointerInfo(), Ty: WideTy);
4495 MIRBuilder.buildStore(Val: ZextInReg, Addr: PtrReg, MMO&: *NewMMO);
4496 StoreMI.eraseFromParent();
4497 return Legalized;
4498 }
4499
4500 if (MemTy.isVector()) {
4501 if (MemTy != SrcTy)
4502 return scalarizeVectorBooleanStore(MI&: StoreMI);
4503
4504 // TODO: We can do better than scalarizing the vector and at least split it
4505 // in half.
4506 return reduceLoadStoreWidth(MI&: StoreMI, TypeIdx: 0, NarrowTy: SrcTy.getElementType());
4507 }
4508
4509 unsigned MemSizeInBits = MemTy.getSizeInBits();
4510 uint64_t LargeSplitSize, SmallSplitSize;
4511
4512 if (!isPowerOf2_32(Value: MemSizeInBits)) {
4513 LargeSplitSize = llvm::bit_floor<uint64_t>(Value: MemTy.getSizeInBits());
4514 SmallSplitSize = MemTy.getSizeInBits() - LargeSplitSize;
4515 } else {
4516 auto &Ctx = MF.getFunction().getContext();
4517 if (TLI.allowsMemoryAccess(Context&: Ctx, DL: MIRBuilder.getDataLayout(), Ty: MemTy, MMO))
4518 return UnableToLegalize; // Don't know what we're being asked to do.
4519
4520 SmallSplitSize = LargeSplitSize = MemSizeInBits / 2;
4521 }
4522
4523 // Extend to the next pow-2. If this store was itself the result of lowering,
4524 // e.g. an s56 store being broken into s32 + s24, we might have a stored type
4525 // that's wider than the stored size.
4526 unsigned AnyExtSize = PowerOf2Ceil(A: MemTy.getSizeInBits());
4527 const LLT NewSrcTy = LLT::integer(SizeInBits: AnyExtSize);
4528
4529 if (SrcTy.isPointer()) {
4530 const LLT IntPtrTy = LLT::integer(SizeInBits: SrcTy.getSizeInBits());
4531 SrcReg = MIRBuilder.buildPtrToInt(Dst: IntPtrTy, Src: SrcReg).getReg(Idx: 0);
4532 }
4533
4534 auto ExtVal = MIRBuilder.buildAnyExtOrTrunc(Res: NewSrcTy, Op: SrcReg);
4535
4536 // Obtain the smaller value by shifting away the larger value.
4537 auto ShiftAmt = MIRBuilder.buildConstant(Res: NewSrcTy, Val: LargeSplitSize);
4538 auto SmallVal = MIRBuilder.buildLShr(Dst: NewSrcTy, Src0: ExtVal, Src1: ShiftAmt);
4539
4540 // Generate the PtrAdd and truncating stores.
4541 LLT PtrTy = MRI.getType(Reg: PtrReg);
4542 auto OffsetCst = MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: PtrTy.getSizeInBits()),
4543 Val: LargeSplitSize / 8);
4544 auto SmallPtr = MIRBuilder.buildObjectPtrOffset(Res: PtrTy, Op0: PtrReg, Op1: OffsetCst);
4545
4546 MachineMemOperand *LargeMMO =
4547 MF.getMachineMemOperand(MMO: &MMO, Offset: 0, Size: LargeSplitSize / 8);
4548 MachineMemOperand *SmallMMO =
4549 MF.getMachineMemOperand(MMO: &MMO, Offset: LargeSplitSize / 8, Size: SmallSplitSize / 8);
4550 MIRBuilder.buildStore(Val: ExtVal, Addr: PtrReg, MMO&: *LargeMMO);
4551 MIRBuilder.buildStore(Val: SmallVal, Addr: SmallPtr, MMO&: *SmallMMO);
4552 StoreMI.eraseFromParent();
4553 return Legalized;
4554}
4555
4556LegalizerHelper::LegalizeResult
4557LegalizerHelper::scalarizeVectorBooleanStore(GStore &StoreMI) {
4558 Register SrcReg = StoreMI.getValueReg();
4559 Register PtrReg = StoreMI.getPointerReg();
4560 LLT SrcTy = MRI.getType(Reg: SrcReg);
4561 MachineMemOperand &MMO = **StoreMI.memoperands_begin();
4562 LLT MemTy = MMO.getMemoryType();
4563 LLT MemScalarTy = MemTy.getElementType();
4564 MachineFunction &MF = MIRBuilder.getMF();
4565
4566 assert(SrcTy.isVector() && "Expect a vector store type");
4567
4568 if (!MemScalarTy.isByteSized()) {
4569 // We need to build an integer scalar of the vector bit pattern.
4570 // It's not legal for us to add padding when storing a vector.
4571 unsigned NumBits = MemTy.getSizeInBits();
4572 LLT IntTy = LLT::integer(SizeInBits: NumBits);
4573 auto CurrVal = MIRBuilder.buildConstant(Res: IntTy, Val: 0);
4574 LLT IdxTy = TLI.getVectorIdxLLT(DL: MF.getDataLayout());
4575
4576 for (unsigned I = 0, E = MemTy.getNumElements(); I < E; ++I) {
4577 auto Elt = MIRBuilder.buildExtractVectorElement(
4578 Res: SrcTy.getElementType(), Val: SrcReg, Idx: MIRBuilder.buildConstant(Res: IdxTy, Val: I));
4579 auto Trunc = MIRBuilder.buildTrunc(Res: MemScalarTy, Op: Elt);
4580 auto ZExt = MIRBuilder.buildZExt(Res: IntTy, Op: Trunc);
4581 unsigned ShiftIntoIdx = MF.getDataLayout().isBigEndian()
4582 ? (MemTy.getNumElements() - 1) - I
4583 : I;
4584 auto ShiftAmt = MIRBuilder.buildConstant(
4585 Res: IntTy, Val: ShiftIntoIdx * MemScalarTy.getSizeInBits());
4586 auto Shifted = MIRBuilder.buildShl(Dst: IntTy, Src0: ZExt, Src1: ShiftAmt);
4587 CurrVal = MIRBuilder.buildOr(Dst: IntTy, Src0: CurrVal, Src1: Shifted);
4588 }
4589 auto PtrInfo = MMO.getPointerInfo();
4590 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo, Ty: IntTy);
4591 MIRBuilder.buildStore(Val: CurrVal, Addr: PtrReg, MMO&: *NewMMO);
4592 StoreMI.eraseFromParent();
4593 return Legalized;
4594 }
4595
4596 // TODO: implement simple scalarization.
4597 return UnableToLegalize;
4598}
4599
4600LegalizerHelper::LegalizeResult
4601LegalizerHelper::bitcast(MachineInstr &MI, unsigned TypeIdx, LLT CastTy) {
4602 switch (MI.getOpcode()) {
4603 case TargetOpcode::G_LOAD: {
4604 if (TypeIdx != 0)
4605 return UnableToLegalize;
4606 MachineMemOperand &MMO = **MI.memoperands_begin();
4607
4608 // Not sure how to interpret a bitcast of an extending load.
4609 if (MMO.getMemoryType().getSizeInBits() != CastTy.getSizeInBits())
4610 return UnableToLegalize;
4611
4612 Observer.changingInstr(MI);
4613 bitcastDst(MI, CastTy, OpIdx: 0);
4614 MMO.setType(CastTy);
4615 // The range metadata is no longer valid when reinterpreted as a different
4616 // type.
4617 MMO.clearRanges();
4618 Observer.changedInstr(MI);
4619 return Legalized;
4620 }
4621 case TargetOpcode::G_STORE: {
4622 if (TypeIdx != 0)
4623 return UnableToLegalize;
4624
4625 MachineMemOperand &MMO = **MI.memoperands_begin();
4626
4627 // Not sure how to interpret a bitcast of a truncating store.
4628 if (MMO.getMemoryType().getSizeInBits() != CastTy.getSizeInBits())
4629 return UnableToLegalize;
4630
4631 Observer.changingInstr(MI);
4632 bitcastSrc(MI, CastTy, OpIdx: 0);
4633 MMO.setType(CastTy);
4634 Observer.changedInstr(MI);
4635 return Legalized;
4636 }
4637 case TargetOpcode::G_SELECT: {
4638 if (TypeIdx != 0)
4639 return UnableToLegalize;
4640
4641 if (MRI.getType(Reg: MI.getOperand(i: 1).getReg()).isVector()) {
4642 LLVM_DEBUG(
4643 dbgs() << "bitcast action not implemented for vector select\n");
4644 return UnableToLegalize;
4645 }
4646
4647 Observer.changingInstr(MI);
4648 bitcastSrc(MI, CastTy, OpIdx: 2);
4649 bitcastSrc(MI, CastTy, OpIdx: 3);
4650 bitcastDst(MI, CastTy, OpIdx: 0);
4651 Observer.changedInstr(MI);
4652 return Legalized;
4653 }
4654 case TargetOpcode::G_AND:
4655 case TargetOpcode::G_OR:
4656 case TargetOpcode::G_XOR: {
4657 Observer.changingInstr(MI);
4658 bitcastSrc(MI, CastTy, OpIdx: 1);
4659 bitcastSrc(MI, CastTy, OpIdx: 2);
4660 bitcastDst(MI, CastTy, OpIdx: 0);
4661 Observer.changedInstr(MI);
4662 return Legalized;
4663 }
4664 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
4665 return bitcastExtractVectorElt(MI, TypeIdx, CastTy);
4666 case TargetOpcode::G_INSERT_VECTOR_ELT:
4667 return bitcastInsertVectorElt(MI, TypeIdx, CastTy);
4668 case TargetOpcode::G_CONCAT_VECTORS:
4669 return bitcastConcatVector(MI, TypeIdx, CastTy);
4670 case TargetOpcode::G_SHUFFLE_VECTOR:
4671 return bitcastShuffleVector(MI, TypeIdx, CastTy);
4672 case TargetOpcode::G_EXTRACT_SUBVECTOR:
4673 return bitcastExtractSubvector(MI, TypeIdx, CastTy);
4674 case TargetOpcode::G_INSERT_SUBVECTOR:
4675 return bitcastInsertSubvector(MI, TypeIdx, CastTy);
4676 default:
4677 return UnableToLegalize;
4678 }
4679}
4680
4681// Legalize an instruction by changing the opcode in place.
4682void LegalizerHelper::changeOpcode(MachineInstr &MI, unsigned NewOpcode) {
4683 Observer.changingInstr(MI);
4684 MI.setDesc(MIRBuilder.getTII().get(Opcode: NewOpcode));
4685 Observer.changedInstr(MI);
4686}
4687
4688LegalizerHelper::LegalizeResult
4689LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) {
4690 using namespace TargetOpcode;
4691 switch(MI.getOpcode()) {
4692 default:
4693 return UnableToLegalize;
4694 case TargetOpcode::G_FCONSTANT:
4695 return lowerFConstant(MI);
4696 case TargetOpcode::G_BITCAST:
4697 return lowerBitcast(MI);
4698 case TargetOpcode::G_SREM:
4699 case TargetOpcode::G_UREM: {
4700 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4701 auto Quot =
4702 MIRBuilder.buildInstr(Opc: MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV, DstOps: {Ty},
4703 SrcOps: {MI.getOperand(i: 1), MI.getOperand(i: 2)});
4704
4705 auto Prod = MIRBuilder.buildMul(Dst: Ty, Src0: Quot, Src1: MI.getOperand(i: 2));
4706 MIRBuilder.buildSub(Dst: MI.getOperand(i: 0), Src0: MI.getOperand(i: 1), Src1: Prod);
4707 MI.eraseFromParent();
4708 return Legalized;
4709 }
4710 case TargetOpcode::G_SADDO:
4711 case TargetOpcode::G_SSUBO:
4712 return lowerSADDO_SSUBO(MI);
4713 case TargetOpcode::G_SADDE:
4714 return lowerSADDE(MI);
4715 case TargetOpcode::G_SSUBE:
4716 return lowerSSUBE(MI);
4717 case TargetOpcode::G_UMULH:
4718 case TargetOpcode::G_SMULH:
4719 return lowerSMULH_UMULH(MI);
4720 case TargetOpcode::G_SMULO:
4721 case TargetOpcode::G_UMULO: {
4722 // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the
4723 // result.
4724 auto [Res, Overflow, LHS, RHS] = MI.getFirst4Regs();
4725 LLT Ty = MRI.getType(Reg: Res);
4726
4727 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO
4728 ? TargetOpcode::G_SMULH
4729 : TargetOpcode::G_UMULH;
4730
4731 Observer.changingInstr(MI);
4732 const auto &TII = MIRBuilder.getTII();
4733 MI.setDesc(TII.get(Opcode: TargetOpcode::G_MUL));
4734 MI.removeOperand(OpNo: 1);
4735 Observer.changedInstr(MI);
4736
4737 auto HiPart = MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Ty}, SrcOps: {LHS, RHS});
4738 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
4739
4740 // Move insert point forward so we can use the Res register if needed.
4741 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
4742
4743 // For *signed* multiply, overflow is detected by checking:
4744 // (hi != (lo >> bitwidth-1))
4745 if (Opcode == TargetOpcode::G_SMULH) {
4746 auto ShiftAmt = MIRBuilder.buildConstant(Res: Ty, Val: Ty.getSizeInBits() - 1);
4747 auto Shifted = MIRBuilder.buildAShr(Dst: Ty, Src0: Res, Src1: ShiftAmt);
4748 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: Overflow, Op0: HiPart, Op1: Shifted);
4749 } else {
4750 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: Overflow, Op0: HiPart, Op1: Zero);
4751 }
4752 return Legalized;
4753 }
4754 case TargetOpcode::G_FNEG: {
4755 auto [Res, ResTy, SubByReg, SubByRegTy] = MI.getFirst2RegLLTs();
4756 LLT TyInt =
4757 ResTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: ResTy.getScalarSizeInBits()));
4758 Register CastedSubByReg = SubByReg;
4759
4760 if (!SubByRegTy.getScalarType().isAnyScalar() &&
4761 !SubByRegTy.getScalarType().isInteger()) {
4762 auto BitcastDst = SubByRegTy.changeElementType(
4763 NewEltTy: LLT::integer(SizeInBits: SubByRegTy.getScalarSizeInBits()));
4764 CastedSubByReg = MIRBuilder.buildBitcast(Dst: BitcastDst, Src: SubByReg).getReg(Idx: 0);
4765 }
4766
4767 auto SignMask = MIRBuilder.buildConstant(
4768 Res: TyInt, Val: APInt::getSignMask(BitWidth: TyInt.getScalarSizeInBits()));
4769
4770 if (ResTy != TyInt) {
4771 Register NewDst =
4772 MIRBuilder.buildXor(Dst: TyInt, Src0: CastedSubByReg, Src1: SignMask).getReg(Idx: 0);
4773 MIRBuilder.buildBitcast(Dst: Res, Src: NewDst);
4774 } else
4775 MIRBuilder.buildXor(Dst: Res, Src0: CastedSubByReg, Src1: SignMask).getReg(Idx: 0);
4776
4777 MI.eraseFromParent();
4778 return Legalized;
4779 }
4780 case TargetOpcode::G_FSUB:
4781 case TargetOpcode::G_STRICT_FSUB: {
4782 auto [Res, LHS, RHS] = MI.getFirst3Regs();
4783 LLT Ty = MRI.getType(Reg: Res);
4784
4785 // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)).
4786 auto Neg = MIRBuilder.buildFNeg(Dst: Ty, Src0: RHS);
4787
4788 if (MI.getOpcode() == TargetOpcode::G_STRICT_FSUB)
4789 MIRBuilder.buildStrictFAdd(Dst: Res, Src0: LHS, Src1: Neg, Flags: MI.getFlags());
4790 else
4791 MIRBuilder.buildFAdd(Dst: Res, Src0: LHS, Src1: Neg, Flags: MI.getFlags());
4792
4793 MI.eraseFromParent();
4794 return Legalized;
4795 }
4796 case TargetOpcode::G_FMAD:
4797 return lowerFMad(MI);
4798 case TargetOpcode::G_FFLOOR:
4799 return lowerFFloor(MI);
4800 case TargetOpcode::G_LROUND:
4801 case TargetOpcode::G_LLROUND: {
4802 Register DstReg = MI.getOperand(i: 0).getReg();
4803 Register SrcReg = MI.getOperand(i: 1).getReg();
4804 LLT SrcTy = MRI.getType(Reg: SrcReg);
4805 auto Round = MIRBuilder.buildInstr(Opc: TargetOpcode::G_INTRINSIC_ROUND, DstOps: {SrcTy},
4806 SrcOps: {SrcReg});
4807 MIRBuilder.buildFPTOSI(Dst: DstReg, Src0: Round);
4808 MI.eraseFromParent();
4809 return Legalized;
4810 }
4811 case TargetOpcode::G_INTRINSIC_ROUND:
4812 return lowerIntrinsicRound(MI);
4813 case TargetOpcode::G_FRINT: {
4814 // Since round even is the assumed rounding mode for unconstrained FP
4815 // operations, rint and roundeven are the same operation.
4816 changeOpcode(MI, NewOpcode: TargetOpcode::G_INTRINSIC_ROUNDEVEN);
4817 return Legalized;
4818 }
4819 case TargetOpcode::G_INTRINSIC_LRINT:
4820 case TargetOpcode::G_INTRINSIC_LLRINT: {
4821 Register DstReg = MI.getOperand(i: 0).getReg();
4822 Register SrcReg = MI.getOperand(i: 1).getReg();
4823 LLT SrcTy = MRI.getType(Reg: SrcReg);
4824 auto Round =
4825 MIRBuilder.buildInstr(Opc: TargetOpcode::G_FRINT, DstOps: {SrcTy}, SrcOps: {SrcReg});
4826 MIRBuilder.buildFPTOSI(Dst: DstReg, Src0: Round);
4827 MI.eraseFromParent();
4828 return Legalized;
4829 }
4830 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
4831 auto [OldValRes, SuccessRes, Addr, CmpVal, NewVal] = MI.getFirst5Regs();
4832 Register NewOldValRes = MRI.cloneVirtualRegister(VReg: OldValRes);
4833 MIRBuilder.buildAtomicCmpXchg(OldValRes: NewOldValRes, Addr, CmpVal, NewVal,
4834 MMO&: **MI.memoperands_begin());
4835 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: SuccessRes, Op0: NewOldValRes, Op1: CmpVal);
4836 MIRBuilder.buildCopy(Res: OldValRes, Op: NewOldValRes);
4837 MI.eraseFromParent();
4838 return Legalized;
4839 }
4840 case TargetOpcode::G_LOAD:
4841 case TargetOpcode::G_SEXTLOAD:
4842 case TargetOpcode::G_ZEXTLOAD:
4843 return lowerLoad(LoadMI&: cast<GAnyLoad>(Val&: MI));
4844 case TargetOpcode::G_STORE:
4845 return lowerStore(StoreMI&: cast<GStore>(Val&: MI));
4846 case TargetOpcode::G_CTLZ_ZERO_POISON:
4847 case TargetOpcode::G_CTTZ_ZERO_POISON:
4848 case TargetOpcode::G_CTLZ:
4849 case TargetOpcode::G_CTTZ:
4850 case TargetOpcode::G_CTPOP:
4851 case TargetOpcode::G_CTLS:
4852 return lowerBitCount(MI);
4853 case G_UADDO: {
4854 auto [Res, CarryOut, LHS, RHS] = MI.getFirst4Regs();
4855
4856 Register NewRes = MRI.cloneVirtualRegister(VReg: Res);
4857
4858 MIRBuilder.buildAdd(Dst: NewRes, Src0: LHS, Src1: RHS);
4859 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_ULT, Res: CarryOut, Op0: NewRes, Op1: RHS);
4860
4861 MIRBuilder.buildCopy(Res, Op: NewRes);
4862
4863 MI.eraseFromParent();
4864 return Legalized;
4865 }
4866 case G_UADDE: {
4867 auto [Res, CarryOut, LHS, RHS, CarryIn] = MI.getFirst5Regs();
4868 const LLT CondTy = MRI.getType(Reg: CarryOut);
4869 const LLT Ty = MRI.getType(Reg: Res);
4870
4871 Register NewRes = MRI.cloneVirtualRegister(VReg: Res);
4872
4873 // Initial add of the two operands.
4874 auto TmpRes = MIRBuilder.buildAdd(Dst: Ty, Src0: LHS, Src1: RHS);
4875
4876 // Initial check for carry.
4877 auto Carry = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_ULT, Res: CondTy, Op0: TmpRes, Op1: LHS);
4878
4879 // Add the sum and the carry.
4880 auto ZExtCarryIn = MIRBuilder.buildZExt(Res: Ty, Op: CarryIn);
4881 MIRBuilder.buildAdd(Dst: NewRes, Src0: TmpRes, Src1: ZExtCarryIn);
4882
4883 // Second check for carry. We can only carry if the initial sum is all 1s
4884 // and the carry is set, resulting in a new sum of 0.
4885 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
4886 auto ResEqZero =
4887 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CondTy, Op0: NewRes, Op1: Zero);
4888 auto Carry2 = MIRBuilder.buildAnd(Dst: CondTy, Src0: ResEqZero, Src1: CarryIn);
4889 MIRBuilder.buildOr(Dst: CarryOut, Src0: Carry, Src1: Carry2);
4890
4891 MIRBuilder.buildCopy(Res, Op: NewRes);
4892
4893 MI.eraseFromParent();
4894 return Legalized;
4895 }
4896 case G_USUBO: {
4897 auto [Res, BorrowOut, LHS, RHS] = MI.getFirst4Regs();
4898
4899 MIRBuilder.buildSub(Dst: Res, Src0: LHS, Src1: RHS);
4900 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_ULT, Res: BorrowOut, Op0: LHS, Op1: RHS);
4901
4902 MI.eraseFromParent();
4903 return Legalized;
4904 }
4905 case G_USUBE: {
4906 auto [Res, BorrowOut, LHS, RHS, BorrowIn] = MI.getFirst5Regs();
4907 const LLT CondTy = MRI.getType(Reg: BorrowOut);
4908 const LLT Ty = MRI.getType(Reg: Res);
4909
4910 // Initial subtract of the two operands.
4911 auto TmpRes = MIRBuilder.buildSub(Dst: Ty, Src0: LHS, Src1: RHS);
4912
4913 // Initial check for borrow.
4914 auto Borrow = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_UGT, Res: CondTy, Op0: TmpRes, Op1: LHS);
4915
4916 // Subtract the borrow from the first subtract.
4917 auto ZExtBorrowIn = MIRBuilder.buildZExt(Res: Ty, Op: BorrowIn);
4918 MIRBuilder.buildSub(Dst: Res, Src0: TmpRes, Src1: ZExtBorrowIn);
4919
4920 // Second check for borrow. We can only borrow if the initial difference is
4921 // 0 and the borrow is set, resulting in a new difference of all 1s.
4922 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
4923 auto TmpResEqZero =
4924 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CondTy, Op0: TmpRes, Op1: Zero);
4925 auto Borrow2 = MIRBuilder.buildAnd(Dst: CondTy, Src0: TmpResEqZero, Src1: BorrowIn);
4926 MIRBuilder.buildOr(Dst: BorrowOut, Src0: Borrow, Src1: Borrow2);
4927
4928 MI.eraseFromParent();
4929 return Legalized;
4930 }
4931 case G_UITOFP:
4932 return lowerUITOFP(MI);
4933 case G_SITOFP:
4934 return lowerSITOFP(MI);
4935 case G_FPTOUI:
4936 return lowerFPTOUI(MI);
4937 case G_FPTOSI:
4938 return lowerFPTOSI(MI);
4939 case G_FPTOUI_SAT:
4940 case G_FPTOSI_SAT:
4941 return lowerFPTOINT_SAT(MI);
4942 case G_FPEXT:
4943 return lowerFPEXT(MI);
4944 case G_FPTRUNC:
4945 return lowerFPTRUNC(MI);
4946 case G_FPOWI:
4947 return lowerFPOWI(MI);
4948 case G_FMODF:
4949 return lowerFMODF(MI);
4950 case G_SMIN:
4951 case G_SMAX:
4952 case G_UMIN:
4953 case G_UMAX:
4954 return lowerMinMax(MI);
4955 case G_SCMP:
4956 case G_UCMP:
4957 return lowerThreewayCompare(MI);
4958 case G_FCOPYSIGN:
4959 return lowerFCopySign(MI);
4960 case G_FMINNUM:
4961 case G_FMAXNUM:
4962 case G_FMINIMUMNUM:
4963 case G_FMAXIMUMNUM:
4964 return lowerFMinNumMaxNum(MI);
4965 case G_FMINIMUM:
4966 case G_FMAXIMUM:
4967 return lowerFMinimumMaximum(MI);
4968 case G_MERGE_VALUES:
4969 return lowerMergeValues(MI);
4970 case G_UNMERGE_VALUES:
4971 return lowerUnmergeValues(MI);
4972 case TargetOpcode::G_SEXT_INREG: {
4973 assert(MI.getOperand(2).isImm() && "Expected immediate");
4974 int64_t SizeInBits = MI.getOperand(i: 2).getImm();
4975
4976 auto [DstReg, SrcReg] = MI.getFirst2Regs();
4977 LLT DstTy = MRI.getType(Reg: DstReg);
4978 Register TmpRes = MRI.createGenericVirtualRegister(Ty: DstTy);
4979
4980 auto MIBSz = MIRBuilder.buildConstant(Res: DstTy, Val: DstTy.getScalarSizeInBits() - SizeInBits);
4981 MIRBuilder.buildShl(Dst: TmpRes, Src0: SrcReg, Src1: MIBSz->getOperand(i: 0));
4982 MIRBuilder.buildAShr(Dst: DstReg, Src0: TmpRes, Src1: MIBSz->getOperand(i: 0));
4983 MI.eraseFromParent();
4984 return Legalized;
4985 }
4986 case G_EXTRACT_VECTOR_ELT:
4987 case G_INSERT_VECTOR_ELT:
4988 return lowerExtractInsertVectorElt(MI);
4989 case G_SHUFFLE_VECTOR:
4990 return lowerShuffleVector(MI);
4991 case G_VECTOR_COMPRESS:
4992 return lowerVECTOR_COMPRESS(MI);
4993 case G_DYN_STACKALLOC:
4994 return lowerDynStackAlloc(MI);
4995 case G_INSERT_SUBVECTOR: {
4996 if (MRI.getType(Reg: MI.getOperand(i: 1).getReg()).isScalable() ||
4997 MRI.getType(Reg: MI.getOperand(i: 2).getReg()).isScalable())
4998 return UnableToLegalize;
4999
5000 // Check that subvector is half size of main vector
5001 Register Vector = MI.getOperand(i: 1).getReg();
5002 Register Subvector = MI.getOperand(i: 2).getReg();
5003 auto InsertionPointImm = MI.getOperand(i: 3).getImm();
5004
5005 LLT VectorTy = MRI.getType(Reg: Vector);
5006 LLT DstTy = MRI.getType(Reg: Subvector);
5007 // If so, -> concat(subvector, extract(half of vector))
5008 // (Operands can be either way round depending on insertion point
5009 if (VectorTy.getSizeInBits() == DstTy.getSizeInBits() * 2) {
5010 bool InsertInLowHalf = InsertionPointImm == 0;
5011 auto Extract = MIRBuilder.buildExtractSubvector(
5012 Res: DstTy, Src: Vector,
5013 Index: (uint64_t)(InsertInLowHalf ? VectorTy.getNumElements() / 2 : 0));
5014
5015 auto LowHalf = InsertInLowHalf ? Subvector : Extract.getReg(Idx: 0);
5016 auto HighHalf = InsertInLowHalf ? Extract.getReg(Idx: 0) : Subvector;
5017
5018 MIRBuilder.buildInstr(Opc: TargetOpcode::G_CONCAT_VECTORS, DstOps: {MI.getOperand(i: 0)},
5019 SrcOps: {LowHalf, HighHalf});
5020 MI.eraseFromParent();
5021 return Legalized;
5022 }
5023 // Else -> shuffle(vector, extend(subvector, size(vector)), mask)
5024 else {
5025 // Extend subvector to same size as vector
5026 Register ExtendedSubvector = MRI.createGenericVirtualRegister(Ty: VectorTy);
5027 MIRBuilder.buildPadVectorWithUndefElements(Res: ExtendedSubvector, Op0: Subvector);
5028
5029 // Calculate mask required for this shuffle
5030 SmallVector<int> Mask;
5031 for (int i = 0; i < VectorTy.getNumElements(); i++) {
5032 // If this index is within bounds, put subvector's index into mask
5033 if (i >= InsertionPointImm &&
5034 i < InsertionPointImm + DstTy.getNumElements())
5035 Mask.push_back(Elt: VectorTy.getNumElements() + i - InsertionPointImm);
5036 else
5037 Mask.push_back(Elt: i);
5038 }
5039
5040 // Build shuffle
5041 MIRBuilder.buildShuffleVector(Res: MI.getOperand(i: 0), Src1: Vector, Src2: ExtendedSubvector,
5042 Mask);
5043 MI.eraseFromParent();
5044 return Legalized;
5045 }
5046 }
5047 case G_EXTRACT_SUBVECTOR: {
5048 Register DstReg = MI.getOperand(i: 0).getReg();
5049 Register SrcReg = MI.getOperand(i: 1).getReg();
5050 uint64_t ExtractionPointImm = MI.getOperand(i: 2).getImm();
5051
5052 LLT SrcTy = MRI.getType(Reg: SrcReg);
5053 LLT DstTy = MRI.getType(Reg: DstReg);
5054
5055 if (SrcTy.isScalable())
5056 return UnableToLegalize;
5057
5058 if (SrcTy.getScalarType() != DstTy.getScalarType())
5059 return UnableToLegalize;
5060
5061 // extract_subvector = build_vector(extract_element, extract_element, ...)
5062 SmallVector<Register> ExtractedElements;
5063 for (uint64_t i = 0; i < DstTy.getNumElements(); i++) {
5064 ExtractedElements.push_back(
5065 Elt: MIRBuilder
5066 .buildExtractVectorElementConstant(Res: SrcTy.getScalarType(), Val: SrcReg,
5067 Idx: ExtractionPointImm + i)
5068 .getReg(Idx: 0));
5069 }
5070
5071 MIRBuilder.buildBuildVector(Res: DstReg, Ops: ExtractedElements);
5072 MI.eraseFromParent();
5073 return Legalized;
5074 }
5075 case G_STACKSAVE:
5076 return lowerStackSave(MI);
5077 case G_STACKRESTORE:
5078 return lowerStackRestore(MI);
5079 case G_EXTRACT:
5080 return lowerExtract(MI);
5081 case G_INSERT:
5082 return lowerInsert(MI);
5083 case G_BSWAP:
5084 return lowerBswap(MI);
5085 case G_BITREVERSE:
5086 return lowerBitreverse(MI);
5087 case G_READ_REGISTER:
5088 case G_WRITE_REGISTER:
5089 return lowerReadWriteRegister(MI);
5090 case G_UADDSAT:
5091 case G_USUBSAT: {
5092 // Try to make a reasonable guess about which lowering strategy to use. The
5093 // target can override this with custom lowering and calling the
5094 // implementation functions.
5095 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5096 if (LI.isLegalOrCustom(Query: {G_UMIN, Ty}))
5097 return lowerAddSubSatToMinMax(MI);
5098 return lowerAddSubSatToAddoSubo(MI);
5099 }
5100 case G_SADDSAT:
5101 case G_SSUBSAT: {
5102 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5103
5104 // FIXME: It would probably make more sense to see if G_SADDO is preferred,
5105 // since it's a shorter expansion. However, we would need to figure out the
5106 // preferred boolean type for the carry out for the query.
5107 if (LI.isLegalOrCustom(Query: {G_SMIN, Ty}) && LI.isLegalOrCustom(Query: {G_SMAX, Ty}))
5108 return lowerAddSubSatToMinMax(MI);
5109 return lowerAddSubSatToAddoSubo(MI);
5110 }
5111 case G_SSHLSAT:
5112 case G_USHLSAT:
5113 return lowerShlSat(MI);
5114 case G_TRUNC_SSAT_S:
5115 case G_TRUNC_USAT_U:
5116 case G_TRUNC_SSAT_U:
5117 return lowerTruncSat(MI);
5118 case G_ABS:
5119 return lowerAbsToAddXor(MI);
5120 case G_ABDS:
5121 case G_ABDU: {
5122 bool IsSigned = MI.getOpcode() == G_ABDS;
5123 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5124 if ((IsSigned && LI.isLegal(Query: {G_SMIN, Ty}) && LI.isLegal(Query: {G_SMAX, Ty})) ||
5125 (!IsSigned && LI.isLegal(Query: {G_UMIN, Ty}) && LI.isLegal(Query: {G_UMAX, Ty}))) {
5126 return lowerAbsDiffToMinMax(MI);
5127 }
5128 return lowerAbsDiffToSelect(MI);
5129 }
5130 case G_FABS:
5131 return lowerFAbs(MI);
5132 case G_SELECT:
5133 return lowerSelect(MI);
5134 case G_IS_FPCLASS:
5135 return lowerISFPCLASS(MI);
5136 case G_SDIVREM:
5137 case G_UDIVREM:
5138 return lowerDIVREM(MI);
5139 case G_FSHL:
5140 case G_FSHR:
5141 return lowerFunnelShift(MI);
5142 case G_ROTL:
5143 case G_ROTR:
5144 return lowerRotate(MI);
5145 case G_MEMSET:
5146 case G_MEMCPY:
5147 case G_MEMMOVE:
5148 case G_MEMCPY_INLINE:
5149 case G_MEMSET_INLINE:
5150 return lowerMemCpyFamily(MI);
5151 case G_ZEXT:
5152 case G_SEXT:
5153 case G_ANYEXT:
5154 return lowerEXT(MI);
5155 case G_TRUNC:
5156 return lowerTRUNC(MI);
5157 GISEL_VECREDUCE_CASES_NONSEQ
5158 return lowerVectorReduction(MI);
5159 case G_VAARG:
5160 return lowerVAArg(MI);
5161 case G_ATOMICRMW_SUB: {
5162 auto [Ret, Mem, Val] = MI.getFirst3Regs();
5163 const LLT ValTy = MRI.getType(Reg: Val);
5164 MachineMemOperand *MMO = *MI.memoperands_begin();
5165
5166 auto VNeg = MIRBuilder.buildNeg(Dst: ValTy, Src0: Val);
5167 MIRBuilder.buildAtomicRMW(Opcode: G_ATOMICRMW_ADD, OldValRes: Ret, Addr: Mem, Val: VNeg, MMO&: *MMO);
5168 MI.eraseFromParent();
5169 return Legalized;
5170 }
5171 case G_SMULFIX:
5172 case G_UMULFIX:
5173 case G_SMULFIXSAT:
5174 case G_UMULFIXSAT:
5175 return lowerMulfix(MI);
5176 }
5177}
5178
5179Align LegalizerHelper::getStackTemporaryAlignment(LLT Ty,
5180 Align MinAlign) const {
5181 // FIXME: We're missing a way to go back from LLT to llvm::Type to query the
5182 // datalayout for the preferred alignment. Also there should be a target hook
5183 // for this to allow targets to reduce the alignment and ignore the
5184 // datalayout. e.g. AMDGPU should always use a 4-byte alignment, regardless of
5185 // the type.
5186 return std::max(a: Align(PowerOf2Ceil(A: Ty.getSizeInBytes())), b: MinAlign);
5187}
5188
5189MachineInstrBuilder
5190LegalizerHelper::createStackTemporary(TypeSize Bytes, Align Alignment,
5191 MachinePointerInfo &PtrInfo) {
5192 MachineFunction &MF = MIRBuilder.getMF();
5193 const DataLayout &DL = MIRBuilder.getDataLayout();
5194 int FrameIdx = MF.getFrameInfo().CreateStackObject(Size: Bytes, Alignment, isSpillSlot: false);
5195
5196 unsigned AddrSpace = DL.getAllocaAddrSpace();
5197 LLT FramePtrTy = LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL.getPointerSizeInBits(AS: AddrSpace));
5198
5199 PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIdx);
5200 return MIRBuilder.buildFrameIndex(Res: FramePtrTy, Idx: FrameIdx);
5201}
5202
5203MachineInstrBuilder LegalizerHelper::createStackStoreLoad(const DstOp &Res,
5204 const SrcOp &Val) {
5205 LLT SrcTy = Val.getLLTTy(MRI);
5206 Align StackTypeAlign =
5207 std::max(a: getStackTemporaryAlignment(Ty: SrcTy),
5208 b: getStackTemporaryAlignment(Ty: Res.getLLTTy(MRI)));
5209 MachinePointerInfo PtrInfo;
5210 auto StackTemp =
5211 createStackTemporary(Bytes: SrcTy.getSizeInBytes(), Alignment: StackTypeAlign, PtrInfo);
5212
5213 MIRBuilder.buildStore(Val, Addr: StackTemp, PtrInfo, Alignment: StackTypeAlign);
5214 return MIRBuilder.buildLoad(Res, Addr: StackTemp, PtrInfo, Alignment: StackTypeAlign);
5215}
5216
5217static Register clampVectorIndex(MachineIRBuilder &B, Register IdxReg,
5218 LLT VecTy) {
5219 LLT IdxTy = B.getMRI()->getType(Reg: IdxReg);
5220 unsigned NElts = VecTy.getNumElements();
5221
5222 int64_t IdxVal;
5223 if (mi_match(R: IdxReg, MRI: *B.getMRI(), P: m_ICst(Cst&: IdxVal))) {
5224 if (IdxVal < VecTy.getNumElements())
5225 return IdxReg;
5226 // If a constant index would be out of bounds, clamp it as well.
5227 }
5228
5229 if (isPowerOf2_32(Value: NElts)) {
5230 APInt Imm = APInt::getLowBitsSet(numBits: IdxTy.getSizeInBits(), loBitsSet: Log2_32(Value: NElts));
5231 return B.buildAnd(Dst: IdxTy, Src0: IdxReg, Src1: B.buildConstant(Res: IdxTy, Val: Imm)).getReg(Idx: 0);
5232 }
5233
5234 return B.buildUMin(Dst: IdxTy, Src0: IdxReg, Src1: B.buildConstant(Res: IdxTy, Val: NElts - 1))
5235 .getReg(Idx: 0);
5236}
5237
5238Register LegalizerHelper::getVectorElementPointer(Register VecPtr, LLT VecTy,
5239 Register Index) {
5240 LLT EltTy = VecTy.getElementType();
5241
5242 // Calculate the element offset and add it to the pointer.
5243 unsigned EltSize = EltTy.getSizeInBits() / 8; // FIXME: should be ABI size.
5244 assert(EltSize * 8 == EltTy.getSizeInBits() &&
5245 "Converting bits to bytes lost precision");
5246
5247 Index = clampVectorIndex(B&: MIRBuilder, IdxReg: Index, VecTy);
5248
5249 // Convert index to the correct size for the address space.
5250 const DataLayout &DL = MIRBuilder.getDataLayout();
5251 unsigned AS = MRI.getType(Reg: VecPtr).getAddressSpace();
5252 unsigned IndexSizeInBits = DL.getIndexSize(AS) * 8;
5253 LLT IdxTy = MRI.getType(Reg: Index).changeElementSize(NewEltSize: IndexSizeInBits);
5254 if (IdxTy != MRI.getType(Reg: Index))
5255 Index = MIRBuilder.buildSExtOrTrunc(Res: IdxTy, Op: Index).getReg(Idx: 0);
5256
5257 auto Mul = MIRBuilder.buildMul(Dst: IdxTy, Src0: Index,
5258 Src1: MIRBuilder.buildConstant(Res: IdxTy, Val: EltSize));
5259
5260 LLT PtrTy = MRI.getType(Reg: VecPtr);
5261 return MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: VecPtr, Op1: Mul).getReg(Idx: 0);
5262}
5263
5264#ifndef NDEBUG
5265/// Check that all vector operands have same number of elements. Other operands
5266/// should be listed in NonVecOp.
5267static bool hasSameNumEltsOnAllVectorOperands(
5268 GenericMachineInstr &MI, MachineRegisterInfo &MRI,
5269 std::initializer_list<unsigned> NonVecOpIndices) {
5270 if (MI.getNumMemOperands() != 0)
5271 return false;
5272
5273 LLT VecTy = MRI.getType(MI.getReg(0));
5274 if (!VecTy.isVector())
5275 return false;
5276 unsigned NumElts = VecTy.getNumElements();
5277
5278 for (unsigned OpIdx = 1; OpIdx < MI.getNumOperands(); ++OpIdx) {
5279 MachineOperand &Op = MI.getOperand(OpIdx);
5280 if (!Op.isReg()) {
5281 if (!is_contained(NonVecOpIndices, OpIdx))
5282 return false;
5283 continue;
5284 }
5285
5286 LLT Ty = MRI.getType(Op.getReg());
5287 if (!Ty.isVector()) {
5288 if (!is_contained(NonVecOpIndices, OpIdx))
5289 return false;
5290 continue;
5291 }
5292
5293 if (Ty.getNumElements() != NumElts)
5294 return false;
5295 }
5296
5297 return true;
5298}
5299#endif
5300
5301/// Fill \p DstOps with DstOps that have same number of elements combined as
5302/// the Ty. These DstOps have either scalar type when \p NumElts = 1 or are
5303/// vectors with \p NumElts elements. When Ty.getNumElements() is not multiple
5304/// of \p NumElts last DstOp (leftover) has fewer then \p NumElts elements.
5305static void makeDstOps(SmallVectorImpl<DstOp> &DstOps, LLT Ty,
5306 unsigned NumElts) {
5307 LLT LeftoverTy;
5308 assert(Ty.isVector() && "Expected vector type");
5309 LLT NarrowTy = Ty.changeElementCount(EC: ElementCount::getFixed(MinVal: NumElts));
5310 int NumParts, NumLeftover;
5311 std::tie(args&: NumParts, args&: NumLeftover) =
5312 getNarrowTypeBreakDown(OrigTy: Ty, NarrowTy, LeftoverTy);
5313
5314 assert(NumParts > 0 && "Error in getNarrowTypeBreakDown");
5315 for (int i = 0; i < NumParts; ++i) {
5316 DstOps.push_back(Elt: NarrowTy);
5317 }
5318
5319 if (LeftoverTy.isValid()) {
5320 assert(NumLeftover == 1 && "expected exactly one leftover");
5321 DstOps.push_back(Elt: LeftoverTy);
5322 }
5323}
5324
5325/// Operand \p Op is used on \p N sub-instructions. Fill \p Ops with \p N SrcOps
5326/// made from \p Op depending on operand type.
5327static void broadcastSrcOp(SmallVectorImpl<SrcOp> &Ops, unsigned N,
5328 MachineOperand &Op) {
5329 for (unsigned i = 0; i < N; ++i) {
5330 if (Op.isReg())
5331 Ops.push_back(Elt: Op.getReg());
5332 else if (Op.isImm())
5333 Ops.push_back(Elt: Op.getImm());
5334 else if (Op.isPredicate())
5335 Ops.push_back(Elt: static_cast<CmpInst::Predicate>(Op.getPredicate()));
5336 else
5337 llvm_unreachable("Unsupported type");
5338 }
5339}
5340
5341// Handle splitting vector operations which need to have the same number of
5342// elements in each type index, but each type index may have a different element
5343// type.
5344//
5345// e.g. <4 x s64> = G_SHL <4 x s64>, <4 x s32> ->
5346// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
5347// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
5348//
5349// Also handles some irregular breakdown cases, e.g.
5350// e.g. <3 x s64> = G_SHL <3 x s64>, <3 x s32> ->
5351// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
5352// s64 = G_SHL s64, s32
5353LegalizerHelper::LegalizeResult
5354LegalizerHelper::fewerElementsVectorMultiEltType(
5355 GenericMachineInstr &MI, unsigned NumElts,
5356 std::initializer_list<unsigned> NonVecOpIndices) {
5357 assert(hasSameNumEltsOnAllVectorOperands(MI, MRI, NonVecOpIndices) &&
5358 "Non-compatible opcode or not specified non-vector operands");
5359 unsigned OrigNumElts = MRI.getType(Reg: MI.getReg(Idx: 0)).getNumElements();
5360
5361 unsigned NumInputs = MI.getNumOperands() - MI.getNumDefs();
5362 unsigned NumDefs = MI.getNumDefs();
5363
5364 // Create DstOps (sub-vectors with NumElts elts + Leftover) for each output.
5365 // Build instructions with DstOps to use instruction found by CSE directly.
5366 // CSE copies found instruction into given vreg when building with vreg dest.
5367 SmallVector<SmallVector<DstOp, 8>, 2> OutputOpsPieces(NumDefs);
5368 // Output registers will be taken from created instructions.
5369 SmallVector<SmallVector<Register, 8>, 2> OutputRegs(NumDefs);
5370 for (unsigned i = 0; i < NumDefs; ++i) {
5371 makeDstOps(DstOps&: OutputOpsPieces[i], Ty: MRI.getType(Reg: MI.getReg(Idx: i)), NumElts);
5372 }
5373
5374 // Split vector input operands into sub-vectors with NumElts elts + Leftover.
5375 // Operands listed in NonVecOpIndices will be used as is without splitting;
5376 // examples: compare predicate in icmp and fcmp (op 1), vector select with i1
5377 // scalar condition (op 1), immediate in sext_inreg (op 2).
5378 SmallVector<SmallVector<SrcOp, 8>, 3> InputOpsPieces(NumInputs);
5379 for (unsigned UseIdx = NumDefs, UseNo = 0; UseIdx < MI.getNumOperands();
5380 ++UseIdx, ++UseNo) {
5381 if (is_contained(Set: NonVecOpIndices, Element: UseIdx)) {
5382 broadcastSrcOp(Ops&: InputOpsPieces[UseNo], N: OutputOpsPieces[0].size(),
5383 Op&: MI.getOperand(i: UseIdx));
5384 } else {
5385 SmallVector<Register, 8> SplitPieces;
5386 extractVectorParts(Reg: MI.getReg(Idx: UseIdx), NumElts, VRegs&: SplitPieces, MIRBuilder,
5387 MRI);
5388 llvm::append_range(C&: InputOpsPieces[UseNo], R&: SplitPieces);
5389 }
5390 }
5391
5392 unsigned NumLeftovers = OrigNumElts % NumElts ? 1 : 0;
5393
5394 // Take i-th piece of each input operand split and build sub-vector/scalar
5395 // instruction. Set i-th DstOp(s) from OutputOpsPieces as destination(s).
5396 for (unsigned i = 0; i < OrigNumElts / NumElts + NumLeftovers; ++i) {
5397 SmallVector<DstOp, 2> Defs;
5398 for (unsigned DstNo = 0; DstNo < NumDefs; ++DstNo)
5399 Defs.push_back(Elt: OutputOpsPieces[DstNo][i]);
5400
5401 SmallVector<SrcOp, 3> Uses;
5402 for (unsigned InputNo = 0; InputNo < NumInputs; ++InputNo)
5403 Uses.push_back(Elt: InputOpsPieces[InputNo][i]);
5404
5405 auto I = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: Defs, SrcOps: Uses, Flags: MI.getFlags());
5406 for (unsigned DstNo = 0; DstNo < NumDefs; ++DstNo)
5407 OutputRegs[DstNo].push_back(Elt: I.getReg(Idx: DstNo));
5408 }
5409
5410 // Merge small outputs into MI's output for each def operand.
5411 if (NumLeftovers) {
5412 for (unsigned i = 0; i < NumDefs; ++i)
5413 mergeMixedSubvectors(DstReg: MI.getReg(Idx: i), PartRegs: OutputRegs[i]);
5414 } else {
5415 for (unsigned i = 0; i < NumDefs; ++i)
5416 MIRBuilder.buildMergeLikeInstr(Res: MI.getReg(Idx: i), Ops: OutputRegs[i]);
5417 }
5418
5419 MI.eraseFromParent();
5420 return Legalized;
5421}
5422
5423LegalizerHelper::LegalizeResult
5424LegalizerHelper::fewerElementsVectorPhi(GenericMachineInstr &MI,
5425 unsigned NumElts) {
5426 unsigned OrigNumElts = MRI.getType(Reg: MI.getReg(Idx: 0)).getNumElements();
5427
5428 unsigned NumInputs = MI.getNumOperands() - MI.getNumDefs();
5429 unsigned NumDefs = MI.getNumDefs();
5430
5431 SmallVector<DstOp, 8> OutputOpsPieces;
5432 SmallVector<Register, 8> OutputRegs;
5433 makeDstOps(DstOps&: OutputOpsPieces, Ty: MRI.getType(Reg: MI.getReg(Idx: 0)), NumElts);
5434
5435 // Instructions that perform register split will be inserted in basic block
5436 // where register is defined (basic block is in the next operand).
5437 SmallVector<SmallVector<Register, 8>, 3> InputOpsPieces(NumInputs / 2);
5438 for (unsigned UseIdx = NumDefs, UseNo = 0; UseIdx < MI.getNumOperands();
5439 UseIdx += 2, ++UseNo) {
5440 MachineBasicBlock &OpMBB = *MI.getOperand(i: UseIdx + 1).getMBB();
5441 MIRBuilder.setInsertPt(MBB&: OpMBB, II: OpMBB.getFirstTerminatorForward());
5442 extractVectorParts(Reg: MI.getReg(Idx: UseIdx), NumElts, VRegs&: InputOpsPieces[UseNo],
5443 MIRBuilder, MRI);
5444 }
5445
5446 // Build PHIs with fewer elements.
5447 unsigned NumLeftovers = OrigNumElts % NumElts ? 1 : 0;
5448 MIRBuilder.setInsertPt(MBB&: *MI.getParent(), II: MI);
5449 for (unsigned i = 0; i < OrigNumElts / NumElts + NumLeftovers; ++i) {
5450 auto Phi = MIRBuilder.buildInstr(Opcode: TargetOpcode::G_PHI);
5451 Phi.addDef(
5452 RegNo: MRI.createGenericVirtualRegister(Ty: OutputOpsPieces[i].getLLTTy(MRI)));
5453 OutputRegs.push_back(Elt: Phi.getReg(Idx: 0));
5454
5455 for (unsigned j = 0; j < NumInputs / 2; ++j) {
5456 Phi.addUse(RegNo: InputOpsPieces[j][i]);
5457 Phi.add(MO: MI.getOperand(i: 1 + j * 2 + 1));
5458 }
5459 }
5460
5461 // Set the insert point after the existing PHIs
5462 MachineBasicBlock &MBB = *MI.getParent();
5463 MIRBuilder.setInsertPt(MBB, II: MBB.getFirstNonPHI());
5464
5465 // Merge small outputs into MI's def.
5466 if (NumLeftovers) {
5467 mergeMixedSubvectors(DstReg: MI.getReg(Idx: 0), PartRegs: OutputRegs);
5468 } else {
5469 MIRBuilder.buildMergeLikeInstr(Res: MI.getReg(Idx: 0), Ops: OutputRegs);
5470 }
5471
5472 MI.eraseFromParent();
5473 return Legalized;
5474}
5475
5476LegalizerHelper::LegalizeResult
5477LegalizerHelper::fewerElementsVectorUnmergeValues(MachineInstr &MI,
5478 unsigned TypeIdx,
5479 LLT NarrowTy) {
5480 const int NumDst = MI.getNumOperands() - 1;
5481 const Register SrcReg = MI.getOperand(i: NumDst).getReg();
5482 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5483 LLT SrcTy = MRI.getType(Reg: SrcReg);
5484
5485 if (TypeIdx != 1 || NarrowTy == DstTy)
5486 return UnableToLegalize;
5487
5488 // Requires compatible types. Otherwise SrcReg should have been defined by
5489 // merge-like instruction that would get artifact combined. Most likely
5490 // instruction that defines SrcReg has to perform more/fewer elements
5491 // legalization compatible with NarrowTy.
5492 assert(SrcTy.isVector() && NarrowTy.isVector() && "Expected vector types");
5493 assert((SrcTy.getScalarType() == NarrowTy.getScalarType()) && "bad type");
5494
5495 if ((SrcTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0) ||
5496 (NarrowTy.getSizeInBits() % DstTy.getSizeInBits() != 0))
5497 return UnableToLegalize;
5498
5499 // This is most likely DstTy (smaller then register size) packed in SrcTy
5500 // (larger then register size) and since unmerge was not combined it will be
5501 // lowered to bit sequence extracts from register. Unpack SrcTy to NarrowTy
5502 // (register size) pieces first. Then unpack each of NarrowTy pieces to DstTy.
5503
5504 // %1:_(DstTy), %2, %3, %4 = G_UNMERGE_VALUES %0:_(SrcTy)
5505 //
5506 // %5:_(NarrowTy), %6 = G_UNMERGE_VALUES %0:_(SrcTy) - reg sequence
5507 // %1:_(DstTy), %2 = G_UNMERGE_VALUES %5:_(NarrowTy) - sequence of bits in reg
5508 // %3:_(DstTy), %4 = G_UNMERGE_VALUES %6:_(NarrowTy)
5509 auto Unmerge = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: SrcReg);
5510 const int NumUnmerge = Unmerge->getNumOperands() - 1;
5511 const int PartsPerUnmerge = NumDst / NumUnmerge;
5512
5513 for (int I = 0; I != NumUnmerge; ++I) {
5514 auto MIB = MIRBuilder.buildInstr(Opcode: TargetOpcode::G_UNMERGE_VALUES);
5515
5516 for (int J = 0; J != PartsPerUnmerge; ++J)
5517 MIB.addDef(RegNo: MI.getOperand(i: I * PartsPerUnmerge + J).getReg());
5518 MIB.addUse(RegNo: Unmerge.getReg(Idx: I));
5519 }
5520
5521 MI.eraseFromParent();
5522 return Legalized;
5523}
5524
5525LegalizerHelper::LegalizeResult
5526LegalizerHelper::fewerElementsVectorMerge(MachineInstr &MI, unsigned TypeIdx,
5527 LLT NarrowTy) {
5528 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
5529 // Requires compatible types. Otherwise user of DstReg did not perform unmerge
5530 // that should have been artifact combined. Most likely instruction that uses
5531 // DstReg has to do more/fewer elements legalization compatible with NarrowTy.
5532 assert(DstTy.isVector() && NarrowTy.isVector() && "Expected vector types");
5533 assert((DstTy.getScalarType() == NarrowTy.getScalarType()) && "bad type");
5534 if (NarrowTy == SrcTy)
5535 return UnableToLegalize;
5536
5537 // This attempts to lower part of LCMTy merge/unmerge sequence. Intended use
5538 // is for old mir tests. Since the changes to more/fewer elements it should no
5539 // longer be possible to generate MIR like this when starting from llvm-ir
5540 // because LCMTy approach was replaced with merge/unmerge to vector elements.
5541 if (TypeIdx == 1) {
5542 assert(SrcTy.isVector() && "Expected vector types");
5543 assert((SrcTy.getScalarType() == NarrowTy.getScalarType()) && "bad type");
5544 if ((DstTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0) ||
5545 (NarrowTy.getNumElements() >= SrcTy.getNumElements()))
5546 return UnableToLegalize;
5547 // %2:_(DstTy) = G_CONCAT_VECTORS %0:_(SrcTy), %1:_(SrcTy)
5548 //
5549 // %3:_(EltTy), %4, %5 = G_UNMERGE_VALUES %0:_(SrcTy)
5550 // %6:_(EltTy), %7, %8 = G_UNMERGE_VALUES %1:_(SrcTy)
5551 // %9:_(NarrowTy) = G_BUILD_VECTOR %3:_(EltTy), %4
5552 // %10:_(NarrowTy) = G_BUILD_VECTOR %5:_(EltTy), %6
5553 // %11:_(NarrowTy) = G_BUILD_VECTOR %7:_(EltTy), %8
5554 // %2:_(DstTy) = G_CONCAT_VECTORS %9:_(NarrowTy), %10, %11
5555
5556 SmallVector<Register, 8> Elts;
5557 LLT EltTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getScalarType();
5558 for (unsigned i = 1; i < MI.getNumOperands(); ++i) {
5559 auto Unmerge = MIRBuilder.buildUnmerge(Res: EltTy, Op: MI.getOperand(i).getReg());
5560 for (unsigned j = 0; j < Unmerge->getNumDefs(); ++j)
5561 Elts.push_back(Elt: Unmerge.getReg(Idx: j));
5562 }
5563
5564 SmallVector<Register, 8> NarrowTyElts;
5565 unsigned NumNarrowTyElts = NarrowTy.getNumElements();
5566 unsigned NumNarrowTyPieces = DstTy.getNumElements() / NumNarrowTyElts;
5567 for (unsigned i = 0, Offset = 0; i < NumNarrowTyPieces;
5568 ++i, Offset += NumNarrowTyElts) {
5569 ArrayRef<Register> Pieces(&Elts[Offset], NumNarrowTyElts);
5570 NarrowTyElts.push_back(
5571 Elt: MIRBuilder.buildMergeLikeInstr(Res: NarrowTy, Ops: Pieces).getReg(Idx: 0));
5572 }
5573
5574 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: NarrowTyElts);
5575 MI.eraseFromParent();
5576 return Legalized;
5577 }
5578
5579 assert(TypeIdx == 0 && "Bad type index");
5580 if ((NarrowTy.getSizeInBits() % SrcTy.getSizeInBits() != 0) ||
5581 (DstTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0))
5582 return UnableToLegalize;
5583
5584 // This is most likely SrcTy (smaller then register size) packed in DstTy
5585 // (larger then register size) and since merge was not combined it will be
5586 // lowered to bit sequence packing into register. Merge SrcTy to NarrowTy
5587 // (register size) pieces first. Then merge each of NarrowTy pieces to DstTy.
5588
5589 // %0:_(DstTy) = G_MERGE_VALUES %1:_(SrcTy), %2, %3, %4
5590 //
5591 // %5:_(NarrowTy) = G_MERGE_VALUES %1:_(SrcTy), %2 - sequence of bits in reg
5592 // %6:_(NarrowTy) = G_MERGE_VALUES %3:_(SrcTy), %4
5593 // %0:_(DstTy) = G_MERGE_VALUES %5:_(NarrowTy), %6 - reg sequence
5594 SmallVector<Register, 8> NarrowTyElts;
5595 unsigned NumParts = DstTy.getNumElements() / NarrowTy.getNumElements();
5596 unsigned NumSrcElts = SrcTy.isVector() ? SrcTy.getNumElements() : 1;
5597 unsigned NumElts = NarrowTy.getNumElements() / NumSrcElts;
5598 for (unsigned i = 0; i < NumParts; ++i) {
5599 SmallVector<Register, 8> Sources;
5600 for (unsigned j = 0; j < NumElts; ++j)
5601 Sources.push_back(Elt: MI.getOperand(i: 1 + i * NumElts + j).getReg());
5602 NarrowTyElts.push_back(
5603 Elt: MIRBuilder.buildMergeLikeInstr(Res: NarrowTy, Ops: Sources).getReg(Idx: 0));
5604 }
5605
5606 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: NarrowTyElts);
5607 MI.eraseFromParent();
5608 return Legalized;
5609}
5610
5611LegalizerHelper::LegalizeResult
5612LegalizerHelper::fewerElementsVectorExtractInsertVectorElt(MachineInstr &MI,
5613 unsigned TypeIdx,
5614 LLT NarrowVecTy) {
5615 auto [DstReg, SrcVec] = MI.getFirst2Regs();
5616 Register InsertVal;
5617 bool IsInsert = MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT;
5618
5619 assert((IsInsert ? TypeIdx == 0 : TypeIdx == 1) && "not a vector type index");
5620 if (IsInsert)
5621 InsertVal = MI.getOperand(i: 2).getReg();
5622
5623 Register Idx = MI.getOperand(i: MI.getNumOperands() - 1).getReg();
5624 LLT VecTy = MRI.getType(Reg: SrcVec);
5625
5626 // If the index is a constant, we can really break this down as you would
5627 // expect, and index into the target size pieces.
5628 auto MaybeCst = getIConstantVRegValWithLookThrough(VReg: Idx, MRI);
5629 if (MaybeCst) {
5630 uint64_t IdxVal = MaybeCst->Value.getZExtValue();
5631 // Avoid out of bounds indexing the pieces.
5632 if (IdxVal >= VecTy.getNumElements()) {
5633 MIRBuilder.buildUndef(Res: DstReg);
5634 MI.eraseFromParent();
5635 return Legalized;
5636 }
5637
5638 if (!NarrowVecTy.isVector()) {
5639 SmallVector<Register, 8> SplitPieces;
5640 extractParts(Reg: MI.getOperand(i: 1).getReg(), Ty: NarrowVecTy,
5641 NumParts: VecTy.getNumElements(), VRegs&: SplitPieces, MIRBuilder, MRI);
5642 if (IsInsert) {
5643 SplitPieces[IdxVal] = InsertVal;
5644 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0).getReg(), Ops: SplitPieces);
5645 } else {
5646 MIRBuilder.buildCopy(Res: MI.getOperand(i: 0).getReg(), Op: SplitPieces[IdxVal]);
5647 }
5648 } else {
5649 SmallVector<Register, 8> VecParts;
5650 LLT GCDTy = extractGCDType(Parts&: VecParts, DstTy: VecTy, NarrowTy: NarrowVecTy, SrcReg: SrcVec);
5651
5652 // Build a sequence of NarrowTy pieces in VecParts for this operand.
5653 LLT LCMTy = buildLCMMergePieces(DstTy: VecTy, NarrowTy: NarrowVecTy, GCDTy, VRegs&: VecParts,
5654 PadStrategy: TargetOpcode::G_ANYEXT);
5655
5656 unsigned NewNumElts = NarrowVecTy.getNumElements();
5657
5658 LLT IdxTy = MRI.getType(Reg: Idx);
5659 int64_t PartIdx = IdxVal / NewNumElts;
5660 auto NewIdx =
5661 MIRBuilder.buildConstant(Res: IdxTy, Val: IdxVal - NewNumElts * PartIdx);
5662
5663 if (IsInsert) {
5664 LLT PartTy = MRI.getType(Reg: VecParts[PartIdx]);
5665
5666 // Use the adjusted index to insert into one of the subvectors.
5667 auto InsertPart = MIRBuilder.buildInsertVectorElement(
5668 Res: PartTy, Val: VecParts[PartIdx], Elt: InsertVal, Idx: NewIdx);
5669 VecParts[PartIdx] = InsertPart.getReg(Idx: 0);
5670
5671 // Recombine the inserted subvector with the others to reform the result
5672 // vector.
5673 buildWidenedRemergeToDst(DstReg, LCMTy, RemergeRegs: VecParts);
5674 } else {
5675 MIRBuilder.buildExtractVectorElement(Res: DstReg, Val: VecParts[PartIdx], Idx: NewIdx);
5676 }
5677 }
5678
5679 MI.eraseFromParent();
5680 return Legalized;
5681 }
5682
5683 // With a variable index, we can't perform the operation in a smaller type, so
5684 // we're forced to expand this.
5685 //
5686 // TODO: We could emit a chain of compare/select to figure out which piece to
5687 // index.
5688 return lowerExtractInsertVectorElt(MI);
5689}
5690
5691LegalizerHelper::LegalizeResult
5692LegalizerHelper::reduceLoadStoreWidth(GLoadStore &LdStMI, unsigned TypeIdx,
5693 LLT NarrowTy) {
5694 // FIXME: Don't know how to handle secondary types yet.
5695 if (TypeIdx != 0)
5696 return UnableToLegalize;
5697
5698 if (!NarrowTy.isByteSized()) {
5699 LLVM_DEBUG(dbgs() << "Can't narrow load/store to non-byte-sized type\n");
5700 return UnableToLegalize;
5701 }
5702
5703 // This implementation doesn't work for atomics. Give up instead of doing
5704 // something invalid.
5705 if (LdStMI.isAtomic())
5706 return UnableToLegalize;
5707
5708 bool IsLoad = isa<GLoad>(Val: LdStMI);
5709 Register ValReg = LdStMI.getReg(Idx: 0);
5710 Register AddrReg = LdStMI.getPointerReg();
5711 LLT ValTy = MRI.getType(Reg: ValReg);
5712
5713 // FIXME: Do we need a distinct NarrowMemory legalize action?
5714 if (ValTy.getSizeInBits() != 8 * LdStMI.getMemSize().getValue()) {
5715 LLVM_DEBUG(dbgs() << "Can't narrow extload/truncstore\n");
5716 return UnableToLegalize;
5717 }
5718
5719 int NumParts = -1;
5720 int NumLeftover = -1;
5721 LLT LeftoverTy;
5722 SmallVector<Register, 8> NarrowRegs, NarrowLeftoverRegs;
5723 if (IsLoad) {
5724 std::tie(args&: NumParts, args&: NumLeftover) = getNarrowTypeBreakDown(OrigTy: ValTy, NarrowTy, LeftoverTy);
5725 } else {
5726 if (extractParts(Reg: ValReg, RegTy: ValTy, MainTy: NarrowTy, LeftoverTy, VRegs&: NarrowRegs,
5727 LeftoverVRegs&: NarrowLeftoverRegs, MIRBuilder, MRI)) {
5728 NumParts = NarrowRegs.size();
5729 NumLeftover = NarrowLeftoverRegs.size();
5730 }
5731 }
5732
5733 if (NumParts == -1)
5734 return UnableToLegalize;
5735
5736 LLT PtrTy = MRI.getType(Reg: AddrReg);
5737 const LLT OffsetTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
5738
5739 unsigned TotalSize = ValTy.getSizeInBits();
5740
5741 // Split the load/store into PartTy sized pieces starting at Offset. If this
5742 // is a load, return the new registers in ValRegs. For a store, each elements
5743 // of ValRegs should be PartTy. Returns the next offset that needs to be
5744 // handled.
5745 bool isBigEndian = MIRBuilder.getDataLayout().isBigEndian();
5746 auto MMO = LdStMI.getMMO();
5747 auto splitTypePieces = [=](LLT PartTy, SmallVectorImpl<Register> &ValRegs,
5748 unsigned NumParts, unsigned Offset) -> unsigned {
5749 MachineFunction &MF = MIRBuilder.getMF();
5750 unsigned PartSize = PartTy.getSizeInBits();
5751 for (unsigned Idx = 0, E = NumParts; Idx != E && Offset < TotalSize;
5752 ++Idx) {
5753 unsigned ByteOffset = Offset / 8;
5754 Register NewAddrReg;
5755
5756 MIRBuilder.materializeObjectPtrOffset(Res&: NewAddrReg, Op0: AddrReg, ValueTy: OffsetTy,
5757 Value: ByteOffset);
5758
5759 MachineMemOperand *NewMMO =
5760 MF.getMachineMemOperand(MMO: &MMO, Offset: ByteOffset, Ty: PartTy);
5761
5762 if (IsLoad) {
5763 Register Dst = MRI.createGenericVirtualRegister(Ty: PartTy);
5764 ValRegs.push_back(Elt: Dst);
5765 MIRBuilder.buildLoad(Res: Dst, Addr: NewAddrReg, MMO&: *NewMMO);
5766 } else {
5767 MIRBuilder.buildStore(Val: ValRegs[Idx], Addr: NewAddrReg, MMO&: *NewMMO);
5768 }
5769 Offset = isBigEndian ? Offset - PartSize : Offset + PartSize;
5770 }
5771
5772 return Offset;
5773 };
5774
5775 unsigned Offset = isBigEndian ? TotalSize - NarrowTy.getSizeInBits() : 0;
5776 unsigned HandledOffset =
5777 splitTypePieces(NarrowTy, NarrowRegs, NumParts, Offset);
5778
5779 // Handle the rest of the register if this isn't an even type breakdown.
5780 if (LeftoverTy.isValid())
5781 splitTypePieces(LeftoverTy, NarrowLeftoverRegs, NumLeftover, HandledOffset);
5782
5783 if (IsLoad) {
5784 insertParts(DstReg: ValReg, ResultTy: ValTy, PartTy: NarrowTy, PartRegs: NarrowRegs,
5785 LeftoverTy, LeftoverRegs: NarrowLeftoverRegs);
5786 }
5787
5788 LdStMI.eraseFromParent();
5789 return Legalized;
5790}
5791
5792LegalizerHelper::LegalizeResult
5793LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
5794 LLT NarrowTy) {
5795 using namespace TargetOpcode;
5796 GenericMachineInstr &GMI = cast<GenericMachineInstr>(Val&: MI);
5797 unsigned NumElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
5798
5799 switch (MI.getOpcode()) {
5800 case G_IMPLICIT_DEF:
5801 case G_TRUNC:
5802 case G_AND:
5803 case G_OR:
5804 case G_XOR:
5805 case G_ADD:
5806 case G_SUB:
5807 case G_MUL:
5808 case G_PTR_ADD:
5809 case G_SMULH:
5810 case G_UMULH:
5811 case G_FADD:
5812 case G_FMUL:
5813 case G_FSUB:
5814 case G_FNEG:
5815 case G_FABS:
5816 case G_FCANONICALIZE:
5817 case G_FDIV:
5818 case G_FREM:
5819 case G_FMA:
5820 case G_FMAD:
5821 case G_FPOW:
5822 case G_FEXP:
5823 case G_FEXP2:
5824 case G_FEXP10:
5825 case G_FLOG:
5826 case G_FLOG2:
5827 case G_FLOG10:
5828 case G_FLDEXP:
5829 case G_FNEARBYINT:
5830 case G_FCEIL:
5831 case G_FFLOOR:
5832 case G_FRINT:
5833 case G_INTRINSIC_LRINT:
5834 case G_INTRINSIC_LLRINT:
5835 case G_INTRINSIC_ROUND:
5836 case G_INTRINSIC_ROUNDEVEN:
5837 case G_LROUND:
5838 case G_LLROUND:
5839 case G_INTRINSIC_TRUNC:
5840 case G_FMODF:
5841 case G_FCOS:
5842 case G_FSIN:
5843 case G_FTAN:
5844 case G_FACOS:
5845 case G_FASIN:
5846 case G_FATAN:
5847 case G_FATAN2:
5848 case G_FCOSH:
5849 case G_FSINH:
5850 case G_FTANH:
5851 case G_FSQRT:
5852 case G_BSWAP:
5853 case G_BITREVERSE:
5854 case G_SDIV:
5855 case G_UDIV:
5856 case G_SREM:
5857 case G_UREM:
5858 case G_SDIVREM:
5859 case G_UDIVREM:
5860 case G_SMIN:
5861 case G_SMAX:
5862 case G_UMIN:
5863 case G_UMAX:
5864 case G_ABS:
5865 case G_FMINNUM:
5866 case G_FMAXNUM:
5867 case G_FMINNUM_IEEE:
5868 case G_FMAXNUM_IEEE:
5869 case G_FMINIMUM:
5870 case G_FMAXIMUM:
5871 case G_FMINIMUMNUM:
5872 case G_FMAXIMUMNUM:
5873 case G_FSHL:
5874 case G_FSHR:
5875 case G_ROTL:
5876 case G_ROTR:
5877 case G_FREEZE:
5878 case G_SADDSAT:
5879 case G_SSUBSAT:
5880 case G_UADDSAT:
5881 case G_USUBSAT:
5882 case G_UMULO:
5883 case G_SMULO:
5884 case G_SHL:
5885 case G_LSHR:
5886 case G_ASHR:
5887 case G_SSHLSAT:
5888 case G_USHLSAT:
5889 case G_CTLZ:
5890 case G_CTLZ_ZERO_POISON:
5891 case G_CTTZ:
5892 case G_CTTZ_ZERO_POISON:
5893 case G_CTPOP:
5894 case G_CTLS:
5895 case G_FCOPYSIGN:
5896 case G_ZEXT:
5897 case G_SEXT:
5898 case G_ANYEXT:
5899 case G_FPEXT:
5900 case G_FPTRUNC:
5901 case G_SITOFP:
5902 case G_UITOFP:
5903 case G_FPTOSI:
5904 case G_FPTOUI:
5905 case G_FPTOSI_SAT:
5906 case G_FPTOUI_SAT:
5907 case G_INTTOPTR:
5908 case G_PTRTOINT:
5909 case G_ADDRSPACE_CAST:
5910 case G_UADDO:
5911 case G_USUBO:
5912 case G_UADDE:
5913 case G_USUBE:
5914 case G_SADDO:
5915 case G_SSUBO:
5916 case G_SADDE:
5917 case G_SSUBE:
5918 case G_STRICT_FADD:
5919 case G_STRICT_FSUB:
5920 case G_STRICT_FMUL:
5921 case G_STRICT_FMA:
5922 case G_STRICT_FLDEXP:
5923 case G_FFREXP:
5924 case G_TRUNC_SSAT_S:
5925 case G_TRUNC_SSAT_U:
5926 case G_TRUNC_USAT_U:
5927 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts);
5928 case G_ICMP:
5929 case G_FCMP:
5930 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {1 /*cpm predicate*/});
5931 case G_IS_FPCLASS:
5932 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {2, 3 /*mask,fpsem*/});
5933 case G_SELECT:
5934 if (MRI.getType(Reg: MI.getOperand(i: 1).getReg()).isVector())
5935 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts);
5936 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {1 /*scalar cond*/});
5937 case G_PHI:
5938 return fewerElementsVectorPhi(MI&: GMI, NumElts);
5939 case G_UNMERGE_VALUES:
5940 return fewerElementsVectorUnmergeValues(MI, TypeIdx, NarrowTy);
5941 case G_BUILD_VECTOR:
5942 assert(TypeIdx == 0 && "not a vector type index");
5943 return fewerElementsVectorMerge(MI, TypeIdx, NarrowTy);
5944 case G_CONCAT_VECTORS:
5945 if (TypeIdx != 1) // TODO: This probably does work as expected already.
5946 return UnableToLegalize;
5947 return fewerElementsVectorMerge(MI, TypeIdx, NarrowTy);
5948 case G_EXTRACT_SUBVECTOR: {
5949 Register DstReg = MI.getOperand(i: 0).getReg();
5950 LLT DstTy = MRI.getType(Reg: DstReg);
5951 Register SrcReg = MI.getOperand(i: 1).getReg();
5952 uint64_t InsertionPointImm = MI.getOperand(i: 2).getImm();
5953
5954 // If Dst > NarrowTy bits, then cannot legalize
5955 if (DstTy.getSizeInBits() > NarrowTy.getSizeInBits())
5956 return UnableToLegalize;
5957
5958 // If DstTy's size is not a multiple of NarrowTy's, then cannot legalize
5959 if (!DstTy.getElementCount().isKnownMultipleOf(RHS: NarrowTy.getElementCount()))
5960 return UnableToLegalize;
5961
5962 auto Unmerge = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: SrcReg);
5963 uint64_t RequiredSubvectorIndex =
5964 InsertionPointImm / NarrowTy.getNumElements();
5965 // If Dst and Narrow are both same size, convert to a copy
5966 if (DstTy.getNumElements() == NarrowTy.getNumElements())
5967 MIRBuilder.buildCopy(Res: DstReg, Op: Unmerge.getReg(Idx: RequiredSubvectorIndex));
5968 else
5969 MIRBuilder.buildExtractSubvector(
5970 Res: DstReg, Src: Unmerge.getReg(Idx: RequiredSubvectorIndex),
5971 Index: InsertionPointImm % NarrowTy.getNumElements());
5972
5973 MI.eraseFromParent();
5974 return Legalized;
5975 }
5976 case G_EXTRACT_VECTOR_ELT:
5977 case G_INSERT_VECTOR_ELT:
5978 return fewerElementsVectorExtractInsertVectorElt(MI, TypeIdx, NarrowVecTy: NarrowTy);
5979 case G_LOAD:
5980 case G_STORE:
5981 return reduceLoadStoreWidth(LdStMI&: cast<GLoadStore>(Val&: MI), TypeIdx, NarrowTy);
5982 case G_SEXT_INREG:
5983 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {2 /*imm*/});
5984 GISEL_VECREDUCE_CASES_NONSEQ
5985 return fewerElementsVectorReductions(MI, TypeIdx, NarrowTy);
5986 case TargetOpcode::G_VECREDUCE_SEQ_FADD:
5987 case TargetOpcode::G_VECREDUCE_SEQ_FMUL:
5988 return fewerElementsVectorSeqReductions(MI, TypeIdx, NarrowTy);
5989 case G_SHUFFLE_VECTOR:
5990 return fewerElementsVectorShuffle(MI, TypeIdx, NarrowTy);
5991 case G_FPOWI:
5992 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {2 /*pow*/});
5993 case G_BITCAST:
5994 return fewerElementsBitcast(MI, TypeIdx, NarrowTy);
5995 case G_INTRINSIC_FPTRUNC_ROUND:
5996 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {2});
5997 default:
5998 return UnableToLegalize;
5999 }
6000}
6001
6002LegalizerHelper::LegalizeResult
6003LegalizerHelper::fewerElementsBitcast(MachineInstr &MI, unsigned int TypeIdx,
6004 LLT NarrowTy) {
6005 assert(MI.getOpcode() == TargetOpcode::G_BITCAST &&
6006 "Not a bitcast operation");
6007
6008 if (TypeIdx != 0)
6009 return UnableToLegalize;
6010
6011 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
6012
6013 unsigned NewElemCount =
6014 NarrowTy.getSizeInBits() / SrcTy.getScalarSizeInBits();
6015 SmallVector<Register> SrcVRegs, BitcastVRegs;
6016 if (NewElemCount == 1) {
6017 LLT SrcNarrowTy = SrcTy.getElementType();
6018
6019 auto Unmerge = MIRBuilder.buildUnmerge(Res: SrcNarrowTy, Op: SrcReg);
6020 getUnmergeResults(Regs&: SrcVRegs, MI: *Unmerge);
6021 } else {
6022 LLT SrcNarrowTy =
6023 SrcTy.changeVectorElementCount(EC: ElementCount::getFixed(MinVal: NewElemCount));
6024
6025 // Split the Src and Dst Reg into smaller registers
6026 if (extractGCDType(Parts&: SrcVRegs, DstTy, NarrowTy: SrcNarrowTy, SrcReg) != SrcNarrowTy)
6027 return UnableToLegalize;
6028 }
6029
6030 // Build new smaller bitcast instructions
6031 // Not supporting Leftover types for now but will have to
6032 for (Register Reg : SrcVRegs)
6033 BitcastVRegs.push_back(Elt: MIRBuilder.buildBitcast(Dst: NarrowTy, Src: Reg).getReg(Idx: 0));
6034
6035 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: BitcastVRegs);
6036 MI.eraseFromParent();
6037 return Legalized;
6038}
6039
6040LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorShuffle(
6041 MachineInstr &MI, unsigned int TypeIdx, LLT NarrowTy) {
6042 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
6043 if (TypeIdx != 0)
6044 return UnableToLegalize;
6045
6046 auto [DstReg, DstTy, Src1Reg, Src1Ty, Src2Reg, Src2Ty] =
6047 MI.getFirst3RegLLTs();
6048 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
6049 // The shuffle should be canonicalized by now.
6050 if (DstTy != Src1Ty)
6051 return UnableToLegalize;
6052 if (DstTy != Src2Ty)
6053 return UnableToLegalize;
6054
6055 if (!isPowerOf2_32(Value: DstTy.getNumElements()))
6056 return UnableToLegalize;
6057
6058 // We only support splitting a shuffle into 2, so adjust NarrowTy accordingly.
6059 // Further legalization attempts will be needed to do split further.
6060 NarrowTy =
6061 DstTy.changeElementCount(EC: DstTy.getElementCount().divideCoefficientBy(RHS: 2));
6062 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
6063
6064 SmallVector<Register> SplitSrc1Regs, SplitSrc2Regs;
6065 extractParts(Reg: Src1Reg, Ty: NarrowTy, NumParts: 2, VRegs&: SplitSrc1Regs, MIRBuilder, MRI);
6066 extractParts(Reg: Src2Reg, Ty: NarrowTy, NumParts: 2, VRegs&: SplitSrc2Regs, MIRBuilder, MRI);
6067 Register Inputs[4] = {SplitSrc1Regs[0], SplitSrc1Regs[1], SplitSrc2Regs[0],
6068 SplitSrc2Regs[1]};
6069
6070 Register Hi, Lo;
6071
6072 // If Lo or Hi uses elements from at most two of the four input vectors, then
6073 // express it as a vector shuffle of those two inputs. Otherwise extract the
6074 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
6075 SmallVector<int, 16> Ops;
6076 for (unsigned High = 0; High < 2; ++High) {
6077 Register &Output = High ? Hi : Lo;
6078
6079 // Build a shuffle mask for the output, discovering on the fly which
6080 // input vectors to use as shuffle operands (recorded in InputUsed).
6081 // If building a suitable shuffle vector proves too hard, then bail
6082 // out with useBuildVector set.
6083 unsigned InputUsed[2] = {-1U, -1U}; // Not yet discovered.
6084 unsigned FirstMaskIdx = High * NewElts;
6085 bool UseBuildVector = false;
6086 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
6087 // The mask element. This indexes into the input.
6088 int Idx = Mask[FirstMaskIdx + MaskOffset];
6089
6090 // The input vector this mask element indexes into.
6091 unsigned Input = (unsigned)Idx / NewElts;
6092
6093 if (Input >= std::size(Inputs)) {
6094 // The mask element does not index into any input vector.
6095 Ops.push_back(Elt: -1);
6096 continue;
6097 }
6098
6099 // Turn the index into an offset from the start of the input vector.
6100 Idx -= Input * NewElts;
6101
6102 // Find or create a shuffle vector operand to hold this input.
6103 unsigned OpNo;
6104 for (OpNo = 0; OpNo < std::size(InputUsed); ++OpNo) {
6105 if (InputUsed[OpNo] == Input) {
6106 // This input vector is already an operand.
6107 break;
6108 } else if (InputUsed[OpNo] == -1U) {
6109 // Create a new operand for this input vector.
6110 InputUsed[OpNo] = Input;
6111 break;
6112 }
6113 }
6114
6115 if (OpNo >= std::size(InputUsed)) {
6116 // More than two input vectors used! Give up on trying to create a
6117 // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
6118 UseBuildVector = true;
6119 break;
6120 }
6121
6122 // Add the mask index for the new shuffle vector.
6123 Ops.push_back(Elt: Idx + OpNo * NewElts);
6124 }
6125
6126 if (UseBuildVector) {
6127 LLT EltTy = NarrowTy.getElementType();
6128 SmallVector<Register, 16> SVOps;
6129
6130 // Extract the input elements by hand.
6131 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
6132 // The mask element. This indexes into the input.
6133 int Idx = Mask[FirstMaskIdx + MaskOffset];
6134
6135 // The input vector this mask element indexes into.
6136 unsigned Input = (unsigned)Idx / NewElts;
6137
6138 if (Input >= std::size(Inputs)) {
6139 // The mask element is "undef" or indexes off the end of the input.
6140 SVOps.push_back(Elt: MIRBuilder.buildUndef(Res: EltTy).getReg(Idx: 0));
6141 continue;
6142 }
6143
6144 // Turn the index into an offset from the start of the input vector.
6145 Idx -= Input * NewElts;
6146
6147 // Extract the vector element by hand.
6148 SVOps.push_back(Elt: MIRBuilder
6149 .buildExtractVectorElement(
6150 Res: EltTy, Val: Inputs[Input],
6151 Idx: MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: 32), Val: Idx))
6152 .getReg(Idx: 0));
6153 }
6154
6155 // Construct the Lo/Hi output using a G_BUILD_VECTOR.
6156 Output = MIRBuilder.buildBuildVector(Res: NarrowTy, Ops: SVOps).getReg(Idx: 0);
6157 } else if (InputUsed[0] == -1U) {
6158 // No input vectors were used! The result is undefined.
6159 Output = MIRBuilder.buildUndef(Res: NarrowTy).getReg(Idx: 0);
6160 } else if (NewElts == 1) {
6161 Output = MIRBuilder.buildCopy(Res: NarrowTy, Op: Inputs[InputUsed[0]]).getReg(Idx: 0);
6162 } else {
6163 Register Op0 = Inputs[InputUsed[0]];
6164 // If only one input was used, use an undefined vector for the other.
6165 Register Op1 = InputUsed[1] == -1U
6166 ? MIRBuilder.buildUndef(Res: NarrowTy).getReg(Idx: 0)
6167 : Inputs[InputUsed[1]];
6168 // At least one input vector was used. Create a new shuffle vector.
6169 Output = MIRBuilder.buildShuffleVector(Res: NarrowTy, Src1: Op0, Src2: Op1, Mask: Ops).getReg(Idx: 0);
6170 }
6171
6172 Ops.clear();
6173 }
6174
6175 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: {Lo, Hi});
6176 MI.eraseFromParent();
6177 return Legalized;
6178}
6179
6180LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorReductions(
6181 MachineInstr &MI, unsigned int TypeIdx, LLT NarrowTy) {
6182 auto &RdxMI = cast<GVecReduce>(Val&: MI);
6183
6184 if (TypeIdx != 1)
6185 return UnableToLegalize;
6186
6187 // The semantics of the normal non-sequential reductions allow us to freely
6188 // re-associate the operation.
6189 auto [DstReg, DstTy, SrcReg, SrcTy] = RdxMI.getFirst2RegLLTs();
6190
6191 if (NarrowTy.isVector() &&
6192 (SrcTy.getNumElements() % NarrowTy.getNumElements() != 0))
6193 return UnableToLegalize;
6194
6195 unsigned ScalarOpc = RdxMI.getScalarOpcForReduction();
6196 SmallVector<Register> SplitSrcs;
6197 // If NarrowTy is a scalar then we're being asked to scalarize.
6198 const unsigned NumParts =
6199 NarrowTy.isVector() ? SrcTy.getNumElements() / NarrowTy.getNumElements()
6200 : SrcTy.getNumElements();
6201
6202 extractParts(Reg: SrcReg, Ty: NarrowTy, NumParts, VRegs&: SplitSrcs, MIRBuilder, MRI);
6203 if (NarrowTy.isScalar()) {
6204 if (DstTy != NarrowTy)
6205 return UnableToLegalize; // FIXME: handle implicit extensions.
6206
6207 if (isPowerOf2_32(Value: NumParts)) {
6208 // Generate a tree of scalar operations to reduce the critical path.
6209 SmallVector<Register> PartialResults;
6210 unsigned NumPartsLeft = NumParts;
6211 while (NumPartsLeft > 1) {
6212 for (unsigned Idx = 0; Idx < NumPartsLeft - 1; Idx += 2) {
6213 PartialResults.emplace_back(
6214 Args: MIRBuilder
6215 .buildInstr(Opc: ScalarOpc, DstOps: {NarrowTy},
6216 SrcOps: {SplitSrcs[Idx], SplitSrcs[Idx + 1]},
6217 Flags: MI.getFlags())
6218 .getReg(Idx: 0));
6219 }
6220 SplitSrcs = PartialResults;
6221 PartialResults.clear();
6222 NumPartsLeft = SplitSrcs.size();
6223 }
6224 assert(SplitSrcs.size() == 1);
6225 MIRBuilder.buildCopy(Res: DstReg, Op: SplitSrcs[0]);
6226 MI.eraseFromParent();
6227 return Legalized;
6228 }
6229 // If we can't generate a tree, then just do sequential operations.
6230 Register Acc = SplitSrcs[0];
6231 for (unsigned Idx = 1; Idx < NumParts; ++Idx)
6232 Acc = MIRBuilder
6233 .buildInstr(Opc: ScalarOpc, DstOps: {NarrowTy}, SrcOps: {Acc, SplitSrcs[Idx]},
6234 Flags: MI.getFlags())
6235 .getReg(Idx: 0);
6236 MIRBuilder.buildCopy(Res: DstReg, Op: Acc);
6237 MI.eraseFromParent();
6238 return Legalized;
6239 }
6240 SmallVector<Register> PartialReductions;
6241 for (unsigned Part = 0; Part < NumParts; ++Part) {
6242 PartialReductions.push_back(Elt: MIRBuilder
6243 .buildInstr(Opc: RdxMI.getOpcode(), DstOps: {DstTy},
6244 SrcOps: {SplitSrcs[Part]},
6245 Flags: MI.getFlags())
6246 .getReg(Idx: 0));
6247 }
6248
6249 // If the types involved are powers of 2, we can generate intermediate vector
6250 // ops, before generating a final reduction operation.
6251 if (isPowerOf2_32(Value: SrcTy.getNumElements()) &&
6252 isPowerOf2_32(Value: NarrowTy.getNumElements())) {
6253 return tryNarrowPow2Reduction(MI, SrcReg, SrcTy, NarrowTy, ScalarOpc);
6254 }
6255
6256 Register Acc = PartialReductions[0];
6257 for (unsigned Part = 1; Part < NumParts; ++Part) {
6258 if (Part == NumParts - 1) {
6259 MIRBuilder.buildInstr(Opc: ScalarOpc, DstOps: {DstReg}, SrcOps: {Acc, PartialReductions[Part]},
6260 Flags: MI.getFlags());
6261 } else {
6262 Acc = MIRBuilder
6263 .buildInstr(Opc: ScalarOpc, DstOps: {DstTy}, SrcOps: {Acc, PartialReductions[Part]},
6264 Flags: MI.getFlags())
6265 .getReg(Idx: 0);
6266 }
6267 }
6268 MI.eraseFromParent();
6269 return Legalized;
6270}
6271
6272LegalizerHelper::LegalizeResult
6273LegalizerHelper::fewerElementsVectorSeqReductions(MachineInstr &MI,
6274 unsigned int TypeIdx,
6275 LLT NarrowTy) {
6276 auto [DstReg, DstTy, ScalarReg, ScalarTy, SrcReg, SrcTy] =
6277 MI.getFirst3RegLLTs();
6278 if (!NarrowTy.isScalar() || TypeIdx != 2 || DstTy != ScalarTy ||
6279 DstTy != NarrowTy)
6280 return UnableToLegalize;
6281
6282 assert((MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FADD ||
6283 MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FMUL) &&
6284 "Unexpected vecreduce opcode");
6285 unsigned ScalarOpc = MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FADD
6286 ? TargetOpcode::G_FADD
6287 : TargetOpcode::G_FMUL;
6288
6289 SmallVector<Register> SplitSrcs;
6290 unsigned NumParts = SrcTy.getNumElements();
6291 extractParts(Reg: SrcReg, Ty: NarrowTy, NumParts, VRegs&: SplitSrcs, MIRBuilder, MRI);
6292 Register Acc = ScalarReg;
6293 for (unsigned i = 0; i < NumParts; i++)
6294 Acc = MIRBuilder
6295 .buildInstr(Opc: ScalarOpc, DstOps: {NarrowTy}, SrcOps: {Acc, SplitSrcs[i]},
6296 Flags: MI.getFlags())
6297 .getReg(Idx: 0);
6298
6299 MIRBuilder.buildCopy(Res: DstReg, Op: Acc);
6300 MI.eraseFromParent();
6301 return Legalized;
6302}
6303
6304LegalizerHelper::LegalizeResult
6305LegalizerHelper::tryNarrowPow2Reduction(MachineInstr &MI, Register SrcReg,
6306 LLT SrcTy, LLT NarrowTy,
6307 unsigned ScalarOpc) {
6308 SmallVector<Register> SplitSrcs;
6309 // Split the sources into NarrowTy size pieces.
6310 extractParts(Reg: SrcReg, Ty: NarrowTy,
6311 NumParts: SrcTy.getNumElements() / NarrowTy.getNumElements(), VRegs&: SplitSrcs,
6312 MIRBuilder, MRI);
6313 // We're going to do a tree reduction using vector operations until we have
6314 // one NarrowTy size value left.
6315 while (SplitSrcs.size() > 1) {
6316 SmallVector<Register> PartialRdxs;
6317 for (unsigned Idx = 0; Idx < SplitSrcs.size()-1; Idx += 2) {
6318 Register LHS = SplitSrcs[Idx];
6319 Register RHS = SplitSrcs[Idx + 1];
6320 // Create the intermediate vector op.
6321 Register Res =
6322 MIRBuilder
6323 .buildInstr(Opc: ScalarOpc, DstOps: {NarrowTy}, SrcOps: {LHS, RHS}, Flags: MI.getFlags())
6324 .getReg(Idx: 0);
6325 PartialRdxs.push_back(Elt: Res);
6326 }
6327 SplitSrcs = std::move(PartialRdxs);
6328 }
6329 // Finally generate the requested NarrowTy based reduction.
6330 Observer.changingInstr(MI);
6331 MI.getOperand(i: 1).setReg(SplitSrcs[0]);
6332 Observer.changedInstr(MI);
6333 return Legalized;
6334}
6335
6336LegalizerHelper::LegalizeResult
6337LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt,
6338 const LLT HalfTy, const LLT AmtTy) {
6339
6340 Register InL = MRI.createGenericVirtualRegister(Ty: HalfTy);
6341 Register InH = MRI.createGenericVirtualRegister(Ty: HalfTy);
6342 MIRBuilder.buildUnmerge(Res: {InL, InH}, Op: MI.getOperand(i: 1));
6343
6344 if (Amt.isZero()) {
6345 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0), Ops: {InL, InH});
6346 MI.eraseFromParent();
6347 return Legalized;
6348 }
6349
6350 LLT NVT = HalfTy;
6351 unsigned NVTBits = HalfTy.getSizeInBits();
6352 unsigned VTBits = 2 * NVTBits;
6353
6354 SrcOp Lo(Register(0)), Hi(Register(0));
6355 if (MI.getOpcode() == TargetOpcode::G_SHL) {
6356 if (Amt.ugt(RHS: VTBits)) {
6357 Lo = Hi = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6358 } else if (Amt.ugt(RHS: NVTBits)) {
6359 Lo = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6360 Hi = MIRBuilder.buildShl(Dst: NVT, Src0: InL,
6361 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt - NVTBits));
6362 } else if (Amt == NVTBits) {
6363 Lo = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6364 Hi = InL;
6365 } else {
6366 Lo = MIRBuilder.buildShl(Dst: NVT, Src0: InL, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt));
6367 auto OrLHS =
6368 MIRBuilder.buildShl(Dst: NVT, Src0: InH, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt));
6369 auto OrRHS = MIRBuilder.buildLShr(
6370 Dst: NVT, Src0: InL, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: -Amt + NVTBits));
6371 Hi = MIRBuilder.buildOr(Dst: NVT, Src0: OrLHS, Src1: OrRHS);
6372 }
6373 } else if (MI.getOpcode() == TargetOpcode::G_LSHR) {
6374 if (Amt.ugt(RHS: VTBits)) {
6375 Lo = Hi = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6376 } else if (Amt.ugt(RHS: NVTBits)) {
6377 Lo = MIRBuilder.buildLShr(Dst: NVT, Src0: InH,
6378 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt - NVTBits));
6379 Hi = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6380 } else if (Amt == NVTBits) {
6381 Lo = InH;
6382 Hi = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6383 } else {
6384 auto ShiftAmtConst = MIRBuilder.buildConstant(Res: AmtTy, Val: Amt);
6385
6386 auto OrLHS = MIRBuilder.buildLShr(Dst: NVT, Src0: InL, Src1: ShiftAmtConst);
6387 auto OrRHS = MIRBuilder.buildShl(
6388 Dst: NVT, Src0: InH, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: -Amt + NVTBits));
6389
6390 Lo = MIRBuilder.buildOr(Dst: NVT, Src0: OrLHS, Src1: OrRHS);
6391 Hi = MIRBuilder.buildLShr(Dst: NVT, Src0: InH, Src1: ShiftAmtConst);
6392 }
6393 } else {
6394 if (Amt.ugt(RHS: VTBits)) {
6395 Hi = Lo = MIRBuilder.buildAShr(
6396 Dst: NVT, Src0: InH, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: NVTBits - 1));
6397 } else if (Amt.ugt(RHS: NVTBits)) {
6398 Lo = MIRBuilder.buildAShr(Dst: NVT, Src0: InH,
6399 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt - NVTBits));
6400 Hi = MIRBuilder.buildAShr(Dst: NVT, Src0: InH,
6401 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: NVTBits - 1));
6402 } else if (Amt == NVTBits) {
6403 Lo = InH;
6404 Hi = MIRBuilder.buildAShr(Dst: NVT, Src0: InH,
6405 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: NVTBits - 1));
6406 } else {
6407 auto ShiftAmtConst = MIRBuilder.buildConstant(Res: AmtTy, Val: Amt);
6408
6409 auto OrLHS = MIRBuilder.buildLShr(Dst: NVT, Src0: InL, Src1: ShiftAmtConst);
6410 auto OrRHS = MIRBuilder.buildShl(
6411 Dst: NVT, Src0: InH, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: -Amt + NVTBits));
6412
6413 Lo = MIRBuilder.buildOr(Dst: NVT, Src0: OrLHS, Src1: OrRHS);
6414 Hi = MIRBuilder.buildAShr(Dst: NVT, Src0: InH, Src1: ShiftAmtConst);
6415 }
6416 }
6417
6418 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0), Ops: {Lo, Hi});
6419 MI.eraseFromParent();
6420
6421 return Legalized;
6422}
6423
6424LegalizerHelper::LegalizeResult
6425LegalizerHelper::narrowScalarShift(MachineInstr &MI, unsigned TypeIdx,
6426 LLT RequestedTy) {
6427 if (TypeIdx == 1) {
6428 Observer.changingInstr(MI);
6429 narrowScalarSrc(MI, NarrowTy: RequestedTy, OpIdx: 2);
6430 Observer.changedInstr(MI);
6431 return Legalized;
6432 }
6433
6434 Register DstReg = MI.getOperand(i: 0).getReg();
6435 LLT DstTy = MRI.getType(Reg: DstReg);
6436 if (DstTy.isVector())
6437 return UnableToLegalize;
6438
6439 Register Amt = MI.getOperand(i: 2).getReg();
6440 LLT ShiftAmtTy = MRI.getType(Reg: Amt);
6441 const unsigned DstEltSize = DstTy.getScalarSizeInBits();
6442 if (DstEltSize % 2 != 0)
6443 return UnableToLegalize;
6444
6445 // Check if we should use multi-way splitting instead of recursive binary
6446 // splitting.
6447 //
6448 // Multi-way splitting directly decomposes wide shifts (e.g., 128-bit ->
6449 // 4×32-bit) in a single legalization step, avoiding the recursive overhead
6450 // and dependency chains created by usual binary splitting approach
6451 // (128->64->32).
6452 //
6453 // The >= 8 parts threshold ensures we only use this optimization when binary
6454 // splitting would require multiple recursive passes, avoiding overhead for
6455 // simple 2-way splits where binary approach is sufficient.
6456 if (RequestedTy.isValid() && RequestedTy.isScalar() &&
6457 DstEltSize % RequestedTy.getSizeInBits() == 0) {
6458 const unsigned NumParts = DstEltSize / RequestedTy.getSizeInBits();
6459 // Use multiway if we have 8 or more parts (i.e., would need 3+ recursive
6460 // steps).
6461 if (NumParts >= 8)
6462 return narrowScalarShiftMultiway(MI, TargetTy: RequestedTy);
6463 }
6464
6465 // Fall back to binary splitting:
6466 // Ignore the input type. We can only go to exactly half the size of the
6467 // input. If that isn't small enough, the resulting pieces will be further
6468 // legalized.
6469 const unsigned NewBitSize = DstEltSize / 2;
6470 const LLT HalfTy = DstTy.getScalarType().changeElementSize(NewEltSize: NewBitSize);
6471 const LLT CondTy = LLT::integer(SizeInBits: 1);
6472
6473 if (auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: Amt, MRI)) {
6474 return narrowScalarShiftByConstant(MI, Amt: VRegAndVal->Value, HalfTy,
6475 AmtTy: ShiftAmtTy);
6476 }
6477
6478 // TODO: Expand with known bits.
6479
6480 // Handle the fully general expansion by an unknown amount.
6481 auto NewBits = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: NewBitSize);
6482
6483 Register InL = MRI.createGenericVirtualRegister(Ty: HalfTy);
6484 Register InH = MRI.createGenericVirtualRegister(Ty: HalfTy);
6485 MIRBuilder.buildUnmerge(Res: {InL, InH}, Op: MI.getOperand(i: 1));
6486
6487 auto AmtExcess = MIRBuilder.buildSub(Dst: ShiftAmtTy, Src0: Amt, Src1: NewBits);
6488 auto AmtLack = MIRBuilder.buildSub(Dst: ShiftAmtTy, Src0: NewBits, Src1: Amt);
6489
6490 auto Zero = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: 0);
6491 auto IsShort = MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_ULT, Res: CondTy, Op0: Amt, Op1: NewBits);
6492 auto IsZero = MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: CondTy, Op0: Amt, Op1: Zero);
6493
6494 Register ResultRegs[2];
6495 switch (MI.getOpcode()) {
6496 case TargetOpcode::G_SHL: {
6497 // Short: ShAmt < NewBitSize
6498 auto LoS = MIRBuilder.buildShl(Dst: HalfTy, Src0: InL, Src1: Amt);
6499
6500 auto LoOr = MIRBuilder.buildLShr(Dst: HalfTy, Src0: InL, Src1: AmtLack);
6501 auto HiOr = MIRBuilder.buildShl(Dst: HalfTy, Src0: InH, Src1: Amt);
6502 auto HiS = MIRBuilder.buildOr(Dst: HalfTy, Src0: LoOr, Src1: HiOr);
6503
6504 // Long: ShAmt >= NewBitSize
6505 auto LoL = MIRBuilder.buildConstant(Res: HalfTy, Val: 0); // Lo part is zero.
6506 auto HiL = MIRBuilder.buildShl(Dst: HalfTy, Src0: InL, Src1: AmtExcess); // Hi from Lo part.
6507
6508 auto Lo = MIRBuilder.buildSelect(Res: HalfTy, Tst: IsShort, Op0: LoS, Op1: LoL);
6509 auto Hi = MIRBuilder.buildSelect(
6510 Res: HalfTy, Tst: IsZero, Op0: InH, Op1: MIRBuilder.buildSelect(Res: HalfTy, Tst: IsShort, Op0: HiS, Op1: HiL));
6511
6512 ResultRegs[0] = Lo.getReg(Idx: 0);
6513 ResultRegs[1] = Hi.getReg(Idx: 0);
6514 break;
6515 }
6516 case TargetOpcode::G_LSHR:
6517 case TargetOpcode::G_ASHR: {
6518 // Short: ShAmt < NewBitSize
6519 auto HiS = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {HalfTy}, SrcOps: {InH, Amt});
6520
6521 auto LoOr = MIRBuilder.buildLShr(Dst: HalfTy, Src0: InL, Src1: Amt);
6522 auto HiOr = MIRBuilder.buildShl(Dst: HalfTy, Src0: InH, Src1: AmtLack);
6523 auto LoS = MIRBuilder.buildOr(Dst: HalfTy, Src0: LoOr, Src1: HiOr);
6524
6525 // Long: ShAmt >= NewBitSize
6526 MachineInstrBuilder HiL;
6527 if (MI.getOpcode() == TargetOpcode::G_LSHR) {
6528 HiL = MIRBuilder.buildConstant(Res: HalfTy, Val: 0); // Hi part is zero.
6529 } else {
6530 auto ShiftAmt = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: NewBitSize - 1);
6531 HiL = MIRBuilder.buildAShr(Dst: HalfTy, Src0: InH, Src1: ShiftAmt); // Sign of Hi part.
6532 }
6533 auto LoL = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {HalfTy},
6534 SrcOps: {InH, AmtExcess}); // Lo from Hi part.
6535
6536 auto Lo = MIRBuilder.buildSelect(
6537 Res: HalfTy, Tst: IsZero, Op0: InL, Op1: MIRBuilder.buildSelect(Res: HalfTy, Tst: IsShort, Op0: LoS, Op1: LoL));
6538
6539 auto Hi = MIRBuilder.buildSelect(Res: HalfTy, Tst: IsShort, Op0: HiS, Op1: HiL);
6540
6541 ResultRegs[0] = Lo.getReg(Idx: 0);
6542 ResultRegs[1] = Hi.getReg(Idx: 0);
6543 break;
6544 }
6545 default:
6546 llvm_unreachable("not a shift");
6547 }
6548
6549 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: ResultRegs);
6550 MI.eraseFromParent();
6551 return Legalized;
6552}
6553
6554Register LegalizerHelper::buildConstantShiftPart(unsigned Opcode,
6555 unsigned PartIdx,
6556 unsigned NumParts,
6557 ArrayRef<Register> SrcParts,
6558 const ShiftParams &Params,
6559 LLT TargetTy, LLT ShiftAmtTy) {
6560 auto WordShiftConst = getIConstantVRegVal(VReg: Params.WordShift, MRI);
6561 auto BitShiftConst = getIConstantVRegVal(VReg: Params.BitShift, MRI);
6562 assert(WordShiftConst && BitShiftConst && "Expected constants");
6563
6564 const unsigned ShiftWords = WordShiftConst->getZExtValue();
6565 const unsigned ShiftBits = BitShiftConst->getZExtValue();
6566 const bool NeedsInterWordShift = ShiftBits != 0;
6567
6568 switch (Opcode) {
6569 case TargetOpcode::G_SHL: {
6570 // Data moves from lower indices to higher indices
6571 // If this part would come from a source beyond our range, it's zero
6572 if (PartIdx < ShiftWords)
6573 return Params.Zero;
6574
6575 unsigned SrcIdx = PartIdx - ShiftWords;
6576 if (!NeedsInterWordShift)
6577 return SrcParts[SrcIdx];
6578
6579 // Combine shifted main part with carry from previous part
6580 auto Hi = MIRBuilder.buildShl(Dst: TargetTy, Src0: SrcParts[SrcIdx], Src1: Params.BitShift);
6581 if (SrcIdx > 0) {
6582 auto Lo = MIRBuilder.buildLShr(Dst: TargetTy, Src0: SrcParts[SrcIdx - 1],
6583 Src1: Params.InvBitShift);
6584 return MIRBuilder.buildOr(Dst: TargetTy, Src0: Hi, Src1: Lo).getReg(Idx: 0);
6585 }
6586 return Hi.getReg(Idx: 0);
6587 }
6588
6589 case TargetOpcode::G_LSHR: {
6590 unsigned SrcIdx = PartIdx + ShiftWords;
6591 if (SrcIdx >= NumParts)
6592 return Params.Zero;
6593 if (!NeedsInterWordShift)
6594 return SrcParts[SrcIdx];
6595
6596 // Combine shifted main part with carry from next part
6597 auto Lo = MIRBuilder.buildLShr(Dst: TargetTy, Src0: SrcParts[SrcIdx], Src1: Params.BitShift);
6598 if (SrcIdx + 1 < NumParts) {
6599 auto Hi = MIRBuilder.buildShl(Dst: TargetTy, Src0: SrcParts[SrcIdx + 1],
6600 Src1: Params.InvBitShift);
6601 return MIRBuilder.buildOr(Dst: TargetTy, Src0: Lo, Src1: Hi).getReg(Idx: 0);
6602 }
6603 return Lo.getReg(Idx: 0);
6604 }
6605
6606 case TargetOpcode::G_ASHR: {
6607 // Like LSHR but preserves sign bit
6608 unsigned SrcIdx = PartIdx + ShiftWords;
6609 if (SrcIdx >= NumParts)
6610 return Params.SignBit;
6611 if (!NeedsInterWordShift)
6612 return SrcParts[SrcIdx];
6613
6614 // Only the original MSB part uses arithmetic shift to preserve sign. All
6615 // other parts use logical shift since they're just moving data bits.
6616 auto Lo =
6617 (SrcIdx == NumParts - 1)
6618 ? MIRBuilder.buildAShr(Dst: TargetTy, Src0: SrcParts[SrcIdx], Src1: Params.BitShift)
6619 : MIRBuilder.buildLShr(Dst: TargetTy, Src0: SrcParts[SrcIdx], Src1: Params.BitShift);
6620 Register HiSrc =
6621 (SrcIdx + 1 < NumParts) ? SrcParts[SrcIdx + 1] : Params.SignBit;
6622 auto Hi = MIRBuilder.buildShl(Dst: TargetTy, Src0: HiSrc, Src1: Params.InvBitShift);
6623 return MIRBuilder.buildOr(Dst: TargetTy, Src0: Lo, Src1: Hi).getReg(Idx: 0);
6624 }
6625
6626 default:
6627 llvm_unreachable("not a shift");
6628 }
6629}
6630
6631Register LegalizerHelper::buildVariableShiftPart(unsigned Opcode,
6632 Register MainOperand,
6633 Register ShiftAmt,
6634 LLT TargetTy,
6635 Register CarryOperand) {
6636 // This helper generates a single output part for variable shifts by combining
6637 // the main operand (shifted by BitShift) with carry bits from an adjacent
6638 // part.
6639
6640 // For G_ASHR, individual parts don't have their own sign bit, only the
6641 // complete value does. So we use LSHR for the main operand shift in ASHR
6642 // context.
6643 unsigned MainOpcode = (Opcode == TargetOpcode::G_ASHR)
6644 ? static_cast<unsigned>(TargetOpcode::G_LSHR)
6645 : Opcode;
6646
6647 // Perform the primary shift on the main operand
6648 Register MainShifted =
6649 MIRBuilder.buildInstr(Opc: MainOpcode, DstOps: {TargetTy}, SrcOps: {MainOperand, ShiftAmt})
6650 .getReg(Idx: 0);
6651
6652 // No carry operand available
6653 if (!CarryOperand.isValid())
6654 return MainShifted;
6655
6656 // If BitShift is 0 (word-aligned shift), no inter-word bit movement occurs,
6657 // so carry bits aren't needed.
6658 LLT ShiftAmtTy = MRI.getType(Reg: ShiftAmt);
6659 auto ZeroConst = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: 0);
6660 LLT BoolTy = LLT::integer(SizeInBits: 1);
6661 auto IsZeroBitShift =
6662 MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: BoolTy, Op0: ShiftAmt, Op1: ZeroConst);
6663
6664 // Extract bits from the adjacent part that will "carry over" into this part.
6665 // The carry direction is opposite to the main shift direction, so we can
6666 // align the two shifted values before combining them with OR.
6667
6668 // Determine the carry shift opcode (opposite direction)
6669 unsigned CarryOpcode = (Opcode == TargetOpcode::G_SHL) ? TargetOpcode::G_LSHR
6670 : TargetOpcode::G_SHL;
6671
6672 // Calculate inverse shift amount: BitWidth - ShiftAmt
6673 auto TargetBitsConst =
6674 MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetTy.getScalarSizeInBits());
6675 auto InvShiftAmt = MIRBuilder.buildSub(Dst: ShiftAmtTy, Src0: TargetBitsConst, Src1: ShiftAmt);
6676
6677 // Shift the carry operand
6678 Register CarryBits =
6679 MIRBuilder
6680 .buildInstr(Opc: CarryOpcode, DstOps: {TargetTy}, SrcOps: {CarryOperand, InvShiftAmt})
6681 .getReg(Idx: 0);
6682
6683 // If BitShift is 0, don't include carry bits (InvShiftAmt would equal
6684 // TargetBits which would be poison for the individual carry shift operation).
6685 auto ZeroReg = MIRBuilder.buildConstant(Res: TargetTy, Val: 0);
6686 Register SafeCarryBits =
6687 MIRBuilder.buildSelect(Res: TargetTy, Tst: IsZeroBitShift, Op0: ZeroReg, Op1: CarryBits)
6688 .getReg(Idx: 0);
6689
6690 // Combine the main shifted part with the carry bits
6691 return MIRBuilder.buildOr(Dst: TargetTy, Src0: MainShifted, Src1: SafeCarryBits).getReg(Idx: 0);
6692}
6693
6694LegalizerHelper::LegalizeResult
6695LegalizerHelper::narrowScalarShiftByConstantMultiway(MachineInstr &MI,
6696 const APInt &Amt,
6697 LLT TargetTy,
6698 LLT ShiftAmtTy) {
6699 // Any wide shift can be decomposed into WordShift + BitShift components.
6700 // When shift amount is known constant, directly compute the decomposition
6701 // values and generate constant registers.
6702 Register DstReg = MI.getOperand(i: 0).getReg();
6703 Register SrcReg = MI.getOperand(i: 1).getReg();
6704 LLT DstTy = MRI.getType(Reg: DstReg);
6705
6706 const unsigned DstBits = DstTy.getScalarSizeInBits();
6707 const unsigned TargetBits = TargetTy.getScalarSizeInBits();
6708 const unsigned NumParts = DstBits / TargetBits;
6709
6710 assert(DstBits % TargetBits == 0 && "Target type must evenly divide source");
6711
6712 // When the shift amount is known at compile time, we just calculate which
6713 // source parts contribute to each output part.
6714
6715 SmallVector<Register, 8> SrcParts;
6716 extractParts(Reg: SrcReg, Ty: TargetTy, NumParts, VRegs&: SrcParts, MIRBuilder, MRI);
6717
6718 if (Amt.isZero()) {
6719 // No shift needed, just copy
6720 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: SrcParts);
6721 MI.eraseFromParent();
6722 return Legalized;
6723 }
6724
6725 ShiftParams Params;
6726 const unsigned ShiftWords = Amt.getZExtValue() / TargetBits;
6727 const unsigned ShiftBits = Amt.getZExtValue() % TargetBits;
6728
6729 // Generate constants and values needed by all shift types
6730 Params.WordShift = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: ShiftWords).getReg(Idx: 0);
6731 Params.BitShift = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: ShiftBits).getReg(Idx: 0);
6732 Params.InvBitShift =
6733 MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBits - ShiftBits).getReg(Idx: 0);
6734 Params.Zero = MIRBuilder.buildConstant(Res: TargetTy, Val: 0).getReg(Idx: 0);
6735
6736 // For ASHR, we need the sign-extended value to fill shifted-out positions
6737 if (MI.getOpcode() == TargetOpcode::G_ASHR)
6738 Params.SignBit =
6739 MIRBuilder
6740 .buildAShr(Dst: TargetTy, Src0: SrcParts[SrcParts.size() - 1],
6741 Src1: MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBits - 1))
6742 .getReg(Idx: 0);
6743
6744 SmallVector<Register, 8> DstParts(NumParts);
6745 for (unsigned I = 0; I < NumParts; ++I)
6746 DstParts[I] = buildConstantShiftPart(Opcode: MI.getOpcode(), PartIdx: I, NumParts, SrcParts,
6747 Params, TargetTy, ShiftAmtTy);
6748
6749 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstParts);
6750 MI.eraseFromParent();
6751 return Legalized;
6752}
6753
6754LegalizerHelper::LegalizeResult
6755LegalizerHelper::narrowScalarShiftMultiway(MachineInstr &MI, LLT TargetTy) {
6756 Register DstReg = MI.getOperand(i: 0).getReg();
6757 Register SrcReg = MI.getOperand(i: 1).getReg();
6758 Register AmtReg = MI.getOperand(i: 2).getReg();
6759 LLT DstTy = MRI.getType(Reg: DstReg);
6760 LLT ShiftAmtTy = MRI.getType(Reg: AmtReg);
6761
6762 const unsigned DstBits = DstTy.getScalarSizeInBits();
6763 const unsigned TargetBits = TargetTy.getScalarSizeInBits();
6764 const unsigned NumParts = DstBits / TargetBits;
6765
6766 assert(DstBits % TargetBits == 0 && "Target type must evenly divide source");
6767 assert(isPowerOf2_32(TargetBits) && "Target bit width must be power of 2");
6768
6769 // If the shift amount is known at compile time, we can use direct indexing
6770 // instead of generating select chains in the general case.
6771 if (auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: AmtReg, MRI))
6772 return narrowScalarShiftByConstantMultiway(MI, Amt: VRegAndVal->Value, TargetTy,
6773 ShiftAmtTy);
6774
6775 // For runtime-variable shift amounts, we must generate a more complex
6776 // sequence that handles all possible shift values using select chains.
6777
6778 // Split the input into target-sized pieces
6779 SmallVector<Register, 8> SrcParts;
6780 extractParts(Reg: SrcReg, Ty: TargetTy, NumParts, VRegs&: SrcParts, MIRBuilder, MRI);
6781
6782 // Shifting by zero should be a no-op.
6783 auto ZeroAmtConst = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: 0);
6784 LLT BoolTy = LLT::integer(SizeInBits: 1);
6785 auto IsZeroShift =
6786 MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: BoolTy, Op0: AmtReg, Op1: ZeroAmtConst);
6787
6788 // Any wide shift can be decomposed into two components:
6789 // 1. WordShift: number of complete target-sized words to shift
6790 // 2. BitShift: number of bits to shift within each word
6791 //
6792 // Example: 128-bit >> 50 with 32-bit target:
6793 // WordShift = 50 / 32 = 1 (shift right by 1 complete word)
6794 // BitShift = 50 % 32 = 18 (shift each word right by 18 bits)
6795 unsigned TargetBitsLog2 = Log2_32(Value: TargetBits);
6796 auto TargetBitsLog2Const =
6797 MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBitsLog2);
6798 auto TargetBitsMask = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBits - 1);
6799
6800 Register WordShift =
6801 MIRBuilder.buildLShr(Dst: ShiftAmtTy, Src0: AmtReg, Src1: TargetBitsLog2Const).getReg(Idx: 0);
6802 Register BitShift =
6803 MIRBuilder.buildAnd(Dst: ShiftAmtTy, Src0: AmtReg, Src1: TargetBitsMask).getReg(Idx: 0);
6804
6805 // Fill values:
6806 // - SHL/LSHR: fill with zeros
6807 // - ASHR: fill with sign-extended MSB
6808 Register ZeroReg = MIRBuilder.buildConstant(Res: TargetTy, Val: 0).getReg(Idx: 0);
6809
6810 Register FillValue;
6811 if (MI.getOpcode() == TargetOpcode::G_ASHR) {
6812 auto TargetBitsMinusOneConst =
6813 MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBits - 1);
6814 FillValue = MIRBuilder
6815 .buildAShr(Dst: TargetTy, Src0: SrcParts[NumParts - 1],
6816 Src1: TargetBitsMinusOneConst)
6817 .getReg(Idx: 0);
6818 } else {
6819 FillValue = ZeroReg;
6820 }
6821
6822 SmallVector<Register, 8> DstParts(NumParts);
6823
6824 // For each output part, generate a select chain that chooses the correct
6825 // result based on the runtime WordShift value. This handles all possible
6826 // word shift amounts by pre-calculating what each would produce.
6827 for (unsigned I = 0; I < NumParts; ++I) {
6828 // Initialize with appropriate default value for this shift type
6829 Register InBoundsResult = FillValue;
6830
6831 // clang-format off
6832 // Build a branchless select chain by pre-computing results for all possible
6833 // WordShift values (0 to NumParts-1). Each iteration nests a new select:
6834 //
6835 // K=0: select(WordShift==0, result0, FillValue)
6836 // K=1: select(WordShift==1, result1, select(WordShift==0, result0, FillValue))
6837 // K=2: select(WordShift==2, result2, select(WordShift==1, result1, select(...)))
6838 // clang-format on
6839 for (unsigned K = 0; K < NumParts; ++K) {
6840 auto WordShiftKConst = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: K);
6841 auto IsWordShiftK = MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: BoolTy,
6842 Op0: WordShift, Op1: WordShiftKConst);
6843
6844 // Calculate source indices for this word shift
6845 //
6846 // For 4-part 128-bit value with K=1 word shift:
6847 // SHL: [3][2][1][0] << K => [2][1][0][Z]
6848 // -> (MainIdx = I-K, CarryIdx = I-K-1)
6849 // LSHR: [3][2][1][0] >> K => [Z][3][2][1]
6850 // -> (MainIdx = I+K, CarryIdx = I+K+1)
6851 int MainSrcIdx;
6852 int CarrySrcIdx; // Index for the word that provides the carried-in bits.
6853
6854 switch (MI.getOpcode()) {
6855 case TargetOpcode::G_SHL:
6856 MainSrcIdx = (int)I - (int)K;
6857 CarrySrcIdx = MainSrcIdx - 1;
6858 break;
6859 case TargetOpcode::G_LSHR:
6860 case TargetOpcode::G_ASHR:
6861 MainSrcIdx = (int)I + (int)K;
6862 CarrySrcIdx = MainSrcIdx + 1;
6863 break;
6864 default:
6865 llvm_unreachable("Not a shift");
6866 }
6867
6868 // Check bounds and build the result for this word shift
6869 Register ResultForK;
6870 if (MainSrcIdx >= 0 && MainSrcIdx < (int)NumParts) {
6871 Register MainOp = SrcParts[MainSrcIdx];
6872 Register CarryOp;
6873
6874 // Determine carry operand with bounds checking
6875 if (CarrySrcIdx >= 0 && CarrySrcIdx < (int)NumParts)
6876 CarryOp = SrcParts[CarrySrcIdx];
6877 else if (MI.getOpcode() == TargetOpcode::G_ASHR &&
6878 CarrySrcIdx >= (int)NumParts)
6879 CarryOp = FillValue; // Use sign extension
6880
6881 ResultForK = buildVariableShiftPart(Opcode: MI.getOpcode(), MainOperand: MainOp, ShiftAmt: BitShift,
6882 TargetTy, CarryOperand: CarryOp);
6883 } else {
6884 // Out of bounds - use fill value for this k
6885 ResultForK = FillValue;
6886 }
6887
6888 // Select this result if WordShift equals k
6889 InBoundsResult =
6890 MIRBuilder
6891 .buildSelect(Res: TargetTy, Tst: IsWordShiftK, Op0: ResultForK, Op1: InBoundsResult)
6892 .getReg(Idx: 0);
6893 }
6894
6895 // Handle zero-shift special case: if shift is 0, use original input
6896 DstParts[I] =
6897 MIRBuilder
6898 .buildSelect(Res: TargetTy, Tst: IsZeroShift, Op0: SrcParts[I], Op1: InBoundsResult)
6899 .getReg(Idx: 0);
6900 }
6901
6902 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstParts);
6903 MI.eraseFromParent();
6904 return Legalized;
6905}
6906
6907LegalizerHelper::LegalizeResult
6908LegalizerHelper::moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
6909 LLT MoreTy) {
6910 assert(TypeIdx == 0 && "Expecting only Idx 0");
6911
6912 Observer.changingInstr(MI);
6913 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
6914 MachineBasicBlock &OpMBB = *MI.getOperand(i: I + 1).getMBB();
6915 MIRBuilder.setInsertPt(MBB&: OpMBB, II: OpMBB.getFirstTerminator());
6916 moreElementsVectorSrc(MI, MoreTy, OpIdx: I);
6917 }
6918
6919 MachineBasicBlock &MBB = *MI.getParent();
6920 MIRBuilder.setInsertPt(MBB, II: --MBB.getFirstNonPHI());
6921 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
6922 Observer.changedInstr(MI);
6923 return Legalized;
6924}
6925
6926MachineInstrBuilder LegalizerHelper::getNeutralElementForVecReduce(
6927 unsigned Opcode, MachineIRBuilder &MIRBuilder, LLT Ty) {
6928 assert(Ty.isScalar() && "Expected scalar type to make neutral element for");
6929
6930 switch (Opcode) {
6931 default:
6932 llvm_unreachable(
6933 "getNeutralElementForVecReduce called with invalid opcode!");
6934 case TargetOpcode::G_VECREDUCE_ADD:
6935 case TargetOpcode::G_VECREDUCE_OR:
6936 case TargetOpcode::G_VECREDUCE_XOR:
6937 case TargetOpcode::G_VECREDUCE_UMAX:
6938 return MIRBuilder.buildConstant(Res: Ty, Val: 0);
6939 case TargetOpcode::G_VECREDUCE_MUL:
6940 return MIRBuilder.buildConstant(Res: Ty, Val: 1);
6941 case TargetOpcode::G_VECREDUCE_AND:
6942 case TargetOpcode::G_VECREDUCE_UMIN:
6943 return MIRBuilder.buildConstant(
6944 Res: Ty, Val: APInt::getAllOnes(numBits: Ty.getScalarSizeInBits()));
6945 case TargetOpcode::G_VECREDUCE_SMAX:
6946 return MIRBuilder.buildConstant(
6947 Res: Ty, Val: APInt::getSignedMinValue(numBits: Ty.getSizeInBits()));
6948 case TargetOpcode::G_VECREDUCE_SMIN:
6949 return MIRBuilder.buildConstant(
6950 Res: Ty, Val: APInt::getSignedMaxValue(numBits: Ty.getSizeInBits()));
6951 case TargetOpcode::G_VECREDUCE_FADD:
6952 return MIRBuilder.buildFConstant(Res: Ty, Val: -0.0);
6953 case TargetOpcode::G_VECREDUCE_FMUL:
6954 return MIRBuilder.buildFConstant(Res: Ty, Val: 1.0);
6955 case TargetOpcode::G_VECREDUCE_FMINIMUM:
6956 case TargetOpcode::G_VECREDUCE_FMAXIMUM:
6957 assert(false && "getNeutralElementForVecReduce unimplemented for "
6958 "G_VECREDUCE_FMINIMUM and G_VECREDUCE_FMAXIMUM!");
6959 }
6960 llvm_unreachable("switch expected to return!");
6961}
6962
6963LegalizerHelper::LegalizeResult
6964LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
6965 LLT MoreTy) {
6966 unsigned Opc = MI.getOpcode();
6967 switch (Opc) {
6968 case TargetOpcode::G_IMPLICIT_DEF:
6969 case TargetOpcode::G_LOAD: {
6970 if (TypeIdx != 0)
6971 return UnableToLegalize;
6972 Observer.changingInstr(MI);
6973 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
6974 Observer.changedInstr(MI);
6975 return Legalized;
6976 }
6977 case TargetOpcode::G_STORE:
6978 if (TypeIdx != 0)
6979 return UnableToLegalize;
6980 Observer.changingInstr(MI);
6981 moreElementsVectorSrc(MI, MoreTy, OpIdx: 0);
6982 Observer.changedInstr(MI);
6983 return Legalized;
6984 case TargetOpcode::G_AND:
6985 case TargetOpcode::G_OR:
6986 case TargetOpcode::G_XOR:
6987 case TargetOpcode::G_ADD:
6988 case TargetOpcode::G_SUB:
6989 case TargetOpcode::G_MUL:
6990 case TargetOpcode::G_FADD:
6991 case TargetOpcode::G_FSUB:
6992 case TargetOpcode::G_FMUL:
6993 case TargetOpcode::G_FDIV:
6994 case TargetOpcode::G_FCOPYSIGN:
6995 case TargetOpcode::G_UADDSAT:
6996 case TargetOpcode::G_USUBSAT:
6997 case TargetOpcode::G_SADDSAT:
6998 case TargetOpcode::G_SSUBSAT:
6999 case TargetOpcode::G_SMIN:
7000 case TargetOpcode::G_SMAX:
7001 case TargetOpcode::G_UMIN:
7002 case TargetOpcode::G_UMAX:
7003 case TargetOpcode::G_FMINNUM:
7004 case TargetOpcode::G_FMAXNUM:
7005 case TargetOpcode::G_FMINNUM_IEEE:
7006 case TargetOpcode::G_FMAXNUM_IEEE:
7007 case TargetOpcode::G_FMINIMUM:
7008 case TargetOpcode::G_FMAXIMUM:
7009 case TargetOpcode::G_FMINIMUMNUM:
7010 case TargetOpcode::G_FMAXIMUMNUM:
7011 case TargetOpcode::G_STRICT_FADD:
7012 case TargetOpcode::G_STRICT_FSUB:
7013 case TargetOpcode::G_STRICT_FMUL: {
7014 Observer.changingInstr(MI);
7015 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
7016 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
7017 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7018 Observer.changedInstr(MI);
7019 return Legalized;
7020 }
7021 case TargetOpcode::G_SHL:
7022 case TargetOpcode::G_ASHR:
7023 case TargetOpcode::G_LSHR: {
7024 Observer.changingInstr(MI);
7025 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
7026 // The shift operand may have a different scalar type from the source and
7027 // destination operands.
7028 LLT ShiftMoreTy = MoreTy.changeElementType(
7029 NewEltTy: MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getElementType());
7030 moreElementsVectorSrc(MI, MoreTy: ShiftMoreTy, OpIdx: 2);
7031 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7032 Observer.changedInstr(MI);
7033 return Legalized;
7034 }
7035 case TargetOpcode::G_FMA:
7036 case TargetOpcode::G_STRICT_FMA:
7037 case TargetOpcode::G_FSHR:
7038 case TargetOpcode::G_FSHL: {
7039 Observer.changingInstr(MI);
7040 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
7041 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
7042 moreElementsVectorSrc(MI, MoreTy, OpIdx: 3);
7043 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7044 Observer.changedInstr(MI);
7045 return Legalized;
7046 }
7047 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
7048 case TargetOpcode::G_EXTRACT:
7049 if (TypeIdx != 1)
7050 return UnableToLegalize;
7051 Observer.changingInstr(MI);
7052 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
7053 Observer.changedInstr(MI);
7054 return Legalized;
7055 case TargetOpcode::G_INSERT:
7056 case TargetOpcode::G_INSERT_VECTOR_ELT:
7057 case TargetOpcode::G_FREEZE:
7058 case TargetOpcode::G_FNEG:
7059 case TargetOpcode::G_FABS:
7060 case TargetOpcode::G_FSQRT:
7061 case TargetOpcode::G_FCEIL:
7062 case TargetOpcode::G_FFLOOR:
7063 case TargetOpcode::G_FNEARBYINT:
7064 case TargetOpcode::G_FRINT:
7065 case TargetOpcode::G_INTRINSIC_ROUND:
7066 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
7067 case TargetOpcode::G_INTRINSIC_TRUNC:
7068 case TargetOpcode::G_BITREVERSE:
7069 case TargetOpcode::G_BSWAP:
7070 case TargetOpcode::G_FCANONICALIZE:
7071 case TargetOpcode::G_SEXT_INREG:
7072 case TargetOpcode::G_ABS:
7073 case TargetOpcode::G_CTLZ:
7074 case TargetOpcode::G_CTPOP:
7075 if (TypeIdx != 0)
7076 return UnableToLegalize;
7077 Observer.changingInstr(MI);
7078 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
7079 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7080 Observer.changedInstr(MI);
7081 return Legalized;
7082 case TargetOpcode::G_SELECT: {
7083 auto [DstReg, DstTy, CondReg, CondTy] = MI.getFirst2RegLLTs();
7084 if (TypeIdx == 1) {
7085 if (!CondTy.isScalar() ||
7086 DstTy.getElementCount() != MoreTy.getElementCount())
7087 return UnableToLegalize;
7088
7089 // This is turning a scalar select of vectors into a vector
7090 // select. Broadcast the select condition.
7091 auto ShufSplat = MIRBuilder.buildShuffleSplat(Res: MoreTy, Src: CondReg);
7092 Observer.changingInstr(MI);
7093 MI.getOperand(i: 1).setReg(ShufSplat.getReg(Idx: 0));
7094 Observer.changedInstr(MI);
7095 return Legalized;
7096 }
7097
7098 if (CondTy.isVector())
7099 return UnableToLegalize;
7100
7101 Observer.changingInstr(MI);
7102 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
7103 moreElementsVectorSrc(MI, MoreTy, OpIdx: 3);
7104 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7105 Observer.changedInstr(MI);
7106 return Legalized;
7107 }
7108 case TargetOpcode::G_UNMERGE_VALUES:
7109 return UnableToLegalize;
7110 case TargetOpcode::G_PHI:
7111 return moreElementsVectorPhi(MI, TypeIdx, MoreTy);
7112 case TargetOpcode::G_SHUFFLE_VECTOR:
7113 return moreElementsVectorShuffle(MI, TypeIdx, MoreTy);
7114 case TargetOpcode::G_BUILD_VECTOR: {
7115 SmallVector<SrcOp, 8> Elts;
7116 for (auto Op : MI.uses()) {
7117 Elts.push_back(Elt: Op.getReg());
7118 }
7119
7120 for (unsigned i = Elts.size(); i < MoreTy.getNumElements(); ++i) {
7121 Elts.push_back(Elt: MIRBuilder.buildUndef(Res: MoreTy.getScalarType()));
7122 }
7123
7124 MIRBuilder.buildDeleteTrailingVectorElements(
7125 Res: MI.getOperand(i: 0).getReg(), Op0: MIRBuilder.buildInstr(Opc, DstOps: {MoreTy}, SrcOps: Elts));
7126 MI.eraseFromParent();
7127 return Legalized;
7128 }
7129 case TargetOpcode::G_SEXT:
7130 case TargetOpcode::G_ZEXT:
7131 case TargetOpcode::G_ANYEXT:
7132 case TargetOpcode::G_TRUNC:
7133 case TargetOpcode::G_FPTRUNC:
7134 case TargetOpcode::G_FPEXT:
7135 case TargetOpcode::G_FPTOSI:
7136 case TargetOpcode::G_FPTOUI:
7137 case TargetOpcode::G_FPTOSI_SAT:
7138 case TargetOpcode::G_FPTOUI_SAT:
7139 case TargetOpcode::G_SITOFP:
7140 case TargetOpcode::G_UITOFP:
7141 case TargetOpcode::G_TRUNC_SSAT_S:
7142 case TargetOpcode::G_TRUNC_SSAT_U:
7143 case TargetOpcode::G_TRUNC_USAT_U: {
7144 Observer.changingInstr(MI);
7145 LLT SrcExtTy;
7146 LLT DstExtTy;
7147 if (TypeIdx == 0) {
7148 DstExtTy = MoreTy;
7149 SrcExtTy = MoreTy.changeElementType(
7150 NewEltTy: MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getElementType());
7151 } else {
7152 DstExtTy = MoreTy.changeElementType(
7153 NewEltTy: MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getElementType());
7154 SrcExtTy = MoreTy;
7155 }
7156 moreElementsVectorSrc(MI, MoreTy: SrcExtTy, OpIdx: 1);
7157 moreElementsVectorDst(MI, WideTy: DstExtTy, OpIdx: 0);
7158 Observer.changedInstr(MI);
7159 return Legalized;
7160 }
7161 case TargetOpcode::G_ICMP:
7162 case TargetOpcode::G_FCMP: {
7163 if (TypeIdx != 1)
7164 return UnableToLegalize;
7165
7166 Observer.changingInstr(MI);
7167 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
7168 moreElementsVectorSrc(MI, MoreTy, OpIdx: 3);
7169 LLT CondTy = MoreTy.changeVectorElementType(
7170 NewEltTy: MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getElementType());
7171 moreElementsVectorDst(MI, WideTy: CondTy, OpIdx: 0);
7172 Observer.changedInstr(MI);
7173 return Legalized;
7174 }
7175 case TargetOpcode::G_BITCAST: {
7176 if (TypeIdx != 0)
7177 return UnableToLegalize;
7178
7179 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
7180 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7181
7182 unsigned coefficient = SrcTy.getNumElements() * MoreTy.getNumElements();
7183 if (coefficient % DstTy.getNumElements() != 0)
7184 return UnableToLegalize;
7185
7186 coefficient = coefficient / DstTy.getNumElements();
7187
7188 LLT NewTy = SrcTy.changeElementCount(
7189 EC: ElementCount::get(MinVal: coefficient, Scalable: MoreTy.isScalable()));
7190 Observer.changingInstr(MI);
7191 moreElementsVectorSrc(MI, MoreTy: NewTy, OpIdx: 1);
7192 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7193 Observer.changedInstr(MI);
7194 return Legalized;
7195 }
7196 case TargetOpcode::G_VECREDUCE_FADD:
7197 case TargetOpcode::G_VECREDUCE_FMUL:
7198 case TargetOpcode::G_VECREDUCE_ADD:
7199 case TargetOpcode::G_VECREDUCE_MUL:
7200 case TargetOpcode::G_VECREDUCE_AND:
7201 case TargetOpcode::G_VECREDUCE_OR:
7202 case TargetOpcode::G_VECREDUCE_XOR:
7203 case TargetOpcode::G_VECREDUCE_SMAX:
7204 case TargetOpcode::G_VECREDUCE_SMIN:
7205 case TargetOpcode::G_VECREDUCE_UMAX:
7206 case TargetOpcode::G_VECREDUCE_UMIN: {
7207 LLT OrigTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
7208 MachineOperand &MO = MI.getOperand(i: 1);
7209 auto NewVec = MIRBuilder.buildPadVectorWithUndefElements(Res: MoreTy, Op0: MO);
7210 auto NeutralElement = getNeutralElementForVecReduce(
7211 Opcode: MI.getOpcode(), MIRBuilder, Ty: MoreTy.getElementType());
7212
7213 LLT IdxTy(TLI.getVectorIdxLLT(DL: MIRBuilder.getDataLayout()));
7214 for (size_t i = OrigTy.getNumElements(), e = MoreTy.getNumElements();
7215 i != e; i++) {
7216 auto Idx = MIRBuilder.buildConstant(Res: IdxTy, Val: i);
7217 NewVec = MIRBuilder.buildInsertVectorElement(Res: MoreTy, Val: NewVec,
7218 Elt: NeutralElement, Idx);
7219 }
7220
7221 Observer.changingInstr(MI);
7222 MO.setReg(NewVec.getReg(Idx: 0));
7223 Observer.changedInstr(MI);
7224 return Legalized;
7225 }
7226
7227 default:
7228 return UnableToLegalize;
7229 }
7230}
7231
7232LegalizerHelper::LegalizeResult
7233LegalizerHelper::equalizeVectorShuffleLengths(MachineInstr &MI) {
7234 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7235 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
7236 unsigned MaskNumElts = Mask.size();
7237 unsigned SrcNumElts = SrcTy.getNumElements();
7238 LLT DestEltTy = DstTy.getElementType();
7239
7240 if (MaskNumElts == SrcNumElts)
7241 return Legalized;
7242
7243 if (MaskNumElts < SrcNumElts) {
7244 // Extend mask to match new destination vector size with
7245 // undef values.
7246 SmallVector<int, 16> NewMask(SrcNumElts, -1);
7247 llvm::copy(Range&: Mask, Out: NewMask.begin());
7248
7249 moreElementsVectorDst(MI, WideTy: SrcTy, OpIdx: 0);
7250 MIRBuilder.setInstrAndDebugLoc(MI);
7251 MIRBuilder.buildShuffleVector(Res: MI.getOperand(i: 0).getReg(),
7252 Src1: MI.getOperand(i: 1).getReg(),
7253 Src2: MI.getOperand(i: 2).getReg(), Mask: NewMask);
7254 MI.eraseFromParent();
7255
7256 return Legalized;
7257 }
7258
7259 unsigned PaddedMaskNumElts = alignTo(Value: MaskNumElts, Align: SrcNumElts);
7260 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
7261 LLT PaddedTy =
7262 DstTy.changeVectorElementCount(EC: ElementCount::getFixed(MinVal: PaddedMaskNumElts));
7263
7264 // Create new source vectors by concatenating the initial
7265 // source vectors with undefined vectors of the same size.
7266 auto Undef = MIRBuilder.buildUndef(Res: SrcTy);
7267 SmallVector<Register, 8> MOps1(NumConcat, Undef.getReg(Idx: 0));
7268 SmallVector<Register, 8> MOps2(NumConcat, Undef.getReg(Idx: 0));
7269 MOps1[0] = MI.getOperand(i: 1).getReg();
7270 MOps2[0] = MI.getOperand(i: 2).getReg();
7271
7272 auto Src1 = MIRBuilder.buildConcatVectors(Res: PaddedTy, Ops: MOps1);
7273 auto Src2 = MIRBuilder.buildConcatVectors(Res: PaddedTy, Ops: MOps2);
7274
7275 // Readjust mask for new input vector length.
7276 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
7277 for (unsigned I = 0; I != MaskNumElts; ++I) {
7278 int Idx = Mask[I];
7279 if (Idx >= static_cast<int>(SrcNumElts))
7280 Idx += PaddedMaskNumElts - SrcNumElts;
7281 MappedOps[I] = Idx;
7282 }
7283
7284 // If we got more elements than required, extract subvector.
7285 if (MaskNumElts != PaddedMaskNumElts) {
7286 auto Shuffle =
7287 MIRBuilder.buildShuffleVector(Res: PaddedTy, Src1, Src2, Mask: MappedOps);
7288
7289 SmallVector<Register, 16> Elts(MaskNumElts);
7290 for (unsigned I = 0; I < MaskNumElts; ++I) {
7291 Elts[I] =
7292 MIRBuilder.buildExtractVectorElementConstant(Res: DestEltTy, Val: Shuffle, Idx: I)
7293 .getReg(Idx: 0);
7294 }
7295 MIRBuilder.buildBuildVector(Res: DstReg, Ops: Elts);
7296 } else {
7297 MIRBuilder.buildShuffleVector(Res: DstReg, Src1, Src2, Mask: MappedOps);
7298 }
7299
7300 MI.eraseFromParent();
7301 return LegalizerHelper::LegalizeResult::Legalized;
7302}
7303
7304LegalizerHelper::LegalizeResult
7305LegalizerHelper::moreElementsVectorShuffle(MachineInstr &MI,
7306 unsigned int TypeIdx, LLT MoreTy) {
7307 auto [DstTy, Src1Ty, Src2Ty] = MI.getFirst3LLTs();
7308 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
7309 unsigned NumElts = DstTy.getNumElements();
7310 unsigned WidenNumElts = MoreTy.getNumElements();
7311
7312 if (DstTy.isVector() && Src1Ty.isVector() &&
7313 DstTy.getNumElements() != Src1Ty.getNumElements()) {
7314 return equalizeVectorShuffleLengths(MI);
7315 }
7316
7317 if (TypeIdx != 0)
7318 return UnableToLegalize;
7319
7320 // Expect a canonicalized shuffle.
7321 if (DstTy != Src1Ty || DstTy != Src2Ty)
7322 return UnableToLegalize;
7323
7324 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
7325 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
7326
7327 // Adjust mask based on new input vector length.
7328 SmallVector<int, 16> NewMask(WidenNumElts, -1);
7329 for (unsigned I = 0; I != NumElts; ++I) {
7330 int Idx = Mask[I];
7331 if (Idx < static_cast<int>(NumElts))
7332 NewMask[I] = Idx;
7333 else
7334 NewMask[I] = Idx - NumElts + WidenNumElts;
7335 }
7336 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7337 MIRBuilder.setInstrAndDebugLoc(MI);
7338 MIRBuilder.buildShuffleVector(Res: MI.getOperand(i: 0).getReg(),
7339 Src1: MI.getOperand(i: 1).getReg(),
7340 Src2: MI.getOperand(i: 2).getReg(), Mask: NewMask);
7341 MI.eraseFromParent();
7342 return Legalized;
7343}
7344
7345void LegalizerHelper::multiplyRegisters(SmallVectorImpl<Register> &DstRegs,
7346 ArrayRef<Register> Src1Regs,
7347 ArrayRef<Register> Src2Regs,
7348 LLT NarrowTy) {
7349 MachineIRBuilder &B = MIRBuilder;
7350 unsigned SrcParts = Src1Regs.size();
7351 unsigned DstParts = DstRegs.size();
7352
7353 unsigned DstIdx = 0; // Low bits of the result.
7354 Register FactorSum =
7355 B.buildMul(Dst: NarrowTy, Src0: Src1Regs[DstIdx], Src1: Src2Regs[DstIdx]).getReg(Idx: 0);
7356 DstRegs[DstIdx] = FactorSum;
7357
7358 Register CarrySumPrevDstIdx;
7359 SmallVector<Register, 4> Factors;
7360
7361 for (DstIdx = 1; DstIdx < DstParts; DstIdx++) {
7362 // Collect high parts of muls from previous DstIdx.
7363 for (unsigned i = DstIdx < SrcParts ? 0 : DstIdx - SrcParts;
7364 i <= std::min(a: DstIdx - 1, b: SrcParts - 1); ++i) {
7365 MachineInstrBuilder Umulh =
7366 B.buildUMulH(Dst: NarrowTy, Src0: Src1Regs[DstIdx - 1 - i], Src1: Src2Regs[i]);
7367 Factors.push_back(Elt: Umulh.getReg(Idx: 0));
7368 }
7369 // Collect low parts of muls for DstIdx. Visit the diagonal starting with
7370 // the low Src1 part, so multiply-add selectors can use it as the first
7371 // accumulated cross product.
7372 unsigned LowStart = DstIdx + 1 < SrcParts ? 0 : DstIdx - SrcParts + 1;
7373 unsigned LowEnd = std::min(a: DstIdx, b: SrcParts - 1);
7374 for (unsigned RevI = LowEnd + 1; RevI != LowStart; --RevI) {
7375 unsigned i = RevI - 1;
7376 MachineInstrBuilder Mul =
7377 B.buildMul(Dst: NarrowTy, Src0: Src1Regs[DstIdx - i], Src1: Src2Regs[i]);
7378 Factors.push_back(Elt: Mul.getReg(Idx: 0));
7379 }
7380 // Add CarrySum from additions calculated for previous DstIdx.
7381 if (DstIdx != 1) {
7382 Factors.push_back(Elt: CarrySumPrevDstIdx);
7383 }
7384
7385 Register CarrySum;
7386 // Add all factors and accumulate all carries into CarrySum.
7387 if (DstIdx != DstParts - 1) {
7388 MachineInstrBuilder Uaddo =
7389 B.buildUAddo(Res: NarrowTy, CarryOut: LLT::integer(SizeInBits: 1), Op0: Factors[0], Op1: Factors[1]);
7390 FactorSum = Uaddo.getReg(Idx: 0);
7391 CarrySum = B.buildZExt(Res: NarrowTy, Op: Uaddo.getReg(Idx: 1)).getReg(Idx: 0);
7392 for (unsigned i = 2; i < Factors.size(); ++i) {
7393 MachineInstrBuilder Uaddo =
7394 B.buildUAddo(Res: NarrowTy, CarryOut: LLT::integer(SizeInBits: 1), Op0: FactorSum, Op1: Factors[i]);
7395 FactorSum = Uaddo.getReg(Idx: 0);
7396 MachineInstrBuilder Carry = B.buildZExt(Res: NarrowTy, Op: Uaddo.getReg(Idx: 1));
7397 CarrySum = B.buildAdd(Dst: NarrowTy, Src0: CarrySum, Src1: Carry).getReg(Idx: 0);
7398 }
7399 } else {
7400 // Since value for the next index is not calculated, neither is CarrySum.
7401 FactorSum = B.buildAdd(Dst: NarrowTy, Src0: Factors[0], Src1: Factors[1]).getReg(Idx: 0);
7402 for (unsigned i = 2; i < Factors.size(); ++i)
7403 FactorSum = B.buildAdd(Dst: NarrowTy, Src0: FactorSum, Src1: Factors[i]).getReg(Idx: 0);
7404 }
7405
7406 CarrySumPrevDstIdx = CarrySum;
7407 DstRegs[DstIdx] = FactorSum;
7408 Factors.clear();
7409 }
7410}
7411
7412LegalizerHelper::LegalizeResult
7413LegalizerHelper::narrowScalarAddSub(MachineInstr &MI, unsigned TypeIdx,
7414 LLT NarrowTy) {
7415 if (TypeIdx != 0)
7416 return UnableToLegalize;
7417
7418 Register DstReg = MI.getOperand(i: 0).getReg();
7419 LLT DstType = MRI.getType(Reg: DstReg);
7420 // FIXME: add support for vector types
7421 if (DstType.isVector())
7422 return UnableToLegalize;
7423
7424 unsigned Opcode = MI.getOpcode();
7425 unsigned OpO, OpE, OpF;
7426 switch (Opcode) {
7427 case TargetOpcode::G_SADDO:
7428 case TargetOpcode::G_SADDE:
7429 case TargetOpcode::G_UADDO:
7430 case TargetOpcode::G_UADDE:
7431 case TargetOpcode::G_ADD:
7432 OpO = TargetOpcode::G_UADDO;
7433 OpE = TargetOpcode::G_UADDE;
7434 OpF = TargetOpcode::G_UADDE;
7435 if (Opcode == TargetOpcode::G_SADDO || Opcode == TargetOpcode::G_SADDE)
7436 OpF = TargetOpcode::G_SADDE;
7437 break;
7438 case TargetOpcode::G_SSUBO:
7439 case TargetOpcode::G_SSUBE:
7440 case TargetOpcode::G_USUBO:
7441 case TargetOpcode::G_USUBE:
7442 case TargetOpcode::G_SUB:
7443 OpO = TargetOpcode::G_USUBO;
7444 OpE = TargetOpcode::G_USUBE;
7445 OpF = TargetOpcode::G_USUBE;
7446 if (Opcode == TargetOpcode::G_SSUBO || Opcode == TargetOpcode::G_SSUBE)
7447 OpF = TargetOpcode::G_SSUBE;
7448 break;
7449 default:
7450 llvm_unreachable("Unexpected add/sub opcode!");
7451 }
7452
7453 // 1 for a plain add/sub, 2 if this is an operation with a carry-out.
7454 unsigned NumDefs = MI.getNumExplicitDefs();
7455 Register Src1 = MI.getOperand(i: NumDefs).getReg();
7456 Register Src2 = MI.getOperand(i: NumDefs + 1).getReg();
7457 Register CarryDst, CarryIn;
7458 if (NumDefs == 2)
7459 CarryDst = MI.getOperand(i: 1).getReg();
7460 if (MI.getNumOperands() == NumDefs + 3)
7461 CarryIn = MI.getOperand(i: NumDefs + 2).getReg();
7462
7463 LLT RegTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7464 LLT LeftoverTy, DummyTy;
7465 SmallVector<Register, 2> Src1Regs, Src2Regs, Src1Left, Src2Left, DstRegs;
7466 extractParts(Reg: Src1, RegTy, MainTy: NarrowTy, LeftoverTy, VRegs&: Src1Regs, LeftoverVRegs&: Src1Left,
7467 MIRBuilder, MRI);
7468 extractParts(Reg: Src2, RegTy, MainTy: NarrowTy, LeftoverTy&: DummyTy, VRegs&: Src2Regs, LeftoverVRegs&: Src2Left, MIRBuilder,
7469 MRI);
7470
7471 int NarrowParts = Src1Regs.size();
7472 Src1Regs.append(RHS: Src1Left);
7473 Src2Regs.append(RHS: Src2Left);
7474 DstRegs.reserve(N: Src1Regs.size());
7475
7476 for (int i = 0, e = Src1Regs.size(); i != e; ++i) {
7477 Register DstReg =
7478 MRI.createGenericVirtualRegister(Ty: MRI.getType(Reg: Src1Regs[i]));
7479 Register CarryOut;
7480 // Forward the final carry-out to the destination register
7481 if (i == e - 1 && CarryDst)
7482 CarryOut = CarryDst;
7483 else
7484 CarryOut = MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: 1));
7485
7486 if (!CarryIn) {
7487 MIRBuilder.buildInstr(Opc: OpO, DstOps: {DstReg, CarryOut},
7488 SrcOps: {Src1Regs[i], Src2Regs[i]});
7489 } else if (i == e - 1) {
7490 MIRBuilder.buildInstr(Opc: OpF, DstOps: {DstReg, CarryOut},
7491 SrcOps: {Src1Regs[i], Src2Regs[i], CarryIn});
7492 } else {
7493 MIRBuilder.buildInstr(Opc: OpE, DstOps: {DstReg, CarryOut},
7494 SrcOps: {Src1Regs[i], Src2Regs[i], CarryIn});
7495 }
7496
7497 DstRegs.push_back(Elt: DstReg);
7498 CarryIn = CarryOut;
7499 }
7500 insertParts(DstReg: MI.getOperand(i: 0).getReg(), ResultTy: RegTy, PartTy: NarrowTy,
7501 PartRegs: ArrayRef(DstRegs).take_front(N: NarrowParts), LeftoverTy,
7502 LeftoverRegs: ArrayRef(DstRegs).drop_front(N: NarrowParts));
7503
7504 MI.eraseFromParent();
7505 return Legalized;
7506}
7507
7508LegalizerHelper::LegalizeResult
7509LegalizerHelper::narrowScalarMul(MachineInstr &MI, LLT NarrowTy) {
7510 auto [DstReg, Src1, Src2] = MI.getFirst3Regs();
7511
7512 LLT Ty = MRI.getType(Reg: DstReg);
7513 if (Ty.isVector())
7514 return UnableToLegalize;
7515
7516 unsigned Size = Ty.getSizeInBits();
7517 unsigned NarrowSize = NarrowTy.getSizeInBits();
7518 if (Size % NarrowSize != 0)
7519 return UnableToLegalize;
7520
7521 unsigned NumParts = Size / NarrowSize;
7522 bool IsMulHigh = MI.getOpcode() == TargetOpcode::G_UMULH;
7523 unsigned DstTmpParts = NumParts * (IsMulHigh ? 2 : 1);
7524
7525 SmallVector<Register, 2> Src1Parts, Src2Parts;
7526 SmallVector<Register, 2> DstTmpRegs(DstTmpParts);
7527 extractParts(Reg: Src1, Ty: NarrowTy, NumParts, VRegs&: Src1Parts, MIRBuilder, MRI);
7528 extractParts(Reg: Src2, Ty: NarrowTy, NumParts, VRegs&: Src2Parts, MIRBuilder, MRI);
7529 multiplyRegisters(DstRegs&: DstTmpRegs, Src1Regs: Src1Parts, Src2Regs: Src2Parts, NarrowTy);
7530
7531 // Take only high half of registers if this is high mul.
7532 ArrayRef<Register> DstRegs(&DstTmpRegs[DstTmpParts - NumParts], NumParts);
7533 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
7534 MI.eraseFromParent();
7535 return Legalized;
7536}
7537
7538LegalizerHelper::LegalizeResult
7539LegalizerHelper::narrowScalarFPTOI(MachineInstr &MI, unsigned TypeIdx,
7540 LLT NarrowTy) {
7541 if (TypeIdx != 0)
7542 return UnableToLegalize;
7543
7544 bool IsSigned = MI.getOpcode() == TargetOpcode::G_FPTOSI;
7545
7546 Register Src = MI.getOperand(i: 1).getReg();
7547 LLT SrcTy = MRI.getType(Reg: Src);
7548
7549 // If all finite floats fit into the narrowed integer type, we can just swap
7550 // out the result type. Only IEEE half qualifies: bfloat is also 16 bits wide
7551 // but has float's exponent range. LLT::float16() is equivalent to
7552 // LLT::scalar(16) on targets without extended LLTs.
7553 if (SrcTy.getScalarType() != LLT::float16() ||
7554 NarrowTy.getScalarSizeInBits() < (IsSigned ? 17u : 16u))
7555 return UnableToLegalize;
7556
7557 Observer.changingInstr(MI);
7558 narrowScalarDst(MI, NarrowTy, OpIdx: 0,
7559 ExtOpcode: IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT);
7560 Observer.changedInstr(MI);
7561 return Legalized;
7562}
7563
7564LegalizerHelper::LegalizeResult
7565LegalizerHelper::narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx,
7566 LLT NarrowTy) {
7567 if (TypeIdx != 1)
7568 return UnableToLegalize;
7569
7570 uint64_t NarrowSize = NarrowTy.getSizeInBits();
7571
7572 int64_t SizeOp1 = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
7573 // FIXME: add support for when SizeOp1 isn't an exact multiple of
7574 // NarrowSize.
7575 if (SizeOp1 % NarrowSize != 0)
7576 return UnableToLegalize;
7577 int NumParts = SizeOp1 / NarrowSize;
7578
7579 SmallVector<Register, 2> SrcRegs, DstRegs;
7580 extractParts(Reg: MI.getOperand(i: 1).getReg(), Ty: NarrowTy, NumParts, VRegs&: SrcRegs,
7581 MIRBuilder, MRI);
7582
7583 Register OpReg = MI.getOperand(i: 0).getReg();
7584 uint64_t OpStart = MI.getOperand(i: 2).getImm();
7585 uint64_t OpSize = MRI.getType(Reg: OpReg).getSizeInBits();
7586 for (int i = 0; i < NumParts; ++i) {
7587 unsigned SrcStart = i * NarrowSize;
7588
7589 if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
7590 // No part of the extract uses this subregister, ignore it.
7591 continue;
7592 } else if (SrcStart == OpStart && NarrowTy == MRI.getType(Reg: OpReg)) {
7593 // The entire subregister is extracted, forward the value.
7594 DstRegs.push_back(Elt: SrcRegs[i]);
7595 continue;
7596 }
7597
7598 // OpSegStart is where this destination segment would start in OpReg if it
7599 // extended infinitely in both directions.
7600 int64_t ExtractOffset;
7601 uint64_t SegSize;
7602 if (OpStart < SrcStart) {
7603 ExtractOffset = 0;
7604 SegSize = std::min(a: NarrowSize, b: OpStart + OpSize - SrcStart);
7605 } else {
7606 ExtractOffset = OpStart - SrcStart;
7607 SegSize = std::min(a: SrcStart + NarrowSize - OpStart, b: OpSize);
7608 }
7609
7610 Register SegReg = SrcRegs[i];
7611 if (ExtractOffset != 0 || SegSize != NarrowSize) {
7612 // A genuine extract is needed.
7613 SegReg = MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: SegSize));
7614 MIRBuilder.buildExtract(Res: SegReg, Src: SrcRegs[i], Index: ExtractOffset);
7615 }
7616
7617 DstRegs.push_back(Elt: SegReg);
7618 }
7619
7620 Register DstReg = MI.getOperand(i: 0).getReg();
7621 if (MRI.getType(Reg: DstReg).isVector())
7622 MIRBuilder.buildBuildVector(Res: DstReg, Ops: DstRegs);
7623 else if (DstRegs.size() > 1)
7624 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
7625 else
7626 MIRBuilder.buildCopy(Res: DstReg, Op: DstRegs[0]);
7627 MI.eraseFromParent();
7628 return Legalized;
7629}
7630
7631LegalizerHelper::LegalizeResult
7632LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx,
7633 LLT NarrowTy) {
7634 // FIXME: Don't know how to handle secondary types yet.
7635 if (TypeIdx != 0)
7636 return UnableToLegalize;
7637
7638 SmallVector<Register, 2> SrcRegs, LeftoverRegs, DstRegs;
7639 LLT RegTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7640 LLT LeftoverTy;
7641 extractParts(Reg: MI.getOperand(i: 1).getReg(), RegTy, MainTy: NarrowTy, LeftoverTy, VRegs&: SrcRegs,
7642 LeftoverVRegs&: LeftoverRegs, MIRBuilder, MRI);
7643
7644 SrcRegs.append(RHS: LeftoverRegs);
7645
7646 uint64_t NarrowSize = NarrowTy.getSizeInBits();
7647 Register OpReg = MI.getOperand(i: 2).getReg();
7648 uint64_t OpStart = MI.getOperand(i: 3).getImm();
7649 uint64_t OpSize = MRI.getType(Reg: OpReg).getSizeInBits();
7650 for (int I = 0, E = SrcRegs.size(); I != E; ++I) {
7651 unsigned DstStart = I * NarrowSize;
7652
7653 if (DstStart == OpStart && NarrowTy == MRI.getType(Reg: OpReg)) {
7654 // The entire subregister is defined by this insert, forward the new
7655 // value.
7656 DstRegs.push_back(Elt: OpReg);
7657 continue;
7658 }
7659
7660 Register SrcReg = SrcRegs[I];
7661 if (MRI.getType(Reg: SrcRegs[I]) == LeftoverTy) {
7662 // The leftover reg is smaller than NarrowTy, so we need to extend it.
7663 SrcReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
7664 MIRBuilder.buildAnyExt(Res: SrcReg, Op: SrcRegs[I]);
7665 }
7666
7667 if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
7668 // No part of the insert affects this subregister, forward the original.
7669 DstRegs.push_back(Elt: SrcReg);
7670 continue;
7671 }
7672
7673 // OpSegStart is where this destination segment would start in OpReg if it
7674 // extended infinitely in both directions.
7675 int64_t ExtractOffset, InsertOffset;
7676 uint64_t SegSize;
7677 if (OpStart < DstStart) {
7678 InsertOffset = 0;
7679 ExtractOffset = DstStart - OpStart;
7680 SegSize = std::min(a: NarrowSize, b: OpStart + OpSize - DstStart);
7681 } else {
7682 InsertOffset = OpStart - DstStart;
7683 ExtractOffset = 0;
7684 SegSize = std::min(a: NarrowSize - InsertOffset, b: OpSize);
7685 }
7686
7687 Register SegReg = OpReg;
7688 if (ExtractOffset != 0 || SegSize != OpSize) {
7689 // A genuine extract is needed.
7690 SegReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: SegSize));
7691 MIRBuilder.buildExtract(Res: SegReg, Src: OpReg, Index: ExtractOffset);
7692 }
7693
7694 Register DstReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
7695 MIRBuilder.buildInsert(Res: DstReg, Src: SrcReg, Op: SegReg, Index: InsertOffset);
7696 DstRegs.push_back(Elt: DstReg);
7697 }
7698
7699 uint64_t WideSize = DstRegs.size() * NarrowSize;
7700 Register DstReg = MI.getOperand(i: 0).getReg();
7701 if (WideSize > RegTy.getSizeInBits()) {
7702 Register MergeReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: WideSize));
7703 MIRBuilder.buildMergeLikeInstr(Res: MergeReg, Ops: DstRegs);
7704 MIRBuilder.buildTrunc(Res: DstReg, Op: MergeReg);
7705 } else
7706 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
7707
7708 MI.eraseFromParent();
7709 return Legalized;
7710}
7711
7712LegalizerHelper::LegalizeResult
7713LegalizerHelper::narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx,
7714 LLT NarrowTy) {
7715 Register DstReg = MI.getOperand(i: 0).getReg();
7716 LLT DstTy = MRI.getType(Reg: DstReg);
7717
7718 assert(MI.getNumOperands() == 3 && TypeIdx == 0);
7719
7720 SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
7721 SmallVector<Register, 4> Src0Regs, Src0LeftoverRegs;
7722 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
7723 LLT LeftoverTy;
7724 if (!extractParts(Reg: MI.getOperand(i: 1).getReg(), RegTy: DstTy, MainTy: NarrowTy, LeftoverTy,
7725 VRegs&: Src0Regs, LeftoverVRegs&: Src0LeftoverRegs, MIRBuilder, MRI))
7726 return UnableToLegalize;
7727
7728 LLT Unused;
7729 if (!extractParts(Reg: MI.getOperand(i: 2).getReg(), RegTy: DstTy, MainTy: NarrowTy, LeftoverTy&: Unused,
7730 VRegs&: Src1Regs, LeftoverVRegs&: Src1LeftoverRegs, MIRBuilder, MRI))
7731 llvm_unreachable("inconsistent extractParts result");
7732
7733 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
7734 auto Inst = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {NarrowTy},
7735 SrcOps: {Src0Regs[I], Src1Regs[I]});
7736 DstRegs.push_back(Elt: Inst.getReg(Idx: 0));
7737 }
7738
7739 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
7740 auto Inst = MIRBuilder.buildInstr(
7741 Opc: MI.getOpcode(),
7742 DstOps: {LeftoverTy}, SrcOps: {Src0LeftoverRegs[I], Src1LeftoverRegs[I]});
7743 DstLeftoverRegs.push_back(Elt: Inst.getReg(Idx: 0));
7744 }
7745
7746 insertParts(DstReg, ResultTy: DstTy, PartTy: NarrowTy, PartRegs: DstRegs,
7747 LeftoverTy, LeftoverRegs: DstLeftoverRegs);
7748
7749 MI.eraseFromParent();
7750 return Legalized;
7751}
7752
7753LegalizerHelper::LegalizeResult
7754LegalizerHelper::narrowScalarExt(MachineInstr &MI, unsigned TypeIdx,
7755 LLT NarrowTy) {
7756 if (TypeIdx != 0)
7757 return UnableToLegalize;
7758
7759 auto [DstReg, SrcReg] = MI.getFirst2Regs();
7760
7761 LLT DstTy = MRI.getType(Reg: DstReg);
7762 if (DstTy.isVector())
7763 return UnableToLegalize;
7764
7765 SmallVector<Register, 8> Parts;
7766 LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg);
7767 LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, VRegs&: Parts, PadStrategy: MI.getOpcode());
7768 buildWidenedRemergeToDst(DstReg, LCMTy, RemergeRegs: Parts);
7769
7770 MI.eraseFromParent();
7771 return Legalized;
7772}
7773
7774LegalizerHelper::LegalizeResult
7775LegalizerHelper::narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx,
7776 LLT NarrowTy) {
7777 if (TypeIdx != 0)
7778 return UnableToLegalize;
7779
7780 Register CondReg = MI.getOperand(i: 1).getReg();
7781 LLT CondTy = MRI.getType(Reg: CondReg);
7782 if (CondTy.isVector()) // TODO: Handle vselect
7783 return UnableToLegalize;
7784
7785 Register DstReg = MI.getOperand(i: 0).getReg();
7786 LLT DstTy = MRI.getType(Reg: DstReg);
7787
7788 SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
7789 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
7790 SmallVector<Register, 4> Src2Regs, Src2LeftoverRegs;
7791 LLT LeftoverTy;
7792 if (!extractParts(Reg: MI.getOperand(i: 2).getReg(), RegTy: DstTy, MainTy: NarrowTy, LeftoverTy,
7793 VRegs&: Src1Regs, LeftoverVRegs&: Src1LeftoverRegs, MIRBuilder, MRI))
7794 return UnableToLegalize;
7795
7796 LLT Unused;
7797 if (!extractParts(Reg: MI.getOperand(i: 3).getReg(), RegTy: DstTy, MainTy: NarrowTy, LeftoverTy&: Unused,
7798 VRegs&: Src2Regs, LeftoverVRegs&: Src2LeftoverRegs, MIRBuilder, MRI))
7799 llvm_unreachable("inconsistent extractParts result");
7800
7801 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
7802 auto Select = MIRBuilder.buildSelect(Res: NarrowTy,
7803 Tst: CondReg, Op0: Src1Regs[I], Op1: Src2Regs[I]);
7804 DstRegs.push_back(Elt: Select.getReg(Idx: 0));
7805 }
7806
7807 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
7808 auto Select = MIRBuilder.buildSelect(
7809 Res: LeftoverTy, Tst: CondReg, Op0: Src1LeftoverRegs[I], Op1: Src2LeftoverRegs[I]);
7810 DstLeftoverRegs.push_back(Elt: Select.getReg(Idx: 0));
7811 }
7812
7813 insertParts(DstReg, ResultTy: DstTy, PartTy: NarrowTy, PartRegs: DstRegs,
7814 LeftoverTy, LeftoverRegs: DstLeftoverRegs);
7815
7816 MI.eraseFromParent();
7817 return Legalized;
7818}
7819
7820LegalizerHelper::LegalizeResult
7821LegalizerHelper::narrowScalarCTLZ(MachineInstr &MI, unsigned TypeIdx,
7822 LLT NarrowTy) {
7823 if (TypeIdx != 1)
7824 return UnableToLegalize;
7825
7826 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7827 unsigned NarrowSize = NarrowTy.getSizeInBits();
7828
7829 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
7830 const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_POISON;
7831
7832 MachineIRBuilder &B = MIRBuilder;
7833 auto UnmergeSrc = B.buildUnmerge(Res: NarrowTy, Op: SrcReg);
7834 // ctlz(Hi:Lo) -> Hi == 0 ? (NarrowSize + ctlz(Lo)) : ctlz(Hi)
7835 auto C_0 = B.buildConstant(Res: NarrowTy, Val: 0);
7836 auto HiIsZero = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: LLT::integer(SizeInBits: 1),
7837 Op0: UnmergeSrc.getReg(Idx: 1), Op1: C_0);
7838 auto LoCTLZ = IsUndef ? B.buildCTLZ_ZERO_POISON(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 0))
7839 : B.buildCTLZ(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 0));
7840 auto C_NarrowSize = B.buildConstant(Res: DstTy, Val: NarrowSize);
7841 auto HiIsZeroCTLZ = B.buildAdd(Dst: DstTy, Src0: LoCTLZ, Src1: C_NarrowSize);
7842 auto HiCTLZ = B.buildCTLZ_ZERO_POISON(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 1));
7843 B.buildSelect(Res: DstReg, Tst: HiIsZero, Op0: HiIsZeroCTLZ, Op1: HiCTLZ);
7844
7845 MI.eraseFromParent();
7846 return Legalized;
7847 }
7848
7849 return UnableToLegalize;
7850}
7851
7852LegalizerHelper::LegalizeResult
7853LegalizerHelper::narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx,
7854 LLT NarrowTy) {
7855 if (TypeIdx != 1)
7856 return UnableToLegalize;
7857
7858 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7859 unsigned NarrowSize = NarrowTy.getSizeInBits();
7860
7861 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
7862 const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTTZ_ZERO_POISON;
7863
7864 MachineIRBuilder &B = MIRBuilder;
7865 auto UnmergeSrc = B.buildUnmerge(Res: NarrowTy, Op: SrcReg);
7866 // cttz(Hi:Lo) -> Lo == 0 ? (cttz(Hi) + NarrowSize) : cttz(Lo)
7867 auto C_0 = B.buildConstant(Res: NarrowTy, Val: 0);
7868 auto LoIsZero = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: LLT::integer(SizeInBits: 1),
7869 Op0: UnmergeSrc.getReg(Idx: 0), Op1: C_0);
7870 auto HiCTTZ = IsUndef ? B.buildCTTZ_ZERO_POISON(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 1))
7871 : B.buildCTTZ(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 1));
7872 auto C_NarrowSize = B.buildConstant(Res: DstTy, Val: NarrowSize);
7873 auto LoIsZeroCTTZ = B.buildAdd(Dst: DstTy, Src0: HiCTTZ, Src1: C_NarrowSize);
7874 auto LoCTTZ = B.buildCTTZ_ZERO_POISON(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 0));
7875 B.buildSelect(Res: DstReg, Tst: LoIsZero, Op0: LoIsZeroCTTZ, Op1: LoCTTZ);
7876
7877 MI.eraseFromParent();
7878 return Legalized;
7879 }
7880
7881 return UnableToLegalize;
7882}
7883
7884LegalizerHelper::LegalizeResult
7885LegalizerHelper::narrowScalarCTLS(MachineInstr &MI, unsigned TypeIdx,
7886 LLT NarrowTy) {
7887 if (TypeIdx != 1)
7888 return UnableToLegalize;
7889
7890 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7891 unsigned NarrowSize = NarrowTy.getSizeInBits();
7892
7893 if (!SrcTy.isScalar() || SrcTy.getSizeInBits() != 2 * NarrowSize)
7894 return UnableToLegalize;
7895
7896 MachineIRBuilder &B = MIRBuilder;
7897
7898 auto UnmergeSrc = B.buildUnmerge(Res: NarrowTy, Op: SrcReg);
7899 Register Lo = UnmergeSrc.getReg(Idx: 0);
7900 Register Hi = UnmergeSrc.getReg(Idx: 1);
7901
7902 auto ShAmt = B.buildConstant(Res: NarrowTy, Val: NarrowSize - 1);
7903 auto Sign = B.buildAShr(Dst: NarrowTy, Src0: Hi, Src1: ShAmt);
7904
7905 auto HiIsSign = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: LLT::scalar(SizeInBits: 1), Op0: Hi, Op1: Sign);
7906
7907 // Invert Lo if Hi is negative. Then count the leading zeros. If there are no
7908 // leading zeros, then the MSB of Lo is different than the MSB of Hi.
7909 // Otherwise the leading zeros represent additional sign bits of the original
7910 // value.
7911 auto LoInv = B.buildXor(Dst: DstTy, Src0: Lo, Src1: Sign);
7912 auto LoCTLZ = B.buildCTLZ(Dst: DstTy, Src0: LoInv);
7913
7914 // Add NarrowSize-1 to LoCTLZ. This is the full CTLS if Hi is all sign bits.
7915 auto C_NarrowSizeM1 = B.buildConstant(Res: DstTy, Val: NarrowSize - 1);
7916 auto HiIsSignCTLS = B.buildAdd(Dst: DstTy, Src0: LoCTLZ, Src1: C_NarrowSizeM1);
7917
7918 auto HiCTLS = B.buildCTLS(Dst: DstTy, Src0: Hi);
7919
7920 B.buildSelect(Res: DstReg, Tst: HiIsSign, Op0: HiIsSignCTLS, Op1: HiCTLS);
7921
7922 MI.eraseFromParent();
7923 return Legalized;
7924}
7925
7926LegalizerHelper::LegalizeResult
7927LegalizerHelper::narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx,
7928 LLT NarrowTy) {
7929 if (TypeIdx != 1)
7930 return UnableToLegalize;
7931
7932 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7933 unsigned NarrowSize = NarrowTy.getSizeInBits();
7934
7935 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
7936 auto UnmergeSrc = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: MI.getOperand(i: 1));
7937
7938 auto LoCTPOP = MIRBuilder.buildCTPOP(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 0));
7939 auto HiCTPOP = MIRBuilder.buildCTPOP(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 1));
7940 MIRBuilder.buildAdd(Dst: DstReg, Src0: HiCTPOP, Src1: LoCTPOP);
7941
7942 MI.eraseFromParent();
7943 return Legalized;
7944 }
7945
7946 return UnableToLegalize;
7947}
7948
7949LegalizerHelper::LegalizeResult
7950LegalizerHelper::narrowScalarFLDEXP(MachineInstr &MI, unsigned TypeIdx,
7951 LLT NarrowTy) {
7952 if (TypeIdx != 1)
7953 return UnableToLegalize;
7954
7955 MachineIRBuilder &B = MIRBuilder;
7956 Register ExpReg = MI.getOperand(i: 2).getReg();
7957 LLT ExpTy = MRI.getType(Reg: ExpReg);
7958
7959 unsigned ClampSize = NarrowTy.getScalarSizeInBits();
7960
7961 // Clamp the exponent to the range of the target type.
7962 auto MinExp = B.buildConstant(Res: ExpTy, Val: minIntN(N: ClampSize));
7963 auto ClampMin = B.buildSMax(Dst: ExpTy, Src0: ExpReg, Src1: MinExp);
7964 auto MaxExp = B.buildConstant(Res: ExpTy, Val: maxIntN(N: ClampSize));
7965 auto Clamp = B.buildSMin(Dst: ExpTy, Src0: ClampMin, Src1: MaxExp);
7966
7967 auto Trunc = B.buildTrunc(Res: NarrowTy, Op: Clamp);
7968 Observer.changingInstr(MI);
7969 MI.getOperand(i: 2).setReg(Trunc.getReg(Idx: 0));
7970 Observer.changedInstr(MI);
7971 return Legalized;
7972}
7973
7974LegalizerHelper::LegalizeResult
7975LegalizerHelper::lowerBitCount(MachineInstr &MI) {
7976 unsigned Opc = MI.getOpcode();
7977 const auto &TII = MIRBuilder.getTII();
7978 auto isSupported = [this](const LegalityQuery &Q) {
7979 auto QAction = LI.getAction(Query: Q).Action;
7980 return QAction == Legal || QAction == Libcall || QAction == Custom;
7981 };
7982 switch (Opc) {
7983 default:
7984 return UnableToLegalize;
7985 case TargetOpcode::G_CTLZ_ZERO_POISON: {
7986 // This trivially expands to CTLZ.
7987 Observer.changingInstr(MI);
7988 MI.setDesc(TII.get(Opcode: TargetOpcode::G_CTLZ));
7989 Observer.changedInstr(MI);
7990 return Legalized;
7991 }
7992 case TargetOpcode::G_CTLZ: {
7993 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7994 unsigned Len = SrcTy.getScalarSizeInBits();
7995
7996 if (isSupported({TargetOpcode::G_CTLZ_ZERO_POISON, {DstTy, SrcTy}})) {
7997 // If CTLZ_ZERO_POISON is supported, emit that and a select for zero.
7998 auto CtlzZU = MIRBuilder.buildCTLZ_ZERO_POISON(Dst: DstTy, Src0: SrcReg);
7999 auto ZeroSrc = MIRBuilder.buildConstant(Res: SrcTy, Val: 0);
8000 auto ICmp = MIRBuilder.buildICmp(
8001 Pred: CmpInst::ICMP_EQ, Res: SrcTy.changeElementSize(NewEltSize: 1), Op0: SrcReg, Op1: ZeroSrc);
8002 auto LenConst = MIRBuilder.buildConstant(Res: DstTy, Val: Len);
8003 MIRBuilder.buildSelect(Res: DstReg, Tst: ICmp, Op0: LenConst, Op1: CtlzZU);
8004 MI.eraseFromParent();
8005 return Legalized;
8006 }
8007 // for now, we do this:
8008 // NewLen = NextPowerOf2(Len);
8009 // x = x | (x >> 1);
8010 // x = x | (x >> 2);
8011 // ...
8012 // x = x | (x >>16);
8013 // x = x | (x >>32); // for 64-bit input
8014 // Upto NewLen/2
8015 // return Len - popcount(x);
8016 //
8017 // Ref: "Hacker's Delight" by Henry Warren
8018 Register Op = SrcReg;
8019 unsigned NewLen = PowerOf2Ceil(A: Len);
8020 for (unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) {
8021 auto MIBShiftAmt = MIRBuilder.buildConstant(Res: SrcTy, Val: 1ULL << i);
8022 auto MIBOp = MIRBuilder.buildOr(
8023 Dst: SrcTy, Src0: Op, Src1: MIRBuilder.buildLShr(Dst: SrcTy, Src0: Op, Src1: MIBShiftAmt));
8024 Op = MIBOp.getReg(Idx: 0);
8025 }
8026 auto MIBPop = MIRBuilder.buildCTPOP(Dst: DstTy, Src0: Op);
8027 MIRBuilder.buildSub(Dst: MI.getOperand(i: 0), Src0: MIRBuilder.buildConstant(Res: DstTy, Val: Len),
8028 Src1: MIBPop);
8029 MI.eraseFromParent();
8030 return Legalized;
8031 }
8032 case TargetOpcode::G_CTTZ_ZERO_POISON: {
8033 // This trivially expands to CTTZ.
8034 Observer.changingInstr(MI);
8035 MI.setDesc(TII.get(Opcode: TargetOpcode::G_CTTZ));
8036 Observer.changedInstr(MI);
8037 return Legalized;
8038 }
8039 case TargetOpcode::G_CTTZ: {
8040 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
8041
8042 unsigned Len = SrcTy.getScalarSizeInBits();
8043 if (isSupported({TargetOpcode::G_CTTZ_ZERO_POISON, {DstTy, SrcTy}})) {
8044 // If CTTZ_ZERO_POISON is legal or custom, emit that and a select with
8045 // zero.
8046 auto CttzZU = MIRBuilder.buildCTTZ_ZERO_POISON(Dst: DstTy, Src0: SrcReg);
8047 auto Zero = MIRBuilder.buildConstant(Res: SrcTy, Val: 0);
8048 auto ICmp = MIRBuilder.buildICmp(
8049 Pred: CmpInst::ICMP_EQ, Res: DstTy.changeElementSize(NewEltSize: 1), Op0: SrcReg, Op1: Zero);
8050 auto LenConst = MIRBuilder.buildConstant(Res: DstTy, Val: Len);
8051 MIRBuilder.buildSelect(Res: DstReg, Tst: ICmp, Op0: LenConst, Op1: CttzZU);
8052 MI.eraseFromParent();
8053 return Legalized;
8054 }
8055 // for now, we use: { return popcount(~x & (x - 1)); }
8056 // unless the target has ctlz but not ctpop, in which case we use:
8057 // { return 32 - nlz(~x & (x-1)); }
8058 // Ref: "Hacker's Delight" by Henry Warren
8059 auto MIBCstNeg1 = MIRBuilder.buildConstant(Res: SrcTy, Val: -1);
8060 auto MIBNot = MIRBuilder.buildXor(Dst: SrcTy, Src0: SrcReg, Src1: MIBCstNeg1);
8061 auto MIBTmp = MIRBuilder.buildAnd(
8062 Dst: SrcTy, Src0: MIBNot, Src1: MIRBuilder.buildAdd(Dst: SrcTy, Src0: SrcReg, Src1: MIBCstNeg1));
8063 if (!isSupported({TargetOpcode::G_CTPOP, {SrcTy, SrcTy}}) &&
8064 isSupported({TargetOpcode::G_CTLZ, {SrcTy, SrcTy}})) {
8065 auto MIBCstLen = MIRBuilder.buildConstant(Res: SrcTy, Val: Len);
8066 MIRBuilder.buildSub(Dst: MI.getOperand(i: 0), Src0: MIBCstLen,
8067 Src1: MIRBuilder.buildCTLZ(Dst: SrcTy, Src0: MIBTmp));
8068 MI.eraseFromParent();
8069 return Legalized;
8070 }
8071 Observer.changingInstr(MI);
8072 MI.setDesc(TII.get(Opcode: TargetOpcode::G_CTPOP));
8073 MI.getOperand(i: 1).setReg(MIBTmp.getReg(Idx: 0));
8074 Observer.changedInstr(MI);
8075 return Legalized;
8076 }
8077 case TargetOpcode::G_CTPOP: {
8078 Register SrcReg = MI.getOperand(i: 1).getReg();
8079 LLT Ty = MRI.getType(Reg: SrcReg);
8080 unsigned Size = Ty.getScalarSizeInBits();
8081 MachineIRBuilder &B = MIRBuilder;
8082
8083 // Bail out on irregular type lengths.
8084 if (Size > 128 || Size % 8 != 0)
8085 return UnableToLegalize;
8086
8087 // Count set bits in blocks of 2 bits. Default approach would be
8088 // B2Count = { val & 0x55555555 } + { (val >> 1) & 0x55555555 }
8089 // We use following formula instead:
8090 // B2Count = val - { (val >> 1) & 0x55555555 }
8091 // since it gives same result in blocks of 2 with one instruction less.
8092 auto C_1 = B.buildConstant(Res: Ty, Val: 1);
8093 auto B2Set1LoTo1Hi = B.buildLShr(Dst: Ty, Src0: SrcReg, Src1: C_1);
8094 APInt B2Mask1HiTo0 = APInt::getSplat(NewLen: Size, V: APInt(8, 0x55));
8095 auto C_B2Mask1HiTo0 = B.buildConstant(Res: Ty, Val: B2Mask1HiTo0);
8096 auto B2Count1Hi = B.buildAnd(Dst: Ty, Src0: B2Set1LoTo1Hi, Src1: C_B2Mask1HiTo0);
8097 auto B2Count = B.buildSub(Dst: Ty, Src0: SrcReg, Src1: B2Count1Hi);
8098
8099 // In order to get count in blocks of 4 add values from adjacent block of 2.
8100 // B4Count = { B2Count & 0x33333333 } + { (B2Count >> 2) & 0x33333333 }
8101 auto C_2 = B.buildConstant(Res: Ty, Val: 2);
8102 auto B4Set2LoTo2Hi = B.buildLShr(Dst: Ty, Src0: B2Count, Src1: C_2);
8103 APInt B4Mask2HiTo0 = APInt::getSplat(NewLen: Size, V: APInt(8, 0x33));
8104 auto C_B4Mask2HiTo0 = B.buildConstant(Res: Ty, Val: B4Mask2HiTo0);
8105 auto B4HiB2Count = B.buildAnd(Dst: Ty, Src0: B4Set2LoTo2Hi, Src1: C_B4Mask2HiTo0);
8106 auto B4LoB2Count = B.buildAnd(Dst: Ty, Src0: B2Count, Src1: C_B4Mask2HiTo0);
8107 auto B4Count = B.buildAdd(Dst: Ty, Src0: B4HiB2Count, Src1: B4LoB2Count);
8108
8109 // For count in blocks of 8 bits we don't have to mask high 4 bits before
8110 // addition since count value sits in range {0,...,8} and 4 bits are enough
8111 // to hold such binary values. After addition high 4 bits still hold count
8112 // of set bits in high 4 bit block, set them to zero and get 8 bit result.
8113 // B8Count = { B4Count + (B4Count >> 4) } & 0x0F0F0F0F
8114 auto C_4 = B.buildConstant(Res: Ty, Val: 4);
8115 auto B8HiB4Count = B.buildLShr(Dst: Ty, Src0: B4Count, Src1: C_4);
8116 auto B8CountDirty4Hi = B.buildAdd(Dst: Ty, Src0: B8HiB4Count, Src1: B4Count);
8117 APInt B8Mask4HiTo0 = APInt::getSplat(NewLen: Size, V: APInt(8, 0x0F));
8118 auto C_B8Mask4HiTo0 = B.buildConstant(Res: Ty, Val: B8Mask4HiTo0);
8119 auto B8Count = B.buildAnd(Dst: Ty, Src0: B8CountDirty4Hi, Src1: C_B8Mask4HiTo0);
8120
8121 assert(Size <= 128 && "Scalar size is too large for CTPOP lower algorithm");
8122
8123 // Avoid the multiply when shift-add is cheaper.
8124 if (Size == 16 && !Ty.isVector()) {
8125 // v = (v + (v >> 8)) & 0xFF;
8126 auto C_8 = B.buildConstant(Res: Ty, Val: 8);
8127 auto HighSum = B.buildLShr(Dst: Ty, Src0: B8Count, Src1: C_8);
8128 auto Res = B.buildAdd(Dst: Ty, Src0: B8Count, Src1: HighSum);
8129 B.buildAnd(Dst: MI.getOperand(i: 0).getReg(), Src0: Res, Src1: B.buildConstant(Res: Ty, Val: 0xFF));
8130 MI.eraseFromParent();
8131 return Legalized;
8132 }
8133
8134 // 8 bits can hold CTPOP result of 128 bit int or smaller. Mul with this
8135 // bitmask will set 8 msb in ResTmp to sum of all B8Counts in 8 bit blocks.
8136 auto MulMask = B.buildConstant(Res: Ty, Val: APInt::getSplat(NewLen: Size, V: APInt(8, 0x01)));
8137
8138 // Shift count result from 8 high bits to low bits.
8139 auto C_SizeM8 = B.buildConstant(Res: Ty, Val: Size - 8);
8140
8141 auto IsMulSupported = [this](const LLT Ty) {
8142 auto Action = LI.getAction(Query: {TargetOpcode::G_MUL, {Ty}}).Action;
8143 return Action == Legal || Action == WidenScalar || Action == Custom;
8144 };
8145 if (IsMulSupported(Ty)) {
8146 auto ResTmp = B.buildMul(Dst: Ty, Src0: B8Count, Src1: MulMask);
8147 B.buildLShr(Dst: MI.getOperand(i: 0).getReg(), Src0: ResTmp, Src1: C_SizeM8);
8148 } else {
8149 auto ResTmp = B8Count;
8150 for (unsigned Shift = 8; Shift < Size; Shift *= 2) {
8151 auto ShiftC = B.buildConstant(Res: Ty, Val: Shift);
8152 auto Shl = B.buildShl(Dst: Ty, Src0: ResTmp, Src1: ShiftC);
8153 ResTmp = B.buildAdd(Dst: Ty, Src0: ResTmp, Src1: Shl);
8154 }
8155 B.buildLShr(Dst: MI.getOperand(i: 0).getReg(), Src0: ResTmp, Src1: C_SizeM8);
8156 }
8157 MI.eraseFromParent();
8158 return Legalized;
8159 }
8160 case TargetOpcode::G_CTLS: {
8161 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
8162
8163 // ctls(x) -> ctlz(x ^ (x >> (N - 1))) - 1
8164 auto SignIdxC =
8165 MIRBuilder.buildConstant(Res: SrcTy, Val: SrcTy.getScalarSizeInBits() - 1);
8166 auto OneC = MIRBuilder.buildConstant(Res: DstTy, Val: 1);
8167
8168 auto Shr = MIRBuilder.buildAShr(Dst: SrcTy, Src0: SrcReg, Src1: SignIdxC);
8169
8170 auto Xor = MIRBuilder.buildXor(Dst: SrcTy, Src0: SrcReg, Src1: Shr);
8171 auto Ctlz = MIRBuilder.buildCTLZ(Dst: DstTy, Src0: Xor);
8172
8173 MIRBuilder.buildSub(Dst: DstReg, Src0: Ctlz, Src1: OneC);
8174 MI.eraseFromParent();
8175 return Legalized;
8176 }
8177 }
8178}
8179
8180// Check that (every element of) Reg is undef or not an exact multiple of BW.
8181static bool isNonZeroModBitWidthOrUndef(const MachineRegisterInfo &MRI,
8182 Register Reg, unsigned BW) {
8183 return matchUnaryPredicate(
8184 MRI, Reg,
8185 Match: [=](const Constant *C) {
8186 // Null constant here means an undef.
8187 const ConstantInt *CI = dyn_cast_or_null<ConstantInt>(Val: C);
8188 return !CI || CI->getValue().urem(RHS: BW) != 0;
8189 },
8190 /*AllowUndefs*/ true);
8191}
8192
8193LegalizerHelper::LegalizeResult
8194LegalizerHelper::lowerFunnelShiftWithInverse(MachineInstr &MI) {
8195 auto [Dst, X, Y, Z] = MI.getFirst4Regs();
8196 LLT Ty = MRI.getType(Reg: Dst);
8197 LLT ShTy = MRI.getType(Reg: Z);
8198
8199 unsigned BW = Ty.getScalarSizeInBits();
8200
8201 if (!isPowerOf2_32(Value: BW))
8202 return UnableToLegalize;
8203
8204 const bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL;
8205 unsigned RevOpcode = IsFSHL ? TargetOpcode::G_FSHR : TargetOpcode::G_FSHL;
8206
8207 if (isNonZeroModBitWidthOrUndef(MRI, Reg: Z, BW)) {
8208 // fshl X, Y, Z -> fshr X, Y, -Z
8209 // fshr X, Y, Z -> fshl X, Y, -Z
8210 auto Zero = MIRBuilder.buildConstant(Res: ShTy, Val: 0);
8211 Z = MIRBuilder.buildSub(Dst: Ty, Src0: Zero, Src1: Z).getReg(Idx: 0);
8212 } else {
8213 // fshl X, Y, Z -> fshr (srl X, 1), (fshr X, Y, 1), ~Z
8214 // fshr X, Y, Z -> fshl (fshl X, Y, 1), (shl Y, 1), ~Z
8215 auto One = MIRBuilder.buildConstant(Res: ShTy, Val: 1);
8216 if (IsFSHL) {
8217 Y = MIRBuilder.buildInstr(Opc: RevOpcode, DstOps: {Ty}, SrcOps: {X, Y, One}).getReg(Idx: 0);
8218 X = MIRBuilder.buildLShr(Dst: Ty, Src0: X, Src1: One).getReg(Idx: 0);
8219 } else {
8220 X = MIRBuilder.buildInstr(Opc: RevOpcode, DstOps: {Ty}, SrcOps: {X, Y, One}).getReg(Idx: 0);
8221 Y = MIRBuilder.buildShl(Dst: Ty, Src0: Y, Src1: One).getReg(Idx: 0);
8222 }
8223
8224 Z = MIRBuilder.buildNot(Dst: ShTy, Src0: Z).getReg(Idx: 0);
8225 }
8226
8227 MIRBuilder.buildInstr(Opc: RevOpcode, DstOps: {Dst}, SrcOps: {X, Y, Z});
8228 MI.eraseFromParent();
8229 return Legalized;
8230}
8231
8232LegalizerHelper::LegalizeResult
8233LegalizerHelper::lowerFunnelShiftAsShifts(MachineInstr &MI) {
8234 auto [Dst, X, Y, Z] = MI.getFirst4Regs();
8235 LLT Ty = MRI.getType(Reg: Dst);
8236 LLT ShTy = MRI.getType(Reg: Z);
8237
8238 const unsigned BW = Ty.getScalarSizeInBits();
8239 const bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL;
8240
8241 Register ShX, ShY;
8242 Register ShAmt, InvShAmt;
8243
8244 // FIXME: Emit optimized urem by constant instead of letting it expand later.
8245 if (isNonZeroModBitWidthOrUndef(MRI, Reg: Z, BW)) {
8246 // fshl: X << C | Y >> (BW - C)
8247 // fshr: X << (BW - C) | Y >> C
8248 // where C = Z % BW is not zero
8249 auto BitWidthC = MIRBuilder.buildConstant(Res: ShTy, Val: BW);
8250 ShAmt = MIRBuilder.buildURem(Dst: ShTy, Src0: Z, Src1: BitWidthC).getReg(Idx: 0);
8251 InvShAmt = MIRBuilder.buildSub(Dst: ShTy, Src0: BitWidthC, Src1: ShAmt).getReg(Idx: 0);
8252 ShX = MIRBuilder.buildShl(Dst: Ty, Src0: X, Src1: IsFSHL ? ShAmt : InvShAmt).getReg(Idx: 0);
8253 ShY = MIRBuilder.buildLShr(Dst: Ty, Src0: Y, Src1: IsFSHL ? InvShAmt : ShAmt).getReg(Idx: 0);
8254 } else {
8255 // fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))
8256 // fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)
8257 auto Mask = MIRBuilder.buildConstant(Res: ShTy, Val: BW - 1);
8258 if (isPowerOf2_32(Value: BW)) {
8259 // Z % BW -> Z & (BW - 1)
8260 ShAmt = MIRBuilder.buildAnd(Dst: ShTy, Src0: Z, Src1: Mask).getReg(Idx: 0);
8261 // (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
8262 auto NotZ = MIRBuilder.buildNot(Dst: ShTy, Src0: Z);
8263 InvShAmt = MIRBuilder.buildAnd(Dst: ShTy, Src0: NotZ, Src1: Mask).getReg(Idx: 0);
8264 } else {
8265 auto BitWidthC = MIRBuilder.buildConstant(Res: ShTy, Val: BW);
8266 ShAmt = MIRBuilder.buildURem(Dst: ShTy, Src0: Z, Src1: BitWidthC).getReg(Idx: 0);
8267 InvShAmt = MIRBuilder.buildSub(Dst: ShTy, Src0: Mask, Src1: ShAmt).getReg(Idx: 0);
8268 }
8269
8270 auto One = MIRBuilder.buildConstant(Res: ShTy, Val: 1);
8271 if (IsFSHL) {
8272 ShX = MIRBuilder.buildShl(Dst: Ty, Src0: X, Src1: ShAmt).getReg(Idx: 0);
8273 auto ShY1 = MIRBuilder.buildLShr(Dst: Ty, Src0: Y, Src1: One);
8274 ShY = MIRBuilder.buildLShr(Dst: Ty, Src0: ShY1, Src1: InvShAmt).getReg(Idx: 0);
8275 } else {
8276 auto ShX1 = MIRBuilder.buildShl(Dst: Ty, Src0: X, Src1: One);
8277 ShX = MIRBuilder.buildShl(Dst: Ty, Src0: ShX1, Src1: InvShAmt).getReg(Idx: 0);
8278 ShY = MIRBuilder.buildLShr(Dst: Ty, Src0: Y, Src1: ShAmt).getReg(Idx: 0);
8279 }
8280 }
8281
8282 MIRBuilder.buildOr(Dst, Src0: ShX, Src1: ShY, Flags: MachineInstr::Disjoint);
8283 MI.eraseFromParent();
8284 return Legalized;
8285}
8286
8287LegalizerHelper::LegalizeResult
8288LegalizerHelper::lowerFunnelShift(MachineInstr &MI) {
8289 // These operations approximately do the following (while avoiding undefined
8290 // shifts by BW):
8291 // G_FSHL: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
8292 // G_FSHR: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
8293 Register Dst = MI.getOperand(i: 0).getReg();
8294 LLT Ty = MRI.getType(Reg: Dst);
8295 LLT ShTy = MRI.getType(Reg: MI.getOperand(i: 3).getReg());
8296
8297 bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL;
8298 unsigned RevOpcode = IsFSHL ? TargetOpcode::G_FSHR : TargetOpcode::G_FSHL;
8299
8300 // TODO: Use smarter heuristic that accounts for vector legalization.
8301 if (LI.getAction(Query: {RevOpcode, {Ty, ShTy}}).Action == Lower)
8302 return lowerFunnelShiftAsShifts(MI);
8303
8304 // This only works for powers of 2, fallback to shifts if it fails.
8305 LegalizerHelper::LegalizeResult Result = lowerFunnelShiftWithInverse(MI);
8306 if (Result == UnableToLegalize)
8307 return lowerFunnelShiftAsShifts(MI);
8308 return Result;
8309}
8310
8311LegalizerHelper::LegalizeResult LegalizerHelper::lowerEXT(MachineInstr &MI) {
8312 auto [Dst, Src] = MI.getFirst2Regs();
8313 LLT DstTy = MRI.getType(Reg: Dst);
8314 LLT SrcTy = MRI.getType(Reg: Src);
8315
8316 uint32_t DstTySize = DstTy.getSizeInBits();
8317 uint32_t DstTyScalarSize = DstTy.getScalarSizeInBits();
8318 uint32_t SrcTyScalarSize = SrcTy.getScalarSizeInBits();
8319
8320 if (!isPowerOf2_32(Value: DstTySize) || !isPowerOf2_32(Value: DstTyScalarSize) ||
8321 !isPowerOf2_32(Value: SrcTyScalarSize))
8322 return UnableToLegalize;
8323
8324 // The step between extend is too large, split it by creating an intermediate
8325 // extend instruction
8326 if (SrcTyScalarSize * 2 < DstTyScalarSize) {
8327 LLT MidTy = SrcTy.changeElementSize(NewEltSize: SrcTyScalarSize * 2);
8328 // If the destination type is illegal, split it into multiple statements
8329 // zext x -> zext(merge(zext(unmerge), zext(unmerge)))
8330 auto NewExt = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {MidTy}, SrcOps: {Src});
8331 // Unmerge the vector
8332 LLT EltTy = MidTy.changeElementCount(
8333 EC: MidTy.getElementCount().divideCoefficientBy(RHS: 2));
8334 auto UnmergeSrc = MIRBuilder.buildUnmerge(Res: EltTy, Op: NewExt);
8335
8336 // ZExt the vectors
8337 LLT ZExtResTy = DstTy.changeElementCount(
8338 EC: DstTy.getElementCount().divideCoefficientBy(RHS: 2));
8339 auto ZExtRes1 = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {ZExtResTy},
8340 SrcOps: {UnmergeSrc.getReg(Idx: 0)});
8341 auto ZExtRes2 = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {ZExtResTy},
8342 SrcOps: {UnmergeSrc.getReg(Idx: 1)});
8343
8344 // Merge the ending vectors
8345 MIRBuilder.buildMergeLikeInstr(Res: Dst, Ops: {ZExtRes1, ZExtRes2});
8346
8347 MI.eraseFromParent();
8348 return Legalized;
8349 }
8350 return UnableToLegalize;
8351}
8352
8353LegalizerHelper::LegalizeResult LegalizerHelper::lowerTRUNC(MachineInstr &MI) {
8354 // MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
8355 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
8356 // Similar to how operand splitting is done in SelectiondDAG, we can handle
8357 // %res(v8s8) = G_TRUNC %in(v8s32) by generating:
8358 // %inlo(<4x s32>), %inhi(<4 x s32>) = G_UNMERGE %in(<8 x s32>)
8359 // %lo16(<4 x s16>) = G_TRUNC %inlo
8360 // %hi16(<4 x s16>) = G_TRUNC %inhi
8361 // %in16(<8 x s16>) = G_CONCAT_VECTORS %lo16, %hi16
8362 // %res(<8 x s8>) = G_TRUNC %in16
8363
8364 assert(MI.getOpcode() == TargetOpcode::G_TRUNC);
8365
8366 Register DstReg = MI.getOperand(i: 0).getReg();
8367 Register SrcReg = MI.getOperand(i: 1).getReg();
8368 LLT DstTy = MRI.getType(Reg: DstReg);
8369 LLT SrcTy = MRI.getType(Reg: SrcReg);
8370
8371 if (DstTy.isVector() && isPowerOf2_32(Value: DstTy.getNumElements()) &&
8372 isPowerOf2_32(Value: DstTy.getScalarSizeInBits()) &&
8373 isPowerOf2_32(Value: SrcTy.getNumElements()) &&
8374 isPowerOf2_32(Value: SrcTy.getScalarSizeInBits())) {
8375 // Split input type.
8376 LLT SplitSrcTy = SrcTy.changeElementCount(
8377 EC: SrcTy.getElementCount().divideCoefficientBy(RHS: 2));
8378
8379 // First, split the source into two smaller vectors.
8380 SmallVector<Register, 2> SplitSrcs;
8381 extractParts(Reg: SrcReg, Ty: SplitSrcTy, NumParts: 2, VRegs&: SplitSrcs, MIRBuilder, MRI);
8382
8383 // Truncate the splits into intermediate narrower elements.
8384 LLT InterTy;
8385 if (DstTy.getScalarSizeInBits() * 2 < SrcTy.getScalarSizeInBits())
8386 InterTy = SplitSrcTy.changeElementSize(NewEltSize: DstTy.getScalarSizeInBits() * 2);
8387 else
8388 InterTy = SplitSrcTy.changeElementSize(NewEltSize: DstTy.getScalarSizeInBits());
8389 for (Register &Src : SplitSrcs)
8390 Src = MIRBuilder.buildTrunc(Res: InterTy, Op: Src).getReg(Idx: 0);
8391
8392 // Combine the new truncates into one vector
8393 auto Merge = MIRBuilder.buildMergeLikeInstr(
8394 Res: DstTy.changeElementSize(NewEltSize: InterTy.getScalarSizeInBits()), Ops: SplitSrcs);
8395
8396 // Truncate the new vector to the final result type
8397 if (DstTy.getScalarSizeInBits() * 2 < SrcTy.getScalarSizeInBits())
8398 MIRBuilder.buildTrunc(Res: MI.getOperand(i: 0).getReg(), Op: Merge.getReg(Idx: 0));
8399 else
8400 MIRBuilder.buildCopy(Res: MI.getOperand(i: 0).getReg(), Op: Merge.getReg(Idx: 0));
8401
8402 MI.eraseFromParent();
8403
8404 return Legalized;
8405 }
8406 return UnableToLegalize;
8407}
8408
8409LegalizerHelper::LegalizeResult
8410LegalizerHelper::lowerRotateWithReverseRotate(MachineInstr &MI) {
8411 auto [Dst, DstTy, Src, SrcTy, Amt, AmtTy] = MI.getFirst3RegLLTs();
8412 auto Zero = MIRBuilder.buildConstant(Res: AmtTy, Val: 0);
8413 bool IsLeft = MI.getOpcode() == TargetOpcode::G_ROTL;
8414 unsigned RevRot = IsLeft ? TargetOpcode::G_ROTR : TargetOpcode::G_ROTL;
8415 auto Neg = MIRBuilder.buildSub(Dst: AmtTy, Src0: Zero, Src1: Amt);
8416 MIRBuilder.buildInstr(Opc: RevRot, DstOps: {Dst}, SrcOps: {Src, Neg});
8417 MI.eraseFromParent();
8418 return Legalized;
8419}
8420
8421LegalizerHelper::LegalizeResult LegalizerHelper::lowerRotate(MachineInstr &MI) {
8422 auto [Dst, DstTy, Src, SrcTy, Amt, AmtTy] = MI.getFirst3RegLLTs();
8423
8424 unsigned EltSizeInBits = DstTy.getScalarSizeInBits();
8425 bool IsLeft = MI.getOpcode() == TargetOpcode::G_ROTL;
8426
8427 MIRBuilder.setInstrAndDebugLoc(MI);
8428
8429 // If a rotate in the other direction is supported, use it.
8430 unsigned RevRot = IsLeft ? TargetOpcode::G_ROTR : TargetOpcode::G_ROTL;
8431 if (LI.isLegalOrCustom(Query: {RevRot, {DstTy, SrcTy}}) &&
8432 isPowerOf2_32(Value: EltSizeInBits))
8433 return lowerRotateWithReverseRotate(MI);
8434
8435 // If a funnel shift is supported, use it.
8436 unsigned FShOpc = IsLeft ? TargetOpcode::G_FSHL : TargetOpcode::G_FSHR;
8437 unsigned RevFsh = !IsLeft ? TargetOpcode::G_FSHL : TargetOpcode::G_FSHR;
8438 bool IsFShLegal = false;
8439 if ((IsFShLegal = LI.isLegalOrCustom(Query: {FShOpc, {DstTy, AmtTy}})) ||
8440 LI.isLegalOrCustom(Query: {RevFsh, {DstTy, AmtTy}})) {
8441 auto buildFunnelShift = [&](unsigned Opc, Register R1, Register R2,
8442 Register R3) {
8443 MIRBuilder.buildInstr(Opc, DstOps: {R1}, SrcOps: {R2, R2, R3});
8444 MI.eraseFromParent();
8445 return Legalized;
8446 };
8447 // If a funnel shift in the other direction is supported, use it.
8448 if (IsFShLegal) {
8449 return buildFunnelShift(FShOpc, Dst, Src, Amt);
8450 } else if (isPowerOf2_32(Value: EltSizeInBits)) {
8451 Amt = MIRBuilder.buildNeg(Dst: DstTy, Src0: Amt).getReg(Idx: 0);
8452 return buildFunnelShift(RevFsh, Dst, Src, Amt);
8453 }
8454 }
8455
8456 auto Zero = MIRBuilder.buildConstant(Res: AmtTy, Val: 0);
8457 unsigned ShOpc = IsLeft ? TargetOpcode::G_SHL : TargetOpcode::G_LSHR;
8458 unsigned RevShiftOpc = IsLeft ? TargetOpcode::G_LSHR : TargetOpcode::G_SHL;
8459 auto BitWidthMinusOneC = MIRBuilder.buildConstant(Res: AmtTy, Val: EltSizeInBits - 1);
8460 Register ShVal;
8461 Register RevShiftVal;
8462 if (isPowerOf2_32(Value: EltSizeInBits)) {
8463 // (rotl x, c) -> x << (c & (w - 1)) | x >> (-c & (w - 1))
8464 // (rotr x, c) -> x >> (c & (w - 1)) | x << (-c & (w - 1))
8465 auto NegAmt = MIRBuilder.buildSub(Dst: AmtTy, Src0: Zero, Src1: Amt);
8466 auto ShAmt = MIRBuilder.buildAnd(Dst: AmtTy, Src0: Amt, Src1: BitWidthMinusOneC);
8467 ShVal = MIRBuilder.buildInstr(Opc: ShOpc, DstOps: {DstTy}, SrcOps: {Src, ShAmt}).getReg(Idx: 0);
8468 auto RevAmt = MIRBuilder.buildAnd(Dst: AmtTy, Src0: NegAmt, Src1: BitWidthMinusOneC);
8469 RevShiftVal =
8470 MIRBuilder.buildInstr(Opc: RevShiftOpc, DstOps: {DstTy}, SrcOps: {Src, RevAmt}).getReg(Idx: 0);
8471 } else {
8472 // (rotl x, c) -> x << (c % w) | x >> 1 >> (w - 1 - (c % w))
8473 // (rotr x, c) -> x >> (c % w) | x << 1 << (w - 1 - (c % w))
8474 auto BitWidthC = MIRBuilder.buildConstant(Res: AmtTy, Val: EltSizeInBits);
8475 auto ShAmt = MIRBuilder.buildURem(Dst: AmtTy, Src0: Amt, Src1: BitWidthC);
8476 ShVal = MIRBuilder.buildInstr(Opc: ShOpc, DstOps: {DstTy}, SrcOps: {Src, ShAmt}).getReg(Idx: 0);
8477 auto RevAmt = MIRBuilder.buildSub(Dst: AmtTy, Src0: BitWidthMinusOneC, Src1: ShAmt);
8478 auto One = MIRBuilder.buildConstant(Res: AmtTy, Val: 1);
8479 auto Inner = MIRBuilder.buildInstr(Opc: RevShiftOpc, DstOps: {DstTy}, SrcOps: {Src, One});
8480 RevShiftVal =
8481 MIRBuilder.buildInstr(Opc: RevShiftOpc, DstOps: {DstTy}, SrcOps: {Inner, RevAmt}).getReg(Idx: 0);
8482 }
8483 MIRBuilder.buildOr(Dst, Src0: ShVal, Src1: RevShiftVal, Flags: MachineInstr::Disjoint);
8484 MI.eraseFromParent();
8485 return Legalized;
8486}
8487
8488// Expand s32 = G_UITOFP s64 to an IEEE float representation using bit
8489// operations and G_SITOFP
8490LegalizerHelper::LegalizeResult
8491LegalizerHelper::lowerU64ToF32WithSITOFP(MachineInstr &MI) {
8492 auto [Dst, Src] = MI.getFirst2Regs();
8493 const LLT S64 = LLT::scalar(SizeInBits: 64);
8494 const LLT S32 = LLT::scalar(SizeInBits: 32);
8495 const LLT S1 = LLT::scalar(SizeInBits: 1);
8496
8497 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S32);
8498
8499 // For i64 < INT_MAX we simply reuse SITOFP.
8500 // Otherwise, divide i64 by 2, round result by ORing with the lowest bit
8501 // saved before division, convert to float by SITOFP, multiply the result
8502 // by 2.
8503 auto One = MIRBuilder.buildConstant(Res: S64, Val: 1);
8504 auto Zero = MIRBuilder.buildConstant(Res: S64, Val: 0);
8505 // Result if Src < INT_MAX
8506 auto SmallResult = MIRBuilder.buildSITOFP(Dst: S32, Src0: Src);
8507 // Result if Src >= INT_MAX
8508 auto Halved = MIRBuilder.buildLShr(Dst: S64, Src0: Src, Src1: One);
8509 auto LowerBit = MIRBuilder.buildAnd(Dst: S64, Src0: Src, Src1: One);
8510 auto RoundedHalved = MIRBuilder.buildOr(Dst: S64, Src0: Halved, Src1: LowerBit);
8511 auto HalvedFP = MIRBuilder.buildSITOFP(Dst: S32, Src0: RoundedHalved);
8512 auto LargeResult = MIRBuilder.buildFAdd(Dst: S32, Src0: HalvedFP, Src1: HalvedFP);
8513 // Check if the original value is larger than INT_MAX by comparing with
8514 // zero to pick one of the two conversions.
8515 auto IsLarge =
8516 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_SLT, Res: S1, Op0: Src, Op1: Zero);
8517 MIRBuilder.buildSelect(Res: Dst, Tst: IsLarge, Op0: LargeResult, Op1: SmallResult);
8518
8519 MI.eraseFromParent();
8520 return Legalized;
8521}
8522
8523// Expand s64 = G_UITOFP s64 using bit and float arithmetic operations to an
8524// IEEE double representation.
8525LegalizerHelper::LegalizeResult
8526LegalizerHelper::lowerU64ToF64BitFloatOps(MachineInstr &MI) {
8527 auto [Dst, Src] = MI.getFirst2Regs();
8528 const LLT S64 = LLT::scalar(SizeInBits: 64);
8529 const LLT S32 = LLT::scalar(SizeInBits: 32);
8530
8531 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S64);
8532
8533 // We create double value from 32 bit parts with 32 exponent difference.
8534 // Note that + and - are float operations that adjust the implicit leading
8535 // one, the bases 2^52 and 2^84 are for illustrative purposes.
8536 //
8537 // X = 2^52 * 1.0...LowBits
8538 // Y = 2^84 * 1.0...HighBits
8539 // Scratch = 2^84 * 1.0...HighBits - 2^84 * 1.0 - 2^52 * 1.0
8540 // = - 2^52 * 1.0...HighBits
8541 // Result = - 2^52 * 1.0...HighBits + 2^52 * 1.0...LowBits
8542 auto TwoP52 = MIRBuilder.buildConstant(Res: S64, UINT64_C(0x4330000000000000));
8543 auto TwoP84 = MIRBuilder.buildConstant(Res: S64, UINT64_C(0x4530000000000000));
8544 auto TwoP52P84 = llvm::bit_cast<double>(UINT64_C(0x4530000000100000));
8545 auto TwoP52P84FP = MIRBuilder.buildFConstant(Res: S64, Val: TwoP52P84);
8546 auto HalfWidth = MIRBuilder.buildConstant(Res: S64, Val: 32);
8547
8548 auto LowBits = MIRBuilder.buildTrunc(Res: S32, Op: Src);
8549 LowBits = MIRBuilder.buildZExt(Res: S64, Op: LowBits);
8550 auto LowBitsFP = MIRBuilder.buildOr(Dst: S64, Src0: TwoP52, Src1: LowBits);
8551 auto HighBits = MIRBuilder.buildLShr(Dst: S64, Src0: Src, Src1: HalfWidth);
8552 auto HighBitsFP = MIRBuilder.buildOr(Dst: S64, Src0: TwoP84, Src1: HighBits);
8553 auto Scratch = MIRBuilder.buildFSub(Dst: S64, Src0: HighBitsFP, Src1: TwoP52P84FP);
8554 MIRBuilder.buildFAdd(Dst, Src0: Scratch, Src1: LowBitsFP);
8555
8556 MI.eraseFromParent();
8557 return Legalized;
8558}
8559
8560/// i64->fp16 itofp can be lowered to i64->f64,f64->f32,f32->f16. We cannot
8561/// convert fpround f64->f16 without double-rounding, so we manually perform the
8562/// lowering here where we know it is valid.
8563static LegalizerHelper::LegalizeResult
8564loweri64tof16ITOFP(MachineInstr &MI, Register Dst, LLT DstTy, Register Src,
8565 LLT SrcTy, MachineIRBuilder &MIRBuilder) {
8566 auto DstFpTy =
8567 SrcTy.changeElementType(NewEltTy: LLT::floatIEEE(SizeInBits: SrcTy.getScalarSizeInBits()));
8568 auto M1 = MI.getOpcode() == TargetOpcode::G_UITOFP
8569 ? MIRBuilder.buildUITOFP(Dst: DstFpTy, Src0: Src)
8570 : MIRBuilder.buildSITOFP(Dst: DstFpTy, Src0: Src);
8571 LLT F32Ty = DstFpTy.changeElementSize(NewEltSize: 32);
8572 auto M2 = MIRBuilder.buildFPTrunc(Res: F32Ty, Op: M1);
8573 MIRBuilder.buildFPTrunc(Res: Dst, Op: M2);
8574 MI.eraseFromParent();
8575 return LegalizerHelper::Legalized;
8576}
8577
8578LegalizerHelper::LegalizeResult LegalizerHelper::lowerUITOFP(MachineInstr &MI) {
8579 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8580
8581 if (SrcTy == LLT::scalar(SizeInBits: 1)) {
8582 auto True = MIRBuilder.buildFConstant(Res: DstTy, Val: 1.0);
8583 auto False = MIRBuilder.buildFConstant(Res: DstTy, Val: 0.0);
8584 MIRBuilder.buildSelect(Res: Dst, Tst: Src, Op0: True, Op1: False);
8585 MI.eraseFromParent();
8586 return Legalized;
8587 }
8588
8589 if (DstTy.getScalarSizeInBits() == 16 && SrcTy.getScalarSizeInBits() == 64)
8590 return loweri64tof16ITOFP(MI, Dst, DstTy, Src, SrcTy, MIRBuilder);
8591
8592 if (SrcTy != LLT::scalar(SizeInBits: 64))
8593 return UnableToLegalize;
8594
8595 if (DstTy == LLT::scalar(SizeInBits: 32))
8596 // TODO: SelectionDAG has several alternative expansions to port which may
8597 // be more reasonable depending on the available instructions. We also need
8598 // a more advanced mechanism to choose an optimal version depending on
8599 // target features such as sitofp or CTLZ availability.
8600 return lowerU64ToF32WithSITOFP(MI);
8601
8602 if (DstTy == LLT::scalar(SizeInBits: 64))
8603 return lowerU64ToF64BitFloatOps(MI);
8604
8605 return UnableToLegalize;
8606}
8607
8608LegalizerHelper::LegalizeResult LegalizerHelper::lowerSITOFP(MachineInstr &MI) {
8609 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8610
8611 const LLT I64 = LLT::integer(SizeInBits: 64);
8612 const LLT I32 = LLT::integer(SizeInBits: 32);
8613 const LLT I1 = LLT::integer(SizeInBits: 1);
8614
8615 if (SrcTy == I1) {
8616 auto True = MIRBuilder.buildFConstant(Res: DstTy, Val: -1.0);
8617 auto False = MIRBuilder.buildFConstant(Res: DstTy, Val: 0.0);
8618 MIRBuilder.buildSelect(Res: Dst, Tst: Src, Op0: True, Op1: False);
8619 MI.eraseFromParent();
8620 return Legalized;
8621 }
8622
8623 if (DstTy.getScalarSizeInBits() == 16 && SrcTy.getScalarSizeInBits() == 64)
8624 return loweri64tof16ITOFP(MI, Dst, DstTy, Src, SrcTy, MIRBuilder);
8625
8626 if (SrcTy != I64)
8627 return UnableToLegalize;
8628
8629 if (DstTy.getScalarSizeInBits() == 32) {
8630 // signed cl2f(long l) {
8631 // long s = l >> 63;
8632 // float r = cul2f((l + s) ^ s);
8633 // return s ? -r : r;
8634 // }
8635 Register L = Src;
8636 auto SignBit = MIRBuilder.buildConstant(Res: I64, Val: 63);
8637 auto S = MIRBuilder.buildAShr(Dst: I64, Src0: L, Src1: SignBit);
8638
8639 auto LPlusS = MIRBuilder.buildAdd(Dst: I64, Src0: L, Src1: S);
8640 auto Xor = MIRBuilder.buildXor(Dst: I64, Src0: LPlusS, Src1: S);
8641 auto R = MIRBuilder.buildUITOFP(Dst: I32, Src0: Xor);
8642
8643 auto RNeg = MIRBuilder.buildFNeg(Dst: I32, Src0: R);
8644 auto SignNotZero = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: I1, Op0: S,
8645 Op1: MIRBuilder.buildConstant(Res: I64, Val: 0));
8646 MIRBuilder.buildSelect(Res: Dst, Tst: SignNotZero, Op0: RNeg, Op1: R);
8647 MI.eraseFromParent();
8648 return Legalized;
8649 }
8650
8651 return UnableToLegalize;
8652}
8653
8654LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOUI(MachineInstr &MI) {
8655 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8656 const LLT S64 = LLT::scalar(SizeInBits: 64);
8657 const LLT S32 = LLT::scalar(SizeInBits: 32);
8658
8659 if (SrcTy != S64 && SrcTy != S32)
8660 return UnableToLegalize;
8661 if (DstTy != S32 && DstTy != S64)
8662 return UnableToLegalize;
8663
8664 // FPTOSI gives same result as FPTOUI for positive signed integers.
8665 // FPTOUI needs to deal with fp values that convert to unsigned integers
8666 // greater or equal to 2^31 for float or 2^63 for double. For brevity 2^Exp.
8667
8668 APInt TwoPExpInt = APInt::getSignMask(BitWidth: DstTy.getSizeInBits());
8669 APFloat TwoPExpFP(SrcTy.getSizeInBits() == 32 ? APFloat::IEEEsingle()
8670 : APFloat::IEEEdouble(),
8671 APInt::getZero(numBits: SrcTy.getSizeInBits()));
8672 TwoPExpFP.convertFromAPInt(Input: TwoPExpInt, IsSigned: false, RM: APFloat::rmNearestTiesToEven);
8673
8674 MachineInstrBuilder FPTOSI = MIRBuilder.buildFPTOSI(Dst: DstTy, Src0: Src);
8675
8676 MachineInstrBuilder Threshold = MIRBuilder.buildFConstant(Res: SrcTy, Val: TwoPExpFP);
8677 // For fp Value greater or equal to Threshold(2^Exp), we use FPTOSI on
8678 // (Value - 2^Exp) and add 2^Exp by setting highest bit in result to 1.
8679 MachineInstrBuilder FSub = MIRBuilder.buildFSub(Dst: SrcTy, Src0: Src, Src1: Threshold);
8680 MachineInstrBuilder ResLowBits = MIRBuilder.buildFPTOSI(Dst: DstTy, Src0: FSub);
8681 MachineInstrBuilder ResHighBit = MIRBuilder.buildConstant(Res: DstTy, Val: TwoPExpInt);
8682 MachineInstrBuilder Res = MIRBuilder.buildXor(Dst: DstTy, Src0: ResLowBits, Src1: ResHighBit);
8683
8684 const LLT S1 = LLT::scalar(SizeInBits: 1);
8685
8686 MachineInstrBuilder FCMP =
8687 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_ULT, Res: S1, Op0: Src, Op1: Threshold);
8688 MIRBuilder.buildSelect(Res: Dst, Tst: FCMP, Op0: FPTOSI, Op1: Res);
8689
8690 MI.eraseFromParent();
8691 return Legalized;
8692}
8693
8694LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOSI(MachineInstr &MI) {
8695 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8696 const LLT S64 = LLT::scalar(SizeInBits: 64);
8697 const LLT S32 = LLT::scalar(SizeInBits: 32);
8698
8699 // FIXME: Only f32 to i64 conversions are supported.
8700 if (SrcTy.getScalarType() != S32 || DstTy.getScalarType() != S64)
8701 return UnableToLegalize;
8702
8703 // Expand f32 -> i64 conversion
8704 // This algorithm comes from compiler-rt's implementation of fixsfdi:
8705 // https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/fixsfdi.c
8706
8707 unsigned SrcEltBits = SrcTy.getScalarSizeInBits();
8708
8709 auto ExponentMask = MIRBuilder.buildConstant(Res: SrcTy, Val: 0x7F800000);
8710 auto ExponentLoBit = MIRBuilder.buildConstant(Res: SrcTy, Val: 23);
8711
8712 auto AndExpMask = MIRBuilder.buildAnd(Dst: SrcTy, Src0: Src, Src1: ExponentMask);
8713 auto ExponentBits = MIRBuilder.buildLShr(Dst: SrcTy, Src0: AndExpMask, Src1: ExponentLoBit);
8714
8715 auto SignMask = MIRBuilder.buildConstant(Res: SrcTy,
8716 Val: APInt::getSignMask(BitWidth: SrcEltBits));
8717 auto AndSignMask = MIRBuilder.buildAnd(Dst: SrcTy, Src0: Src, Src1: SignMask);
8718 auto SignLowBit = MIRBuilder.buildConstant(Res: SrcTy, Val: SrcEltBits - 1);
8719 auto Sign = MIRBuilder.buildAShr(Dst: SrcTy, Src0: AndSignMask, Src1: SignLowBit);
8720 Sign = MIRBuilder.buildSExt(Res: DstTy, Op: Sign);
8721
8722 auto MantissaMask = MIRBuilder.buildConstant(Res: SrcTy, Val: 0x007FFFFF);
8723 auto AndMantissaMask = MIRBuilder.buildAnd(Dst: SrcTy, Src0: Src, Src1: MantissaMask);
8724 auto K = MIRBuilder.buildConstant(Res: SrcTy, Val: 0x00800000);
8725
8726 auto R = MIRBuilder.buildOr(Dst: SrcTy, Src0: AndMantissaMask, Src1: K);
8727 R = MIRBuilder.buildZExt(Res: DstTy, Op: R);
8728
8729 auto Bias = MIRBuilder.buildConstant(Res: SrcTy, Val: 127);
8730 auto Exponent = MIRBuilder.buildSub(Dst: SrcTy, Src0: ExponentBits, Src1: Bias);
8731 auto SubExponent = MIRBuilder.buildSub(Dst: SrcTy, Src0: Exponent, Src1: ExponentLoBit);
8732 auto ExponentSub = MIRBuilder.buildSub(Dst: SrcTy, Src0: ExponentLoBit, Src1: Exponent);
8733
8734 auto Shl = MIRBuilder.buildShl(Dst: DstTy, Src0: R, Src1: SubExponent);
8735 auto Srl = MIRBuilder.buildLShr(Dst: DstTy, Src0: R, Src1: ExponentSub);
8736
8737 const LLT S1 = LLT::scalar(SizeInBits: 1);
8738 auto CmpGt = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SGT,
8739 Res: S1, Op0: Exponent, Op1: ExponentLoBit);
8740
8741 R = MIRBuilder.buildSelect(Res: DstTy, Tst: CmpGt, Op0: Shl, Op1: Srl);
8742
8743 auto XorSign = MIRBuilder.buildXor(Dst: DstTy, Src0: R, Src1: Sign);
8744 auto Ret = MIRBuilder.buildSub(Dst: DstTy, Src0: XorSign, Src1: Sign);
8745
8746 auto ZeroSrcTy = MIRBuilder.buildConstant(Res: SrcTy, Val: 0);
8747
8748 auto ExponentLt0 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT,
8749 Res: S1, Op0: Exponent, Op1: ZeroSrcTy);
8750
8751 auto ZeroDstTy = MIRBuilder.buildConstant(Res: DstTy, Val: 0);
8752 MIRBuilder.buildSelect(Res: Dst, Tst: ExponentLt0, Op0: ZeroDstTy, Op1: Ret);
8753
8754 MI.eraseFromParent();
8755 return Legalized;
8756}
8757
8758LegalizerHelper::LegalizeResult
8759LegalizerHelper::lowerFPTOINT_SAT(MachineInstr &MI) {
8760 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8761
8762 bool IsSigned = MI.getOpcode() == TargetOpcode::G_FPTOSI_SAT;
8763 unsigned SatWidth = DstTy.getScalarSizeInBits();
8764
8765 // Determine minimum and maximum integer values and their corresponding
8766 // floating-point values.
8767 APInt MinInt, MaxInt;
8768 if (IsSigned) {
8769 MinInt = APInt::getSignedMinValue(numBits: SatWidth);
8770 MaxInt = APInt::getSignedMaxValue(numBits: SatWidth);
8771 } else {
8772 MinInt = APInt::getMinValue(numBits: SatWidth);
8773 MaxInt = APInt::getMaxValue(numBits: SatWidth);
8774 }
8775
8776 const fltSemantics &Semantics = getFltSemanticForLLT(Ty: SrcTy.getScalarType());
8777 APFloat MinFloat(Semantics);
8778 APFloat MaxFloat(Semantics);
8779
8780 APFloat::opStatus MinStatus =
8781 MinFloat.convertFromAPInt(Input: MinInt, IsSigned, RM: APFloat::rmTowardZero);
8782 APFloat::opStatus MaxStatus =
8783 MaxFloat.convertFromAPInt(Input: MaxInt, IsSigned, RM: APFloat::rmTowardZero);
8784 bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact) &&
8785 !(MaxStatus & APFloat::opStatus::opInexact);
8786
8787 // If the integer bounds are exactly representable as floats, emit a
8788 // min+max+fptoi sequence. Otherwise we have to use a sequence of comparisons
8789 // and selects.
8790 if (AreExactFloatBounds) {
8791 // Clamp Src by MinFloat from below. If Src is NaN the result is MinFloat.
8792 auto MaxC = MIRBuilder.buildFConstant(Res: SrcTy, Val: MinFloat);
8793 auto MaxP =
8794 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OGT, Res: LLT::integer(SizeInBits: 1), Op0: Src, Op1: MaxC);
8795 auto Max = MIRBuilder.buildSelect(Res: SrcTy, Tst: MaxP, Op0: Src, Op1: MaxC);
8796 // Clamp by MaxFloat from above. NaN cannot occur.
8797 auto MinC = MIRBuilder.buildFConstant(Res: SrcTy, Val: MaxFloat);
8798 auto MinP = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OLT, Res: LLT::integer(SizeInBits: 1), Op0: Max,
8799 Op1: MinC, Flags: MachineInstr::FmNoNans);
8800 auto Min =
8801 MIRBuilder.buildSelect(Res: SrcTy, Tst: MinP, Op0: Max, Op1: MinC, Flags: MachineInstr::FmNoNans);
8802 // Convert clamped value to integer. In the unsigned case we're done,
8803 // because we mapped NaN to MinFloat, which will cast to zero.
8804 if (!IsSigned) {
8805 MIRBuilder.buildFPTOUI(Dst, Src0: Min);
8806 MI.eraseFromParent();
8807 return Legalized;
8808 }
8809
8810 // Otherwise, select 0 if Src is NaN.
8811 auto FpToInt = MIRBuilder.buildFPTOSI(Dst: DstTy, Src0: Min);
8812 auto IsZero =
8813 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_UNO, Res: LLT::integer(SizeInBits: 1), Op0: Src, Op1: Src);
8814 MIRBuilder.buildSelect(Res: Dst, Tst: IsZero, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: 0),
8815 Op1: FpToInt);
8816 MI.eraseFromParent();
8817 return Legalized;
8818 }
8819
8820 // Result of direct conversion. The assumption here is that the operation is
8821 // non-trapping and it's fine to apply it to an out-of-range value if we
8822 // select it away later.
8823 auto FpToInt = IsSigned ? MIRBuilder.buildFPTOSI(Dst: DstTy, Src0: Src)
8824 : MIRBuilder.buildFPTOUI(Dst: DstTy, Src0: Src);
8825
8826 // If Src ULT MinFloat, select MinInt. In particular, this also selects
8827 // MinInt if Src is NaN.
8828 auto ULT = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_ULT, Res: LLT::integer(SizeInBits: 1), Op0: Src,
8829 Op1: MIRBuilder.buildFConstant(Res: SrcTy, Val: MinFloat));
8830 auto Max = MIRBuilder.buildSelect(
8831 Res: DstTy, Tst: ULT, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: MinInt), Op1: FpToInt);
8832 // If Src OGT MaxFloat, select MaxInt.
8833 auto OGT = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OGT, Res: LLT::integer(SizeInBits: 1), Op0: Src,
8834 Op1: MIRBuilder.buildFConstant(Res: SrcTy, Val: MaxFloat));
8835
8836 // In the unsigned case we are done, because we mapped NaN to MinInt, which
8837 // is already zero.
8838 if (!IsSigned) {
8839 MIRBuilder.buildSelect(Res: Dst, Tst: OGT, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: MaxInt),
8840 Op1: Max);
8841 MI.eraseFromParent();
8842 return Legalized;
8843 }
8844
8845 // Otherwise, select 0 if Src is NaN.
8846 auto Min = MIRBuilder.buildSelect(
8847 Res: DstTy, Tst: OGT, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: MaxInt), Op1: Max);
8848 auto IsZero =
8849 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_UNO, Res: LLT::integer(SizeInBits: 1), Op0: Src, Op1: Src);
8850 MIRBuilder.buildSelect(Res: Dst, Tst: IsZero, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: 0), Op1: Min);
8851 MI.eraseFromParent();
8852 return Legalized;
8853}
8854
8855// Floating-point conversions using truncating and extending loads and stores.
8856LegalizerHelper::LegalizeResult
8857LegalizerHelper::lowerFPExtAndTruncMem(MachineInstr &MI) {
8858 assert((MI.getOpcode() == TargetOpcode::G_FPEXT ||
8859 MI.getOpcode() == TargetOpcode::G_FPTRUNC) &&
8860 "Only G_FPEXT and G_FPTRUNC are expected");
8861
8862 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
8863 MachinePointerInfo PtrInfo;
8864 unsigned StoreOpc;
8865 unsigned LoadOpc;
8866 LLT StackTy;
8867 if (MI.getOpcode() == TargetOpcode::G_FPEXT) {
8868 StackTy = SrcTy;
8869 StoreOpc = TargetOpcode::G_STORE;
8870 LoadOpc = TargetOpcode::G_FPEXTLOAD;
8871 } else {
8872 StackTy = DstTy;
8873 StoreOpc = TargetOpcode::G_FPTRUNCSTORE;
8874 LoadOpc = TargetOpcode::G_LOAD;
8875 }
8876
8877 Align StackTyAlign = getStackTemporaryAlignment(Ty: StackTy);
8878 auto StackTemp =
8879 createStackTemporary(Bytes: StackTy.getSizeInBytes(), Alignment: StackTyAlign, PtrInfo);
8880
8881 MachineFunction &MF = MIRBuilder.getMF();
8882 auto *StoreMMO = MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
8883 MemTy: StackTy, BaseAlignment: StackTyAlign);
8884 MIRBuilder.buildStoreInstr(Opcode: StoreOpc, Val: SrcReg, Addr: StackTemp, MMO&: *StoreMMO);
8885
8886 auto *LoadMMO = MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOLoad,
8887 MemTy: StackTy, BaseAlignment: StackTyAlign);
8888 MIRBuilder.buildLoadInstr(Opcode: LoadOpc, Res: DstReg, Addr: StackTemp, MMO&: *LoadMMO);
8889
8890 MI.eraseFromParent();
8891 return Legalized;
8892}
8893
8894// Expand a bf16 -> f32/f64 fpext with a shift and bitcast. This is based on the
8895// SDAG ISD::BF16_TO_FP lowering.
8896LegalizerHelper::LegalizeResult
8897LegalizerHelper::lowerFPEXT_BF16(MachineInstr &MI) {
8898 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
8899 assert(SrcTy.getScalarType().isBFloat16() &&
8900 "expected a bf16 source for bf16 fpext lowering");
8901
8902 LLT I16Ty = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 16));
8903 LLT I32Ty = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 32));
8904 LLT F32Ty = SrcTy.changeElementType(NewEltTy: LLT::float32());
8905
8906 auto SrcI =
8907 MIRBuilder.buildAnyExt(Res: I32Ty, Op: MIRBuilder.buildBitcast(Dst: I16Ty, Src: SrcReg));
8908 auto Shl =
8909 MIRBuilder.buildShl(Dst: I32Ty, Src0: SrcI, Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 16));
8910
8911 if (DstTy.getScalarType().isFloat32())
8912 MIRBuilder.buildBitcast(Dst: DstReg, Src: Shl);
8913 else
8914 MIRBuilder.buildFPExt(Res: DstReg, Op: MIRBuilder.buildBitcast(Dst: F32Ty, Src: Shl));
8915
8916 MI.eraseFromParent();
8917 return Legalized;
8918}
8919
8920LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPEXT(MachineInstr &MI) {
8921 auto [DstTy, SrcTy] = MI.getFirst2LLTs();
8922 if (SrcTy.getScalarType().isBFloat16() &&
8923 (DstTy.getScalarType().isFloat32() || DstTy.getScalarType().isFloat64()))
8924 return lowerFPEXT_BF16(MI);
8925
8926 return lowerFPExtAndTruncMem(MI);
8927}
8928
8929// f64 -> f16 conversion using round-to-nearest-even rounding mode.
8930LegalizerHelper::LegalizeResult
8931LegalizerHelper::lowerFPTRUNC_F64_TO_F16(MachineInstr &MI) {
8932 const LLT S1 = LLT::scalar(SizeInBits: 1);
8933 const LLT I32 = LLT::integer(SizeInBits: 32);
8934
8935 auto [Dst, Src] = MI.getFirst2Regs();
8936 assert(MRI.getType(Dst).getScalarType() == LLT::float16() &&
8937 MRI.getType(Src).getScalarType() == LLT::float64());
8938
8939 if (MRI.getType(Reg: Src).isVector()) // TODO: Handle vectors directly.
8940 return UnableToLegalize;
8941
8942 if (MI.getFlag(Flag: MachineInstr::FmAfn)) {
8943 unsigned Flags = MI.getFlags();
8944 auto Src32 = MIRBuilder.buildFPTrunc(Res: LLT::float32(), Op: Src, Flags);
8945 MIRBuilder.buildFPTrunc(Res: Dst, Op: Src32, Flags);
8946 MI.eraseFromParent();
8947 return Legalized;
8948 }
8949
8950 const unsigned ExpMask = 0x7ff;
8951 const unsigned ExpBiasf64 = 1023;
8952 const unsigned ExpBiasf16 = 15;
8953
8954 auto Unmerge = MIRBuilder.buildUnmerge(Res: I32, Op: Src);
8955 Register U = Unmerge.getReg(Idx: 0);
8956 Register UH = Unmerge.getReg(Idx: 1);
8957
8958 auto E = MIRBuilder.buildLShr(Dst: I32, Src0: UH, Src1: MIRBuilder.buildConstant(Res: I32, Val: 20));
8959 E = MIRBuilder.buildAnd(Dst: I32, Src0: E, Src1: MIRBuilder.buildConstant(Res: I32, Val: ExpMask));
8960
8961 // Subtract the fp64 exponent bias (1023) to get the real exponent and
8962 // add the f16 bias (15) to get the biased exponent for the f16 format.
8963 E = MIRBuilder.buildAdd(
8964 Dst: I32, Src0: E, Src1: MIRBuilder.buildConstant(Res: I32, Val: -ExpBiasf64 + ExpBiasf16));
8965
8966 auto M = MIRBuilder.buildLShr(Dst: I32, Src0: UH, Src1: MIRBuilder.buildConstant(Res: I32, Val: 8));
8967 M = MIRBuilder.buildAnd(Dst: I32, Src0: M, Src1: MIRBuilder.buildConstant(Res: I32, Val: 0xffe));
8968
8969 auto MaskedSig =
8970 MIRBuilder.buildAnd(Dst: I32, Src0: UH, Src1: MIRBuilder.buildConstant(Res: I32, Val: 0x1ff));
8971 MaskedSig = MIRBuilder.buildOr(Dst: I32, Src0: MaskedSig, Src1: U);
8972
8973 auto Zero = MIRBuilder.buildConstant(Res: I32, Val: 0);
8974 auto SigCmpNE0 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: S1, Op0: MaskedSig, Op1: Zero);
8975 auto Lo40Set = MIRBuilder.buildZExt(Res: I32, Op: SigCmpNE0);
8976 M = MIRBuilder.buildOr(Dst: I32, Src0: M, Src1: Lo40Set);
8977
8978 // (M != 0 ? 0x0200 : 0) | 0x7c00;
8979 auto Bits0x200 = MIRBuilder.buildConstant(Res: I32, Val: 0x0200);
8980 auto CmpM_NE0 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: S1, Op0: M, Op1: Zero);
8981 auto SelectCC = MIRBuilder.buildSelect(Res: I32, Tst: CmpM_NE0, Op0: Bits0x200, Op1: Zero);
8982
8983 auto Bits0x7c00 = MIRBuilder.buildConstant(Res: I32, Val: 0x7c00);
8984 auto I = MIRBuilder.buildOr(Dst: I32, Src0: SelectCC, Src1: Bits0x7c00);
8985
8986 // N = M | (E << 12);
8987 auto EShl12 = MIRBuilder.buildShl(Dst: I32, Src0: E, Src1: MIRBuilder.buildConstant(Res: I32, Val: 12));
8988 auto N = MIRBuilder.buildOr(Dst: I32, Src0: M, Src1: EShl12);
8989
8990 // B = clamp(1-E, 0, 13);
8991 auto One = MIRBuilder.buildConstant(Res: I32, Val: 1);
8992 auto OneSubExp = MIRBuilder.buildSub(Dst: I32, Src0: One, Src1: E);
8993 auto B = MIRBuilder.buildSMax(Dst: I32, Src0: OneSubExp, Src1: Zero);
8994 B = MIRBuilder.buildSMin(Dst: I32, Src0: B, Src1: MIRBuilder.buildConstant(Res: I32, Val: 13));
8995
8996 auto SigSetHigh =
8997 MIRBuilder.buildOr(Dst: I32, Src0: M, Src1: MIRBuilder.buildConstant(Res: I32, Val: 0x1000));
8998
8999 auto D = MIRBuilder.buildLShr(Dst: I32, Src0: SigSetHigh, Src1: B);
9000 auto D0 = MIRBuilder.buildShl(Dst: I32, Src0: D, Src1: B);
9001
9002 auto D0_NE_SigSetHigh = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: S1,
9003 Op0: D0, Op1: SigSetHigh);
9004 auto D1 = MIRBuilder.buildZExt(Res: I32, Op: D0_NE_SigSetHigh);
9005 D = MIRBuilder.buildOr(Dst: I32, Src0: D, Src1: D1);
9006
9007 auto CmpELtOne = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: S1, Op0: E, Op1: One);
9008 auto V = MIRBuilder.buildSelect(Res: I32, Tst: CmpELtOne, Op0: D, Op1: N);
9009
9010 auto VLow3 = MIRBuilder.buildAnd(Dst: I32, Src0: V, Src1: MIRBuilder.buildConstant(Res: I32, Val: 7));
9011 V = MIRBuilder.buildLShr(Dst: I32, Src0: V, Src1: MIRBuilder.buildConstant(Res: I32, Val: 2));
9012
9013 auto VLow3Eq3 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: S1, Op0: VLow3,
9014 Op1: MIRBuilder.buildConstant(Res: I32, Val: 3));
9015 auto V0 = MIRBuilder.buildZExt(Res: I32, Op: VLow3Eq3);
9016
9017 auto VLow3Gt5 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SGT, Res: S1, Op0: VLow3,
9018 Op1: MIRBuilder.buildConstant(Res: I32, Val: 5));
9019 auto V1 = MIRBuilder.buildZExt(Res: I32, Op: VLow3Gt5);
9020
9021 V1 = MIRBuilder.buildOr(Dst: I32, Src0: V0, Src1: V1);
9022 V = MIRBuilder.buildAdd(Dst: I32, Src0: V, Src1: V1);
9023
9024 auto CmpEGt30 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SGT, Res: S1, Op0: E,
9025 Op1: MIRBuilder.buildConstant(Res: I32, Val: 30));
9026 V = MIRBuilder.buildSelect(Res: I32, Tst: CmpEGt30,
9027 Op0: MIRBuilder.buildConstant(Res: I32, Val: 0x7c00), Op1: V);
9028
9029 auto CmpEGt1039 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: S1, Op0: E,
9030 Op1: MIRBuilder.buildConstant(Res: I32, Val: 1039));
9031 V = MIRBuilder.buildSelect(Res: I32, Tst: CmpEGt1039, Op0: I, Op1: V);
9032
9033 // Extract the sign bit.
9034 auto Sign = MIRBuilder.buildLShr(Dst: I32, Src0: UH, Src1: MIRBuilder.buildConstant(Res: I32, Val: 16));
9035 Sign = MIRBuilder.buildAnd(Dst: I32, Src0: Sign, Src1: MIRBuilder.buildConstant(Res: I32, Val: 0x8000));
9036
9037 // Insert the sign bit
9038 V = MIRBuilder.buildOr(Dst: I32, Src0: Sign, Src1: V);
9039
9040 MIRBuilder.buildTrunc(Res: Dst, Op: V);
9041 MI.eraseFromParent();
9042 return Legalized;
9043}
9044
9045// f32 -> bf16 conversion using round-to-nearest-even rounding mode.
9046LegalizerHelper::LegalizeResult
9047LegalizerHelper::lowerFPTRUNC_F32_TO_BF16(MachineInstr &MI) {
9048 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
9049 assert(DstTy.getScalarType() == LLT::bfloat16() &&
9050 SrcTy.getScalarType() == LLT::float32());
9051
9052 LLT I1Ty = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9053 LLT I16Ty = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 16));
9054 LLT I32Ty = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 32));
9055
9056 auto SrcI = MIRBuilder.buildBitcast(Dst: I32Ty, Src: SrcReg);
9057
9058 // Conversions should set NaN's quiet bit. This also prevents NaNs from
9059 // turning into infinities.
9060 auto NaN = MIRBuilder.buildOr(Dst: I32Ty, Src0: SrcI,
9061 Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 0x400000));
9062
9063 // Factor in the contribution of the low 16 bits.
9064 auto Lsb =
9065 MIRBuilder.buildLShr(Dst: I32Ty, Src0: SrcI, Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 16));
9066 Lsb = MIRBuilder.buildAnd(Dst: I32Ty, Src0: Lsb, Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 1));
9067 auto RoundingBias =
9068 MIRBuilder.buildAdd(Dst: I32Ty, Src0: Lsb, Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 0x7fff));
9069 auto Add = MIRBuilder.buildAdd(Dst: I32Ty, Src0: SrcI, Src1: RoundingBias);
9070
9071 // Don't round if we had a NaN, we don't want to turn 0x7fffffff into
9072 // 0x80000000.
9073 if (!MI.getFlag(Flag: MachineInstr::FmNoNans)) {
9074 auto IsNaN = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_UNO, Res: I1Ty, Op0: SrcReg,
9075 Op1: MIRBuilder.buildFConstant(Res: SrcTy, Val: 0));
9076 Add = MIRBuilder.buildSelect(Res: I32Ty, Tst: IsNaN, Op0: NaN, Op1: Add);
9077 }
9078
9079 // Now that we have rounded, shift the bits into position.
9080 auto Srl =
9081 MIRBuilder.buildLShr(Dst: I32Ty, Src0: Add, Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 16));
9082 auto Trunc = MIRBuilder.buildTrunc(Res: I16Ty, Op: Srl);
9083 MIRBuilder.buildBitcast(Dst: DstReg, Src: Trunc);
9084 MI.eraseFromParent();
9085 return Legalized;
9086}
9087
9088// Round a wide fp value to ResultTy's element size, forcing inexact
9089// results to the odd value so a subsequent narrowing round is correct. This
9090// avoids double-rounding when narrowing e.g. f64 -> f32 -> bf16. See Boldo &
9091// Melquiond, "When double rounding is odd" (2005).
9092Register LegalizerHelper::lowerRoundInexactToOdd(LLT ResultTy, Register Op) {
9093 LLT OperandTy = MRI.getType(Reg: Op);
9094 if (OperandTy.getScalarType() == ResultTy.getScalarType())
9095 return Op;
9096
9097 LLT ResultIntTy =
9098 ResultTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: ResultTy.getScalarSizeInBits()));
9099 LLT ResultCCTy = ResultTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9100 LLT OperandCCTy = OperandTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9101
9102 auto Narrow = MIRBuilder.buildFPTrunc(Res: ResultTy, Op);
9103 auto NarrowAsWide = MIRBuilder.buildFPExt(Res: OperandTy, Op: Narrow);
9104
9105 auto NarrowBits = MIRBuilder.buildBitcast(Dst: ResultIntTy, Src: Narrow);
9106 auto One = MIRBuilder.buildConstant(Res: ResultIntTy, Val: 1);
9107 auto NegativeOne = MIRBuilder.buildConstant(Res: ResultIntTy, Val: -1);
9108 auto Zero = MIRBuilder.buildConstant(Res: ResultIntTy, Val: 0);
9109 auto And = MIRBuilder.buildAnd(Dst: ResultIntTy, Src0: NarrowBits, Src1: One);
9110 // The result is already odd so we don't need to do anything.
9111 auto AlreadyOdd =
9112 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: ResultCCTy, Op0: And, Op1: Zero);
9113
9114 // We keep results which are exact, odd or NaN.
9115 auto KeepNarrow =
9116 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_UEQ, Res: OperandCCTy, Op0: Op, Op1: NarrowAsWide);
9117 KeepNarrow = MIRBuilder.buildOr(Dst: OperandCCTy, Src0: KeepNarrow, Src1: AlreadyOdd);
9118 // We morally performed a round-down if AbsNarrow is smaller than AbsWide.
9119 auto AbsWide = MIRBuilder.buildFAbs(Dst: OperandTy, Src0: Op);
9120 auto AbsNarrowAsWide = MIRBuilder.buildFAbs(Dst: OperandTy, Src0: NarrowAsWide);
9121 auto NarrowIsRd = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OGT, Res: OperandCCTy,
9122 Op0: AbsWide, Op1: AbsNarrowAsWide);
9123 // If narrow is the rounded-down value, pick the rounded-up value as it will
9124 // be odd; otherwise adjust down.
9125 auto Adjust =
9126 MIRBuilder.buildSelect(Res: ResultIntTy, Tst: NarrowIsRd, Op0: One, Op1: NegativeOne);
9127 auto Adjusted = MIRBuilder.buildAdd(Dst: ResultIntTy, Src0: NarrowBits, Src1: Adjust);
9128 auto Res =
9129 MIRBuilder.buildSelect(Res: ResultIntTy, Tst: KeepNarrow, Op0: NarrowBits, Op1: Adjusted);
9130 return MIRBuilder.buildBitcast(Dst: ResultTy, Src: Res).getReg(Idx: 0);
9131}
9132
9133// f64 -> bf16 conversion, correcting for double rounding.
9134LegalizerHelper::LegalizeResult
9135LegalizerHelper::lowerFPTRUNC_F64_TO_BF16(MachineInstr &MI) {
9136 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
9137 assert(DstTy.getScalarType() == LLT::bfloat16() &&
9138 SrcTy.getScalarType() == LLT::float64());
9139
9140 LLT F32Ty = SrcTy.changeElementType(NewEltTy: LLT::float32());
9141 Register OddF32 = lowerRoundInexactToOdd(ResultTy: F32Ty, Op: SrcReg);
9142 MIRBuilder.buildFPTrunc(Res: DstReg, Op: OddF32, Flags: MI.getFlags());
9143 MI.eraseFromParent();
9144 return Legalized;
9145}
9146
9147LegalizerHelper::LegalizeResult
9148LegalizerHelper::lowerFPTRUNC(MachineInstr &MI) {
9149 auto [DstTy, SrcTy] = MI.getFirst2LLTs();
9150 if (DstTy.getScalarType().isFloat16() && SrcTy.getScalarType().isFloat64())
9151 return lowerFPTRUNC_F64_TO_F16(MI);
9152
9153 if (DstTy.getScalarType().isBFloat16() && SrcTy.getScalarType().isFloat32())
9154 return lowerFPTRUNC_F32_TO_BF16(MI);
9155
9156 if (DstTy.getScalarType().isBFloat16() && SrcTy.getScalarType().isFloat64())
9157 return lowerFPTRUNC_F64_TO_BF16(MI);
9158
9159 return lowerFPExtAndTruncMem(MI);
9160}
9161
9162LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPOWI(MachineInstr &MI) {
9163 auto [Dst, Src0, Src1] = MI.getFirst3Regs();
9164 LLT Ty = MRI.getType(Reg: Dst);
9165
9166 auto CvtSrc1 = MIRBuilder.buildSITOFP(Dst: Ty, Src0: Src1);
9167 MIRBuilder.buildFPow(Dst, Src0, Src1: CvtSrc1, Flags: MI.getFlags());
9168 MI.eraseFromParent();
9169 return Legalized;
9170}
9171
9172LegalizerHelper::LegalizeResult LegalizerHelper::lowerFMODF(MachineInstr &MI) {
9173 auto [DstFrac, DstInt, Src] = MI.getFirst3Regs();
9174 LLT Ty = MRI.getType(Reg: Src);
9175 auto Flags = MI.getFlags();
9176 const LLT CondTy = Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9177
9178 auto IntPart = MIRBuilder.buildIntrinsicTrunc(Dst: Ty, Src0: Src, Flags);
9179 auto FracPart = MIRBuilder.buildFSub(Dst: Ty, Src0: Src, Src1: IntPart, Flags);
9180
9181 Register FracToUse;
9182 if (MI.getFlag(Flag: MachineInstr::FmNoInfs)) {
9183 FracToUse = FracPart.getReg(Idx: 0);
9184 } else {
9185 auto Abs = MIRBuilder.buildFAbs(Dst: Ty, Src0: Src, Flags);
9186 const fltSemantics &Semantics = getFltSemanticForLLT(Ty: Ty.getScalarType());
9187 auto Inf = MIRBuilder.buildFConstant(Res: Ty, Val: APFloat::getInf(Sem: Semantics));
9188 auto IsInf = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OEQ, Res: CondTy, Op0: Abs, Op1: Inf);
9189 auto Zero = MIRBuilder.buildFConstant(Res: Ty, Val: 0.0);
9190 auto Select = MIRBuilder.buildSelect(Res: Ty, Tst: IsInf, Op0: Zero, Op1: FracPart);
9191 FracToUse = Select.getReg(Idx: 0);
9192 }
9193
9194 MIRBuilder.buildFCopysign(Dst: DstFrac, Src0: FracToUse, Src1: Src, Flags);
9195 MIRBuilder.buildCopy(Res: DstInt, Op: IntPart.getReg(Idx: 0));
9196
9197 MI.eraseFromParent();
9198 return Legalized;
9199}
9200
9201static CmpInst::Predicate minMaxToCompare(unsigned Opc) {
9202 switch (Opc) {
9203 case TargetOpcode::G_SMIN:
9204 return CmpInst::ICMP_SLT;
9205 case TargetOpcode::G_SMAX:
9206 return CmpInst::ICMP_SGT;
9207 case TargetOpcode::G_UMIN:
9208 return CmpInst::ICMP_ULT;
9209 case TargetOpcode::G_UMAX:
9210 return CmpInst::ICMP_UGT;
9211 default:
9212 llvm_unreachable("not in integer min/max");
9213 }
9214}
9215
9216LegalizerHelper::LegalizeResult LegalizerHelper::lowerMinMax(MachineInstr &MI) {
9217 auto [Dst, Src0, Src1] = MI.getFirst3Regs();
9218
9219 const CmpInst::Predicate Pred = minMaxToCompare(Opc: MI.getOpcode());
9220 LLT CmpType = MRI.getType(Reg: Dst).changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9221
9222 auto Cmp = MIRBuilder.buildICmp(Pred, Res: CmpType, Op0: Src0, Op1: Src1);
9223 MIRBuilder.buildSelect(Res: Dst, Tst: Cmp, Op0: Src0, Op1: Src1);
9224
9225 MI.eraseFromParent();
9226 return Legalized;
9227}
9228
9229LegalizerHelper::LegalizeResult
9230LegalizerHelper::lowerThreewayCompare(MachineInstr &MI) {
9231 GSUCmp *Cmp = cast<GSUCmp>(Val: &MI);
9232
9233 Register Dst = Cmp->getReg(Idx: 0);
9234 LLT DstTy = MRI.getType(Reg: Dst);
9235 LLT SrcTy = MRI.getType(Reg: Cmp->getReg(Idx: 1));
9236 LLT CmpTy = DstTy.changeElementSize(NewEltSize: 1);
9237
9238 CmpInst::Predicate LTPredicate = Cmp->isSigned()
9239 ? CmpInst::Predicate::ICMP_SLT
9240 : CmpInst::Predicate::ICMP_ULT;
9241 CmpInst::Predicate GTPredicate = Cmp->isSigned()
9242 ? CmpInst::Predicate::ICMP_SGT
9243 : CmpInst::Predicate::ICMP_UGT;
9244
9245 auto Zero = MIRBuilder.buildConstant(Res: DstTy, Val: 0);
9246 auto IsGT = MIRBuilder.buildICmp(Pred: GTPredicate, Res: CmpTy, Op0: Cmp->getLHSReg(),
9247 Op1: Cmp->getRHSReg());
9248 auto IsLT = MIRBuilder.buildICmp(Pred: LTPredicate, Res: CmpTy, Op0: Cmp->getLHSReg(),
9249 Op1: Cmp->getRHSReg());
9250
9251 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
9252 auto BC = TLI.getBooleanContents(isVec: DstTy.isVector(), /*isFP=*/isFloat: false);
9253 if (TLI.preferSelectsOverBooleanArithmetic(
9254 VT: getApproximateEVTForLLT(Ty: SrcTy, Ctx)) ||
9255 BC == TargetLowering::UndefinedBooleanContent) {
9256 auto One = MIRBuilder.buildConstant(Res: DstTy, Val: 1);
9257 auto SelectZeroOrOne = MIRBuilder.buildSelect(Res: DstTy, Tst: IsGT, Op0: One, Op1: Zero);
9258
9259 auto MinusOne = MIRBuilder.buildConstant(Res: DstTy, Val: -1);
9260 MIRBuilder.buildSelect(Res: Dst, Tst: IsLT, Op0: MinusOne, Op1: SelectZeroOrOne);
9261 } else {
9262 if (BC == TargetLowering::ZeroOrNegativeOneBooleanContent)
9263 std::swap(a&: IsGT, b&: IsLT);
9264 // Extend boolean results to DstTy, which is at least i2, before subtracting
9265 // them.
9266 unsigned BoolExtOp =
9267 MIRBuilder.getBoolExtOp(IsVec: DstTy.isVector(), /*isFP=*/IsFP: false);
9268 IsGT = MIRBuilder.buildInstr(Opc: BoolExtOp, DstOps: {DstTy}, SrcOps: {IsGT});
9269 IsLT = MIRBuilder.buildInstr(Opc: BoolExtOp, DstOps: {DstTy}, SrcOps: {IsLT});
9270 MIRBuilder.buildSub(Dst, Src0: IsGT, Src1: IsLT);
9271 }
9272
9273 MI.eraseFromParent();
9274 return Legalized;
9275}
9276
9277LegalizerHelper::LegalizeResult
9278LegalizerHelper::lowerFCopySign(MachineInstr &MI) {
9279 auto [Dst, DstTy, Src0, Src0Ty, Src1, Src1Ty] = MI.getFirst3RegLLTs();
9280 const int Src0Size = Src0Ty.getScalarSizeInBits();
9281 const int Src1Size = Src1Ty.getScalarSizeInBits();
9282
9283 LLT DstIntTy =
9284 DstTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: DstTy.getScalarSizeInBits()));
9285 LLT Src0IntTy = Src0Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: Src0Size));
9286 LLT Src1IntTy = Src1Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: Src1Size));
9287
9288 Register Src0Int = Src0;
9289 Register Src1Int = Src1;
9290
9291 if (!(Src0Ty.getScalarType().isAnyScalar() ||
9292 Src0Ty.getScalarType().isInteger()))
9293 Src0Int = MIRBuilder.buildBitcast(Dst: Src0IntTy, Src: Src0).getReg(Idx: 0);
9294
9295 if (!(Src1Ty.getScalarType().isAnyScalar() ||
9296 Src1Ty.getScalarType().isInteger()))
9297 Src1Int = MIRBuilder.buildBitcast(Dst: Src1IntTy, Src: Src1).getReg(Idx: 0);
9298
9299 auto SignBitMask =
9300 MIRBuilder.buildConstant(Res: Src0IntTy, Val: APInt::getSignMask(BitWidth: Src0Size));
9301
9302 auto NotSignBitMask = MIRBuilder.buildConstant(
9303 Res: Src0IntTy, Val: APInt::getLowBitsSet(numBits: Src0Size, loBitsSet: Src0Size - 1));
9304
9305 Register And0 =
9306 MIRBuilder.buildAnd(Dst: Src0IntTy, Src0: Src0Int, Src1: NotSignBitMask).getReg(Idx: 0);
9307 Register And1;
9308 if (Src0Ty == Src1Ty) {
9309 And1 = MIRBuilder.buildAnd(Dst: Src1IntTy, Src0: Src1Int, Src1: SignBitMask).getReg(Idx: 0);
9310 } else if (Src0Size > Src1Size) {
9311 auto ShiftAmt = MIRBuilder.buildConstant(Res: Src0IntTy, Val: Src0Size - Src1Size);
9312 auto Zext = MIRBuilder.buildZExt(Res: Src0IntTy, Op: Src1Int);
9313 auto Shift = MIRBuilder.buildShl(Dst: Src0IntTy, Src0: Zext, Src1: ShiftAmt);
9314 And1 = MIRBuilder.buildAnd(Dst: Src0IntTy, Src0: Shift, Src1: SignBitMask).getReg(Idx: 0);
9315 } else {
9316 auto ShiftAmt = MIRBuilder.buildConstant(Res: Src1IntTy, Val: Src1Size - Src0Size);
9317 auto Shift = MIRBuilder.buildLShr(Dst: Src1IntTy, Src0: Src1Int, Src1: ShiftAmt);
9318 auto Trunc = MIRBuilder.buildTrunc(Res: Src0IntTy, Op: Shift);
9319 And1 = MIRBuilder.buildAnd(Dst: Src0IntTy, Src0: Trunc, Src1: SignBitMask).getReg(Idx: 0);
9320 }
9321
9322 // Be careful about setting nsz/nnan/ninf on every instruction, since the
9323 // constants are a nan and -0.0, but the final result should preserve
9324 // everything.
9325 unsigned Flags = MI.getFlags();
9326
9327 // We masked the sign bit and the not-sign bit, so these are disjoint.
9328 Flags |= MachineInstr::Disjoint;
9329
9330 if (DstTy == DstIntTy)
9331 MIRBuilder.buildOr(Dst, Src0: And0, Src1: And1, Flags).getReg(Idx: 0);
9332 else {
9333 Register NewDst = MIRBuilder.buildOr(Dst: DstIntTy, Src0: And0, Src1: And1, Flags).getReg(Idx: 0);
9334 MIRBuilder.buildBitcast(Dst, Src: NewDst);
9335 }
9336
9337 MI.eraseFromParent();
9338 return Legalized;
9339}
9340
9341LegalizerHelper::LegalizeResult
9342LegalizerHelper::lowerFMinNumMaxNum(MachineInstr &MI) {
9343 // FIXME: fminnum/fmaxnum and fminimumnum/fmaximumnum should not have
9344 // identical handling. fminimumnum/fmaximumnum also need a path that do not
9345 // depend on fminnum/fmaxnum.
9346
9347 unsigned NewOp;
9348 switch (MI.getOpcode()) {
9349 case TargetOpcode::G_FMINNUM:
9350 NewOp = TargetOpcode::G_FMINNUM_IEEE;
9351 break;
9352 case TargetOpcode::G_FMINIMUMNUM:
9353 NewOp = TargetOpcode::G_FMINNUM;
9354 break;
9355 case TargetOpcode::G_FMAXNUM:
9356 NewOp = TargetOpcode::G_FMAXNUM_IEEE;
9357 break;
9358 case TargetOpcode::G_FMAXIMUMNUM:
9359 NewOp = TargetOpcode::G_FMAXNUM;
9360 break;
9361 default:
9362 llvm_unreachable("unexpected min/max opcode");
9363 }
9364
9365 auto [Dst, Src0, Src1] = MI.getFirst3Regs();
9366 LLT Ty = MRI.getType(Reg: Dst);
9367
9368 if (!MI.getFlag(Flag: MachineInstr::FmNoNans)) {
9369 // Insert canonicalizes if it's possible we need to quiet to get correct
9370 // sNaN behavior.
9371
9372 // Note this must be done here, and not as an optimization combine in the
9373 // absence of a dedicate quiet-snan instruction as we're using an
9374 // omni-purpose G_FCANONICALIZE.
9375 if (!VT->isKnownNeverSNaN(Val: Src0))
9376 Src0 = MIRBuilder.buildFCanonicalize(Dst: Ty, Src0, Flags: MI.getFlags()).getReg(Idx: 0);
9377
9378 if (!VT->isKnownNeverSNaN(Val: Src1))
9379 Src1 = MIRBuilder.buildFCanonicalize(Dst: Ty, Src0: Src1, Flags: MI.getFlags()).getReg(Idx: 0);
9380 }
9381
9382 // If there are no nans, it's safe to simply replace this with the non-IEEE
9383 // version.
9384 MIRBuilder.buildInstr(Opc: NewOp, DstOps: {Dst}, SrcOps: {Src0, Src1}, Flags: MI.getFlags());
9385 MI.eraseFromParent();
9386 return Legalized;
9387}
9388
9389LegalizerHelper::LegalizeResult
9390LegalizerHelper::lowerFMinimumMaximum(MachineInstr &MI) {
9391 unsigned Opc = MI.getOpcode();
9392 auto [Dst, Src0, Src1] = MI.getFirst3Regs();
9393 LLT Ty = MRI.getType(Reg: Dst);
9394 const LLT CmpTy = Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9395
9396 bool IsMax = (Opc == TargetOpcode::G_FMAXIMUM);
9397 unsigned OpcIeee =
9398 IsMax ? TargetOpcode::G_FMAXNUM_IEEE : TargetOpcode::G_FMINNUM_IEEE;
9399 unsigned OpcNonIeee =
9400 IsMax ? TargetOpcode::G_FMAXNUM : TargetOpcode::G_FMINNUM;
9401 bool MinMaxMustRespectOrderedZero = false;
9402 Register Res;
9403
9404 // IEEE variants don't need canonicalization
9405 if (LI.isLegalOrCustom(Query: {OpcIeee, Ty})) {
9406 Res = MIRBuilder.buildInstr(Opc: OpcIeee, DstOps: {Ty}, SrcOps: {Src0, Src1}).getReg(Idx: 0);
9407 MinMaxMustRespectOrderedZero = true;
9408 } else if (LI.isLegalOrCustom(Query: {OpcNonIeee, Ty})) {
9409 Res = MIRBuilder.buildInstr(Opc: OpcNonIeee, DstOps: {Ty}, SrcOps: {Src0, Src1}).getReg(Idx: 0);
9410 } else {
9411 auto Compare = MIRBuilder.buildFCmp(
9412 Pred: IsMax ? CmpInst::FCMP_OGT : CmpInst::FCMP_OLT, Res: CmpTy, Op0: Src0, Op1: Src1);
9413 Res = MIRBuilder.buildSelect(Res: Ty, Tst: Compare, Op0: Src0, Op1: Src1).getReg(Idx: 0);
9414 }
9415
9416 // Propagate any NaN of both operands
9417 if (!MI.getFlag(Flag: MachineInstr::FmNoNans) &&
9418 (!VT->isKnownNeverNaN(Val: Src0) || !VT->isKnownNeverNaN(Val: Src1))) {
9419 auto IsOrdered = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_ORD, Res: CmpTy, Op0: Src0, Op1: Src1);
9420
9421 LLT ElementTy = Ty.isScalar() ? Ty : Ty.getElementType();
9422 APFloat NaNValue = APFloat::getNaN(Sem: getFltSemanticForLLT(Ty: ElementTy));
9423 Register NaN = MIRBuilder.buildFConstant(Res: ElementTy, Val: NaNValue).getReg(Idx: 0);
9424 if (Ty.isVector())
9425 NaN = MIRBuilder.buildSplatBuildVector(Res: Ty, Src: NaN).getReg(Idx: 0);
9426
9427 Res = MIRBuilder.buildSelect(Res: Ty, Tst: IsOrdered, Op0: Res, Op1: NaN).getReg(Idx: 0);
9428 }
9429
9430 // fminimum/fmaximum requires -0.0 less than +0.0
9431 if (!MinMaxMustRespectOrderedZero && !MI.getFlag(Flag: MachineInstr::FmNsz)) {
9432 GISelValueTracking VT(MIRBuilder.getMF());
9433 KnownFPClass Src0Info = VT.computeKnownFPClass(R: Src0, InterestedClasses: fcZero);
9434 KnownFPClass Src1Info = VT.computeKnownFPClass(R: Src1, InterestedClasses: fcZero);
9435
9436 if (!Src0Info.isKnownNeverZero() && !Src1Info.isKnownNeverZero()) {
9437 const unsigned Flags = MI.getFlags();
9438 Register Zero = MIRBuilder.buildFConstant(Res: Ty, Val: 0.0).getReg(Idx: 0);
9439 auto IsZero = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OEQ, Res: CmpTy, Op0: Res, Op1: Zero);
9440
9441 unsigned TestClass = IsMax ? fcPosZero : fcNegZero;
9442
9443 auto LHSTestZero = MIRBuilder.buildIsFPClass(Res: CmpTy, Src: Src0, Mask: TestClass);
9444 auto LHSSelect =
9445 MIRBuilder.buildSelect(Res: Ty, Tst: LHSTestZero, Op0: Src0, Op1: Res, Flags);
9446
9447 auto RHSTestZero = MIRBuilder.buildIsFPClass(Res: CmpTy, Src: Src1, Mask: TestClass);
9448 auto RHSSelect =
9449 MIRBuilder.buildSelect(Res: Ty, Tst: RHSTestZero, Op0: Src1, Op1: LHSSelect, Flags);
9450
9451 Res = MIRBuilder.buildSelect(Res: Ty, Tst: IsZero, Op0: RHSSelect, Op1: Res, Flags).getReg(Idx: 0);
9452 }
9453 }
9454
9455 MIRBuilder.buildCopy(Res: Dst, Op: Res);
9456 MI.eraseFromParent();
9457 return Legalized;
9458}
9459
9460LegalizerHelper::LegalizeResult LegalizerHelper::lowerFMad(MachineInstr &MI) {
9461 // Expand G_FMAD a, b, c -> G_FADD (G_FMUL a, b), c
9462 Register DstReg = MI.getOperand(i: 0).getReg();
9463 LLT Ty = MRI.getType(Reg: DstReg);
9464 unsigned Flags = MI.getFlags();
9465
9466 auto Mul = MIRBuilder.buildFMul(Dst: Ty, Src0: MI.getOperand(i: 1), Src1: MI.getOperand(i: 2),
9467 Flags);
9468 MIRBuilder.buildFAdd(Dst: DstReg, Src0: Mul, Src1: MI.getOperand(i: 3), Flags);
9469 MI.eraseFromParent();
9470 return Legalized;
9471}
9472
9473LegalizerHelper::LegalizeResult
9474LegalizerHelper::lowerIntrinsicRound(MachineInstr &MI) {
9475 auto [DstReg, X] = MI.getFirst2Regs();
9476 const unsigned Flags = MI.getFlags();
9477 const LLT Ty = MRI.getType(Reg: DstReg);
9478 const LLT CondTy = Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9479
9480 // round(x) =>
9481 // t = trunc(x);
9482 // d = fabs(x - t);
9483 // o = copysign(d >= 0.5 ? 1.0 : 0.0, x);
9484 // return t + o;
9485
9486 auto T = MIRBuilder.buildIntrinsicTrunc(Dst: Ty, Src0: X, Flags);
9487
9488 auto Diff = MIRBuilder.buildFSub(Dst: Ty, Src0: X, Src1: T, Flags);
9489 auto AbsDiff = MIRBuilder.buildFAbs(Dst: Ty, Src0: Diff, Flags);
9490
9491 auto Half = MIRBuilder.buildFConstant(Res: Ty, Val: 0.5);
9492 auto Cmp =
9493 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OGE, Res: CondTy, Op0: AbsDiff, Op1: Half, Flags);
9494
9495 // Could emit G_UITOFP instead
9496 auto One = MIRBuilder.buildFConstant(Res: Ty, Val: 1.0);
9497 auto Zero = MIRBuilder.buildFConstant(Res: Ty, Val: 0.0);
9498 auto BoolFP = MIRBuilder.buildSelect(Res: Ty, Tst: Cmp, Op0: One, Op1: Zero);
9499 auto SignedOffset = MIRBuilder.buildFCopysign(Dst: Ty, Src0: BoolFP, Src1: X);
9500
9501 MIRBuilder.buildFAdd(Dst: DstReg, Src0: T, Src1: SignedOffset, Flags);
9502
9503 MI.eraseFromParent();
9504 return Legalized;
9505}
9506
9507LegalizerHelper::LegalizeResult LegalizerHelper::lowerFFloor(MachineInstr &MI) {
9508 auto [DstReg, SrcReg] = MI.getFirst2Regs();
9509 unsigned Flags = MI.getFlags();
9510 LLT Ty = MRI.getType(Reg: DstReg);
9511 const LLT CondTy = Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9512
9513 // result = trunc(src);
9514 // if (src < 0.0 && src != result)
9515 // result += -1.0.
9516
9517 auto Trunc = MIRBuilder.buildIntrinsicTrunc(Dst: Ty, Src0: SrcReg, Flags);
9518 auto Zero = MIRBuilder.buildFConstant(Res: Ty, Val: 0.0);
9519
9520 auto Lt0 = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OLT, Res: CondTy,
9521 Op0: SrcReg, Op1: Zero, Flags);
9522 auto NeTrunc = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_ONE, Res: CondTy,
9523 Op0: SrcReg, Op1: Trunc, Flags);
9524 auto And = MIRBuilder.buildAnd(Dst: CondTy, Src0: Lt0, Src1: NeTrunc);
9525 auto AddVal = MIRBuilder.buildSITOFP(Dst: Ty, Src0: And);
9526
9527 MIRBuilder.buildFAdd(Dst: DstReg, Src0: Trunc, Src1: AddVal, Flags);
9528 MI.eraseFromParent();
9529 return Legalized;
9530}
9531
9532LegalizerHelper::LegalizeResult
9533LegalizerHelper::lowerMergeValues(MachineInstr &MI) {
9534 const unsigned NumOps = MI.getNumOperands();
9535 auto [DstReg, DstTy, Src0Reg, Src0Ty] = MI.getFirst2RegLLTs();
9536 unsigned PartSize = Src0Ty.getSizeInBits();
9537
9538 LLT WideTy = LLT::integer(SizeInBits: DstTy.getSizeInBits());
9539 Register ResultReg = MIRBuilder.buildZExt(Res: WideTy, Op: Src0Reg).getReg(Idx: 0);
9540
9541 for (unsigned I = 2; I != NumOps; ++I) {
9542 const unsigned Offset = (I - 1) * PartSize;
9543
9544 Register SrcReg = MI.getOperand(i: I).getReg();
9545 auto ZextInput = MIRBuilder.buildZExt(Res: WideTy, Op: SrcReg);
9546
9547 Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg :
9548 MRI.createGenericVirtualRegister(Ty: WideTy);
9549
9550 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: Offset);
9551 auto Shl = MIRBuilder.buildShl(Dst: WideTy, Src0: ZextInput, Src1: ShiftAmt);
9552 MIRBuilder.buildOr(Dst: NextResult, Src0: ResultReg, Src1: Shl);
9553 ResultReg = NextResult;
9554 }
9555
9556 if (DstTy.isPointer()) {
9557 if (MIRBuilder.getDataLayout().isNonIntegralAddressSpace(
9558 AddrSpace: DstTy.getAddressSpace())) {
9559 LLVM_DEBUG(dbgs() << "Not casting nonintegral address space\n");
9560 return UnableToLegalize;
9561 }
9562
9563 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: ResultReg);
9564 } else if (WideTy != DstTy) {
9565 MIRBuilder.buildBitcast(Dst: DstReg, Src: ResultReg);
9566 }
9567
9568 MI.eraseFromParent();
9569 return Legalized;
9570}
9571
9572LegalizerHelper::LegalizeResult
9573LegalizerHelper::lowerUnmergeValues(MachineInstr &MI) {
9574 const unsigned NumDst = MI.getNumOperands() - 1;
9575 Register SrcReg = MI.getOperand(i: NumDst).getReg();
9576 Register Dst0Reg = MI.getOperand(i: 0).getReg();
9577 LLT DstTy = MRI.getType(Reg: Dst0Reg);
9578 if (DstTy.isPointer())
9579 return UnableToLegalize; // TODO
9580
9581 SrcReg = coerceToInteger(Val: SrcReg);
9582 if (!SrcReg)
9583 return UnableToLegalize;
9584
9585 // Expand scalarizing unmerge as bitcast to integer and shift.
9586 LLT IntTy = MRI.getType(Reg: SrcReg);
9587
9588 MIRBuilder.buildTrunc(Res: Dst0Reg, Op: SrcReg);
9589
9590 const unsigned DstSize = DstTy.getSizeInBits();
9591 unsigned Offset = DstSize;
9592 for (unsigned I = 1; I != NumDst; ++I, Offset += DstSize) {
9593 auto ShiftAmt = MIRBuilder.buildConstant(Res: IntTy, Val: Offset);
9594 auto Shift = MIRBuilder.buildLShr(Dst: IntTy, Src0: SrcReg, Src1: ShiftAmt);
9595 MIRBuilder.buildTrunc(Res: MI.getOperand(i: I), Op: Shift);
9596 }
9597
9598 MI.eraseFromParent();
9599 return Legalized;
9600}
9601
9602/// Lower a vector extract or insert by writing the vector to a stack temporary
9603/// and reloading the element or vector.
9604///
9605/// %dst = G_EXTRACT_VECTOR_ELT %vec, %idx
9606/// =>
9607/// %stack_temp = G_FRAME_INDEX
9608/// G_STORE %vec, %stack_temp
9609/// %idx = clamp(%idx, %vec.getNumElements())
9610/// %element_ptr = G_PTR_ADD %stack_temp, %idx
9611/// %dst = G_LOAD %element_ptr
9612LegalizerHelper::LegalizeResult
9613LegalizerHelper::lowerExtractInsertVectorElt(MachineInstr &MI) {
9614 Register DstReg = MI.getOperand(i: 0).getReg();
9615 Register SrcVec = MI.getOperand(i: 1).getReg();
9616 Register InsertVal;
9617 if (MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)
9618 InsertVal = MI.getOperand(i: 2).getReg();
9619
9620 Register Idx = MI.getOperand(i: MI.getNumOperands() - 1).getReg();
9621
9622 LLT VecTy = MRI.getType(Reg: SrcVec);
9623 LLT EltTy = VecTy.getElementType();
9624 unsigned NumElts = VecTy.getNumElements();
9625
9626 int64_t IdxVal;
9627 if (mi_match(R: Idx, MRI, P: m_ICst(Cst&: IdxVal)) && IdxVal <= NumElts) {
9628 SmallVector<Register, 8> SrcRegs;
9629 extractParts(Reg: SrcVec, Ty: EltTy, NumParts: NumElts, VRegs&: SrcRegs, MIRBuilder, MRI);
9630
9631 if (InsertVal) {
9632 SrcRegs[IdxVal] = MI.getOperand(i: 2).getReg();
9633 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: SrcRegs);
9634 } else {
9635 MIRBuilder.buildCopy(Res: DstReg, Op: SrcRegs[IdxVal]);
9636 }
9637
9638 MI.eraseFromParent();
9639 return Legalized;
9640 }
9641
9642 if (!EltTy.isByteSized()) { // Not implemented.
9643 LLVM_DEBUG(dbgs() << "Can't handle non-byte element vectors yet\n");
9644 return UnableToLegalize;
9645 }
9646
9647 unsigned EltBytes = EltTy.getSizeInBytes();
9648 Align VecAlign = getStackTemporaryAlignment(Ty: VecTy);
9649 Align EltAlign;
9650
9651 MachinePointerInfo PtrInfo;
9652 auto StackTemp = createStackTemporary(
9653 Bytes: TypeSize::getFixed(ExactSize: VecTy.getSizeInBytes()), Alignment: VecAlign, PtrInfo);
9654 MIRBuilder.buildStore(Val: SrcVec, Addr: StackTemp, PtrInfo, Alignment: VecAlign);
9655
9656 // Get the pointer to the element, and be sure not to hit undefined behavior
9657 // if the index is out of bounds.
9658 Register EltPtr = getVectorElementPointer(VecPtr: StackTemp.getReg(Idx: 0), VecTy, Index: Idx);
9659
9660 if (mi_match(R: Idx, MRI, P: m_ICst(Cst&: IdxVal))) {
9661 int64_t Offset = IdxVal * EltBytes;
9662 PtrInfo = PtrInfo.getWithOffset(O: Offset);
9663 EltAlign = commonAlignment(A: VecAlign, Offset);
9664 } else {
9665 // We lose information with a variable offset.
9666 EltAlign = getStackTemporaryAlignment(Ty: EltTy);
9667 PtrInfo = MachinePointerInfo(MRI.getType(Reg: EltPtr).getAddressSpace());
9668 }
9669
9670 if (InsertVal) {
9671 // Write the inserted element
9672 MIRBuilder.buildStore(Val: InsertVal, Addr: EltPtr, PtrInfo, Alignment: EltAlign);
9673
9674 // Reload the whole vector.
9675 MIRBuilder.buildLoad(Res: DstReg, Addr: StackTemp, PtrInfo, Alignment: VecAlign);
9676 } else {
9677 MIRBuilder.buildLoad(Res: DstReg, Addr: EltPtr, PtrInfo, Alignment: EltAlign);
9678 }
9679
9680 MI.eraseFromParent();
9681 return Legalized;
9682}
9683
9684LegalizerHelper::LegalizeResult
9685LegalizerHelper::lowerShuffleVector(MachineInstr &MI) {
9686 auto [DstReg, DstTy, Src0Reg, Src0Ty, Src1Reg, Src1Ty] =
9687 MI.getFirst3RegLLTs();
9688 LLT IdxTy = LLT::scalar(SizeInBits: 32);
9689
9690 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
9691 Register Undef;
9692 SmallVector<Register, 32> BuildVec;
9693 LLT EltTy = DstTy.getScalarType();
9694
9695 DenseMap<unsigned, Register> CachedExtract;
9696
9697 for (int Idx : Mask) {
9698 if (Idx < 0) {
9699 if (!Undef.isValid())
9700 Undef = MIRBuilder.buildUndef(Res: EltTy).getReg(Idx: 0);
9701 BuildVec.push_back(Elt: Undef);
9702 continue;
9703 }
9704
9705 assert(!Src0Ty.isScalar() && "Unexpected scalar G_SHUFFLE_VECTOR");
9706
9707 int NumElts = Src0Ty.getNumElements();
9708 Register SrcVec = Idx < NumElts ? Src0Reg : Src1Reg;
9709 int ExtractIdx = Idx < NumElts ? Idx : Idx - NumElts;
9710 auto [It, Inserted] = CachedExtract.try_emplace(Key: Idx);
9711 if (Inserted) {
9712 auto IdxK = MIRBuilder.buildConstant(Res: IdxTy, Val: ExtractIdx);
9713 It->second =
9714 MIRBuilder.buildExtractVectorElement(Res: EltTy, Val: SrcVec, Idx: IdxK).getReg(Idx: 0);
9715 }
9716 BuildVec.push_back(Elt: It->second);
9717 }
9718
9719 assert(DstTy.isVector() && "Unexpected scalar G_SHUFFLE_VECTOR");
9720 MIRBuilder.buildBuildVector(Res: DstReg, Ops: BuildVec);
9721 MI.eraseFromParent();
9722 return Legalized;
9723}
9724
9725LegalizerHelper::LegalizeResult
9726LegalizerHelper::lowerVECTOR_COMPRESS(llvm::MachineInstr &MI) {
9727 auto [Dst, DstTy, Vec, VecTy, Mask, MaskTy, Passthru, PassthruTy] =
9728 MI.getFirst4RegLLTs();
9729
9730 if (VecTy.isScalableVector())
9731 report_fatal_error(reason: "Cannot expand masked_compress for scalable vectors.");
9732
9733 Align VecAlign = getStackTemporaryAlignment(Ty: VecTy);
9734 MachinePointerInfo PtrInfo;
9735 Register StackPtr =
9736 createStackTemporary(Bytes: TypeSize::getFixed(ExactSize: VecTy.getSizeInBytes()), Alignment: VecAlign,
9737 PtrInfo)
9738 .getReg(Idx: 0);
9739 MachinePointerInfo ValPtrInfo =
9740 MachinePointerInfo::getUnknownStack(MF&: *MI.getMF());
9741
9742 LLT IdxTy = LLT::integer(SizeInBits: 32);
9743 LLT ValTy = VecTy.getElementType();
9744 Align ValAlign = getStackTemporaryAlignment(Ty: ValTy);
9745
9746 auto OutPos = MIRBuilder.buildConstant(Res: IdxTy, Val: 0);
9747
9748 bool HasPassthru = !mi_match(R: Passthru, MRI, P: m_GImplicitDef());
9749
9750 if (HasPassthru)
9751 MIRBuilder.buildStore(Val: Passthru, Addr: StackPtr, PtrInfo, Alignment: VecAlign);
9752
9753 Register LastWriteVal;
9754 std::optional<APInt> PassthruSplatVal =
9755 isConstantOrConstantSplatVector(Def: Passthru, MRI);
9756
9757 if (PassthruSplatVal.has_value()) {
9758 LastWriteVal =
9759 MIRBuilder.buildConstant(Res: ValTy, Val: PassthruSplatVal.value()).getReg(Idx: 0);
9760 } else if (HasPassthru) {
9761 auto Popcount = MIRBuilder.buildZExt(Res: MaskTy.changeElementSize(NewEltSize: 32), Op: Mask);
9762 Popcount = MIRBuilder.buildInstr(Opc: TargetOpcode::G_VECREDUCE_ADD,
9763 DstOps: {LLT::integer(SizeInBits: 32)}, SrcOps: {Popcount});
9764
9765 Register LastElmtPtr =
9766 getVectorElementPointer(VecPtr: StackPtr, VecTy, Index: Popcount.getReg(Idx: 0));
9767 LastWriteVal =
9768 MIRBuilder.buildLoad(Res: ValTy, Addr: LastElmtPtr, PtrInfo: ValPtrInfo, Alignment: ValAlign)
9769 .getReg(Idx: 0);
9770 }
9771
9772 unsigned NumElmts = VecTy.getNumElements();
9773 for (unsigned I = 0; I < NumElmts; ++I) {
9774 auto Idx = MIRBuilder.buildConstant(Res: IdxTy, Val: I);
9775 auto Val = MIRBuilder.buildExtractVectorElement(Res: ValTy, Val: Vec, Idx);
9776 Register ElmtPtr =
9777 getVectorElementPointer(VecPtr: StackPtr, VecTy, Index: OutPos.getReg(Idx: 0));
9778 MIRBuilder.buildStore(Val, Addr: ElmtPtr, PtrInfo: ValPtrInfo, Alignment: ValAlign);
9779
9780 LLT MaskITy = MaskTy.getElementType();
9781 auto MaskI = MIRBuilder.buildExtractVectorElement(Res: MaskITy, Val: Mask, Idx);
9782 if (MaskITy.getSizeInBits() > 1)
9783 MaskI = MIRBuilder.buildTrunc(Res: LLT::integer(SizeInBits: 1), Op: MaskI);
9784
9785 MaskI = MIRBuilder.buildZExt(Res: IdxTy, Op: MaskI);
9786 OutPos = MIRBuilder.buildAdd(Dst: IdxTy, Src0: OutPos, Src1: MaskI);
9787
9788 if (HasPassthru && I == NumElmts - 1) {
9789 auto EndOfVector =
9790 MIRBuilder.buildConstant(Res: IdxTy, Val: VecTy.getNumElements() - 1);
9791 auto AllLanesSelected = MIRBuilder.buildICmp(
9792 Pred: CmpInst::ICMP_UGT, Res: LLT::integer(SizeInBits: 1), Op0: OutPos, Op1: EndOfVector);
9793 OutPos = MIRBuilder.buildInstr(Opc: TargetOpcode::G_UMIN, DstOps: {IdxTy},
9794 SrcOps: {OutPos, EndOfVector});
9795 ElmtPtr = getVectorElementPointer(VecPtr: StackPtr, VecTy, Index: OutPos.getReg(Idx: 0));
9796
9797 LastWriteVal =
9798 MIRBuilder.buildSelect(Res: ValTy, Tst: AllLanesSelected, Op0: Val, Op1: LastWriteVal)
9799 .getReg(Idx: 0);
9800 MIRBuilder.buildStore(Val: LastWriteVal, Addr: ElmtPtr, PtrInfo: ValPtrInfo, Alignment: ValAlign);
9801 }
9802 }
9803
9804 // TODO: Use StackPtr's FrameIndex alignment.
9805 MIRBuilder.buildLoad(Res: Dst, Addr: StackPtr, PtrInfo, Alignment: VecAlign);
9806
9807 MI.eraseFromParent();
9808 return Legalized;
9809}
9810
9811Register LegalizerHelper::getDynStackAllocTargetPtr(Register SPReg,
9812 Register AllocSize,
9813 Align Alignment,
9814 LLT PtrTy) {
9815 LLT IntPtrTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
9816
9817 auto SPTmp = MIRBuilder.buildCopy(Res: PtrTy, Op: SPReg);
9818 SPTmp = MIRBuilder.buildCast(Dst: IntPtrTy, Src: SPTmp);
9819
9820 // Subtract the final alloc from the SP. We use G_PTRTOINT here so we don't
9821 // have to generate an extra instruction to negate the alloc and then use
9822 // G_PTR_ADD to add the negative offset.
9823 auto Alloc = MIRBuilder.buildSub(Dst: IntPtrTy, Src0: SPTmp, Src1: AllocSize);
9824 if (Alignment > Align(1)) {
9825 APInt AlignMask(IntPtrTy.getSizeInBits(), Alignment.value(), true);
9826 AlignMask.negate();
9827 auto AlignCst = MIRBuilder.buildConstant(Res: IntPtrTy, Val: AlignMask);
9828 Alloc = MIRBuilder.buildAnd(Dst: IntPtrTy, Src0: Alloc, Src1: AlignCst);
9829 }
9830
9831 return MIRBuilder.buildCast(Dst: PtrTy, Src: Alloc).getReg(Idx: 0);
9832}
9833
9834LegalizerHelper::LegalizeResult
9835LegalizerHelper::lowerDynStackAlloc(MachineInstr &MI) {
9836 const auto &MF = *MI.getMF();
9837 const auto &TFI = *MF.getSubtarget().getFrameLowering();
9838 if (TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp)
9839 return UnableToLegalize;
9840
9841 Register Dst = MI.getOperand(i: 0).getReg();
9842 Register AllocSize = MI.getOperand(i: 1).getReg();
9843 Align Alignment = assumeAligned(Value: MI.getOperand(i: 2).getImm());
9844
9845 LLT PtrTy = MRI.getType(Reg: Dst);
9846 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
9847 Register SPTmp =
9848 getDynStackAllocTargetPtr(SPReg, AllocSize, Alignment, PtrTy);
9849
9850 MIRBuilder.buildCopy(Res: SPReg, Op: SPTmp);
9851 MIRBuilder.buildCopy(Res: Dst, Op: SPTmp);
9852
9853 MI.eraseFromParent();
9854 return Legalized;
9855}
9856
9857LegalizerHelper::LegalizeResult
9858LegalizerHelper::lowerStackSave(MachineInstr &MI) {
9859 Register StackPtr = TLI.getStackPointerRegisterToSaveRestore();
9860 if (!StackPtr)
9861 return UnableToLegalize;
9862
9863 MIRBuilder.buildCopy(Res: MI.getOperand(i: 0), Op: StackPtr);
9864 MI.eraseFromParent();
9865 return Legalized;
9866}
9867
9868LegalizerHelper::LegalizeResult
9869LegalizerHelper::lowerStackRestore(MachineInstr &MI) {
9870 Register StackPtr = TLI.getStackPointerRegisterToSaveRestore();
9871 if (!StackPtr)
9872 return UnableToLegalize;
9873
9874 MIRBuilder.buildCopy(Res: StackPtr, Op: MI.getOperand(i: 0));
9875 MI.eraseFromParent();
9876 return Legalized;
9877}
9878
9879LegalizerHelper::LegalizeResult
9880LegalizerHelper::lowerExtract(MachineInstr &MI) {
9881 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
9882 unsigned Offset = MI.getOperand(i: 2).getImm();
9883
9884 // Extract sub-vector or one element
9885 if (SrcTy.isVector()) {
9886 unsigned SrcEltSize = SrcTy.getElementType().getSizeInBits();
9887 unsigned DstSize = DstTy.getSizeInBits();
9888
9889 if ((Offset % SrcEltSize == 0) && (DstSize % SrcEltSize == 0) &&
9890 (Offset + DstSize <= SrcTy.getSizeInBits())) {
9891 // Unmerge and allow access to each Src element for the artifact combiner.
9892 auto Unmerge = MIRBuilder.buildUnmerge(Res: SrcTy.getElementType(), Op: SrcReg);
9893
9894 // Take element(s) we need to extract and copy it (merge them).
9895 SmallVector<Register, 8> SubVectorElts;
9896 for (unsigned Idx = Offset / SrcEltSize;
9897 Idx < (Offset + DstSize) / SrcEltSize; ++Idx) {
9898 SubVectorElts.push_back(Elt: Unmerge.getReg(Idx));
9899 }
9900 if (SubVectorElts.size() == 1)
9901 MIRBuilder.buildCopy(Res: DstReg, Op: SubVectorElts[0]);
9902 else
9903 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: SubVectorElts);
9904
9905 MI.eraseFromParent();
9906 return Legalized;
9907 }
9908 }
9909
9910 const DataLayout &DL = MIRBuilder.getDataLayout();
9911 if ((SrcTy.isPointer() &&
9912 DL.isNonIntegralAddressSpace(AddrSpace: SrcTy.getAddressSpace())) ||
9913 (DstTy.isPointer() &&
9914 DL.isNonIntegralAddressSpace(AddrSpace: DstTy.getAddressSpace()))) {
9915 LLVM_DEBUG(dbgs() << "Not casting non-integral address space integer\n");
9916 return UnableToLegalize;
9917 }
9918
9919 if ((DstTy.isScalar() || DstTy.isPointer()) &&
9920 (SrcTy.isScalar() || SrcTy.isPointer() ||
9921 (SrcTy.isVector() && DstTy == SrcTy.getElementType()))) {
9922 LLT SrcIntTy = SrcTy;
9923 if (!SrcTy.isScalar()) {
9924 SrcIntTy = LLT::integer(SizeInBits: SrcTy.getSizeInBits());
9925 SrcReg = MIRBuilder.buildCast(Dst: SrcIntTy, Src: SrcReg).getReg(Idx: 0);
9926 }
9927
9928 Register ResultReg = DstReg;
9929 if (DstTy.isPointer())
9930 ResultReg =
9931 MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: DstTy.getSizeInBits()));
9932
9933 if (Offset == 0)
9934 MIRBuilder.buildTrunc(Res: ResultReg, Op: SrcReg);
9935 else {
9936 auto ShiftAmt = MIRBuilder.buildConstant(Res: SrcIntTy, Val: Offset);
9937 auto Shr = MIRBuilder.buildLShr(Dst: SrcIntTy, Src0: SrcReg, Src1: ShiftAmt);
9938 MIRBuilder.buildTrunc(Res: ResultReg, Op: Shr);
9939 }
9940
9941 if (DstTy.isPointer())
9942 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: ResultReg);
9943
9944 MI.eraseFromParent();
9945 return Legalized;
9946 }
9947
9948 return UnableToLegalize;
9949}
9950
9951LegalizerHelper::LegalizeResult LegalizerHelper::lowerInsert(MachineInstr &MI) {
9952 auto [Dst, Src, InsertSrc] = MI.getFirst3Regs();
9953 uint64_t Offset = MI.getOperand(i: 3).getImm();
9954
9955 LLT DstTy = MRI.getType(Reg: Src);
9956 LLT InsertTy = MRI.getType(Reg: InsertSrc);
9957
9958 const DataLayout &DL = MIRBuilder.getDataLayout();
9959 bool IsNonIntegralInsert =
9960 InsertTy.isPointerOrPointerVector() &&
9961 DL.isNonIntegralAddressSpace(AddrSpace: InsertTy.getAddressSpace());
9962 bool IsNonIntegralDst = DstTy.isPointerOrPointerVector() &&
9963 DL.isNonIntegralAddressSpace(AddrSpace: DstTy.getAddressSpace());
9964
9965 // Insert sub-vector or one element
9966 if (DstTy.isVector()) {
9967 LLT EltTy = DstTy.getElementType();
9968
9969 if ((IsNonIntegralInsert || IsNonIntegralDst) && InsertTy != EltTy) {
9970 LLVM_DEBUG(dbgs() << "Not casting non-integral address space integer\n");
9971 return UnableToLegalize;
9972 }
9973
9974 unsigned EltSize = EltTy.getSizeInBits();
9975 unsigned InsertSize = InsertTy.getSizeInBits();
9976
9977 if ((Offset % EltSize == 0) && (InsertSize % EltSize == 0) &&
9978 (Offset + InsertSize <= DstTy.getSizeInBits())) {
9979 auto UnmergeSrc = MIRBuilder.buildUnmerge(Res: EltTy, Op: Src);
9980 SmallVector<Register, 8> DstElts;
9981 unsigned Idx = 0;
9982 // Elements from Src before insert start Offset
9983 for (; Idx < Offset / EltSize; ++Idx) {
9984 DstElts.push_back(Elt: UnmergeSrc.getReg(Idx));
9985 }
9986
9987 // Replace elements in Src with elements from InsertSrc
9988 if (InsertTy.getSizeInBits() > EltSize) {
9989 auto UnmergeInsertSrc = MIRBuilder.buildUnmerge(Res: EltTy, Op: InsertSrc);
9990 for (unsigned i = 0; Idx < (Offset + InsertSize) / EltSize;
9991 ++Idx, ++i) {
9992 DstElts.push_back(Elt: UnmergeInsertSrc.getReg(Idx: i));
9993 }
9994 } else {
9995 if (InsertTy.isPointer() && !EltTy.isPointer())
9996 InsertSrc = MIRBuilder.buildPtrToInt(Dst: EltTy, Src: InsertSrc).getReg(Idx: 0);
9997 else if (!InsertTy.isPointer() && EltTy.isPointer())
9998 InsertSrc = MIRBuilder.buildIntToPtr(Dst: EltTy, Src: InsertSrc).getReg(Idx: 0);
9999 DstElts.push_back(Elt: InsertSrc);
10000 ++Idx;
10001 }
10002
10003 // Remaining elements from Src after insert
10004 for (; Idx < DstTy.getNumElements(); ++Idx) {
10005 DstElts.push_back(Elt: UnmergeSrc.getReg(Idx));
10006 }
10007
10008 MIRBuilder.buildMergeLikeInstr(Res: Dst, Ops: DstElts);
10009 MI.eraseFromParent();
10010 return Legalized;
10011 }
10012 }
10013
10014 if (InsertTy.isVector() ||
10015 (DstTy.isVector() && DstTy.getElementType() != InsertTy))
10016 return UnableToLegalize;
10017
10018 if (IsNonIntegralDst || IsNonIntegralInsert) {
10019 LLVM_DEBUG(dbgs() << "Not casting non-integral address space integer\n");
10020 return UnableToLegalize;
10021 }
10022
10023 LLT IntDstTy = DstTy;
10024
10025 if (!DstTy.isScalar()) {
10026 IntDstTy = LLT::scalar(SizeInBits: DstTy.getSizeInBits());
10027 Src = MIRBuilder.buildCast(Dst: IntDstTy, Src).getReg(Idx: 0);
10028 }
10029
10030 if (!InsertTy.isScalar()) {
10031 const LLT IntInsertTy = LLT::scalar(SizeInBits: InsertTy.getSizeInBits());
10032 InsertSrc = MIRBuilder.buildPtrToInt(Dst: IntInsertTy, Src: InsertSrc).getReg(Idx: 0);
10033 }
10034
10035 Register ExtInsSrc = MIRBuilder.buildZExt(Res: IntDstTy, Op: InsertSrc).getReg(Idx: 0);
10036 if (Offset != 0) {
10037 auto ShiftAmt = MIRBuilder.buildConstant(Res: IntDstTy, Val: Offset);
10038 ExtInsSrc = MIRBuilder.buildShl(Dst: IntDstTy, Src0: ExtInsSrc, Src1: ShiftAmt).getReg(Idx: 0);
10039 }
10040
10041 APInt MaskVal = APInt::getBitsSetWithWrap(
10042 numBits: DstTy.getSizeInBits(), loBit: Offset + InsertTy.getSizeInBits(), hiBit: Offset);
10043
10044 auto Mask = MIRBuilder.buildConstant(Res: IntDstTy, Val: MaskVal);
10045 auto MaskedSrc = MIRBuilder.buildAnd(Dst: IntDstTy, Src0: Src, Src1: Mask);
10046 auto Or = MIRBuilder.buildOr(Dst: IntDstTy, Src0: MaskedSrc, Src1: ExtInsSrc);
10047
10048 MIRBuilder.buildCast(Dst, Src: Or);
10049 MI.eraseFromParent();
10050 return Legalized;
10051}
10052
10053LegalizerHelper::LegalizeResult
10054LegalizerHelper::lowerSADDO_SSUBO(MachineInstr &MI) {
10055 auto [Dst0, Dst0Ty, Dst1, Dst1Ty, LHS, LHSTy, RHS, RHSTy] =
10056 MI.getFirst4RegLLTs();
10057 const bool IsAdd = MI.getOpcode() == TargetOpcode::G_SADDO;
10058
10059 LLT Ty = Dst0Ty;
10060 LLT BoolTy = Dst1Ty;
10061
10062 Register NewDst0 = MRI.cloneVirtualRegister(VReg: Dst0);
10063
10064 if (IsAdd)
10065 MIRBuilder.buildAdd(Dst: NewDst0, Src0: LHS, Src1: RHS);
10066 else
10067 MIRBuilder.buildSub(Dst: NewDst0, Src0: LHS, Src1: RHS);
10068
10069 // TODO: If SADDSAT/SSUBSAT is legal, compare results to detect overflow.
10070
10071 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
10072
10073 if (IsAdd) {
10074 // For an addition, the result should be less than one of the operands (LHS)
10075 // if and only if the other operand (RHS) is negative, otherwise there will
10076 // be overflow.
10077 auto ResultLowerThanLHS =
10078 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: NewDst0, Op1: LHS);
10079 auto RHSNegative =
10080 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: RHS, Op1: Zero);
10081 MIRBuilder.buildXor(Dst: Dst1, Src0: RHSNegative, Src1: ResultLowerThanLHS);
10082 } else {
10083 // For subtraction, overflow occurs when the signed comparison of operands
10084 // doesn't match the sign of the result.
10085 auto LHSLessThanRHS =
10086 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: LHS, Op1: RHS);
10087 auto ResultNegative =
10088 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: NewDst0, Op1: Zero);
10089 MIRBuilder.buildXor(Dst: Dst1, Src0: LHSLessThanRHS, Src1: ResultNegative);
10090 }
10091
10092 MIRBuilder.buildCopy(Res: Dst0, Op: NewDst0);
10093 MI.eraseFromParent();
10094
10095 return Legalized;
10096}
10097
10098LegalizerHelper::LegalizeResult LegalizerHelper::lowerSADDE(MachineInstr &MI) {
10099 auto [Res, OvOut, LHS, RHS, CarryIn] = MI.getFirst5Regs();
10100 const LLT Ty = MRI.getType(Reg: Res);
10101
10102 // sum = LHS + RHS + zext(CarryIn)
10103 auto Tmp = MIRBuilder.buildAdd(Dst: Ty, Src0: LHS, Src1: RHS);
10104 auto CarryZ = MIRBuilder.buildZExt(Res: Ty, Op: CarryIn);
10105 auto Sum = MIRBuilder.buildAdd(Dst: Ty, Src0: Tmp, Src1: CarryZ);
10106 MIRBuilder.buildCopy(Res, Op: Sum);
10107
10108 // OvOut = icmp slt ((sum ^ lhs) & (sum ^ rhs)), 0
10109 auto AX = MIRBuilder.buildXor(Dst: Ty, Src0: Sum, Src1: LHS);
10110 auto BX = MIRBuilder.buildXor(Dst: Ty, Src0: Sum, Src1: RHS);
10111 auto T = MIRBuilder.buildAnd(Dst: Ty, Src0: AX, Src1: BX);
10112
10113 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
10114 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: OvOut, Op0: T, Op1: Zero);
10115
10116 MI.eraseFromParent();
10117 return Legalized;
10118}
10119
10120LegalizerHelper::LegalizeResult LegalizerHelper::lowerSSUBE(MachineInstr &MI) {
10121 auto [Res, OvOut, LHS, RHS, CarryIn] = MI.getFirst5Regs();
10122 const LLT Ty = MRI.getType(Reg: Res);
10123
10124 // Diff = LHS - (RHS + zext(CarryIn))
10125 auto CarryZ = MIRBuilder.buildZExt(Res: Ty, Op: CarryIn);
10126 auto RHSPlusCI = MIRBuilder.buildAdd(Dst: Ty, Src0: RHS, Src1: CarryZ);
10127 auto Diff = MIRBuilder.buildSub(Dst: Ty, Src0: LHS, Src1: RHSPlusCI);
10128 MIRBuilder.buildCopy(Res, Op: Diff);
10129
10130 // ov = msb((LHS ^ RHS) & (LHS ^ Diff))
10131 auto X1 = MIRBuilder.buildXor(Dst: Ty, Src0: LHS, Src1: RHS);
10132 auto X2 = MIRBuilder.buildXor(Dst: Ty, Src0: LHS, Src1: Diff);
10133 auto T = MIRBuilder.buildAnd(Dst: Ty, Src0: X1, Src1: X2);
10134 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
10135 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: OvOut, Op0: T, Op1: Zero);
10136
10137 MI.eraseFromParent();
10138 return Legalized;
10139}
10140
10141LegalizerHelper::LegalizeResult
10142LegalizerHelper::lowerAddSubSatToMinMax(MachineInstr &MI) {
10143 auto [Res, LHS, RHS] = MI.getFirst3Regs();
10144 LLT Ty = MRI.getType(Reg: Res);
10145 bool IsSigned;
10146 bool IsAdd;
10147 unsigned BaseOp;
10148 switch (MI.getOpcode()) {
10149 default:
10150 llvm_unreachable("unexpected addsat/subsat opcode");
10151 case TargetOpcode::G_UADDSAT:
10152 IsSigned = false;
10153 IsAdd = true;
10154 BaseOp = TargetOpcode::G_ADD;
10155 break;
10156 case TargetOpcode::G_SADDSAT:
10157 IsSigned = true;
10158 IsAdd = true;
10159 BaseOp = TargetOpcode::G_ADD;
10160 break;
10161 case TargetOpcode::G_USUBSAT:
10162 IsSigned = false;
10163 IsAdd = false;
10164 BaseOp = TargetOpcode::G_SUB;
10165 break;
10166 case TargetOpcode::G_SSUBSAT:
10167 IsSigned = true;
10168 IsAdd = false;
10169 BaseOp = TargetOpcode::G_SUB;
10170 break;
10171 }
10172
10173 if (IsSigned) {
10174 // sadd.sat(a, b) ->
10175 // hi = 0x7fffffff - smax(a, 0)
10176 // lo = 0x80000000 - smin(a, 0)
10177 // a + smin(smax(lo, b), hi)
10178 // ssub.sat(a, b) ->
10179 // lo = smax(a, -1) - 0x7fffffff
10180 // hi = smin(a, -1) - 0x80000000
10181 // a - smin(smax(lo, b), hi)
10182 // TODO: AMDGPU can use a "median of 3" instruction here:
10183 // a +/- med3(lo, b, hi)
10184 uint64_t NumBits = Ty.getScalarSizeInBits();
10185 auto MaxVal =
10186 MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMaxValue(numBits: NumBits));
10187 auto MinVal =
10188 MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMinValue(numBits: NumBits));
10189 MachineInstrBuilder Hi, Lo;
10190 if (IsAdd) {
10191 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
10192 Hi = MIRBuilder.buildSub(Dst: Ty, Src0: MaxVal, Src1: MIRBuilder.buildSMax(Dst: Ty, Src0: LHS, Src1: Zero));
10193 Lo = MIRBuilder.buildSub(Dst: Ty, Src0: MinVal, Src1: MIRBuilder.buildSMin(Dst: Ty, Src0: LHS, Src1: Zero));
10194 } else {
10195 auto NegOne = MIRBuilder.buildConstant(Res: Ty, Val: -1);
10196 Lo = MIRBuilder.buildSub(Dst: Ty, Src0: MIRBuilder.buildSMax(Dst: Ty, Src0: LHS, Src1: NegOne),
10197 Src1: MaxVal);
10198 Hi = MIRBuilder.buildSub(Dst: Ty, Src0: MIRBuilder.buildSMin(Dst: Ty, Src0: LHS, Src1: NegOne),
10199 Src1: MinVal);
10200 }
10201 auto RHSClamped =
10202 MIRBuilder.buildSMin(Dst: Ty, Src0: MIRBuilder.buildSMax(Dst: Ty, Src0: Lo, Src1: RHS), Src1: Hi);
10203 MIRBuilder.buildInstr(Opc: BaseOp, DstOps: {Res}, SrcOps: {LHS, RHSClamped});
10204 } else {
10205 // uadd.sat(a, b) -> a + umin(~a, b)
10206 // usub.sat(a, b) -> a - umin(a, b)
10207 Register Not = IsAdd ? MIRBuilder.buildNot(Dst: Ty, Src0: LHS).getReg(Idx: 0) : LHS;
10208 auto Min = MIRBuilder.buildUMin(Dst: Ty, Src0: Not, Src1: RHS);
10209 MIRBuilder.buildInstr(Opc: BaseOp, DstOps: {Res}, SrcOps: {LHS, Min});
10210 }
10211
10212 MI.eraseFromParent();
10213 return Legalized;
10214}
10215
10216LegalizerHelper::LegalizeResult
10217LegalizerHelper::lowerAddSubSatToAddoSubo(MachineInstr &MI) {
10218 auto [Res, LHS, RHS] = MI.getFirst3Regs();
10219 LLT Ty = MRI.getType(Reg: Res);
10220 LLT BoolTy = Ty.changeElementSize(NewEltSize: 1);
10221 bool IsSigned;
10222 bool IsAdd;
10223 unsigned OverflowOp;
10224 switch (MI.getOpcode()) {
10225 default:
10226 llvm_unreachable("unexpected addsat/subsat opcode");
10227 case TargetOpcode::G_UADDSAT:
10228 IsSigned = false;
10229 IsAdd = true;
10230 OverflowOp = TargetOpcode::G_UADDO;
10231 break;
10232 case TargetOpcode::G_SADDSAT:
10233 IsSigned = true;
10234 IsAdd = true;
10235 OverflowOp = TargetOpcode::G_SADDO;
10236 break;
10237 case TargetOpcode::G_USUBSAT:
10238 IsSigned = false;
10239 IsAdd = false;
10240 OverflowOp = TargetOpcode::G_USUBO;
10241 break;
10242 case TargetOpcode::G_SSUBSAT:
10243 IsSigned = true;
10244 IsAdd = false;
10245 OverflowOp = TargetOpcode::G_SSUBO;
10246 break;
10247 }
10248
10249 auto OverflowRes =
10250 MIRBuilder.buildInstr(Opc: OverflowOp, DstOps: {Ty, BoolTy}, SrcOps: {LHS, RHS});
10251 Register Tmp = OverflowRes.getReg(Idx: 0);
10252 Register Ov = OverflowRes.getReg(Idx: 1);
10253 MachineInstrBuilder Clamp;
10254 if (IsSigned) {
10255 // sadd.sat(a, b) ->
10256 // {tmp, ov} = saddo(a, b)
10257 // ov ? (tmp >>s 31) + 0x80000000 : r
10258 // ssub.sat(a, b) ->
10259 // {tmp, ov} = ssubo(a, b)
10260 // ov ? (tmp >>s 31) + 0x80000000 : r
10261 uint64_t NumBits = Ty.getScalarSizeInBits();
10262 auto ShiftAmount = MIRBuilder.buildConstant(Res: Ty, Val: NumBits - 1);
10263 auto Sign = MIRBuilder.buildAShr(Dst: Ty, Src0: Tmp, Src1: ShiftAmount);
10264 auto MinVal =
10265 MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMinValue(numBits: NumBits));
10266 Clamp = MIRBuilder.buildAdd(Dst: Ty, Src0: Sign, Src1: MinVal);
10267 } else {
10268 // uadd.sat(a, b) ->
10269 // {tmp, ov} = uaddo(a, b)
10270 // ov ? 0xffffffff : tmp
10271 // usub.sat(a, b) ->
10272 // {tmp, ov} = usubo(a, b)
10273 // ov ? 0 : tmp
10274 Clamp = MIRBuilder.buildConstant(Res: Ty, Val: IsAdd ? -1 : 0);
10275 }
10276 MIRBuilder.buildSelect(Res, Tst: Ov, Op0: Clamp, Op1: Tmp);
10277
10278 MI.eraseFromParent();
10279 return Legalized;
10280}
10281
10282LegalizerHelper::LegalizeResult
10283LegalizerHelper::lowerShlSat(MachineInstr &MI) {
10284 assert((MI.getOpcode() == TargetOpcode::G_SSHLSAT ||
10285 MI.getOpcode() == TargetOpcode::G_USHLSAT) &&
10286 "Expected shlsat opcode!");
10287 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SSHLSAT;
10288 auto [Res, LHS, RHS] = MI.getFirst3Regs();
10289 LLT Ty = MRI.getType(Reg: Res);
10290 LLT BoolTy = Ty.changeElementSize(NewEltSize: 1);
10291
10292 unsigned BW = Ty.getScalarSizeInBits();
10293 auto Result = MIRBuilder.buildShl(Dst: Ty, Src0: LHS, Src1: RHS);
10294 auto Orig = IsSigned ? MIRBuilder.buildAShr(Dst: Ty, Src0: Result, Src1: RHS)
10295 : MIRBuilder.buildLShr(Dst: Ty, Src0: Result, Src1: RHS);
10296
10297 MachineInstrBuilder SatVal;
10298 if (IsSigned) {
10299 auto SatMin = MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMinValue(numBits: BW));
10300 auto SatMax = MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMaxValue(numBits: BW));
10301 auto Cmp = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: LHS,
10302 Op1: MIRBuilder.buildConstant(Res: Ty, Val: 0));
10303 SatVal = MIRBuilder.buildSelect(Res: Ty, Tst: Cmp, Op0: SatMin, Op1: SatMax);
10304 } else {
10305 SatVal = MIRBuilder.buildConstant(Res: Ty, Val: APInt::getMaxValue(numBits: BW));
10306 }
10307 auto Ov = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: BoolTy, Op0: LHS, Op1: Orig);
10308 MIRBuilder.buildSelect(Res, Tst: Ov, Op0: SatVal, Op1: Result);
10309
10310 MI.eraseFromParent();
10311 return Legalized;
10312}
10313
10314LegalizerHelper::LegalizeResult
10315LegalizerHelper::lowerTruncSat(MachineInstr &MI) {
10316 unsigned Opc = MI.getOpcode();
10317 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
10318 unsigned DstSize = DstTy.getScalarSizeInBits();
10319 unsigned SrcSize = SrcTy.getScalarSizeInBits();
10320
10321 if (Opc == TargetOpcode::G_TRUNC_SSAT_S) {
10322 auto Max = MIRBuilder.buildConstant(
10323 Res: SrcTy, Val: APInt::getSignedMaxValue(numBits: DstSize).sext(width: SrcSize));
10324 Src = MIRBuilder.buildSMin(Dst: SrcTy, Src0: Src, Src1: Max).getReg(Idx: 0);
10325 auto Min = MIRBuilder.buildConstant(
10326 Res: SrcTy, Val: APInt::getSignedMinValue(numBits: DstSize).sext(width: SrcSize));
10327 Src = MIRBuilder.buildSMax(Dst: SrcTy, Src0: Src, Src1: Min).getReg(Idx: 0);
10328 } else if (Opc == TargetOpcode::G_TRUNC_USAT_U) {
10329 auto Max = MIRBuilder.buildConstant(
10330 Res: SrcTy, Val: APInt::getAllOnes(numBits: DstSize).zext(width: SrcSize));
10331 Src = MIRBuilder.buildUMin(Dst: SrcTy, Src0: Src, Src1: Max).getReg(Idx: 0);
10332 } else if (Opc == TargetOpcode::G_TRUNC_SSAT_U) {
10333 auto Max = MIRBuilder.buildConstant(
10334 Res: SrcTy, Val: APInt::getAllOnes(numBits: DstSize).zext(width: SrcSize));
10335 Src = MIRBuilder.buildSMin(Dst: SrcTy, Src0: Src, Src1: Max).getReg(Idx: 0);
10336 auto Min = MIRBuilder.buildConstant(Res: SrcTy, Val: APInt::getZero(numBits: SrcSize));
10337 Src = MIRBuilder.buildSMax(Dst: SrcTy, Src0: Src, Src1: Min).getReg(Idx: 0);
10338 } else {
10339 llvm_unreachable("Expected truncsat opcode!");
10340 }
10341
10342 MIRBuilder.buildTrunc(Res: Dst, Op: Src);
10343 MI.eraseFromParent();
10344 return Legalized;
10345}
10346
10347LegalizerHelper::LegalizeResult LegalizerHelper::lowerBswap(MachineInstr &MI) {
10348 auto [Dst, Src] = MI.getFirst2Regs();
10349 const LLT Ty = MRI.getType(Reg: Src);
10350 unsigned SizeInBytes = (Ty.getScalarSizeInBits() + 7) / 8;
10351 unsigned BaseShiftAmt = (SizeInBytes - 1) * 8;
10352
10353 // Swap most and least significant byte, set remaining bytes in Res to zero.
10354 auto ShiftAmt = MIRBuilder.buildConstant(Res: Ty, Val: BaseShiftAmt);
10355 auto LSByteShiftedLeft = MIRBuilder.buildShl(Dst: Ty, Src0: Src, Src1: ShiftAmt);
10356 auto MSByteShiftedRight = MIRBuilder.buildLShr(Dst: Ty, Src0: Src, Src1: ShiftAmt);
10357 auto Res = MIRBuilder.buildOr(Dst: Ty, Src0: MSByteShiftedRight, Src1: LSByteShiftedLeft);
10358
10359 // Set i-th high/low byte in Res to i-th low/high byte from Src.
10360 for (unsigned i = 1; i < SizeInBytes / 2; ++i) {
10361 // AND with Mask leaves byte i unchanged and sets remaining bytes to 0.
10362 APInt APMask = APInt::getBitsSet(numBits: SizeInBytes * 8, loBit: i * 8, hiBit: i * 8 + 8);
10363 auto Mask = MIRBuilder.buildConstant(Res: Ty, Val: APMask);
10364 auto ShiftAmt = MIRBuilder.buildConstant(Res: Ty, Val: BaseShiftAmt - 16 * i);
10365 // Low byte shifted left to place of high byte: (Src & Mask) << ShiftAmt.
10366 auto LoByte = MIRBuilder.buildAnd(Dst: Ty, Src0: Src, Src1: Mask);
10367 auto LoShiftedLeft = MIRBuilder.buildShl(Dst: Ty, Src0: LoByte, Src1: ShiftAmt);
10368 Res = MIRBuilder.buildOr(Dst: Ty, Src0: Res, Src1: LoShiftedLeft);
10369 // High byte shifted right to place of low byte: (Src >> ShiftAmt) & Mask.
10370 auto SrcShiftedRight = MIRBuilder.buildLShr(Dst: Ty, Src0: Src, Src1: ShiftAmt);
10371 auto HiShiftedRight = MIRBuilder.buildAnd(Dst: Ty, Src0: SrcShiftedRight, Src1: Mask);
10372 Res = MIRBuilder.buildOr(Dst: Ty, Src0: Res, Src1: HiShiftedRight);
10373 }
10374 Res.getInstr()->getOperand(i: 0).setReg(Dst);
10375
10376 MI.eraseFromParent();
10377 return Legalized;
10378}
10379
10380//{ (Src & Mask) >> N } | { (Src << N) & Mask }
10381static MachineInstrBuilder SwapN(unsigned N, DstOp Dst, MachineIRBuilder &B,
10382 MachineInstrBuilder Src, const APInt &Mask) {
10383 const LLT Ty = Dst.getLLTTy(MRI: *B.getMRI());
10384 MachineInstrBuilder C_N = B.buildConstant(Res: Ty, Val: N);
10385 MachineInstrBuilder MaskLoNTo0 = B.buildConstant(Res: Ty, Val: Mask);
10386 auto LHS = B.buildLShr(Dst: Ty, Src0: B.buildAnd(Dst: Ty, Src0: Src, Src1: MaskLoNTo0), Src1: C_N);
10387 auto RHS = B.buildAnd(Dst: Ty, Src0: B.buildShl(Dst: Ty, Src0: Src, Src1: C_N), Src1: MaskLoNTo0);
10388 return B.buildOr(Dst, Src0: LHS, Src1: RHS);
10389}
10390
10391LegalizerHelper::LegalizeResult
10392LegalizerHelper::lowerBitreverse(MachineInstr &MI) {
10393 auto [Dst, Src] = MI.getFirst2Regs();
10394 const LLT SrcTy = MRI.getType(Reg: Src);
10395 unsigned Size = SrcTy.getScalarSizeInBits();
10396 unsigned VSize = SrcTy.getSizeInBits();
10397
10398 if (Size >= 8) {
10399 if (SrcTy.isVector() && (VSize % 8 == 0) &&
10400 (LI.isLegal(Query: {TargetOpcode::G_BITREVERSE,
10401 {LLT::fixed_vector(NumElements: VSize / 8, ScalarTy: LLT::integer(SizeInBits: 8)),
10402 LLT::fixed_vector(NumElements: VSize / 8, ScalarTy: LLT::integer(SizeInBits: 8))}}))) {
10403 // If bitreverse is legal for i8 vector of the same size, then cast
10404 // to i8 vector type.
10405 // e.g. v4s32 -> v16s8
10406 LLT VTy = LLT::fixed_vector(NumElements: VSize / 8, ScalarTy: LLT::integer(SizeInBits: 8));
10407 auto BSWAP = MIRBuilder.buildBSwap(Dst: SrcTy, Src0: Src);
10408 auto Cast = MIRBuilder.buildBitcast(Dst: VTy, Src: BSWAP);
10409 auto RBIT = MIRBuilder.buildBitReverse(Dst: VTy, Src: Cast);
10410 MIRBuilder.buildBitcast(Dst, Src: RBIT);
10411 } else {
10412 MachineInstrBuilder BSWAP =
10413 MIRBuilder.buildInstr(Opc: TargetOpcode::G_BSWAP, DstOps: {SrcTy}, SrcOps: {Src});
10414
10415 // swap high and low 4 bits in 8 bit blocks 7654|3210 -> 3210|7654
10416 // [(val & 0xF0F0F0F0) >> 4] | [(val & 0x0F0F0F0F) << 4]
10417 // -> [(val & 0xF0F0F0F0) >> 4] | [(val << 4) & 0xF0F0F0F0]
10418 MachineInstrBuilder Swap4 = SwapN(N: 4, Dst: SrcTy, B&: MIRBuilder, Src: BSWAP,
10419 Mask: APInt::getSplat(NewLen: Size, V: APInt(8, 0xF0)));
10420
10421 // swap high and low 2 bits in 4 bit blocks 32|10 76|54 -> 10|32 54|76
10422 // [(val & 0xCCCCCCCC) >> 2] & [(val & 0x33333333) << 2]
10423 // -> [(val & 0xCCCCCCCC) >> 2] & [(val << 2) & 0xCCCCCCCC]
10424 MachineInstrBuilder Swap2 = SwapN(N: 2, Dst: SrcTy, B&: MIRBuilder, Src: Swap4,
10425 Mask: APInt::getSplat(NewLen: Size, V: APInt(8, 0xCC)));
10426
10427 // swap high and low 1 bit in 2 bit blocks 1|0 3|2 5|4 7|6 -> 0|1 2|3 4|5
10428 // 6|7
10429 // [(val & 0xAAAAAAAA) >> 1] & [(val & 0x55555555) << 1]
10430 // -> [(val & 0xAAAAAAAA) >> 1] & [(val << 1) & 0xAAAAAAAA]
10431 SwapN(N: 1, Dst, B&: MIRBuilder, Src: Swap2, Mask: APInt::getSplat(NewLen: Size, V: APInt(8, 0xAA)));
10432 }
10433 } else {
10434 // Expand bitreverse for types smaller than 8 bits.
10435 MachineInstrBuilder Tmp;
10436 for (unsigned I = 0, J = Size - 1; I < Size; ++I, --J) {
10437 MachineInstrBuilder Tmp2;
10438 if (I < J) {
10439 auto ShAmt = MIRBuilder.buildConstant(Res: SrcTy, Val: J - I);
10440 Tmp2 = MIRBuilder.buildShl(Dst: SrcTy, Src0: Src, Src1: ShAmt);
10441 } else {
10442 auto ShAmt = MIRBuilder.buildConstant(Res: SrcTy, Val: I - J);
10443 Tmp2 = MIRBuilder.buildLShr(Dst: SrcTy, Src0: Src, Src1: ShAmt);
10444 }
10445
10446 auto Mask = MIRBuilder.buildConstant(Res: SrcTy, Val: 1ULL << J);
10447 Tmp2 = MIRBuilder.buildAnd(Dst: SrcTy, Src0: Tmp2, Src1: Mask);
10448 if (I == 0)
10449 Tmp = Tmp2;
10450 else
10451 Tmp = MIRBuilder.buildOr(Dst: SrcTy, Src0: Tmp, Src1: Tmp2);
10452 }
10453 MIRBuilder.buildCopy(Res: Dst, Op: Tmp);
10454 }
10455
10456 MI.eraseFromParent();
10457 return Legalized;
10458}
10459
10460LegalizerHelper::LegalizeResult
10461LegalizerHelper::lowerReadWriteRegister(MachineInstr &MI) {
10462 MachineFunction &MF = MIRBuilder.getMF();
10463
10464 bool IsRead = MI.getOpcode() == TargetOpcode::G_READ_REGISTER;
10465 int NameOpIdx = IsRead ? 1 : 0;
10466 int ValRegIndex = IsRead ? 0 : 1;
10467
10468 Register ValReg = MI.getOperand(i: ValRegIndex).getReg();
10469 const LLT Ty = MRI.getType(Reg: ValReg);
10470 const MDString *RegStr = cast<MDString>(
10471 Val: cast<MDNode>(Val: MI.getOperand(i: NameOpIdx).getMetadata())->getOperand(I: 0));
10472
10473 Register PhysReg = TLI.getRegisterByName(RegName: RegStr->getString().data(), Ty, MF);
10474 if (!PhysReg) {
10475 const Function &Fn = MF.getFunction();
10476 Fn.getContext().diagnose(DI: DiagnosticInfoGenericWithLoc(
10477 "invalid register \"" + Twine(RegStr->getString().data()) + "\" for " +
10478 (IsRead ? "llvm.read_register" : "llvm.write_register"),
10479 Fn, MI.getDebugLoc()));
10480 if (IsRead)
10481 MIRBuilder.buildUndef(Res: ValReg);
10482
10483 MI.eraseFromParent();
10484 return Legalized;
10485 }
10486
10487 if (IsRead)
10488 MIRBuilder.buildCopy(Res: ValReg, Op: PhysReg);
10489 else
10490 MIRBuilder.buildCopy(Res: PhysReg, Op: ValReg);
10491
10492 MI.eraseFromParent();
10493 return Legalized;
10494}
10495
10496LegalizerHelper::LegalizeResult
10497LegalizerHelper::lowerSMULH_UMULH(MachineInstr &MI) {
10498 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SMULH;
10499 unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
10500 Register Result = MI.getOperand(i: 0).getReg();
10501 LLT OrigTy = MRI.getType(Reg: Result);
10502 auto SizeInBits = OrigTy.getScalarSizeInBits();
10503 LLT WideTy = OrigTy.changeElementSize(NewEltSize: SizeInBits * 2);
10504
10505 auto LHS = MIRBuilder.buildInstr(Opc: ExtOp, DstOps: {WideTy}, SrcOps: {MI.getOperand(i: 1)});
10506 auto RHS = MIRBuilder.buildInstr(Opc: ExtOp, DstOps: {WideTy}, SrcOps: {MI.getOperand(i: 2)});
10507 auto Mul = MIRBuilder.buildMul(Dst: WideTy, Src0: LHS, Src1: RHS);
10508 unsigned ShiftOp = IsSigned ? TargetOpcode::G_ASHR : TargetOpcode::G_LSHR;
10509
10510 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: SizeInBits);
10511 auto Shifted = MIRBuilder.buildInstr(Opc: ShiftOp, DstOps: {WideTy}, SrcOps: {Mul, ShiftAmt});
10512 MIRBuilder.buildTrunc(Res: Result, Op: Shifted);
10513
10514 MI.eraseFromParent();
10515 return Legalized;
10516}
10517
10518LegalizerHelper::LegalizeResult
10519LegalizerHelper::lowerISFPCLASS(MachineInstr &MI) {
10520 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
10521 FPClassTest Mask = static_cast<FPClassTest>(MI.getOperand(i: 2).getImm());
10522
10523 if (Mask == fcNone) {
10524 MIRBuilder.buildConstant(Res: DstReg, Val: 0);
10525 MI.eraseFromParent();
10526 return Legalized;
10527 }
10528 if (Mask == fcAllFlags) {
10529 MIRBuilder.buildConstant(Res: DstReg, Val: 1);
10530 MI.eraseFromParent();
10531 return Legalized;
10532 }
10533
10534 // TODO: Try inverting the test with getInvertedFPClassTest like the DAG
10535 // version
10536
10537 unsigned BitSize = SrcTy.getScalarSizeInBits();
10538 const fltSemantics &Semantics = getFltSemanticForLLT(Ty: SrcTy.getScalarType());
10539
10540 LLT IntTy = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: BitSize));
10541 auto AsInt = SrcTy == IntTy ? MIRBuilder.buildCopy(Res: IntTy, Op: SrcReg)
10542 : MIRBuilder.buildBitcast(Dst: IntTy, Src: SrcReg);
10543
10544 // Various masks.
10545 APInt SignBit = APInt::getSignMask(BitWidth: BitSize);
10546 APInt ValueMask = APInt::getSignedMaxValue(numBits: BitSize); // All bits but sign.
10547 APInt Inf = APFloat::getInf(Sem: Semantics).bitcastToAPInt(); // Exp and int bit.
10548 APInt ExpMask = Inf;
10549 APInt AllOneMantissa = APFloat::getLargest(Sem: Semantics).bitcastToAPInt() & ~Inf;
10550 APInt QNaNBitMask =
10551 APInt::getOneBitSet(numBits: BitSize, BitNo: AllOneMantissa.getActiveBits() - 1);
10552 APInt InversionMask = APInt::getAllOnes(numBits: DstTy.getScalarSizeInBits());
10553
10554 auto SignBitC = MIRBuilder.buildConstant(Res: IntTy, Val: SignBit);
10555 auto ValueMaskC = MIRBuilder.buildConstant(Res: IntTy, Val: ValueMask);
10556 auto InfC = MIRBuilder.buildConstant(Res: IntTy, Val: Inf);
10557 auto ExpMaskC = MIRBuilder.buildConstant(Res: IntTy, Val: ExpMask);
10558 auto ZeroC = MIRBuilder.buildConstant(Res: IntTy, Val: 0);
10559
10560 auto Abs = MIRBuilder.buildAnd(Dst: IntTy, Src0: AsInt, Src1: ValueMaskC);
10561 auto Sign =
10562 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_NE, Res: DstTy, Op0: AsInt, Op1: Abs);
10563
10564 auto Res = MIRBuilder.buildConstant(Res: DstTy, Val: 0);
10565 // Clang doesn't support capture of structured bindings:
10566 LLT DstTyCopy = DstTy;
10567 const auto appendToRes = [&](MachineInstrBuilder ToAppend) {
10568 Res = MIRBuilder.buildOr(Dst: DstTyCopy, Src0: Res, Src1: ToAppend);
10569 };
10570
10571 // Tests that involve more than one class should be processed first.
10572 if ((Mask & fcFinite) == fcFinite) {
10573 // finite(V) ==> abs(V) u< exp_mask
10574 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: Abs,
10575 Op1: ExpMaskC));
10576 Mask &= ~fcFinite;
10577 } else if ((Mask & fcFinite) == fcPosFinite) {
10578 // finite(V) && V > 0 ==> V u< exp_mask
10579 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: AsInt,
10580 Op1: ExpMaskC));
10581 Mask &= ~fcPosFinite;
10582 } else if ((Mask & fcFinite) == fcNegFinite) {
10583 // finite(V) && V < 0 ==> abs(V) u< exp_mask && signbit == 1
10584 auto Cmp = MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: Abs,
10585 Op1: ExpMaskC);
10586 auto And = MIRBuilder.buildAnd(Dst: DstTy, Src0: Cmp, Src1: Sign);
10587 appendToRes(And);
10588 Mask &= ~fcNegFinite;
10589 }
10590
10591 if (FPClassTest PartialCheck = Mask & (fcZero | fcSubnormal)) {
10592 // fcZero | fcSubnormal => test all exponent bits are 0
10593 // TODO: Handle sign bit specific cases
10594 // TODO: Handle inverted case
10595 if (PartialCheck == (fcZero | fcSubnormal)) {
10596 auto ExpBits = MIRBuilder.buildAnd(Dst: IntTy, Src0: AsInt, Src1: ExpMaskC);
10597 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10598 Op0: ExpBits, Op1: ZeroC));
10599 Mask &= ~PartialCheck;
10600 }
10601 }
10602
10603 // Check for individual classes.
10604 if (FPClassTest PartialCheck = Mask & fcZero) {
10605 if (PartialCheck == fcPosZero)
10606 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10607 Op0: AsInt, Op1: ZeroC));
10608 else if (PartialCheck == fcZero)
10609 appendToRes(
10610 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy, Op0: Abs, Op1: ZeroC));
10611 else // fcNegZero
10612 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10613 Op0: AsInt, Op1: SignBitC));
10614 }
10615
10616 if (FPClassTest PartialCheck = Mask & fcSubnormal) {
10617 // issubnormal(V) ==> unsigned(abs(V) - 1) u< (all mantissa bits set)
10618 // issubnormal(V) && V>0 ==> unsigned(V - 1) u< (all mantissa bits set)
10619 auto V = (PartialCheck == fcPosSubnormal) ? AsInt : Abs;
10620 auto OneC = MIRBuilder.buildConstant(Res: IntTy, Val: 1);
10621 auto VMinusOne = MIRBuilder.buildSub(Dst: IntTy, Src0: V, Src1: OneC);
10622 auto SubnormalRes =
10623 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: VMinusOne,
10624 Op1: MIRBuilder.buildConstant(Res: IntTy, Val: AllOneMantissa));
10625 if (PartialCheck == fcNegSubnormal)
10626 SubnormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: SubnormalRes, Src1: Sign);
10627 appendToRes(SubnormalRes);
10628 }
10629
10630 if (FPClassTest PartialCheck = Mask & fcInf) {
10631 if (PartialCheck == fcPosInf)
10632 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10633 Op0: AsInt, Op1: InfC));
10634 else if (PartialCheck == fcInf)
10635 appendToRes(
10636 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy, Op0: Abs, Op1: InfC));
10637 else { // fcNegInf
10638 APInt NegInf = APFloat::getInf(Sem: Semantics, Negative: true).bitcastToAPInt();
10639 auto NegInfC = MIRBuilder.buildConstant(Res: IntTy, Val: NegInf);
10640 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10641 Op0: AsInt, Op1: NegInfC));
10642 }
10643 }
10644
10645 if (FPClassTest PartialCheck = Mask & fcNan) {
10646 auto InfWithQnanBitC = MIRBuilder.buildConstant(Res: IntTy, Val: Inf | QNaNBitMask);
10647 if (PartialCheck == fcNan) {
10648 // isnan(V) ==> abs(V) u> int(inf)
10649 appendToRes(
10650 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: DstTy, Op0: Abs, Op1: InfC));
10651 } else if (PartialCheck == fcQNan) {
10652 // isquiet(V) ==> abs(V) u>= (unsigned(Inf) | quiet_bit)
10653 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGE, Res: DstTy, Op0: Abs,
10654 Op1: InfWithQnanBitC));
10655 } else { // fcSNan
10656 // issignaling(V) ==> abs(V) u> unsigned(Inf) &&
10657 // abs(V) u< (unsigned(Inf) | quiet_bit)
10658 auto IsNan =
10659 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: DstTy, Op0: Abs, Op1: InfC);
10660 auto IsNotQnan = MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy,
10661 Op0: Abs, Op1: InfWithQnanBitC);
10662 appendToRes(MIRBuilder.buildAnd(Dst: DstTy, Src0: IsNan, Src1: IsNotQnan));
10663 }
10664 }
10665
10666 if (FPClassTest PartialCheck = Mask & fcNormal) {
10667 // isnormal(V) ==> (0 u< exp u< max_exp) ==> (unsigned(exp-1) u<
10668 // (max_exp-1))
10669 APInt ExpLSB = ExpMask & ~(ExpMask.shl(shiftAmt: 1));
10670 auto ExpMinusOne = MIRBuilder.buildSub(
10671 Dst: IntTy, Src0: Abs, Src1: MIRBuilder.buildConstant(Res: IntTy, Val: ExpLSB));
10672 APInt MaxExpMinusOne = ExpMask - ExpLSB;
10673 auto NormalRes =
10674 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: ExpMinusOne,
10675 Op1: MIRBuilder.buildConstant(Res: IntTy, Val: MaxExpMinusOne));
10676 if (PartialCheck == fcNegNormal)
10677 NormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: NormalRes, Src1: Sign);
10678 else if (PartialCheck == fcPosNormal) {
10679 auto PosSign = MIRBuilder.buildXor(
10680 Dst: DstTy, Src0: Sign, Src1: MIRBuilder.buildConstant(Res: DstTy, Val: InversionMask));
10681 NormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: NormalRes, Src1: PosSign);
10682 }
10683 appendToRes(NormalRes);
10684 }
10685
10686 MIRBuilder.buildCopy(Res: DstReg, Op: Res);
10687 MI.eraseFromParent();
10688 return Legalized;
10689}
10690
10691LegalizerHelper::LegalizeResult LegalizerHelper::lowerSelect(MachineInstr &MI) {
10692 // Implement G_SELECT in terms of XOR, AND, OR.
10693 auto [DstReg, DstTy, MaskReg, MaskTy, Op1Reg, Op1Ty, Op2Reg, Op2Ty] =
10694 MI.getFirst4RegLLTs();
10695
10696 LLT Op1TyInt =
10697 Op1Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: Op1Ty.getScalarSizeInBits()));
10698
10699 bool IsEltPtr = DstTy.isPointerOrPointerVector();
10700 if (IsEltPtr) {
10701 LLT ScalarPtrTy = LLT::integer(SizeInBits: DstTy.getScalarSizeInBits());
10702 LLT NewTy = DstTy.changeElementType(NewEltTy: ScalarPtrTy);
10703 Op1Reg = MIRBuilder.buildPtrToInt(Dst: NewTy, Src: Op1Reg).getReg(Idx: 0);
10704 Op1Ty = MRI.getType(Reg: Op1Reg);
10705 Op2Reg = MIRBuilder.buildPtrToInt(Dst: NewTy, Src: Op2Reg).getReg(Idx: 0);
10706 Op2Ty = MRI.getType(Reg: Op2Reg);
10707 DstTy = NewTy;
10708 }
10709
10710 if (MaskTy.isScalar()) {
10711 // Turn the scalar condition into a vector condition mask if needed.
10712
10713 Register MaskElt = MaskReg;
10714
10715 // The condition was potentially zero extended before, but we want a sign
10716 // extended boolean.
10717 if (MaskTy != LLT::scalar(SizeInBits: 1))
10718 MaskElt = MIRBuilder.buildSExtInReg(Res: MaskTy, Op: MaskElt, ImmOp: 1).getReg(Idx: 0);
10719
10720 // Continue the sign extension (or truncate) to match the data type.
10721 MaskTy = DstTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: DstTy.getScalarSizeInBits()));
10722 MaskElt =
10723 MIRBuilder.buildSExtOrTrunc(Res: MaskTy.getScalarType(), Op: MaskElt).getReg(Idx: 0);
10724
10725 if (DstTy.isVector()) {
10726 // Generate a vector splat idiom.
10727 auto ShufSplat = MIRBuilder.buildShuffleSplat(Res: MaskTy, Src: MaskElt);
10728 MaskReg = ShufSplat.getReg(Idx: 0);
10729 } else {
10730 MaskReg = MaskElt;
10731 }
10732 } else if (!DstTy.isVector()) {
10733 // Cannot handle the case that mask is a vector and dst is a scalar.
10734 return UnableToLegalize;
10735 }
10736
10737 if (MaskTy.getSizeInBits() != DstTy.getSizeInBits()) {
10738 return UnableToLegalize;
10739 }
10740
10741 if (!Op1Ty.getScalarType().isAnyScalar() &&
10742 !Op1Ty.getScalarType().isInteger())
10743 Op1Reg = MIRBuilder.buildBitcast(Dst: Op1TyInt, Src: Op1Reg).getReg(Idx: 0);
10744
10745 if (!Op2Ty.getScalarType().isAnyScalar() &&
10746 !Op2Ty.getScalarType().isInteger()) {
10747 auto Op2TyInt =
10748 Op2Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: Op2Ty.getScalarSizeInBits()));
10749 Op2Reg = MIRBuilder.buildBitcast(Dst: Op2TyInt, Src: Op2Reg).getReg(Idx: 0);
10750 }
10751
10752 auto NotMask = MIRBuilder.buildNot(Dst: MaskTy, Src0: MaskReg);
10753 auto NewOp1 = MIRBuilder.buildAnd(Dst: MaskTy, Src0: Op1Reg, Src1: MaskReg);
10754 auto NewOp2 = MIRBuilder.buildAnd(Dst: MaskTy, Src0: Op2Reg, Src1: NotMask);
10755 if (IsEltPtr) {
10756 auto Or = MIRBuilder.buildOr(Dst: DstTy, Src0: NewOp1, Src1: NewOp2);
10757 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: Or);
10758 } else {
10759 if (DstTy == Op1TyInt)
10760 MIRBuilder.buildOr(Dst: DstReg, Src0: NewOp1, Src1: NewOp2);
10761 else {
10762 auto Or = MIRBuilder.buildOr(Dst: Op1TyInt, Src0: NewOp1, Src1: NewOp2);
10763 MIRBuilder.buildBitcast(Dst: DstReg, Src: Or.getReg(Idx: 0));
10764 }
10765 }
10766 MI.eraseFromParent();
10767 return Legalized;
10768}
10769
10770LegalizerHelper::LegalizeResult LegalizerHelper::lowerDIVREM(MachineInstr &MI) {
10771 // Split DIVREM into individual instructions.
10772 unsigned Opcode = MI.getOpcode();
10773
10774 MIRBuilder.buildInstr(
10775 Opc: Opcode == TargetOpcode::G_SDIVREM ? TargetOpcode::G_SDIV
10776 : TargetOpcode::G_UDIV,
10777 DstOps: {MI.getOperand(i: 0).getReg()}, SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3)});
10778 MIRBuilder.buildInstr(
10779 Opc: Opcode == TargetOpcode::G_SDIVREM ? TargetOpcode::G_SREM
10780 : TargetOpcode::G_UREM,
10781 DstOps: {MI.getOperand(i: 1).getReg()}, SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3)});
10782 MI.eraseFromParent();
10783 return Legalized;
10784}
10785
10786LegalizerHelper::LegalizeResult
10787LegalizerHelper::lowerAbsToAddXor(MachineInstr &MI) {
10788 // Expand %res = G_ABS %a into:
10789 // %v1 = G_ASHR %a, scalar_size-1
10790 // %v2 = G_ADD %a, %v1
10791 // %res = G_XOR %v2, %v1
10792 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
10793 Register OpReg = MI.getOperand(i: 1).getReg();
10794 auto ShiftAmt =
10795 MIRBuilder.buildConstant(Res: DstTy, Val: DstTy.getScalarSizeInBits() - 1);
10796 auto Shift = MIRBuilder.buildAShr(Dst: DstTy, Src0: OpReg, Src1: ShiftAmt);
10797 auto Add = MIRBuilder.buildAdd(Dst: DstTy, Src0: OpReg, Src1: Shift);
10798 MIRBuilder.buildXor(Dst: MI.getOperand(i: 0).getReg(), Src0: Add, Src1: Shift);
10799 MI.eraseFromParent();
10800 return Legalized;
10801}
10802
10803LegalizerHelper::LegalizeResult
10804LegalizerHelper::lowerAbsToMaxNeg(MachineInstr &MI) {
10805 // Expand %res = G_ABS %a into:
10806 // %v1 = G_CONSTANT 0
10807 // %v2 = G_SUB %v1, %a
10808 // %res = G_SMAX %a, %v2
10809 Register SrcReg = MI.getOperand(i: 1).getReg();
10810 LLT Ty = MRI.getType(Reg: SrcReg);
10811 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
10812 auto Sub = MIRBuilder.buildSub(Dst: Ty, Src0: Zero, Src1: SrcReg);
10813 MIRBuilder.buildSMax(Dst: MI.getOperand(i: 0), Src0: SrcReg, Src1: Sub);
10814 MI.eraseFromParent();
10815 return Legalized;
10816}
10817
10818LegalizerHelper::LegalizeResult
10819LegalizerHelper::lowerAbsToCNeg(MachineInstr &MI) {
10820 Register SrcReg = MI.getOperand(i: 1).getReg();
10821 Register DestReg = MI.getOperand(i: 0).getReg();
10822 LLT Ty = MRI.getType(Reg: SrcReg), IType = LLT::integer(SizeInBits: 1);
10823 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0).getReg(Idx: 0);
10824 auto Sub = MIRBuilder.buildSub(Dst: Ty, Src0: Zero, Src1: SrcReg).getReg(Idx: 0);
10825 auto ICmp = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SGT, Res: IType, Op0: SrcReg, Op1: Zero);
10826 MIRBuilder.buildSelect(Res: DestReg, Tst: ICmp, Op0: SrcReg, Op1: Sub);
10827 MI.eraseFromParent();
10828 return Legalized;
10829}
10830
10831LegalizerHelper::LegalizeResult
10832LegalizerHelper::lowerAbsDiffToSelect(MachineInstr &MI) {
10833 assert((MI.getOpcode() == TargetOpcode::G_ABDS ||
10834 MI.getOpcode() == TargetOpcode::G_ABDU) &&
10835 "Expected G_ABDS or G_ABDU instruction");
10836
10837 auto [DstReg, LHS, RHS] = MI.getFirst3Regs();
10838 LLT Ty = MRI.getType(Reg: LHS);
10839
10840 // abds(lhs, rhs) -> select(sgt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
10841 // abdu(lhs, rhs) -> select(ugt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
10842 Register LHSSub = MIRBuilder.buildSub(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10843 Register RHSSub = MIRBuilder.buildSub(Dst: Ty, Src0: RHS, Src1: LHS).getReg(Idx: 0);
10844 CmpInst::Predicate Pred = (MI.getOpcode() == TargetOpcode::G_ABDS)
10845 ? CmpInst::ICMP_SGT
10846 : CmpInst::ICMP_UGT;
10847 auto ICmp = MIRBuilder.buildICmp(Pred, Res: LLT::scalar(SizeInBits: 1), Op0: LHS, Op1: RHS);
10848 MIRBuilder.buildSelect(Res: DstReg, Tst: ICmp, Op0: LHSSub, Op1: RHSSub);
10849
10850 MI.eraseFromParent();
10851 return Legalized;
10852}
10853
10854LegalizerHelper::LegalizeResult
10855LegalizerHelper::lowerAbsDiffToMinMax(MachineInstr &MI) {
10856 assert((MI.getOpcode() == TargetOpcode::G_ABDS ||
10857 MI.getOpcode() == TargetOpcode::G_ABDU) &&
10858 "Expected G_ABDS or G_ABDU instruction");
10859
10860 auto [DstReg, LHS, RHS] = MI.getFirst3Regs();
10861 LLT Ty = MRI.getType(Reg: LHS);
10862
10863 // abds(lhs, rhs) -→ sub(smax(lhs, rhs), smin(lhs, rhs))
10864 // abdu(lhs, rhs) -→ sub(umax(lhs, rhs), umin(lhs, rhs))
10865 Register MaxReg, MinReg;
10866 if (MI.getOpcode() == TargetOpcode::G_ABDS) {
10867 MaxReg = MIRBuilder.buildSMax(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10868 MinReg = MIRBuilder.buildSMin(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10869 } else {
10870 MaxReg = MIRBuilder.buildUMax(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10871 MinReg = MIRBuilder.buildUMin(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10872 }
10873 MIRBuilder.buildSub(Dst: DstReg, Src0: MaxReg, Src1: MinReg);
10874
10875 MI.eraseFromParent();
10876 return Legalized;
10877}
10878
10879LegalizerHelper::LegalizeResult LegalizerHelper::lowerFAbs(MachineInstr &MI) {
10880 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
10881 LLT TyInt =
10882 DstTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: DstTy.getScalarSizeInBits()));
10883 Register CastedSrc = SrcReg;
10884
10885 if (!(SrcTy.getScalarType().isAnyScalar() ||
10886 SrcTy.getScalarType().isInteger())) {
10887 auto SrcTyInt =
10888 SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: SrcTy.getScalarSizeInBits()));
10889 CastedSrc = MIRBuilder.buildBitcast(Dst: SrcTyInt, Src: SrcReg).getReg(Idx: 0);
10890 }
10891
10892 if (MRI.getType(Reg: DstReg) != TyInt) {
10893 // Reset sign bit
10894 Register NewDst =
10895 MIRBuilder
10896 .buildAnd(Dst: TyInt, Src0: CastedSrc,
10897 Src1: MIRBuilder.buildConstant(
10898 Res: TyInt, Val: APInt::getSignedMaxValue(
10899 numBits: DstTy.getScalarSizeInBits())))
10900 .getReg(Idx: 0);
10901
10902 MIRBuilder.buildBitcast(Dst: DstReg, Src: NewDst);
10903 } else
10904 MIRBuilder
10905 .buildAnd(
10906 Dst: DstReg, Src0: CastedSrc,
10907 Src1: MIRBuilder.buildConstant(
10908 Res: TyInt, Val: APInt::getSignedMaxValue(numBits: DstTy.getScalarSizeInBits())))
10909 .getReg(Idx: 0);
10910
10911 MI.eraseFromParent();
10912 return Legalized;
10913}
10914
10915LegalizerHelper::LegalizeResult
10916LegalizerHelper::lowerVectorReduction(MachineInstr &MI) {
10917 Register SrcReg = MI.getOperand(i: 1).getReg();
10918 LLT SrcTy = MRI.getType(Reg: SrcReg);
10919 LLT DstTy = MRI.getType(Reg: SrcReg);
10920
10921 // The source could be a scalar if the IR type was <1 x sN>.
10922 if (SrcTy.isScalar()) {
10923 if (DstTy.getSizeInBits() > SrcTy.getSizeInBits())
10924 return UnableToLegalize; // FIXME: handle extension.
10925 // This can be just a plain copy.
10926 Observer.changingInstr(MI);
10927 MI.setDesc(MIRBuilder.getTII().get(Opcode: TargetOpcode::COPY));
10928 Observer.changedInstr(MI);
10929 return Legalized;
10930 }
10931 return UnableToLegalize;
10932}
10933
10934LegalizerHelper::LegalizeResult LegalizerHelper::lowerVAArg(MachineInstr &MI) {
10935 MachineFunction &MF = *MI.getMF();
10936 const DataLayout &DL = MIRBuilder.getDataLayout();
10937 LLVMContext &Ctx = MF.getFunction().getContext();
10938 Register ListPtr = MI.getOperand(i: 1).getReg();
10939 LLT PtrTy = MRI.getType(Reg: ListPtr);
10940
10941 // LstPtr is a pointer to the head of the list. Get the address
10942 // of the head of the list.
10943 Align PtrAlignment = DL.getABITypeAlign(Ty: getTypeForLLT(Ty: PtrTy, C&: Ctx));
10944 MachineMemOperand *PtrLoadMMO = MF.getMachineMemOperand(
10945 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOLoad, MemTy: PtrTy, BaseAlignment: PtrAlignment);
10946 auto VAList = MIRBuilder.buildLoad(Res: PtrTy, Addr: ListPtr, MMO&: *PtrLoadMMO).getReg(Idx: 0);
10947
10948 const Align A(MI.getOperand(i: 2).getImm());
10949 LLT PtrTyAsScalarTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
10950 if (A > TLI.getMinStackArgumentAlignment()) {
10951 Register AlignAmt =
10952 MIRBuilder.buildConstant(Res: PtrTyAsScalarTy, Val: A.value() - 1).getReg(Idx: 0);
10953 auto AddDst = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: VAList, Op1: AlignAmt);
10954 auto AndDst = MIRBuilder.buildMaskLowPtrBits(Res: PtrTy, Op0: AddDst, NumBits: Log2(A));
10955 VAList = AndDst.getReg(Idx: 0);
10956 }
10957
10958 // Increment the pointer, VAList, to the next vaarg
10959 // The list should be bumped by the size of element in the current head of
10960 // list.
10961 Register Dst = MI.getOperand(i: 0).getReg();
10962 LLT LLTTy = MRI.getType(Reg: Dst);
10963 Type *Ty = getTypeForLLT(Ty: LLTTy, C&: Ctx);
10964 auto IncAmt =
10965 MIRBuilder.buildConstant(Res: PtrTyAsScalarTy, Val: DL.getTypeAllocSize(Ty));
10966 auto Succ = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: VAList, Op1: IncAmt);
10967
10968 // Store the increment VAList to the legalized pointer
10969 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
10970 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOStore, MemTy: PtrTy, BaseAlignment: PtrAlignment);
10971 MIRBuilder.buildStore(Val: Succ, Addr: ListPtr, MMO&: *StoreMMO);
10972 // Load the actual argument out of the pointer VAList
10973 Align EltAlignment = DL.getABITypeAlign(Ty);
10974 MachineMemOperand *EltLoadMMO = MF.getMachineMemOperand(
10975 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOLoad, MemTy: LLTTy, BaseAlignment: EltAlignment);
10976 MIRBuilder.buildLoad(Res: Dst, Addr: VAList, MMO&: *EltLoadMMO);
10977
10978 MI.eraseFromParent();
10979 return Legalized;
10980}
10981
10982LegalizerHelper::LegalizeResult LegalizerHelper::lowerMulfix(MachineInstr &MI) {
10983 unsigned OpCode = MI.getOpcode();
10984 assert((OpCode == TargetOpcode::G_SMULFIX ||
10985 OpCode == TargetOpcode::G_UMULFIX ||
10986 OpCode == TargetOpcode::G_SMULFIXSAT ||
10987 OpCode == TargetOpcode::G_UMULFIXSAT) &&
10988 "Operator must be either G_SMULFIX[SAT] or G_UMULFIX[SAT]!");
10989 auto [Dst, LHS, RHS] = MI.getFirst3Regs();
10990 LLT Ty = MRI.getType(Reg: Dst);
10991 unsigned Scale = MI.getOperand(i: 3).getImm();
10992
10993 bool Saturating = (OpCode == TargetOpcode::G_SMULFIXSAT ||
10994 OpCode == TargetOpcode::G_UMULFIXSAT);
10995 bool IsSigned = (OpCode == TargetOpcode::G_SMULFIX ||
10996 OpCode == TargetOpcode::G_SMULFIXSAT);
10997
10998 if (!Saturating && Scale == 0) {
10999 MIRBuilder.buildMul(Dst, Src0: LHS, Src1: RHS);
11000 MI.eraseFromParent();
11001 return Legalized;
11002 }
11003
11004 // TODO: Port other lowering paths from SelectionDAG.
11005 LLT WideTy = Ty.changeElementSize(NewEltSize: Ty.getScalarSizeInBits() * 2);
11006 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: Scale);
11007 MachineInstrBuilder ExtLHS{}, ExtRHS{}, Shift{};
11008 if (IsSigned) {
11009 ExtLHS = MIRBuilder.buildSExt(Res: WideTy, Op: LHS);
11010 ExtRHS = MIRBuilder.buildSExt(Res: WideTy, Op: RHS);
11011 } else {
11012 ExtLHS = MIRBuilder.buildZExt(Res: WideTy, Op: LHS);
11013 ExtRHS = MIRBuilder.buildZExt(Res: WideTy, Op: RHS);
11014 }
11015
11016 auto Mul = MIRBuilder.buildMul(Dst: WideTy, Src0: ExtLHS, Src1: ExtRHS);
11017 if (IsSigned)
11018 Shift = MIRBuilder.buildAShr(Dst: WideTy, Src0: Mul, Src1: ShiftAmt);
11019 else
11020 Shift = MIRBuilder.buildLShr(Dst: WideTy, Src0: Mul, Src1: ShiftAmt);
11021
11022 if (!Saturating)
11023 MIRBuilder.buildTrunc(Res: Dst, Op: Shift);
11024 else if (IsSigned)
11025 MIRBuilder.buildTruncSSatS(Res: Dst, Op: Shift);
11026 else
11027 MIRBuilder.buildTruncUSatU(Res: Dst, Op: Shift);
11028
11029 MI.eraseFromParent();
11030 return Legalized;
11031}
11032
11033// Get a vectorized representation of the memset value operand, GISel edition.
11034static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB) {
11035 MachineRegisterInfo &MRI = *MIB.getMRI();
11036 unsigned NumBits = Ty.getScalarSizeInBits();
11037 auto ValVRegAndVal = getIConstantVRegValWithLookThrough(VReg: Val, MRI);
11038 if (!Ty.isVector() && ValVRegAndVal) {
11039 APInt Scalar = ValVRegAndVal->Value.trunc(width: 8);
11040 APInt SplatVal = APInt::getSplat(NewLen: NumBits, V: Scalar);
11041 return MIB.buildConstant(Res: Ty, Val: SplatVal).getReg(Idx: 0);
11042 }
11043
11044 // Extend the byte value to the larger type, and then multiply by a magic
11045 // value 0x010101... in order to replicate it across every byte.
11046 // Unless it's zero, in which case just emit a larger G_CONSTANT 0.
11047 if (ValVRegAndVal && ValVRegAndVal->Value == 0) {
11048 return MIB.buildConstant(Res: Ty, Val: 0).getReg(Idx: 0);
11049 }
11050
11051 LLT ExtType = Ty.getScalarType();
11052 auto ZExt = MIB.buildZExtOrTrunc(Res: ExtType, Op: Val);
11053 if (NumBits > 8) {
11054 APInt Magic = APInt::getSplat(NewLen: NumBits, V: APInt(8, 0x01));
11055 auto MagicMI = MIB.buildConstant(Res: ExtType, Val: Magic);
11056 Val = MIB.buildMul(Dst: ExtType, Src0: ZExt, Src1: MagicMI).getReg(Idx: 0);
11057 }
11058
11059 // For vector types create a G_BUILD_VECTOR.
11060 if (Ty.isVector())
11061 Val = MIB.buildSplatBuildVector(Res: Ty, Src: Val).getReg(Idx: 0);
11062
11063 return Val;
11064}
11065
11066LegalizerHelper::LegalizeResult
11067LegalizerHelper::lowerMemset(MachineInstr &MI, Register Dst, Register Val,
11068 uint64_t KnownLen, Align Alignment,
11069 bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
11070 auto &MF = *MI.getParent()->getParent();
11071 const auto &TLI = *MF.getSubtarget().getTargetLowering();
11072 auto &DL = MF.getDataLayout();
11073 LLVMContext &C = MF.getFunction().getContext();
11074
11075 assert(KnownLen != 0 && "Have a zero length memset length!");
11076 assert(!MemOps.empty() && "Expected at least one memory op");
11077
11078 MachineFrameInfo &MFI = MF.getFrameInfo();
11079 MachineInstr *FIDef = getOpcodeDef(Opcode: TargetOpcode::G_FRAME_INDEX, Reg: Dst, MRI);
11080 const auto &DstMMO = **MI.memoperands_begin();
11081
11082 if (DstAlignCanChange) {
11083 // Get an estimate of the type from the LLT.
11084 Type *IRTy = getTypeForLLT(Ty: MemOps[0], C);
11085 Align NewAlign = DL.getABITypeAlign(Ty: IRTy);
11086 if (NewAlign > Alignment) {
11087 Alignment = NewAlign;
11088 unsigned FI = FIDef->getOperand(i: 1).getIndex();
11089 // Give the stack frame object a larger alignment if needed.
11090 if (MFI.getObjectAlign(ObjectIdx: FI) < Alignment)
11091 MFI.setObjectAlignment(ObjectIdx: FI, Alignment);
11092 }
11093 }
11094
11095 MachineIRBuilder MIB(MI);
11096 // Find the largest store and generate the bit pattern for it.
11097 LLT LargestTy = MemOps[0];
11098 for (unsigned i = 1; i < MemOps.size(); i++)
11099 if (MemOps[i].getSizeInBits() > LargestTy.getSizeInBits())
11100 LargestTy = MemOps[i];
11101
11102 // The memset stored value is always defined as an s8, so in order to make it
11103 // work with larger store types we need to repeat the bit pattern across the
11104 // wider type.
11105 Register MemSetValue = getMemsetValue(Val, Ty: LargestTy, MIB);
11106
11107 if (!MemSetValue)
11108 return UnableToLegalize;
11109
11110 // Generate the stores. For each store type in the list, we generate the
11111 // matching store of that type to the destination address.
11112 LLT PtrTy = MRI.getType(Reg: Dst);
11113 unsigned DstOff = 0;
11114 unsigned Size = KnownLen;
11115 for (unsigned I = 0; I < MemOps.size(); I++) {
11116 LLT Ty = MemOps[I];
11117 unsigned TySize = Ty.getSizeInBytes();
11118
11119 if (TySize > Size) {
11120 // Issuing a load / store pair that overlaps with the previous pair.
11121 // Adjust the offset accordingly.
11122 assert(I == MemOps.size() - 1 && I != 0);
11123 DstOff -= TySize - Size;
11124 }
11125
11126 // If this store is smaller than the largest store see whether we can get
11127 // the smaller value for free with a truncate.
11128 Register Value = MemSetValue;
11129 if (Ty.getSizeInBits() < LargestTy.getSizeInBits()) {
11130 MVT VT = getMVTForLLT(Ty);
11131 MVT LargestVT = getMVTForLLT(Ty: LargestTy);
11132 if (!LargestTy.isVector() && !Ty.isVector() &&
11133 TLI.isTruncateFree(FromVT: LargestVT, ToVT: VT))
11134 Value = MIB.buildTrunc(Res: Ty, Op: MemSetValue).getReg(Idx: 0);
11135 else
11136 Value = getMemsetValue(Val, Ty, MIB);
11137 if (!Value)
11138 return UnableToLegalize;
11139 }
11140
11141 auto *StoreMMO = MF.getMachineMemOperand(MMO: &DstMMO, Offset: DstOff, Ty);
11142
11143 Register Ptr = Dst;
11144 if (DstOff != 0) {
11145 auto Offset =
11146 MIB.buildConstant(Res: LLT::integer(SizeInBits: PtrTy.getSizeInBits()), Val: DstOff);
11147 Ptr = MIB.buildObjectPtrOffset(Res: PtrTy, Op0: Dst, Op1: Offset).getReg(Idx: 0);
11148 }
11149
11150 MIB.buildStore(Val: Value, Addr: Ptr, MMO&: *StoreMMO);
11151 DstOff += Ty.getSizeInBytes();
11152 Size -= TySize;
11153 }
11154
11155 MI.eraseFromParent();
11156 return Legalized;
11157}
11158
11159LegalizerHelper::LegalizeResult
11160LegalizerHelper::lowerMemcpy(MachineInstr &MI, Register Dst, Register Src,
11161 uint64_t KnownLen, Align Alignment,
11162 bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
11163 auto &MF = *MI.getParent()->getParent();
11164 auto &DL = MF.getDataLayout();
11165 LLVMContext &C = MF.getFunction().getContext();
11166
11167 assert(KnownLen != 0 && "Have a zero length memcpy length!");
11168 assert(!MemOps.empty() && "Expected at least one memory op");
11169
11170 MachineFrameInfo &MFI = MF.getFrameInfo();
11171 MachineInstr *FIDef = getOpcodeDef(Opcode: TargetOpcode::G_FRAME_INDEX, Reg: Dst, MRI);
11172
11173 // FIXME: infer better src pointer alignment like SelectionDAG does here.
11174 // FIXME: also use the equivalent of isMemSrcFromConstant and alwaysinlining
11175 // if the memcpy is in a tail call position.
11176
11177 const auto &DstMMO = **MI.memoperands_begin();
11178 const auto &SrcMMO = **std::next(x: MI.memoperands_begin());
11179
11180 if (DstAlignCanChange) {
11181 // Get an estimate of the type from the LLT.
11182 Type *IRTy = getTypeForLLT(Ty: MemOps[0], C);
11183 Align NewAlign = DL.getABITypeAlign(Ty: IRTy);
11184
11185 // Don't promote to an alignment that would require dynamic stack
11186 // realignment.
11187 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
11188 if (!TRI->hasStackRealignment(MF))
11189 if (MaybeAlign StackAlign = DL.getStackAlignment())
11190 NewAlign = std::min(a: NewAlign, b: *StackAlign);
11191
11192 if (NewAlign > Alignment) {
11193 Alignment = NewAlign;
11194 unsigned FI = FIDef->getOperand(i: 1).getIndex();
11195 // Give the stack frame object a larger alignment if needed.
11196 if (MFI.getObjectAlign(ObjectIdx: FI) < Alignment)
11197 MFI.setObjectAlignment(ObjectIdx: FI, Alignment);
11198 }
11199 }
11200
11201 LLVM_DEBUG(dbgs() << "Inlining memcpy: " << MI << " into loads & stores\n");
11202
11203 MachineIRBuilder MIB(MI);
11204 // Now we need to emit a pair of load and stores for each of the types we've
11205 // collected. I.e. for each type, generate a load from the source pointer of
11206 // that type width, and then generate a corresponding store to the dest buffer
11207 // of that value loaded. This can result in a sequence of loads and stores
11208 // mixed types, depending on what the target specifies as good types to use.
11209 unsigned CurrOffset = 0;
11210 unsigned Size = KnownLen;
11211 for (auto CopyTy : MemOps) {
11212 TypeSize TySize = CopyTy.getSizeInBytes();
11213
11214 // Issuing a load / store pair that overlaps with the previous pair. Adjust
11215 // the offset accordingly.
11216 if (TySize > Size) {
11217 unsigned Overlap = TySize - Size;
11218 assert(Overlap < CurrOffset &&
11219 "overlapping memcpy load/store spans the whole region or more");
11220 CurrOffset -= Overlap;
11221 }
11222
11223 // Construct MMOs for the accesses.
11224 auto *LoadMMO = MF.getMachineMemOperand(MMO: &SrcMMO, Offset: CurrOffset, Size: TySize);
11225 auto *StoreMMO = MF.getMachineMemOperand(MMO: &DstMMO, Offset: CurrOffset, Size: TySize);
11226
11227 // Create the load.
11228 Register LoadPtr = Src;
11229 Register Offset;
11230 if (CurrOffset != 0) {
11231 LLT SrcTy = MRI.getType(Reg: Src);
11232 Offset =
11233 MIB.buildConstant(Res: LLT::integer(SizeInBits: SrcTy.getSizeInBits()), Val: CurrOffset)
11234 .getReg(Idx: 0);
11235 LoadPtr = MIB.buildObjectPtrOffset(Res: SrcTy, Op0: Src, Op1: Offset).getReg(Idx: 0);
11236 }
11237 auto LdVal = MIB.buildLoad(Res: CopyTy, Addr: LoadPtr, MMO&: *LoadMMO);
11238
11239 // Create the store.
11240 Register StorePtr = Dst;
11241 if (CurrOffset != 0) {
11242 LLT DstTy = MRI.getType(Reg: Dst);
11243 StorePtr = MIB.buildObjectPtrOffset(Res: DstTy, Op0: Dst, Op1: Offset).getReg(Idx: 0);
11244 }
11245 MIB.buildStore(Val: LdVal, Addr: StorePtr, MMO&: *StoreMMO);
11246 CurrOffset += TySize;
11247 Size -= TySize;
11248 }
11249
11250 MI.eraseFromParent();
11251 return Legalized;
11252}
11253
11254LegalizerHelper::LegalizeResult
11255LegalizerHelper::lowerMemmove(MachineInstr &MI, Register Dst, Register Src,
11256 uint64_t KnownLen, Align Alignment,
11257 bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
11258 auto &MF = *MI.getParent()->getParent();
11259 auto &DL = MF.getDataLayout();
11260 LLVMContext &C = MF.getFunction().getContext();
11261
11262 assert(KnownLen != 0 && "Have a zero length memmove length!");
11263 assert(!MemOps.empty() && "Expected at least one memory op");
11264
11265 MachineFrameInfo &MFI = MF.getFrameInfo();
11266 MachineInstr *FIDef = getOpcodeDef(Opcode: TargetOpcode::G_FRAME_INDEX, Reg: Dst, MRI);
11267 const auto &DstMMO = **MI.memoperands_begin();
11268 const auto &SrcMMO = **std::next(x: MI.memoperands_begin());
11269
11270 if (DstAlignCanChange) {
11271 // Get an estimate of the type from the LLT.
11272 Type *IRTy = getTypeForLLT(Ty: MemOps[0], C);
11273 Align NewAlign = DL.getABITypeAlign(Ty: IRTy);
11274
11275 // Don't promote to an alignment that would require dynamic stack
11276 // realignment.
11277 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
11278 if (!TRI->hasStackRealignment(MF))
11279 if (MaybeAlign StackAlign = DL.getStackAlignment())
11280 NewAlign = std::min(a: NewAlign, b: *StackAlign);
11281
11282 if (NewAlign > Alignment) {
11283 Alignment = NewAlign;
11284 unsigned FI = FIDef->getOperand(i: 1).getIndex();
11285 // Give the stack frame object a larger alignment if needed.
11286 if (MFI.getObjectAlign(ObjectIdx: FI) < Alignment)
11287 MFI.setObjectAlignment(ObjectIdx: FI, Alignment);
11288 }
11289 }
11290
11291 LLVM_DEBUG(dbgs() << "Inlining memmove: " << MI << " into loads & stores\n");
11292
11293 MachineIRBuilder MIB(MI);
11294 // Memmove requires that we perform the loads first before issuing the stores.
11295 // Apart from that, this loop is pretty much doing the same thing as the
11296 // memcpy codegen function.
11297 unsigned CurrOffset = 0;
11298 unsigned Size = KnownLen;
11299 SmallVector<Register, 16> LoadVals;
11300 for (auto CopyTy : MemOps) {
11301 TypeSize TySize = CopyTy.getSizeInBytes();
11302
11303 // Issuing a load that overlaps with the previous load. Adjust the offset
11304 // accordingly.
11305 if (TySize > Size) {
11306 unsigned Overlap = TySize - Size;
11307 assert(Overlap < CurrOffset &&
11308 "overlapping memmove load spans the whole region or more");
11309 CurrOffset -= Overlap;
11310 }
11311
11312 // Construct MMO for the load.
11313 auto *LoadMMO = MF.getMachineMemOperand(MMO: &SrcMMO, Offset: CurrOffset, Size: TySize);
11314
11315 // Create the load.
11316 Register LoadPtr = Src;
11317 if (CurrOffset != 0) {
11318 LLT SrcTy = MRI.getType(Reg: Src);
11319 auto Offset =
11320 MIB.buildConstant(Res: LLT::integer(SizeInBits: SrcTy.getSizeInBits()), Val: CurrOffset);
11321 LoadPtr = MIB.buildObjectPtrOffset(Res: SrcTy, Op0: Src, Op1: Offset).getReg(Idx: 0);
11322 }
11323 LoadVals.push_back(Elt: MIB.buildLoad(Res: CopyTy, Addr: LoadPtr, MMO&: *LoadMMO).getReg(Idx: 0));
11324 CurrOffset += TySize;
11325 Size -= TySize;
11326 }
11327
11328 CurrOffset = 0;
11329 Size = KnownLen;
11330 for (unsigned I = 0; I < MemOps.size(); ++I) {
11331 LLT CopyTy = MemOps[I];
11332 TypeSize TySize = CopyTy.getSizeInBytes();
11333
11334 // Issuing a store that overlaps with the previous store. Adjust the offset
11335 // accordingly.
11336 if (TySize > Size) {
11337 unsigned Overlap = TySize - Size;
11338 assert(Overlap < CurrOffset &&
11339 "overlapping memmove store spans the whole region or more");
11340 CurrOffset -= Overlap;
11341 }
11342
11343 // Now store the values loaded.
11344 auto *StoreMMO = MF.getMachineMemOperand(MMO: &DstMMO, Offset: CurrOffset, Size: TySize);
11345
11346 Register StorePtr = Dst;
11347 if (CurrOffset != 0) {
11348 LLT DstTy = MRI.getType(Reg: Dst);
11349 auto Offset =
11350 MIB.buildConstant(Res: LLT::integer(SizeInBits: DstTy.getSizeInBits()), Val: CurrOffset);
11351 StorePtr = MIB.buildObjectPtrOffset(Res: DstTy, Op0: Dst, Op1: Offset).getReg(Idx: 0);
11352 }
11353 MIB.buildStore(Val: LoadVals[I], Addr: StorePtr, MMO&: *StoreMMO);
11354 CurrOffset += TySize;
11355 Size -= TySize;
11356 }
11357 MI.eraseFromParent();
11358 return Legalized;
11359}
11360
11361LegalizerHelper::LegalizeResult LegalizerHelper::lowerMemCpyFamily(
11362 MachineInstr &MI, Register Dst, Register Src, uint64_t KnownLen,
11363 Align Alignment, bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
11364 const unsigned Opc = MI.getOpcode();
11365 assert((Opc == TargetOpcode::G_MEMCPY ||
11366 Opc == TargetOpcode::G_MEMCPY_INLINE ||
11367 Opc == TargetOpcode::G_MEMMOVE || Opc == TargetOpcode::G_MEMSET ||
11368 Opc == TargetOpcode::G_MEMSET_INLINE) &&
11369 "Expected memcpy like instruction");
11370
11371 if (KnownLen == 0) {
11372 MI.eraseFromParent();
11373 return Legalized;
11374 }
11375
11376 if (Opc == TargetOpcode::G_MEMCPY || Opc == TargetOpcode::G_MEMCPY_INLINE) {
11377 return lowerMemcpy(MI, Dst, Src, KnownLen, Alignment, DstAlignCanChange,
11378 MemOps);
11379 }
11380 if (Opc == TargetOpcode::G_MEMMOVE)
11381 return lowerMemmove(MI, Dst, Src, KnownLen, Alignment, DstAlignCanChange,
11382 MemOps);
11383 if (Opc == TargetOpcode::G_MEMSET || Opc == TargetOpcode::G_MEMSET_INLINE)
11384 return lowerMemset(MI, Dst, Val: Src, KnownLen, Alignment, DstAlignCanChange,
11385 MemOps);
11386 return UnableToLegalize;
11387}
11388
11389LegalizerHelper::LegalizeResult
11390LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen) {
11391 Register Dst, Src;
11392 uint64_t KnownLen;
11393 Align Alignment;
11394 bool DstAlignCanChange;
11395 std::vector<LLT> MemOps;
11396 if (!canLowerMemCpyFamily(MI, MRI, MaxLen, Dst, Src, KnownLen, Alignment,
11397 DstAlignCanChange, MemOps))
11398 return UnableToLegalize;
11399 return lowerMemCpyFamily(MI, Dst, Src, KnownLen, Alignment, DstAlignCanChange,
11400 MemOps);
11401}
11402