1//===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file implements the LegalizerHelper class to legalize
10/// individual instructions and the LegalizeMachineIR wrapper pass for the
11/// primary legalization.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
16#include "llvm/CodeGen/GlobalISel/CallLowering.h"
17#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
18#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
19#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
20#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
21#include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h"
22#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
23#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24#include "llvm/CodeGen/GlobalISel/Utils.h"
25#include "llvm/CodeGen/LowLevelTypeUtils.h"
26#include "llvm/CodeGen/MachineConstantPool.h"
27#include "llvm/CodeGen/MachineFrameInfo.h"
28#include "llvm/CodeGen/MachineRegisterInfo.h"
29#include "llvm/CodeGen/RuntimeLibcallUtil.h"
30#include "llvm/CodeGen/TargetFrameLowering.h"
31#include "llvm/CodeGen/TargetInstrInfo.h"
32#include "llvm/CodeGen/TargetLowering.h"
33#include "llvm/CodeGen/TargetOpcodes.h"
34#include "llvm/CodeGen/TargetSubtargetInfo.h"
35#include "llvm/IR/Instructions.h"
36#include "llvm/Support/Debug.h"
37#include "llvm/Support/MathExtras.h"
38#include "llvm/Support/raw_ostream.h"
39#include "llvm/Target/TargetMachine.h"
40#include <cassert>
41#include <numeric>
42#include <optional>
43
44#define DEBUG_TYPE "legalizer"
45
46using namespace llvm;
47using namespace LegalizeActions;
48using namespace MIPatternMatch;
49
50/// Try to break down \p OrigTy into \p NarrowTy sized pieces.
51///
52/// Returns the number of \p NarrowTy elements needed to reconstruct \p OrigTy,
53/// with any leftover piece as type \p LeftoverTy
54///
55/// Returns -1 in the first element of the pair if the breakdown is not
56/// satisfiable.
57static std::pair<int, int>
58getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) {
59 assert(!LeftoverTy.isValid() && "this is an out argument");
60
61 unsigned Size = OrigTy.getSizeInBits();
62 unsigned NarrowSize = NarrowTy.getSizeInBits();
63 unsigned NumParts = Size / NarrowSize;
64 unsigned LeftoverSize = Size - NumParts * NarrowSize;
65 assert(Size > NarrowSize);
66
67 if (LeftoverSize == 0)
68 return {NumParts, 0};
69
70 if (NarrowTy.isVector()) {
71 unsigned EltSize = OrigTy.getScalarSizeInBits();
72 if (LeftoverSize % EltSize != 0)
73 return {-1, -1};
74 LeftoverTy = OrigTy.changeElementCount(
75 EC: ElementCount::getFixed(MinVal: LeftoverSize / EltSize));
76 } else {
77 LeftoverTy = LLT::integer(SizeInBits: LeftoverSize);
78 }
79
80 int NumLeftover = LeftoverSize / LeftoverTy.getSizeInBits();
81 return std::make_pair(x&: NumParts, y&: NumLeftover);
82}
83
84static Type *getFloatTypeForLLT(LLVMContext &Ctx, LLT Ty) {
85
86 if (!Ty.isScalar())
87 return nullptr;
88
89 switch (Ty.getSizeInBits()) {
90 case 16:
91 return Type::getHalfTy(C&: Ctx);
92 case 32:
93 return Type::getFloatTy(C&: Ctx);
94 case 64:
95 return Type::getDoubleTy(C&: Ctx);
96 case 80:
97 return Type::getX86_FP80Ty(C&: Ctx);
98 case 128:
99 return Type::getFP128Ty(C&: Ctx);
100 default:
101 return nullptr;
102 }
103}
104
105LegalizerHelper::LegalizerHelper(MachineFunction &MF,
106 GISelChangeObserver &Observer,
107 MachineIRBuilder &Builder,
108 const LibcallLoweringInfo *Libcalls)
109 : MIRBuilder(Builder), Observer(Observer), MRI(MF.getRegInfo()),
110 LI(*MF.getSubtarget().getLegalizerInfo()),
111 TLI(*MF.getSubtarget().getTargetLowering()), Libcalls(Libcalls) {}
112
113LegalizerHelper::LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI,
114 GISelChangeObserver &Observer,
115 MachineIRBuilder &B,
116 const LibcallLoweringInfo *Libcalls,
117 GISelValueTracking *VT)
118 : MIRBuilder(B), Observer(Observer), MRI(MF.getRegInfo()), LI(LI),
119 TLI(*MF.getSubtarget().getTargetLowering()), Libcalls(Libcalls), VT(VT) {}
120
121LegalizerHelper::LegalizeResult
122LegalizerHelper::legalizeInstrStep(MachineInstr &MI,
123 LostDebugLocObserver &LocObserver) {
124 LLVM_DEBUG(dbgs() << "\nLegalizing: " << MI);
125
126 MIRBuilder.setInstrAndDebugLoc(MI);
127
128 if (isa<GIntrinsic>(Val: MI))
129 return LI.legalizeIntrinsic(Helper&: *this, MI) ? Legalized : UnableToLegalize;
130 auto Step = LI.getAction(MI, MRI);
131 switch (Step.Action) {
132 case Legal:
133 LLVM_DEBUG(dbgs() << ".. Already legal\n");
134 return AlreadyLegal;
135 case Libcall:
136 LLVM_DEBUG(dbgs() << ".. Convert to libcall\n");
137 return libcall(MI, LocObserver);
138 case NarrowScalar:
139 LLVM_DEBUG(dbgs() << ".. Narrow scalar\n");
140 return narrowScalar(MI, TypeIdx: Step.TypeIdx, NarrowTy: Step.NewType);
141 case WidenScalar:
142 LLVM_DEBUG(dbgs() << ".. Widen scalar\n");
143 return widenScalar(MI, TypeIdx: Step.TypeIdx, WideTy: Step.NewType);
144 case Bitcast:
145 LLVM_DEBUG(dbgs() << ".. Bitcast type\n");
146 return bitcast(MI, TypeIdx: Step.TypeIdx, Ty: Step.NewType);
147 case Lower:
148 LLVM_DEBUG(dbgs() << ".. Lower\n");
149 return lower(MI, TypeIdx: Step.TypeIdx, Ty: Step.NewType);
150 case FewerElements:
151 LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n");
152 return fewerElementsVector(MI, TypeIdx: Step.TypeIdx, NarrowTy: Step.NewType);
153 case MoreElements:
154 LLVM_DEBUG(dbgs() << ".. Increase number of elements\n");
155 return moreElementsVector(MI, TypeIdx: Step.TypeIdx, MoreTy: Step.NewType);
156 case Custom:
157 LLVM_DEBUG(dbgs() << ".. Custom legalization\n");
158 return LI.legalizeCustom(Helper&: *this, MI, LocObserver) ? Legalized
159 : UnableToLegalize;
160 default:
161 LLVM_DEBUG(dbgs() << ".. Unable to legalize\n");
162 return UnableToLegalize;
163 }
164}
165
166void LegalizerHelper::insertParts(Register DstReg,
167 LLT ResultTy, LLT PartTy,
168 ArrayRef<Register> PartRegs,
169 LLT LeftoverTy,
170 ArrayRef<Register> LeftoverRegs) {
171 if (!LeftoverTy.isValid()) {
172 assert(LeftoverRegs.empty());
173
174 if (!ResultTy.isVector()) {
175 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: PartRegs);
176 return;
177 }
178
179 if (PartTy.isVector())
180 MIRBuilder.buildConcatVectors(Res: DstReg, Ops: PartRegs);
181 else
182 MIRBuilder.buildBuildVector(Res: DstReg, Ops: PartRegs);
183 return;
184 }
185
186 // Merge sub-vectors with different number of elements and insert into DstReg.
187 if (ResultTy.isVector()) {
188 assert(LeftoverRegs.size() == 1 && "Expected one leftover register");
189 SmallVector<Register, 8> AllRegs(PartRegs);
190 AllRegs.append(in_start: LeftoverRegs.begin(), in_end: LeftoverRegs.end());
191 return mergeMixedSubvectors(DstReg, PartRegs: AllRegs);
192 }
193
194 SmallVector<Register> GCDRegs;
195 LLT GCDTy = getGCDType(OrigTy: getGCDType(OrigTy: ResultTy, TargetTy: LeftoverTy), TargetTy: PartTy);
196 for (auto PartReg : concat<const Register>(Ranges&: PartRegs, Ranges&: LeftoverRegs))
197 extractGCDType(Parts&: GCDRegs, GCDTy, SrcReg: PartReg);
198 LLT ResultLCMTy = buildLCMMergePieces(DstTy: ResultTy, NarrowTy: LeftoverTy, GCDTy, VRegs&: GCDRegs);
199 buildWidenedRemergeToDst(DstReg, LCMTy: ResultLCMTy, RemergeRegs: GCDRegs);
200}
201
202void LegalizerHelper::appendVectorElts(SmallVectorImpl<Register> &Elts,
203 Register Reg) {
204 LLT Ty = MRI.getType(Reg);
205 SmallVector<Register, 8> RegElts;
206 extractParts(Reg, Ty: Ty.getScalarType(), NumParts: Ty.getNumElements(), VRegs&: RegElts,
207 MIRBuilder, MRI);
208 Elts.append(RHS: RegElts);
209}
210
211/// Merge \p PartRegs with different types into \p DstReg.
212void LegalizerHelper::mergeMixedSubvectors(Register DstReg,
213 ArrayRef<Register> PartRegs) {
214 SmallVector<Register, 8> AllElts;
215 for (unsigned i = 0; i < PartRegs.size() - 1; ++i)
216 appendVectorElts(Elts&: AllElts, Reg: PartRegs[i]);
217
218 Register Leftover = PartRegs[PartRegs.size() - 1];
219 if (!MRI.getType(Reg: Leftover).isVector())
220 AllElts.push_back(Elt: Leftover);
221 else
222 appendVectorElts(Elts&: AllElts, Reg: Leftover);
223
224 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: AllElts);
225}
226
227/// Append the result registers of G_UNMERGE_VALUES \p MI to \p Regs.
228static void getUnmergeResults(SmallVectorImpl<Register> &Regs,
229 const MachineInstr &MI) {
230 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
231
232 const int StartIdx = Regs.size();
233 const int NumResults = MI.getNumOperands() - 1;
234 Regs.resize(N: Regs.size() + NumResults);
235 for (int I = 0; I != NumResults; ++I)
236 Regs[StartIdx + I] = MI.getOperand(i: I).getReg();
237}
238
239void LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts,
240 LLT GCDTy, Register SrcReg) {
241 LLT SrcTy = MRI.getType(Reg: SrcReg);
242 if (SrcTy == GCDTy) {
243 // If the source already evenly divides the result type, we don't need to do
244 // anything.
245 Parts.push_back(Elt: SrcReg);
246 } else {
247 // Need to split into common type sized pieces.
248 auto Unmerge = MIRBuilder.buildUnmerge(Res: GCDTy, Op: SrcReg);
249 getUnmergeResults(Regs&: Parts, MI: *Unmerge);
250 }
251}
252
253LLT LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts, LLT DstTy,
254 LLT NarrowTy, Register SrcReg) {
255 LLT SrcTy = MRI.getType(Reg: SrcReg);
256 LLT GCDTy = getGCDType(OrigTy: getGCDType(OrigTy: SrcTy, TargetTy: NarrowTy), TargetTy: DstTy);
257 extractGCDType(Parts, GCDTy, SrcReg);
258 return GCDTy;
259}
260
261LLT LegalizerHelper::buildLCMMergePieces(LLT DstTy, LLT NarrowTy, LLT GCDTy,
262 SmallVectorImpl<Register> &VRegs,
263 unsigned PadStrategy) {
264 LLT LCMTy = getLCMType(OrigTy: DstTy, TargetTy: NarrowTy);
265
266 int NumParts = LCMTy.getSizeInBits() / NarrowTy.getSizeInBits();
267 int NumSubParts = NarrowTy.getSizeInBits() / GCDTy.getSizeInBits();
268 int NumOrigSrc = VRegs.size();
269
270 Register PadReg;
271
272 // Get a value we can use to pad the source value if the sources won't evenly
273 // cover the result type.
274 if (NumOrigSrc < NumParts * NumSubParts) {
275 if (PadStrategy == TargetOpcode::G_ZEXT)
276 PadReg = MIRBuilder.buildConstant(Res: GCDTy, Val: 0).getReg(Idx: 0);
277 else if (PadStrategy == TargetOpcode::G_ANYEXT)
278 PadReg = MIRBuilder.buildUndef(Res: GCDTy).getReg(Idx: 0);
279 else {
280 assert(PadStrategy == TargetOpcode::G_SEXT);
281
282 // Shift the sign bit of the low register through the high register.
283 auto ShiftAmt =
284 MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: GCDTy.getSizeInBits() - 1);
285 PadReg = MIRBuilder.buildAShr(Dst: GCDTy, Src0: VRegs.back(), Src1: ShiftAmt).getReg(Idx: 0);
286 }
287 }
288
289 // Registers for the final merge to be produced.
290 SmallVector<Register, 4> Remerge(NumParts);
291
292 // Registers needed for intermediate merges, which will be merged into a
293 // source for Remerge.
294 SmallVector<Register, 4> SubMerge(NumSubParts);
295
296 // Once we've fully read off the end of the original source bits, we can reuse
297 // the same high bits for remaining padding elements.
298 Register AllPadReg;
299
300 // Build merges to the LCM type to cover the original result type.
301 for (int I = 0; I != NumParts; ++I) {
302 bool AllMergePartsArePadding = true;
303
304 // Build the requested merges to the requested type.
305 for (int J = 0; J != NumSubParts; ++J) {
306 int Idx = I * NumSubParts + J;
307 if (Idx >= NumOrigSrc) {
308 SubMerge[J] = PadReg;
309 continue;
310 }
311
312 SubMerge[J] = VRegs[Idx];
313
314 // There are meaningful bits here we can't reuse later.
315 AllMergePartsArePadding = false;
316 }
317
318 // If we've filled up a complete piece with padding bits, we can directly
319 // emit the natural sized constant if applicable, rather than a merge of
320 // smaller constants.
321 if (AllMergePartsArePadding && !AllPadReg) {
322 if (PadStrategy == TargetOpcode::G_ANYEXT)
323 AllPadReg = MIRBuilder.buildUndef(Res: NarrowTy).getReg(Idx: 0);
324 else if (PadStrategy == TargetOpcode::G_ZEXT)
325 AllPadReg = MIRBuilder.buildConstant(Res: NarrowTy, Val: 0).getReg(Idx: 0);
326
327 // If this is a sign extension, we can't materialize a trivial constant
328 // with the right type and have to produce a merge.
329 }
330
331 if (AllPadReg) {
332 // Avoid creating additional instructions if we're just adding additional
333 // copies of padding bits.
334 Remerge[I] = AllPadReg;
335 continue;
336 }
337
338 if (NumSubParts == 1)
339 Remerge[I] = SubMerge[0];
340 else
341 Remerge[I] = MIRBuilder.buildMergeLikeInstr(Res: NarrowTy, Ops: SubMerge).getReg(Idx: 0);
342
343 // In the sign extend padding case, re-use the first all-signbit merge.
344 if (AllMergePartsArePadding && !AllPadReg)
345 AllPadReg = Remerge[I];
346 }
347
348 VRegs = std::move(Remerge);
349 return LCMTy;
350}
351
352void LegalizerHelper::buildWidenedRemergeToDst(Register DstReg, LLT LCMTy,
353 ArrayRef<Register> RemergeRegs) {
354 LLT DstTy = MRI.getType(Reg: DstReg);
355
356 // Create the merge to the widened source, and extract the relevant bits into
357 // the result.
358
359 if (DstTy == LCMTy) {
360 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: RemergeRegs);
361 return;
362 }
363
364 auto Remerge = MIRBuilder.buildMergeLikeInstr(Res: LCMTy, Ops: RemergeRegs);
365 if (DstTy.isScalar() && LCMTy.isScalar()) {
366 MIRBuilder.buildTrunc(Res: DstReg, Op: Remerge);
367 return;
368 }
369
370 if (LCMTy.isVector()) {
371 unsigned NumDefs = LCMTy.getSizeInBits() / DstTy.getSizeInBits();
372 SmallVector<Register, 8> UnmergeDefs(NumDefs);
373 UnmergeDefs[0] = DstReg;
374 for (unsigned I = 1; I != NumDefs; ++I)
375 UnmergeDefs[I] = MRI.createGenericVirtualRegister(Ty: DstTy);
376
377 MIRBuilder.buildUnmerge(Res: UnmergeDefs,
378 Op: MIRBuilder.buildMergeLikeInstr(Res: LCMTy, Ops: RemergeRegs));
379 return;
380 }
381
382 llvm_unreachable("unhandled case");
383}
384
385static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
386#define RTLIBCASE_INT(LibcallPrefix) \
387 do { \
388 switch (Size) { \
389 case 32: \
390 return RTLIB::LibcallPrefix##32; \
391 case 64: \
392 return RTLIB::LibcallPrefix##64; \
393 case 128: \
394 return RTLIB::LibcallPrefix##128; \
395 default: \
396 llvm_unreachable("unexpected size"); \
397 } \
398 } while (0)
399
400#define RTLIBCASE(LibcallPrefix) \
401 do { \
402 switch (Size) { \
403 case 32: \
404 return RTLIB::LibcallPrefix##32; \
405 case 64: \
406 return RTLIB::LibcallPrefix##64; \
407 case 80: \
408 return RTLIB::LibcallPrefix##80; \
409 case 128: \
410 return RTLIB::LibcallPrefix##128; \
411 default: \
412 llvm_unreachable("unexpected size"); \
413 } \
414 } while (0)
415
416 switch (Opcode) {
417 case TargetOpcode::G_LROUND:
418 RTLIBCASE(LROUND_F);
419 case TargetOpcode::G_LLROUND:
420 RTLIBCASE(LLROUND_F);
421 case TargetOpcode::G_MUL:
422 RTLIBCASE_INT(MUL_I);
423 case TargetOpcode::G_SDIV:
424 RTLIBCASE_INT(SDIV_I);
425 case TargetOpcode::G_UDIV:
426 RTLIBCASE_INT(UDIV_I);
427 case TargetOpcode::G_SREM:
428 RTLIBCASE_INT(SREM_I);
429 case TargetOpcode::G_UREM:
430 RTLIBCASE_INT(UREM_I);
431 case TargetOpcode::G_CTLZ_ZERO_POISON:
432 RTLIBCASE_INT(CTLZ_I);
433 case TargetOpcode::G_FADD:
434 RTLIBCASE(ADD_F);
435 case TargetOpcode::G_FSUB:
436 RTLIBCASE(SUB_F);
437 case TargetOpcode::G_FMUL:
438 RTLIBCASE(MUL_F);
439 case TargetOpcode::G_FDIV:
440 RTLIBCASE(DIV_F);
441 case TargetOpcode::G_FEXP:
442 RTLIBCASE(EXP_F);
443 case TargetOpcode::G_FEXP2:
444 RTLIBCASE(EXP2_F);
445 case TargetOpcode::G_FEXP10:
446 RTLIBCASE(EXP10_F);
447 case TargetOpcode::G_FREM:
448 RTLIBCASE(REM_F);
449 case TargetOpcode::G_FPOW:
450 RTLIBCASE(POW_F);
451 case TargetOpcode::G_FPOWI:
452 RTLIBCASE(POWI_F);
453 case TargetOpcode::G_FMA:
454 RTLIBCASE(FMA_F);
455 case TargetOpcode::G_FSIN:
456 RTLIBCASE(SIN_F);
457 case TargetOpcode::G_FCOS:
458 RTLIBCASE(COS_F);
459 case TargetOpcode::G_FTAN:
460 RTLIBCASE(TAN_F);
461 case TargetOpcode::G_FASIN:
462 RTLIBCASE(ASIN_F);
463 case TargetOpcode::G_FACOS:
464 RTLIBCASE(ACOS_F);
465 case TargetOpcode::G_FATAN:
466 RTLIBCASE(ATAN_F);
467 case TargetOpcode::G_FATAN2:
468 RTLIBCASE(ATAN2_F);
469 case TargetOpcode::G_FSINH:
470 RTLIBCASE(SINH_F);
471 case TargetOpcode::G_FCOSH:
472 RTLIBCASE(COSH_F);
473 case TargetOpcode::G_FTANH:
474 RTLIBCASE(TANH_F);
475 case TargetOpcode::G_FSINCOS:
476 RTLIBCASE(SINCOS_F);
477 case TargetOpcode::G_FMODF:
478 RTLIBCASE(MODF_F);
479 case TargetOpcode::G_FLOG10:
480 RTLIBCASE(LOG10_F);
481 case TargetOpcode::G_FLOG:
482 RTLIBCASE(LOG_F);
483 case TargetOpcode::G_FLOG2:
484 RTLIBCASE(LOG2_F);
485 case TargetOpcode::G_FLDEXP:
486 RTLIBCASE(LDEXP_F);
487 case TargetOpcode::G_FCEIL:
488 RTLIBCASE(CEIL_F);
489 case TargetOpcode::G_FFLOOR:
490 RTLIBCASE(FLOOR_F);
491 case TargetOpcode::G_FMINNUM:
492 RTLIBCASE(FMIN_F);
493 case TargetOpcode::G_FMAXNUM:
494 RTLIBCASE(FMAX_F);
495 case TargetOpcode::G_FMINIMUMNUM:
496 RTLIBCASE(FMINIMUM_NUM_F);
497 case TargetOpcode::G_FMAXIMUMNUM:
498 RTLIBCASE(FMAXIMUM_NUM_F);
499 case TargetOpcode::G_FSQRT:
500 RTLIBCASE(SQRT_F);
501 case TargetOpcode::G_FRINT:
502 RTLIBCASE(RINT_F);
503 case TargetOpcode::G_FNEARBYINT:
504 RTLIBCASE(NEARBYINT_F);
505 case TargetOpcode::G_INTRINSIC_TRUNC:
506 RTLIBCASE(TRUNC_F);
507 case TargetOpcode::G_INTRINSIC_ROUND:
508 RTLIBCASE(ROUND_F);
509 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
510 RTLIBCASE(ROUNDEVEN_F);
511 case TargetOpcode::G_INTRINSIC_LRINT:
512 RTLIBCASE(LRINT_F);
513 case TargetOpcode::G_INTRINSIC_LLRINT:
514 RTLIBCASE(LLRINT_F);
515 }
516 llvm_unreachable("Unknown libcall function");
517#undef RTLIBCASE_INT
518#undef RTLIBCASE
519}
520
521static bool hasSwiftErrorArg(MachineFunction &MF) {
522 const TargetLowering &TLI = *MF.getSubtarget().getTargetLowering();
523 return TLI.supportSwiftError() &&
524 MF.getFunction().getAttributes().hasAttrSomewhere(
525 Kind: Attribute::SwiftError);
526}
527
528/// True if an instruction is in tail position in its caller. Intended for
529/// legalizing libcalls as tail calls when possible.
530static bool isLibCallInTailPosition(const CallLowering::ArgInfo &Result,
531 MachineInstr &MI,
532 const TargetInstrInfo &TII,
533 MachineRegisterInfo &MRI) {
534 MachineBasicBlock &MBB = *MI.getParent();
535 const Function &F = MBB.getParent()->getFunction();
536
537 // Conservatively require the attributes of the call to match those of
538 // the return. Ignore NoAlias and NonNull because they don't affect the
539 // call sequence.
540 AttributeList CallerAttrs = F.getAttributes();
541 if (AttrBuilder(F.getContext(), CallerAttrs.getRetAttrs())
542 .removeAttribute(Val: Attribute::NoAlias)
543 .removeAttribute(Val: Attribute::NonNull)
544 .hasAttributes())
545 return false;
546
547 // It's not safe to eliminate the sign / zero extension of the return value.
548 if (CallerAttrs.hasRetAttr(Kind: Attribute::ZExt) ||
549 CallerAttrs.hasRetAttr(Kind: Attribute::SExt))
550 return false;
551
552 // Only tail call if the following instruction is a standard return or if we
553 // have a `thisreturn` callee, and a sequence like:
554 //
555 // G_MEMCPY %0, %1, %2
556 // $x0 = COPY %0
557 // RET_ReallyLR implicit $x0
558 auto Next = next_nodbg(It: MI.getIterator(), End: MBB.instr_end());
559 if (Next != MBB.instr_end() && Next->isCopy()) {
560 if (MI.getOpcode() == TargetOpcode::G_BZERO)
561 return false;
562
563 // For MEMCPY/MOMMOVE/MEMSET these will be the first use (the dst), as the
564 // mempy/etc routines return the same parameter. For other it will be the
565 // returned value.
566 Register VReg = MI.getOperand(i: 0).getReg();
567 if (!VReg.isVirtual() || VReg != Next->getOperand(i: 1).getReg())
568 return false;
569
570 Register PReg = Next->getOperand(i: 0).getReg();
571 if (!PReg.isPhysical())
572 return false;
573
574 auto Ret = next_nodbg(It: Next, End: MBB.instr_end());
575 if (Ret == MBB.instr_end() || !Ret->isReturn())
576 return false;
577
578 if (Ret->getNumImplicitOperands() != 1)
579 return false;
580
581 if (!Ret->getOperand(i: 0).isReg() || PReg != Ret->getOperand(i: 0).getReg())
582 return false;
583
584 // Skip over the COPY that we just validated.
585 Next = Ret;
586 }
587
588 if (Next == MBB.instr_end() || TII.isTailCall(Inst: *Next) || !Next->isReturn())
589 return false;
590
591 return true;
592}
593
594LegalizerHelper::LegalizeResult LegalizerHelper::createLibcall(
595 const char *Name, const CallLowering::ArgInfo &Result,
596 ArrayRef<CallLowering::ArgInfo> Args, const CallingConv::ID CC,
597 LostDebugLocObserver &LocObserver, MachineInstr *MI) const {
598 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
599
600 CallLowering::CallLoweringInfo Info;
601 Info.CallConv = CC;
602 Info.Callee = MachineOperand::CreateES(SymName: Name);
603 Info.OrigRet = Result;
604 if (MI)
605 Info.IsTailCall =
606 (Result.Ty->isVoidTy() ||
607 Result.Ty == MIRBuilder.getMF().getFunction().getReturnType()) &&
608 isLibCallInTailPosition(Result, MI&: *MI, TII: MIRBuilder.getTII(),
609 MRI&: *MIRBuilder.getMRI()) &&
610 // Lowering doesn't support tail calling inside a function with
611 // a swifterror argument yet.
612 !hasSwiftErrorArg(MF&: MIRBuilder.getMF());
613
614 llvm::append_range(C&: Info.OrigArgs, R&: Args);
615 if (!CLI.lowerCall(MIRBuilder, Info))
616 return LegalizerHelper::UnableToLegalize;
617
618 if (MI && Info.LoweredTailCall) {
619 assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?");
620
621 // Check debug locations before removing the return.
622 LocObserver.checkpoint(CheckDebugLocs: true);
623
624 // We must have a return following the call (or debug insts) to get past
625 // isLibCallInTailPosition.
626 do {
627 MachineInstr *Next = MI->getNextNode();
628 assert(Next &&
629 (Next->isCopy() || Next->isReturn() || Next->isDebugInstr()) &&
630 "Expected instr following MI to be return or debug inst?");
631 // We lowered a tail call, so the call is now the return from the block.
632 // Delete the old return.
633 Next->eraseFromParent();
634 } while (MI->getNextNode());
635
636 // We expect to lose the debug location from the return.
637 LocObserver.checkpoint(CheckDebugLocs: false);
638 }
639 return LegalizerHelper::Legalized;
640}
641
642LegalizerHelper::LegalizeResult LegalizerHelper::createLibcall(
643 RTLIB::Libcall Libcall, const CallLowering::ArgInfo &Result,
644 ArrayRef<CallLowering::ArgInfo> Args, LostDebugLocObserver &LocObserver,
645 MachineInstr *MI) const {
646 if (!Libcalls)
647 return LegalizerHelper::UnableToLegalize;
648
649 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: Libcall);
650 if (LibcallImpl == RTLIB::Unsupported)
651 return LegalizerHelper::UnableToLegalize;
652
653 StringRef Name = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: LibcallImpl);
654 const CallingConv::ID CC = Libcalls->getLibcallImplCallingConv(Call: LibcallImpl);
655 return createLibcall(Name: Name.data(), Result, Args, CC, LocObserver, MI);
656}
657
658// Useful for libcalls where all operands have the same type.
659LegalizerHelper::LegalizeResult
660LegalizerHelper::simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder,
661 unsigned Size, Type *OpType,
662 LostDebugLocObserver &LocObserver) const {
663 auto Libcall = getRTLibDesc(Opcode: MI.getOpcode(), Size);
664
665 // FIXME: What does the original arg index mean here?
666 SmallVector<CallLowering::ArgInfo, 3> Args;
667 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands()))
668 Args.push_back(Elt: {MO.getReg(), OpType, 0});
669 return createLibcall(Libcall, Result: {MI.getOperand(i: 0).getReg(), OpType, 0}, Args,
670 LocObserver, MI: &MI);
671}
672
673LegalizerHelper::LegalizeResult LegalizerHelper::emitSincosLibcall(
674 MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size, Type *OpType,
675 LostDebugLocObserver &LocObserver) {
676 MachineFunction &MF = *MI.getMF();
677 MachineRegisterInfo &MRI = MF.getRegInfo();
678
679 Register DstSin = MI.getOperand(i: 0).getReg();
680 Register DstCos = MI.getOperand(i: 1).getReg();
681 Register Src = MI.getOperand(i: 2).getReg();
682 LLT DstTy = MRI.getType(Reg: DstSin);
683
684 int MemSize = DstTy.getSizeInBytes();
685 Align Alignment = getStackTemporaryAlignment(Type: DstTy);
686 const DataLayout &DL = MIRBuilder.getDataLayout();
687 unsigned AddrSpace = DL.getAllocaAddrSpace();
688 MachinePointerInfo PtrInfo;
689
690 Register StackPtrSin =
691 createStackTemporary(Bytes: TypeSize::getFixed(ExactSize: MemSize), Alignment, PtrInfo)
692 .getReg(Idx: 0);
693 Register StackPtrCos =
694 createStackTemporary(Bytes: TypeSize::getFixed(ExactSize: MemSize), Alignment, PtrInfo)
695 .getReg(Idx: 0);
696
697 auto &Ctx = MF.getFunction().getContext();
698 auto LibcallResult = createLibcall(
699 Libcall: getRTLibDesc(Opcode: MI.getOpcode(), Size), Result: {{0}, Type::getVoidTy(C&: Ctx), 0},
700 Args: {{Src, OpType, 0},
701 {StackPtrSin, PointerType::get(C&: Ctx, AddressSpace: AddrSpace), 1},
702 {StackPtrCos, PointerType::get(C&: Ctx, AddressSpace: AddrSpace), 2}},
703 LocObserver, MI: &MI);
704
705 if (LibcallResult != LegalizeResult::Legalized)
706 return LegalizerHelper::UnableToLegalize;
707
708 MachineMemOperand *LoadMMOSin = MF.getMachineMemOperand(
709 PtrInfo, F: MachineMemOperand::MOLoad, Size: MemSize, BaseAlignment: Alignment);
710 MachineMemOperand *LoadMMOCos = MF.getMachineMemOperand(
711 PtrInfo, F: MachineMemOperand::MOLoad, Size: MemSize, BaseAlignment: Alignment);
712
713 MIRBuilder.buildLoad(Res: DstSin, Addr: StackPtrSin, MMO&: *LoadMMOSin);
714 MIRBuilder.buildLoad(Res: DstCos, Addr: StackPtrCos, MMO&: *LoadMMOCos);
715 MI.eraseFromParent();
716
717 return LegalizerHelper::Legalized;
718}
719
720LegalizerHelper::LegalizeResult
721LegalizerHelper::emitModfLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder,
722 unsigned Size, Type *OpType,
723 LostDebugLocObserver &LocObserver) {
724 MachineFunction &MF = MIRBuilder.getMF();
725 MachineRegisterInfo &MRI = MF.getRegInfo();
726
727 Register DstFrac = MI.getOperand(i: 0).getReg();
728 Register DstInt = MI.getOperand(i: 1).getReg();
729 Register Src = MI.getOperand(i: 2).getReg();
730 LLT DstTy = MRI.getType(Reg: DstFrac);
731
732 int MemSize = DstTy.getSizeInBytes();
733 Align Alignment = getStackTemporaryAlignment(Type: DstTy);
734 const DataLayout &DL = MIRBuilder.getDataLayout();
735 unsigned AddrSpace = DL.getAllocaAddrSpace();
736 MachinePointerInfo PtrInfo;
737
738 Register StackPtrInt =
739 createStackTemporary(Bytes: TypeSize::getFixed(ExactSize: MemSize), Alignment, PtrInfo)
740 .getReg(Idx: 0);
741
742 auto &Ctx = MF.getFunction().getContext();
743 auto LibcallResult = createLibcall(
744 Libcall: getRTLibDesc(Opcode: MI.getOpcode(), Size), Result: {DstFrac, OpType, 0},
745 Args: {{Src, OpType, 0}, {StackPtrInt, PointerType::get(C&: Ctx, AddressSpace: AddrSpace), 1}},
746 LocObserver, MI: &MI);
747
748 if (LibcallResult != LegalizeResult::Legalized)
749 return LegalizerHelper::UnableToLegalize;
750
751 MachineMemOperand *LoadMMOInt = MF.getMachineMemOperand(
752 PtrInfo, F: MachineMemOperand::MOLoad, Size: MemSize, BaseAlignment: Alignment);
753
754 MIRBuilder.buildLoad(Res: DstInt, Addr: StackPtrInt, MMO&: *LoadMMOInt);
755 MI.eraseFromParent();
756
757 return LegalizerHelper::Legalized;
758}
759
760static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
761 Type *FromType) {
762 auto ToMVT = MVT::getVT(Ty: ToType);
763 auto FromMVT = MVT::getVT(Ty: FromType);
764
765 switch (Opcode) {
766 case TargetOpcode::G_FPEXT:
767 return RTLIB::getFPEXT(OpVT: FromMVT, RetVT: ToMVT);
768 case TargetOpcode::G_FPTRUNC:
769 return RTLIB::getFPROUND(OpVT: FromMVT, RetVT: ToMVT);
770 case TargetOpcode::G_FPTOSI:
771 return RTLIB::getFPTOSINT(OpVT: FromMVT, RetVT: ToMVT);
772 case TargetOpcode::G_FPTOUI:
773 return RTLIB::getFPTOUINT(OpVT: FromMVT, RetVT: ToMVT);
774 case TargetOpcode::G_SITOFP:
775 return RTLIB::getSINTTOFP(OpVT: FromMVT, RetVT: ToMVT);
776 case TargetOpcode::G_UITOFP:
777 return RTLIB::getUINTTOFP(OpVT: FromMVT, RetVT: ToMVT);
778 }
779 llvm_unreachable("Unsupported libcall function");
780}
781
782LegalizerHelper::LegalizeResult LegalizerHelper::conversionLibcall(
783 MachineInstr &MI, Type *ToType, Type *FromType,
784 LostDebugLocObserver &LocObserver, bool IsSigned) const {
785 CallLowering::ArgInfo Arg = {MI.getOperand(i: 1).getReg(), FromType, 0};
786 if (FromType->isIntegerTy()) {
787 if (TLI.shouldSignExtendTypeInLibCall(Ty: FromType, IsSigned))
788 Arg.Flags[0].setSExt();
789 else
790 Arg.Flags[0].setZExt();
791 }
792
793 RTLIB::Libcall Libcall = getConvRTLibDesc(Opcode: MI.getOpcode(), ToType, FromType);
794 return createLibcall(Libcall, Result: {MI.getOperand(i: 0).getReg(), ToType, 0}, Args: Arg,
795 LocObserver, MI: &MI);
796}
797
798LegalizerHelper::LegalizeResult
799LegalizerHelper::createMemLibcall(MachineRegisterInfo &MRI, MachineInstr &MI,
800 LostDebugLocObserver &LocObserver) const {
801 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
802
803 SmallVector<CallLowering::ArgInfo, 3> Args;
804 // Add all the args, except for the last which is an imm denoting 'tail'.
805 for (unsigned i = 0; i < MI.getNumOperands() - 1; ++i) {
806 Register Reg = MI.getOperand(i).getReg();
807
808 // Need derive an IR type for call lowering.
809 LLT OpLLT = MRI.getType(Reg);
810 Type *OpTy = nullptr;
811 if (OpLLT.isPointer())
812 OpTy = PointerType::get(C&: Ctx, AddressSpace: OpLLT.getAddressSpace());
813 else
814 OpTy = IntegerType::get(C&: Ctx, NumBits: OpLLT.getSizeInBits());
815 Args.push_back(Elt: {Reg, OpTy, 0});
816 }
817
818 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
819 RTLIB::Libcall RTLibcall;
820 unsigned Opc = MI.getOpcode();
821 switch (Opc) {
822 case TargetOpcode::G_BZERO:
823 RTLibcall = RTLIB::BZERO;
824 break;
825 case TargetOpcode::G_MEMCPY:
826 RTLibcall = RTLIB::MEMCPY;
827 Args[0].Flags[0].setReturned();
828 break;
829 case TargetOpcode::G_MEMMOVE:
830 RTLibcall = RTLIB::MEMMOVE;
831 Args[0].Flags[0].setReturned();
832 break;
833 case TargetOpcode::G_MEMSET:
834 RTLibcall = RTLIB::MEMSET;
835 Args[0].Flags[0].setReturned();
836 break;
837 default:
838 llvm_unreachable("unsupported opcode");
839 }
840
841 if (!Libcalls) // FIXME: Should be mandatory
842 return LegalizerHelper::UnableToLegalize;
843
844 RTLIB::LibcallImpl RTLibcallImpl = Libcalls->getLibcallImpl(Call: RTLibcall);
845
846 // Unsupported libcall on the target.
847 if (RTLibcallImpl == RTLIB::Unsupported) {
848 LLVM_DEBUG(dbgs() << ".. .. Could not find libcall name for "
849 << MIRBuilder.getTII().getName(Opc) << "\n");
850 return LegalizerHelper::UnableToLegalize;
851 }
852
853 CallLowering::CallLoweringInfo Info;
854 Info.CallConv = Libcalls->getLibcallImplCallingConv(Call: RTLibcallImpl);
855
856 StringRef LibcallName =
857 RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: RTLibcallImpl);
858 Info.Callee = MachineOperand::CreateES(SymName: LibcallName.data());
859 Info.OrigRet = CallLowering::ArgInfo({0}, Type::getVoidTy(C&: Ctx), 0);
860 Info.IsTailCall =
861 MI.getOperand(i: MI.getNumOperands() - 1).getImm() &&
862 isLibCallInTailPosition(Result: Info.OrigRet, MI, TII: MIRBuilder.getTII(), MRI) &&
863 // Lowering doesn't support tail calling inside a function with
864 // a swifterror argument yet.
865 !hasSwiftErrorArg(MF&: MIRBuilder.getMF());
866
867 llvm::append_range(C&: Info.OrigArgs, R&: Args);
868 if (!CLI.lowerCall(MIRBuilder, Info))
869 return LegalizerHelper::UnableToLegalize;
870
871 if (Info.LoweredTailCall) {
872 assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?");
873
874 // Check debug locations before removing the return.
875 LocObserver.checkpoint(CheckDebugLocs: true);
876
877 // We must have a return following the call (or debug insts) to get past
878 // isLibCallInTailPosition.
879 do {
880 MachineInstr *Next = MI.getNextNode();
881 assert(Next &&
882 (Next->isCopy() || Next->isReturn() || Next->isDebugInstr()) &&
883 "Expected instr following MI to be return or debug inst?");
884 // We lowered a tail call, so the call is now the return from the block.
885 // Delete the old return.
886 Next->eraseFromParent();
887 } while (MI.getNextNode());
888
889 // We expect to lose the debug location from the return.
890 LocObserver.checkpoint(CheckDebugLocs: false);
891 }
892
893 return LegalizerHelper::Legalized;
894}
895
896static RTLIB::Libcall getOutlineAtomicLibcall(MachineInstr &MI) {
897 unsigned Opc = MI.getOpcode();
898 auto &AtomicMI = cast<GMemOperation>(Val&: MI);
899 auto &MMO = AtomicMI.getMMO();
900 auto Ordering = MMO.getMergedOrdering();
901 LLT MemType = MMO.getMemoryType();
902 uint64_t MemSize = MemType.getSizeInBytes();
903 if (MemType.isVector())
904 return RTLIB::UNKNOWN_LIBCALL;
905
906#define LCALLS(A, B) {A##B##_RELAX, A##B##_ACQ, A##B##_REL, A##B##_ACQ_REL}
907#define LCALL5(A) \
908 LCALLS(A, 1), LCALLS(A, 2), LCALLS(A, 4), LCALLS(A, 8), LCALLS(A, 16)
909 switch (Opc) {
910 case TargetOpcode::G_ATOMIC_CMPXCHG:
911 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
912 const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_CAS)};
913 return getOutlineAtomicHelper(LC, Order: Ordering, MemSize);
914 }
915 case TargetOpcode::G_ATOMICRMW_XCHG: {
916 const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_SWP)};
917 return getOutlineAtomicHelper(LC, Order: Ordering, MemSize);
918 }
919 case TargetOpcode::G_ATOMICRMW_ADD:
920 case TargetOpcode::G_ATOMICRMW_SUB: {
921 const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDADD)};
922 return getOutlineAtomicHelper(LC, Order: Ordering, MemSize);
923 }
924 case TargetOpcode::G_ATOMICRMW_AND: {
925 const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDCLR)};
926 return getOutlineAtomicHelper(LC, Order: Ordering, MemSize);
927 }
928 case TargetOpcode::G_ATOMICRMW_OR: {
929 const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDSET)};
930 return getOutlineAtomicHelper(LC, Order: Ordering, MemSize);
931 }
932 case TargetOpcode::G_ATOMICRMW_XOR: {
933 const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDEOR)};
934 return getOutlineAtomicHelper(LC, Order: Ordering, MemSize);
935 }
936 default:
937 return RTLIB::UNKNOWN_LIBCALL;
938 }
939#undef LCALLS
940#undef LCALL5
941}
942
943LegalizerHelper::LegalizeResult
944LegalizerHelper::createAtomicLibcall(MachineInstr &MI) const {
945 auto &Ctx = MIRBuilder.getContext();
946
947 Type *RetTy;
948 SmallVector<Register> RetRegs;
949 SmallVector<CallLowering::ArgInfo, 3> Args;
950 unsigned Opc = MI.getOpcode();
951 switch (Opc) {
952 case TargetOpcode::G_ATOMIC_CMPXCHG:
953 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
954 Register Success;
955 LLT SuccessLLT;
956 auto [Ret, RetLLT, Mem, MemLLT, Cmp, CmpLLT, New, NewLLT] =
957 MI.getFirst4RegLLTs();
958 RetRegs.push_back(Elt: Ret);
959 RetTy = IntegerType::get(C&: Ctx, NumBits: RetLLT.getSizeInBits());
960 if (Opc == TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS) {
961 std::tie(args&: Ret, args&: RetLLT, args&: Success, args&: SuccessLLT, args&: Mem, args&: MemLLT, args&: Cmp, args&: CmpLLT, args&: New,
962 args&: NewLLT) = MI.getFirst5RegLLTs();
963 RetRegs.push_back(Elt: Success);
964 RetTy = StructType::get(
965 Context&: Ctx, Elements: {RetTy, IntegerType::get(C&: Ctx, NumBits: SuccessLLT.getSizeInBits())});
966 }
967 Args.push_back(Elt: {Cmp, IntegerType::get(C&: Ctx, NumBits: CmpLLT.getSizeInBits()), 0});
968 Args.push_back(Elt: {New, IntegerType::get(C&: Ctx, NumBits: NewLLT.getSizeInBits()), 0});
969 Args.push_back(Elt: {Mem, PointerType::get(C&: Ctx, AddressSpace: MemLLT.getAddressSpace()), 0});
970 break;
971 }
972 case TargetOpcode::G_ATOMICRMW_XCHG:
973 case TargetOpcode::G_ATOMICRMW_ADD:
974 case TargetOpcode::G_ATOMICRMW_SUB:
975 case TargetOpcode::G_ATOMICRMW_AND:
976 case TargetOpcode::G_ATOMICRMW_OR:
977 case TargetOpcode::G_ATOMICRMW_XOR: {
978 auto [Ret, RetLLT, Mem, MemLLT, Val, ValLLT] = MI.getFirst3RegLLTs();
979 RetRegs.push_back(Elt: Ret);
980 RetTy = IntegerType::get(C&: Ctx, NumBits: RetLLT.getSizeInBits());
981 if (Opc == TargetOpcode::G_ATOMICRMW_AND)
982 Val =
983 MIRBuilder.buildXor(Dst: ValLLT, Src0: MIRBuilder.buildConstant(Res: ValLLT, Val: -1), Src1: Val)
984 .getReg(Idx: 0);
985 else if (Opc == TargetOpcode::G_ATOMICRMW_SUB)
986 Val =
987 MIRBuilder.buildSub(Dst: ValLLT, Src0: MIRBuilder.buildConstant(Res: ValLLT, Val: 0), Src1: Val)
988 .getReg(Idx: 0);
989 Args.push_back(Elt: {Val, IntegerType::get(C&: Ctx, NumBits: ValLLT.getSizeInBits()), 0});
990 Args.push_back(Elt: {Mem, PointerType::get(C&: Ctx, AddressSpace: MemLLT.getAddressSpace()), 0});
991 break;
992 }
993 default:
994 llvm_unreachable("unsupported opcode");
995 }
996
997 if (!Libcalls) // FIXME: Should be mandatory
998 return LegalizerHelper::UnableToLegalize;
999
1000 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
1001 RTLIB::Libcall RTLibcall = getOutlineAtomicLibcall(MI);
1002 RTLIB::LibcallImpl RTLibcallImpl = Libcalls->getLibcallImpl(Call: RTLibcall);
1003
1004 // Unsupported libcall on the target.
1005 if (RTLibcallImpl == RTLIB::Unsupported) {
1006 LLVM_DEBUG(dbgs() << ".. .. Could not find libcall name for "
1007 << MIRBuilder.getTII().getName(Opc) << "\n");
1008 return LegalizerHelper::UnableToLegalize;
1009 }
1010
1011 CallLowering::CallLoweringInfo Info;
1012 Info.CallConv = Libcalls->getLibcallImplCallingConv(Call: RTLibcallImpl);
1013
1014 StringRef LibcallName =
1015 RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: RTLibcallImpl);
1016 Info.Callee = MachineOperand::CreateES(SymName: LibcallName.data());
1017 Info.OrigRet = CallLowering::ArgInfo(RetRegs, RetTy, 0);
1018
1019 llvm::append_range(C&: Info.OrigArgs, R&: Args);
1020 if (!CLI.lowerCall(MIRBuilder, Info))
1021 return LegalizerHelper::UnableToLegalize;
1022
1023 return LegalizerHelper::Legalized;
1024}
1025
1026static RTLIB::Libcall
1027getStateLibraryFunctionFor(MachineInstr &MI, const TargetLowering &TLI) {
1028 RTLIB::Libcall RTLibcall;
1029 switch (MI.getOpcode()) {
1030 case TargetOpcode::G_GET_FPENV:
1031 RTLibcall = RTLIB::FEGETENV;
1032 break;
1033 case TargetOpcode::G_SET_FPENV:
1034 case TargetOpcode::G_RESET_FPENV:
1035 RTLibcall = RTLIB::FESETENV;
1036 break;
1037 case TargetOpcode::G_GET_FPMODE:
1038 RTLibcall = RTLIB::FEGETMODE;
1039 break;
1040 case TargetOpcode::G_SET_FPMODE:
1041 case TargetOpcode::G_RESET_FPMODE:
1042 RTLibcall = RTLIB::FESETMODE;
1043 break;
1044 default:
1045 llvm_unreachable("Unexpected opcode");
1046 }
1047 return RTLibcall;
1048}
1049
1050// Some library functions that read FP state (fegetmode, fegetenv) write the
1051// state into a region in memory. IR intrinsics that do the same operations
1052// (get_fpmode, get_fpenv) return the state as integer value. To implement these
1053// intrinsics via the library functions, we need to use temporary variable,
1054// for example:
1055//
1056// %0:_(s32) = G_GET_FPMODE
1057//
1058// is transformed to:
1059//
1060// %1:_(p0) = G_FRAME_INDEX %stack.0
1061// BL &fegetmode
1062// %0:_(s32) = G_LOAD % 1
1063//
1064LegalizerHelper::LegalizeResult
1065LegalizerHelper::createGetStateLibcall(MachineInstr &MI,
1066 LostDebugLocObserver &LocObserver) {
1067 const DataLayout &DL = MIRBuilder.getDataLayout();
1068 auto &MF = MIRBuilder.getMF();
1069 auto &MRI = *MIRBuilder.getMRI();
1070 auto &Ctx = MF.getFunction().getContext();
1071
1072 // Create temporary, where library function will put the read state.
1073 Register Dst = MI.getOperand(i: 0).getReg();
1074 LLT StateTy = MRI.getType(Reg: Dst);
1075 TypeSize StateSize = StateTy.getSizeInBytes();
1076 Align TempAlign = getStackTemporaryAlignment(Type: StateTy);
1077 MachinePointerInfo TempPtrInfo;
1078 auto Temp = createStackTemporary(Bytes: StateSize, Alignment: TempAlign, PtrInfo&: TempPtrInfo);
1079
1080 // Create a call to library function, with the temporary as an argument.
1081 unsigned TempAddrSpace = DL.getAllocaAddrSpace();
1082 Type *StatePtrTy = PointerType::get(C&: Ctx, AddressSpace: TempAddrSpace);
1083 RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI);
1084 auto Res = createLibcall(
1085 Libcall: RTLibcall, Result: CallLowering::ArgInfo({0}, Type::getVoidTy(C&: Ctx), 0),
1086 Args: CallLowering::ArgInfo({Temp.getReg(Idx: 0), StatePtrTy, 0}), LocObserver,
1087 MI: nullptr);
1088 if (Res != LegalizerHelper::Legalized)
1089 return Res;
1090
1091 // Create a load from the temporary.
1092 MachineMemOperand *MMO = MF.getMachineMemOperand(
1093 PtrInfo: TempPtrInfo, F: MachineMemOperand::MOLoad, MemTy: StateTy, BaseAlignment: TempAlign);
1094 MIRBuilder.buildLoadInstr(Opcode: TargetOpcode::G_LOAD, Res: Dst, Addr: Temp, MMO&: *MMO);
1095
1096 return LegalizerHelper::Legalized;
1097}
1098
1099// Similar to `createGetStateLibcall` the function calls a library function
1100// using transient space in stack. In this case the library function reads
1101// content of memory region.
1102LegalizerHelper::LegalizeResult
1103LegalizerHelper::createSetStateLibcall(MachineInstr &MI,
1104 LostDebugLocObserver &LocObserver) {
1105 const DataLayout &DL = MIRBuilder.getDataLayout();
1106 auto &MF = MIRBuilder.getMF();
1107 auto &MRI = *MIRBuilder.getMRI();
1108 auto &Ctx = MF.getFunction().getContext();
1109
1110 // Create temporary, where library function will get the new state.
1111 Register Src = MI.getOperand(i: 0).getReg();
1112 LLT StateTy = MRI.getType(Reg: Src);
1113 TypeSize StateSize = StateTy.getSizeInBytes();
1114 Align TempAlign = getStackTemporaryAlignment(Type: StateTy);
1115 MachinePointerInfo TempPtrInfo;
1116 auto Temp = createStackTemporary(Bytes: StateSize, Alignment: TempAlign, PtrInfo&: TempPtrInfo);
1117
1118 // Put the new state into the temporary.
1119 MachineMemOperand *MMO = MF.getMachineMemOperand(
1120 PtrInfo: TempPtrInfo, F: MachineMemOperand::MOStore, MemTy: StateTy, BaseAlignment: TempAlign);
1121 MIRBuilder.buildStore(Val: Src, Addr: Temp, MMO&: *MMO);
1122
1123 // Create a call to library function, with the temporary as an argument.
1124 unsigned TempAddrSpace = DL.getAllocaAddrSpace();
1125 Type *StatePtrTy = PointerType::get(C&: Ctx, AddressSpace: TempAddrSpace);
1126 RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI);
1127 return createLibcall(Libcall: RTLibcall,
1128 Result: CallLowering::ArgInfo({0}, Type::getVoidTy(C&: Ctx), 0),
1129 Args: CallLowering::ArgInfo({Temp.getReg(Idx: 0), StatePtrTy, 0}),
1130 LocObserver, MI: nullptr);
1131}
1132
1133/// Returns the corresponding libcall for the given Pred and
1134/// the ICMP predicate that should be generated to compare with #0
1135/// after the libcall.
1136static std::pair<RTLIB::Libcall, CmpInst::Predicate>
1137getFCMPLibcallDesc(const CmpInst::Predicate Pred, unsigned Size) {
1138#define RTLIBCASE_CMP(LibcallPrefix, ICmpPred) \
1139 do { \
1140 switch (Size) { \
1141 case 32: \
1142 return {RTLIB::LibcallPrefix##32, ICmpPred}; \
1143 case 64: \
1144 return {RTLIB::LibcallPrefix##64, ICmpPred}; \
1145 case 128: \
1146 return {RTLIB::LibcallPrefix##128, ICmpPred}; \
1147 default: \
1148 llvm_unreachable("unexpected size"); \
1149 } \
1150 } while (0)
1151
1152 // These use the three-way (-1/0/1) compare libcalls, whose result is tested
1153 // against 0 with a signed integer predicate. Unordered (UO) is a boolean.
1154 switch (Pred) {
1155 case CmpInst::FCMP_OEQ:
1156 RTLIBCASE_CMP(FCMP3_PRED_OEQ_F, CmpInst::ICMP_EQ);
1157 case CmpInst::FCMP_UNE:
1158 RTLIBCASE_CMP(FCMP3_PRED_UNE_F, CmpInst::ICMP_NE);
1159 case CmpInst::FCMP_OGE:
1160 RTLIBCASE_CMP(FCMP3_PRED_OGE_F, CmpInst::ICMP_SGE);
1161 case CmpInst::FCMP_OLT:
1162 RTLIBCASE_CMP(FCMP3_PRED_OLT_F, CmpInst::ICMP_SLT);
1163 case CmpInst::FCMP_OLE:
1164 RTLIBCASE_CMP(FCMP3_PRED_OLE_F, CmpInst::ICMP_SLE);
1165 case CmpInst::FCMP_OGT:
1166 RTLIBCASE_CMP(FCMP3_PRED_OGT_F, CmpInst::ICMP_SGT);
1167 case CmpInst::FCMP_UNO:
1168 RTLIBCASE_CMP(UO_F, CmpInst::ICMP_NE);
1169 default:
1170 return {RTLIB::UNKNOWN_LIBCALL, CmpInst::BAD_ICMP_PREDICATE};
1171 }
1172}
1173
1174LegalizerHelper::LegalizeResult
1175LegalizerHelper::createFCMPLibcall(MachineInstr &MI,
1176 LostDebugLocObserver &LocObserver) {
1177 auto &MF = MIRBuilder.getMF();
1178 auto &Ctx = MF.getFunction().getContext();
1179 const GFCmp *Cmp = cast<GFCmp>(Val: &MI);
1180
1181 LLT OpLLT = MRI.getType(Reg: Cmp->getLHSReg());
1182 unsigned Size = OpLLT.getSizeInBits();
1183 if ((Size != 32 && Size != 64 && Size != 128) ||
1184 OpLLT != MRI.getType(Reg: Cmp->getRHSReg()))
1185 return UnableToLegalize;
1186
1187 Type *OpType = getFloatTypeForLLT(Ctx, Ty: OpLLT);
1188
1189 // DstReg type is s32
1190 const Register DstReg = Cmp->getReg(Idx: 0);
1191 LLT DstTy = MRI.getType(Reg: DstReg);
1192 const auto Cond = Cmp->getCond();
1193
1194 // Reference:
1195 // https://gcc.gnu.org/onlinedocs/gccint/Soft-float-library-routines.html#Comparison-functions-1
1196 // Generates a libcall followed by ICMP.
1197 const auto BuildLibcall = [&](const RTLIB::Libcall Libcall,
1198 const CmpInst::Predicate ICmpPred,
1199 const DstOp &Res) -> Register {
1200 // FCMP libcall always returns an i32, and needs an ICMP with #0.
1201 LLT TempLLT = LLT::integer(SizeInBits: 32);
1202 Register Temp = MRI.createGenericVirtualRegister(Ty: TempLLT);
1203 // Generate libcall, holding result in Temp
1204 const auto Status = createLibcall(
1205 Libcall, Result: {Temp, Type::getInt32Ty(C&: Ctx), 0},
1206 Args: {{Cmp->getLHSReg(), OpType, 0}, {Cmp->getRHSReg(), OpType, 1}},
1207 LocObserver, MI: &MI);
1208 if (Status != Legalized)
1209 return {};
1210
1211 // Compare temp with #0 to get the final result.
1212 return MIRBuilder
1213 .buildICmp(Pred: ICmpPred, Res, Op0: Temp, Op1: MIRBuilder.buildConstant(Res: TempLLT, Val: 0))
1214 .getReg(Idx: 0);
1215 };
1216
1217 // Simple case if we have a direct mapping from predicate to libcall
1218 if (const auto [Libcall, ICmpPred] = getFCMPLibcallDesc(Pred: Cond, Size);
1219 Libcall != RTLIB::UNKNOWN_LIBCALL &&
1220 ICmpPred != CmpInst::BAD_ICMP_PREDICATE) {
1221 if (BuildLibcall(Libcall, ICmpPred, DstReg)) {
1222 return Legalized;
1223 }
1224 return UnableToLegalize;
1225 }
1226
1227 // No direct mapping found, should be generated as combination of libcalls.
1228
1229 switch (Cond) {
1230 case CmpInst::FCMP_UEQ: {
1231 // FCMP_UEQ: unordered or equal
1232 // Convert into (FCMP_OEQ || FCMP_UNO).
1233
1234 const auto [OeqLibcall, OeqPred] =
1235 getFCMPLibcallDesc(Pred: CmpInst::FCMP_OEQ, Size);
1236 const auto Oeq = BuildLibcall(OeqLibcall, OeqPred, DstTy);
1237
1238 const auto [UnoLibcall, UnoPred] =
1239 getFCMPLibcallDesc(Pred: CmpInst::FCMP_UNO, Size);
1240 const auto Uno = BuildLibcall(UnoLibcall, UnoPred, DstTy);
1241 if (Oeq && Uno)
1242 MIRBuilder.buildOr(Dst: DstReg, Src0: Oeq, Src1: Uno);
1243 else
1244 return UnableToLegalize;
1245
1246 break;
1247 }
1248 case CmpInst::FCMP_ONE: {
1249 // FCMP_ONE: ordered and operands are unequal
1250 // Convert into (!FCMP_OEQ && !FCMP_UNO).
1251
1252 // We inverse the predicate instead of generating a NOT
1253 // to save one instruction.
1254 // On AArch64 isel can even select two cmp into a single ccmp.
1255 const auto [OeqLibcall, OeqPred] =
1256 getFCMPLibcallDesc(Pred: CmpInst::FCMP_OEQ, Size);
1257 const auto NotOeq =
1258 BuildLibcall(OeqLibcall, CmpInst::getInversePredicate(pred: OeqPred), DstTy);
1259
1260 const auto [UnoLibcall, UnoPred] =
1261 getFCMPLibcallDesc(Pred: CmpInst::FCMP_UNO, Size);
1262 const auto NotUno =
1263 BuildLibcall(UnoLibcall, CmpInst::getInversePredicate(pred: UnoPred), DstTy);
1264
1265 if (NotOeq && NotUno)
1266 MIRBuilder.buildAnd(Dst: DstReg, Src0: NotOeq, Src1: NotUno);
1267 else
1268 return UnableToLegalize;
1269
1270 break;
1271 }
1272 case CmpInst::FCMP_ULT:
1273 case CmpInst::FCMP_UGE:
1274 case CmpInst::FCMP_UGT:
1275 case CmpInst::FCMP_ULE:
1276 case CmpInst::FCMP_ORD: {
1277 // Convert into: !(inverse(Pred))
1278 // E.g. FCMP_ULT becomes !FCMP_OGE
1279 // This is equivalent to the following, but saves some instructions.
1280 // MIRBuilder.buildNot(
1281 // PredTy,
1282 // MIRBuilder.buildFCmp(CmpInst::getInversePredicate(Pred), PredTy,
1283 // Op1, Op2));
1284 const auto [InversedLibcall, InversedPred] =
1285 getFCMPLibcallDesc(Pred: CmpInst::getInversePredicate(pred: Cond), Size);
1286 if (!BuildLibcall(InversedLibcall,
1287 CmpInst::getInversePredicate(pred: InversedPred), DstReg))
1288 return UnableToLegalize;
1289 break;
1290 }
1291 default:
1292 return UnableToLegalize;
1293 }
1294
1295 return Legalized;
1296}
1297
1298// The function is used to legalize operations that set default environment
1299// state. In C library a call like `fesetmode(FE_DFL_MODE)` is used for that.
1300// On most targets supported in glibc FE_DFL_MODE is defined as
1301// `((const femode_t *) -1)`. Such assumption is used here. If for some target
1302// it is not true, the target must provide custom lowering.
1303LegalizerHelper::LegalizeResult
1304LegalizerHelper::createResetStateLibcall(MachineInstr &MI,
1305 LostDebugLocObserver &LocObserver) {
1306 const DataLayout &DL = MIRBuilder.getDataLayout();
1307 auto &MF = MIRBuilder.getMF();
1308 auto &Ctx = MF.getFunction().getContext();
1309
1310 // Create an argument for the library function.
1311 unsigned AddrSpace = DL.getDefaultGlobalsAddressSpace();
1312 Type *StatePtrTy = PointerType::get(C&: Ctx, AddressSpace: AddrSpace);
1313 unsigned PtrSize = DL.getPointerSizeInBits(AS: AddrSpace);
1314 LLT MemTy = LLT::pointer(AddressSpace: AddrSpace, SizeInBits: PtrSize);
1315 auto DefValue = MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: PtrSize), Val: -1LL);
1316 DstOp Dest(MRI.createGenericVirtualRegister(Ty: MemTy));
1317 MIRBuilder.buildIntToPtr(Dst: Dest, Src: DefValue);
1318
1319 RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI);
1320 return createLibcall(
1321 Libcall: RTLibcall, Result: CallLowering::ArgInfo({0}, Type::getVoidTy(C&: Ctx), 0),
1322 Args: CallLowering::ArgInfo({Dest.getReg(), StatePtrTy, 0}), LocObserver, MI: &MI);
1323}
1324
1325LegalizerHelper::LegalizeResult
1326LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
1327 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
1328
1329 switch (MI.getOpcode()) {
1330 default:
1331 return UnableToLegalize;
1332 case TargetOpcode::G_MUL:
1333 case TargetOpcode::G_SDIV:
1334 case TargetOpcode::G_UDIV:
1335 case TargetOpcode::G_SREM:
1336 case TargetOpcode::G_UREM:
1337 case TargetOpcode::G_CTLZ_ZERO_POISON: {
1338 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1339 unsigned Size = LLTy.getSizeInBits();
1340 Type *HLTy = IntegerType::get(C&: Ctx, NumBits: Size);
1341 auto Status = simpleLibcall(MI, MIRBuilder, Size, OpType: HLTy, LocObserver);
1342 if (Status != Legalized)
1343 return Status;
1344 break;
1345 }
1346 case TargetOpcode::G_FADD:
1347 case TargetOpcode::G_FSUB:
1348 case TargetOpcode::G_FMUL:
1349 case TargetOpcode::G_FDIV:
1350 case TargetOpcode::G_FMA:
1351 case TargetOpcode::G_FPOW:
1352 case TargetOpcode::G_FREM:
1353 case TargetOpcode::G_FCOS:
1354 case TargetOpcode::G_FSIN:
1355 case TargetOpcode::G_FTAN:
1356 case TargetOpcode::G_FACOS:
1357 case TargetOpcode::G_FASIN:
1358 case TargetOpcode::G_FATAN:
1359 case TargetOpcode::G_FATAN2:
1360 case TargetOpcode::G_FCOSH:
1361 case TargetOpcode::G_FSINH:
1362 case TargetOpcode::G_FTANH:
1363 case TargetOpcode::G_FLOG10:
1364 case TargetOpcode::G_FLOG:
1365 case TargetOpcode::G_FLOG2:
1366 case TargetOpcode::G_FEXP:
1367 case TargetOpcode::G_FEXP2:
1368 case TargetOpcode::G_FEXP10:
1369 case TargetOpcode::G_FCEIL:
1370 case TargetOpcode::G_FFLOOR:
1371 case TargetOpcode::G_FMINNUM:
1372 case TargetOpcode::G_FMAXNUM:
1373 case TargetOpcode::G_FMINIMUMNUM:
1374 case TargetOpcode::G_FMAXIMUMNUM:
1375 case TargetOpcode::G_FSQRT:
1376 case TargetOpcode::G_FRINT:
1377 case TargetOpcode::G_FNEARBYINT:
1378 case TargetOpcode::G_INTRINSIC_TRUNC:
1379 case TargetOpcode::G_INTRINSIC_ROUND:
1380 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: {
1381 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1382 unsigned Size = LLTy.getSizeInBits();
1383 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1384 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1385 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1386 return UnableToLegalize;
1387 }
1388 auto Status = simpleLibcall(MI, MIRBuilder, Size, OpType: HLTy, LocObserver);
1389 if (Status != Legalized)
1390 return Status;
1391 break;
1392 }
1393 case TargetOpcode::G_FSINCOS: {
1394 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1395 unsigned Size = LLTy.getSizeInBits();
1396 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1397 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1398 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1399 return UnableToLegalize;
1400 }
1401 return emitSincosLibcall(MI, MIRBuilder, Size, OpType: HLTy, LocObserver);
1402 }
1403 case TargetOpcode::G_FMODF: {
1404 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1405 unsigned Size = LLTy.getSizeInBits();
1406 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1407 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1408 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1409 return UnableToLegalize;
1410 }
1411 return emitModfLibcall(MI, MIRBuilder, Size, OpType: HLTy, LocObserver);
1412 }
1413 case TargetOpcode::G_LROUND:
1414 case TargetOpcode::G_LLROUND:
1415 case TargetOpcode::G_INTRINSIC_LRINT:
1416 case TargetOpcode::G_INTRINSIC_LLRINT: {
1417 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
1418 unsigned Size = LLTy.getSizeInBits();
1419 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1420 Type *ITy = IntegerType::get(
1421 C&: Ctx, NumBits: MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits());
1422 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1423 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1424 return UnableToLegalize;
1425 }
1426 auto Libcall = getRTLibDesc(Opcode: MI.getOpcode(), Size);
1427 LegalizeResult Status =
1428 createLibcall(Libcall, Result: {MI.getOperand(i: 0).getReg(), ITy, 0},
1429 Args: {{MI.getOperand(i: 1).getReg(), HLTy, 0}}, LocObserver, MI: &MI);
1430 if (Status != Legalized)
1431 return Status;
1432 MI.eraseFromParent();
1433 return Legalized;
1434 }
1435 case TargetOpcode::G_FPOWI:
1436 case TargetOpcode::G_FLDEXP: {
1437 LLT LLTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1438 unsigned Size = LLTy.getSizeInBits();
1439 Type *HLTy = getFloatTypeForLLT(Ctx, Ty: LLTy);
1440 Type *ITy = IntegerType::get(
1441 C&: Ctx, NumBits: MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits());
1442 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
1443 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
1444 return UnableToLegalize;
1445 }
1446 auto Libcall = getRTLibDesc(Opcode: MI.getOpcode(), Size);
1447 SmallVector<CallLowering::ArgInfo, 2> Args = {
1448 {MI.getOperand(i: 1).getReg(), HLTy, 0},
1449 {MI.getOperand(i: 2).getReg(), ITy, 1}};
1450 Args[1].Flags[0].setSExt();
1451 LegalizeResult Status = createLibcall(
1452 Libcall, Result: {MI.getOperand(i: 0).getReg(), HLTy, 0}, Args, LocObserver, MI: &MI);
1453 if (Status != Legalized)
1454 return Status;
1455 break;
1456 }
1457 case TargetOpcode::G_FPEXT:
1458 case TargetOpcode::G_FPTRUNC: {
1459 Type *FromTy = getFloatTypeForLLT(Ctx, Ty: MRI.getType(Reg: MI.getOperand(i: 1).getReg()));
1460 Type *ToTy = getFloatTypeForLLT(Ctx, Ty: MRI.getType(Reg: MI.getOperand(i: 0).getReg()));
1461 if (!FromTy || !ToTy)
1462 return UnableToLegalize;
1463 LegalizeResult Status = conversionLibcall(MI, ToType: ToTy, FromType: FromTy, LocObserver);
1464 if (Status != Legalized)
1465 return Status;
1466 break;
1467 }
1468 case TargetOpcode::G_FCMP: {
1469 LegalizeResult Status = createFCMPLibcall(MI, LocObserver);
1470 if (Status != Legalized)
1471 return Status;
1472 MI.eraseFromParent();
1473 return Status;
1474 }
1475 case TargetOpcode::G_FPTOSI:
1476 case TargetOpcode::G_FPTOUI: {
1477 // FIXME: Support other types
1478 Type *FromTy =
1479 getFloatTypeForLLT(Ctx, Ty: MRI.getType(Reg: MI.getOperand(i: 1).getReg()));
1480 unsigned ToSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
1481 if ((ToSize != 32 && ToSize != 64 && ToSize != 128) || !FromTy)
1482 return UnableToLegalize;
1483 LegalizeResult Status = conversionLibcall(MI, ToType: Type::getIntNTy(C&: Ctx, N: ToSize),
1484 FromType: FromTy, LocObserver);
1485 if (Status != Legalized)
1486 return Status;
1487 break;
1488 }
1489 case TargetOpcode::G_SITOFP:
1490 case TargetOpcode::G_UITOFP: {
1491 unsigned FromSize = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
1492 Type *ToTy =
1493 getFloatTypeForLLT(Ctx, Ty: MRI.getType(Reg: MI.getOperand(i: 0).getReg()));
1494 if ((FromSize != 32 && FromSize != 64 && FromSize != 128) || !ToTy)
1495 return UnableToLegalize;
1496 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SITOFP;
1497 LegalizeResult Status = conversionLibcall(
1498 MI, ToType: ToTy, FromType: Type::getIntNTy(C&: Ctx, N: FromSize), LocObserver, IsSigned);
1499 if (Status != Legalized)
1500 return Status;
1501 break;
1502 }
1503 case TargetOpcode::G_ATOMICRMW_XCHG:
1504 case TargetOpcode::G_ATOMICRMW_ADD:
1505 case TargetOpcode::G_ATOMICRMW_SUB:
1506 case TargetOpcode::G_ATOMICRMW_AND:
1507 case TargetOpcode::G_ATOMICRMW_OR:
1508 case TargetOpcode::G_ATOMICRMW_XOR:
1509 case TargetOpcode::G_ATOMIC_CMPXCHG:
1510 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
1511 auto Status = createAtomicLibcall(MI);
1512 if (Status != Legalized)
1513 return Status;
1514 break;
1515 }
1516 case TargetOpcode::G_BZERO:
1517 case TargetOpcode::G_MEMCPY:
1518 case TargetOpcode::G_MEMMOVE:
1519 case TargetOpcode::G_MEMSET: {
1520 LegalizeResult Result =
1521 createMemLibcall(MRI&: *MIRBuilder.getMRI(), MI, LocObserver);
1522 if (Result != Legalized)
1523 return Result;
1524 MI.eraseFromParent();
1525 return Result;
1526 }
1527 case TargetOpcode::G_GET_FPENV:
1528 case TargetOpcode::G_GET_FPMODE: {
1529 LegalizeResult Result = createGetStateLibcall(MI, LocObserver);
1530 if (Result != Legalized)
1531 return Result;
1532 break;
1533 }
1534 case TargetOpcode::G_SET_FPENV:
1535 case TargetOpcode::G_SET_FPMODE: {
1536 LegalizeResult Result = createSetStateLibcall(MI, LocObserver);
1537 if (Result != Legalized)
1538 return Result;
1539 break;
1540 }
1541 case TargetOpcode::G_RESET_FPENV:
1542 case TargetOpcode::G_RESET_FPMODE: {
1543 LegalizeResult Result = createResetStateLibcall(MI, LocObserver);
1544 if (Result != Legalized)
1545 return Result;
1546 break;
1547 }
1548 }
1549
1550 MI.eraseFromParent();
1551 return Legalized;
1552}
1553
1554LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
1555 unsigned TypeIdx,
1556 LLT NarrowTy) {
1557 uint64_t SizeOp0 = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
1558 uint64_t NarrowSize = NarrowTy.getSizeInBits();
1559
1560 switch (MI.getOpcode()) {
1561 default:
1562 return UnableToLegalize;
1563 case TargetOpcode::G_IMPLICIT_DEF: {
1564 Register DstReg = MI.getOperand(i: 0).getReg();
1565 LLT DstTy = MRI.getType(Reg: DstReg);
1566
1567 // If SizeOp0 is not an exact multiple of NarrowSize, emit
1568 // G_ANYEXT(G_IMPLICIT_DEF). Cast result to vector if needed.
1569 // FIXME: Although this would also be legal for the general case, it causes
1570 // a lot of regressions in the emitted code (superfluous COPYs, artifact
1571 // combines not being hit). This seems to be a problem related to the
1572 // artifact combiner.
1573 if (SizeOp0 % NarrowSize != 0) {
1574 LLT ImplicitTy = DstTy.changeElementType(NewEltTy: NarrowTy);
1575 Register ImplicitReg = MIRBuilder.buildUndef(Res: ImplicitTy).getReg(Idx: 0);
1576 MIRBuilder.buildAnyExt(Res: DstReg, Op: ImplicitReg);
1577
1578 MI.eraseFromParent();
1579 return Legalized;
1580 }
1581
1582 int NumParts = SizeOp0 / NarrowSize;
1583
1584 SmallVector<Register, 2> DstRegs;
1585 for (int i = 0; i < NumParts; ++i)
1586 DstRegs.push_back(Elt: MIRBuilder.buildUndef(Res: NarrowTy).getReg(Idx: 0));
1587
1588 if (DstTy.isVector())
1589 MIRBuilder.buildBuildVector(Res: DstReg, Ops: DstRegs);
1590 else
1591 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
1592 MI.eraseFromParent();
1593 return Legalized;
1594 }
1595 case TargetOpcode::G_CONSTANT: {
1596 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1597 const APInt &Val = MI.getOperand(i: 1).getCImm()->getValue();
1598 unsigned TotalSize = Ty.getSizeInBits();
1599 unsigned NarrowSize = NarrowTy.getSizeInBits();
1600 int NumParts = TotalSize / NarrowSize;
1601
1602 SmallVector<Register, 4> PartRegs;
1603 for (int I = 0; I != NumParts; ++I) {
1604 unsigned Offset = I * NarrowSize;
1605 auto K = MIRBuilder.buildConstant(Res: NarrowTy,
1606 Val: Val.lshr(shiftAmt: Offset).trunc(width: NarrowSize));
1607 PartRegs.push_back(Elt: K.getReg(Idx: 0));
1608 }
1609
1610 LLT LeftoverTy;
1611 unsigned LeftoverBits = TotalSize - NumParts * NarrowSize;
1612 SmallVector<Register, 1> LeftoverRegs;
1613 if (LeftoverBits != 0) {
1614 LeftoverTy = LLT::scalar(SizeInBits: LeftoverBits);
1615 auto K = MIRBuilder.buildConstant(
1616 Res: LeftoverTy,
1617 Val: Val.lshr(shiftAmt: NumParts * NarrowSize).trunc(width: LeftoverBits));
1618 LeftoverRegs.push_back(Elt: K.getReg(Idx: 0));
1619 }
1620
1621 insertParts(DstReg: MI.getOperand(i: 0).getReg(),
1622 ResultTy: Ty, PartTy: NarrowTy, PartRegs, LeftoverTy, LeftoverRegs);
1623
1624 MI.eraseFromParent();
1625 return Legalized;
1626 }
1627 case TargetOpcode::G_SEXT:
1628 case TargetOpcode::G_ZEXT:
1629 case TargetOpcode::G_ANYEXT:
1630 return narrowScalarExt(MI, TypeIdx, Ty: NarrowTy);
1631 case TargetOpcode::G_TRUNC: {
1632 if (TypeIdx != 1)
1633 return UnableToLegalize;
1634
1635 uint64_t SizeOp1 = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
1636 if (NarrowTy.getSizeInBits() * 2 != SizeOp1) {
1637 LLVM_DEBUG(dbgs() << "Can't narrow trunc to type " << NarrowTy << "\n");
1638 return UnableToLegalize;
1639 }
1640
1641 auto Unmerge = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: MI.getOperand(i: 1));
1642 MIRBuilder.buildCopy(Res: MI.getOperand(i: 0), Op: Unmerge.getReg(Idx: 0));
1643 MI.eraseFromParent();
1644 return Legalized;
1645 }
1646 case TargetOpcode::G_CONSTANT_FOLD_BARRIER:
1647 case TargetOpcode::G_FREEZE: {
1648 if (TypeIdx != 0)
1649 return UnableToLegalize;
1650
1651 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1652 // Should widen scalar first
1653 if (Ty.getSizeInBits() % NarrowTy.getSizeInBits() != 0)
1654 return UnableToLegalize;
1655
1656 auto Unmerge = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: MI.getOperand(i: 1).getReg());
1657 SmallVector<Register, 8> Parts;
1658 for (unsigned i = 0; i < Unmerge->getNumDefs(); ++i) {
1659 Parts.push_back(
1660 Elt: MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {NarrowTy}, SrcOps: {Unmerge.getReg(Idx: i)})
1661 .getReg(Idx: 0));
1662 }
1663
1664 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0).getReg(), Ops: Parts);
1665 MI.eraseFromParent();
1666 return Legalized;
1667 }
1668 case TargetOpcode::G_ADD:
1669 case TargetOpcode::G_SUB:
1670 case TargetOpcode::G_SADDO:
1671 case TargetOpcode::G_SSUBO:
1672 case TargetOpcode::G_SADDE:
1673 case TargetOpcode::G_SSUBE:
1674 case TargetOpcode::G_UADDO:
1675 case TargetOpcode::G_USUBO:
1676 case TargetOpcode::G_UADDE:
1677 case TargetOpcode::G_USUBE:
1678 return narrowScalarAddSub(MI, TypeIdx, NarrowTy);
1679 case TargetOpcode::G_MUL:
1680 case TargetOpcode::G_UMULH:
1681 return narrowScalarMul(MI, Ty: NarrowTy);
1682 case TargetOpcode::G_EXTRACT:
1683 return narrowScalarExtract(MI, TypeIdx, Ty: NarrowTy);
1684 case TargetOpcode::G_INSERT:
1685 return narrowScalarInsert(MI, TypeIdx, Ty: NarrowTy);
1686 case TargetOpcode::G_LOAD: {
1687 auto &LoadMI = cast<GLoad>(Val&: MI);
1688 Register DstReg = LoadMI.getDstReg();
1689 LLT DstTy = MRI.getType(Reg: DstReg);
1690 if (DstTy.isVector())
1691 return UnableToLegalize;
1692
1693 if (8 * LoadMI.getMemSize().getValue() != DstTy.getSizeInBits()) {
1694 Register TmpReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
1695 MIRBuilder.buildLoad(Res: TmpReg, Addr: LoadMI.getPointerReg(), MMO&: LoadMI.getMMO());
1696 MIRBuilder.buildAnyExt(Res: DstReg, Op: TmpReg);
1697 LoadMI.eraseFromParent();
1698 return Legalized;
1699 }
1700
1701 return reduceLoadStoreWidth(MI&: LoadMI, TypeIdx, NarrowTy);
1702 }
1703 case TargetOpcode::G_ZEXTLOAD:
1704 case TargetOpcode::G_SEXTLOAD:
1705 case TargetOpcode::G_FPEXTLOAD: {
1706 auto &LoadMI = cast<GExtLoad>(Val&: MI);
1707 Register DstReg = LoadMI.getDstReg();
1708 Register PtrReg = LoadMI.getPointerReg();
1709
1710 Register TmpReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
1711 auto &MMO = LoadMI.getMMO();
1712 unsigned MemSize = MMO.getSizeInBits().getValue();
1713
1714 if (MemSize == NarrowSize) {
1715 MIRBuilder.buildLoad(Res: TmpReg, Addr: PtrReg, MMO);
1716 } else if (MemSize < NarrowSize) {
1717 MIRBuilder.buildLoadInstr(Opcode: LoadMI.getOpcode(), Res: TmpReg, Addr: PtrReg, MMO);
1718 } else if (MemSize > NarrowSize) {
1719 // FIXME: Need to split the load.
1720 return UnableToLegalize;
1721 }
1722
1723 if (isa<GZExtLoad>(Val: LoadMI))
1724 MIRBuilder.buildZExt(Res: DstReg, Op: TmpReg);
1725 else if (isa<GSExtLoad>(Val: LoadMI))
1726 MIRBuilder.buildSExt(Res: DstReg, Op: TmpReg);
1727 else
1728 MIRBuilder.buildFPExt(Res: DstReg, Op: TmpReg);
1729
1730 LoadMI.eraseFromParent();
1731 return Legalized;
1732 }
1733 case TargetOpcode::G_STORE: {
1734 auto &StoreMI = cast<GStore>(Val&: MI);
1735
1736 Register SrcReg = StoreMI.getValueReg();
1737 LLT SrcTy = MRI.getType(Reg: SrcReg);
1738 if (SrcTy.isVector())
1739 return UnableToLegalize;
1740
1741 int NumParts = SizeOp0 / NarrowSize;
1742 unsigned HandledSize = NumParts * NarrowTy.getSizeInBits();
1743 unsigned LeftoverBits = SrcTy.getSizeInBits() - HandledSize;
1744 if (SrcTy.isVector() && LeftoverBits != 0)
1745 return UnableToLegalize;
1746
1747 if (8 * StoreMI.getMemSize().getValue() != SrcTy.getSizeInBits()) {
1748 Register TmpReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
1749 MIRBuilder.buildTrunc(Res: TmpReg, Op: SrcReg);
1750 MIRBuilder.buildStore(Val: TmpReg, Addr: StoreMI.getPointerReg(), MMO&: StoreMI.getMMO());
1751 StoreMI.eraseFromParent();
1752 return Legalized;
1753 }
1754
1755 return reduceLoadStoreWidth(MI&: StoreMI, TypeIdx: 0, NarrowTy);
1756 }
1757 case TargetOpcode::G_FPTRUNCSTORE: {
1758 auto &StoreMI = cast<GFPTruncStore>(Val&: MI);
1759 Register SrcReg = StoreMI.getValueReg();
1760 Register PtrReg = StoreMI.getPointerReg();
1761
1762 auto &MMO = StoreMI.getMMO();
1763 unsigned MemSize = MMO.getSizeInBits().getValue();
1764 if (MemSize > NarrowSize) {
1765 return UnableToLegalize;
1766 }
1767
1768 auto TmpReg = MIRBuilder.buildFPTrunc(Res: NarrowTy, Op: SrcReg);
1769 if (MemSize == NarrowSize) {
1770 MIRBuilder.buildStore(Val: TmpReg, Addr: PtrReg, MMO);
1771 } else if (MemSize < NarrowSize) {
1772 MIRBuilder.buildStoreInstr(Opcode: TargetOpcode::G_FPTRUNCSTORE, Val: TmpReg, Addr: PtrReg,
1773 MMO);
1774 }
1775
1776 StoreMI.eraseFromParent();
1777 return Legalized;
1778 }
1779 case TargetOpcode::G_SELECT:
1780 return narrowScalarSelect(MI, TypeIdx, Ty: NarrowTy);
1781 case TargetOpcode::G_AND:
1782 case TargetOpcode::G_OR:
1783 case TargetOpcode::G_XOR: {
1784 // Legalize bitwise operation:
1785 // A = BinOp<Ty> B, C
1786 // into:
1787 // B1, ..., BN = G_UNMERGE_VALUES B
1788 // C1, ..., CN = G_UNMERGE_VALUES C
1789 // A1 = BinOp<Ty/N> B1, C2
1790 // ...
1791 // AN = BinOp<Ty/N> BN, CN
1792 // A = G_MERGE_VALUES A1, ..., AN
1793 return narrowScalarBasic(MI, TypeIdx, Ty: NarrowTy);
1794 }
1795 case TargetOpcode::G_SHL:
1796 case TargetOpcode::G_LSHR:
1797 case TargetOpcode::G_ASHR:
1798 return narrowScalarShift(MI, TypeIdx, Ty: NarrowTy);
1799 case TargetOpcode::G_CTLZ:
1800 case TargetOpcode::G_CTLZ_ZERO_POISON:
1801 case TargetOpcode::G_CTTZ:
1802 case TargetOpcode::G_CTTZ_ZERO_POISON:
1803 case TargetOpcode::G_CTLS:
1804 case TargetOpcode::G_CTPOP:
1805 if (TypeIdx == 1)
1806 switch (MI.getOpcode()) {
1807 case TargetOpcode::G_CTLZ:
1808 case TargetOpcode::G_CTLZ_ZERO_POISON:
1809 return narrowScalarCTLZ(MI, TypeIdx, Ty: NarrowTy);
1810 case TargetOpcode::G_CTTZ:
1811 case TargetOpcode::G_CTTZ_ZERO_POISON:
1812 return narrowScalarCTTZ(MI, TypeIdx, Ty: NarrowTy);
1813 case TargetOpcode::G_CTPOP:
1814 return narrowScalarCTPOP(MI, TypeIdx, Ty: NarrowTy);
1815 case TargetOpcode::G_CTLS:
1816 return narrowScalarCTLS(MI, TypeIdx, Ty: NarrowTy);
1817 default:
1818 return UnableToLegalize;
1819 }
1820
1821 Observer.changingInstr(MI);
1822 narrowScalarDst(MI, NarrowTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_ZEXT);
1823 Observer.changedInstr(MI);
1824 return Legalized;
1825 case TargetOpcode::G_INTTOPTR:
1826 if (TypeIdx != 1)
1827 return UnableToLegalize;
1828
1829 Observer.changingInstr(MI);
1830 narrowScalarSrc(MI, NarrowTy, OpIdx: 1);
1831 Observer.changedInstr(MI);
1832 return Legalized;
1833 case TargetOpcode::G_PTRTOINT:
1834 if (TypeIdx != 0)
1835 return UnableToLegalize;
1836
1837 Observer.changingInstr(MI);
1838 narrowScalarDst(MI, NarrowTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_ZEXT);
1839 Observer.changedInstr(MI);
1840 return Legalized;
1841 case TargetOpcode::G_PHI: {
1842 // FIXME: add support for when SizeOp0 isn't an exact multiple of
1843 // NarrowSize.
1844 if (SizeOp0 % NarrowSize != 0)
1845 return UnableToLegalize;
1846
1847 unsigned NumParts = SizeOp0 / NarrowSize;
1848 SmallVector<Register, 2> DstRegs(NumParts);
1849 SmallVector<SmallVector<Register, 2>, 2> SrcRegs(MI.getNumOperands() / 2);
1850 Observer.changingInstr(MI);
1851 for (unsigned i = 1; i < MI.getNumOperands(); i += 2) {
1852 MachineBasicBlock &OpMBB = *MI.getOperand(i: i + 1).getMBB();
1853 MIRBuilder.setInsertPt(MBB&: OpMBB, II: OpMBB.getFirstTerminatorForward());
1854 extractParts(Reg: MI.getOperand(i).getReg(), Ty: NarrowTy, NumParts,
1855 VRegs&: SrcRegs[i / 2], MIRBuilder, MRI);
1856 }
1857 MachineBasicBlock &MBB = *MI.getParent();
1858 MIRBuilder.setInsertPt(MBB, II: MI);
1859 for (unsigned i = 0; i < NumParts; ++i) {
1860 DstRegs[i] = MRI.createGenericVirtualRegister(Ty: NarrowTy);
1861 MachineInstrBuilder MIB =
1862 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_PHI).addDef(RegNo: DstRegs[i]);
1863 for (unsigned j = 1; j < MI.getNumOperands(); j += 2)
1864 MIB.addUse(RegNo: SrcRegs[j / 2][i]).add(MO: MI.getOperand(i: j + 1));
1865 }
1866 MIRBuilder.setInsertPt(MBB, II: MBB.getFirstNonPHI());
1867 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0), Ops: DstRegs);
1868 Observer.changedInstr(MI);
1869 MI.eraseFromParent();
1870 return Legalized;
1871 }
1872 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1873 case TargetOpcode::G_INSERT_VECTOR_ELT: {
1874 if (TypeIdx != 2)
1875 return UnableToLegalize;
1876
1877 int OpIdx = MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
1878 Observer.changingInstr(MI);
1879 narrowScalarSrc(MI, NarrowTy, OpIdx);
1880 Observer.changedInstr(MI);
1881 return Legalized;
1882 }
1883 case TargetOpcode::G_ICMP: {
1884 Register LHS = MI.getOperand(i: 2).getReg();
1885 LLT SrcTy = MRI.getType(Reg: LHS);
1886 CmpInst::Predicate Pred =
1887 static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
1888
1889 LLT LeftoverTy; // Example: s88 -> s64 (NarrowTy) + s24 (leftover)
1890 SmallVector<Register, 4> LHSPartRegs, LHSLeftoverRegs;
1891 if (!extractParts(Reg: LHS, RegTy: SrcTy, MainTy: NarrowTy, LeftoverTy, VRegs&: LHSPartRegs,
1892 LeftoverVRegs&: LHSLeftoverRegs, MIRBuilder, MRI))
1893 return UnableToLegalize;
1894
1895 LLT Unused; // Matches LeftoverTy; G_ICMP LHS and RHS are the same type.
1896 SmallVector<Register, 4> RHSPartRegs, RHSLeftoverRegs;
1897 if (!extractParts(Reg: MI.getOperand(i: 3).getReg(), RegTy: SrcTy, MainTy: NarrowTy, LeftoverTy&: Unused,
1898 VRegs&: RHSPartRegs, LeftoverVRegs&: RHSLeftoverRegs, MIRBuilder, MRI))
1899 return UnableToLegalize;
1900
1901 // We now have the LHS and RHS of the compare split into narrow-type
1902 // registers, plus potentially some leftover type.
1903 Register Dst = MI.getOperand(i: 0).getReg();
1904 LLT ResTy = MRI.getType(Reg: Dst);
1905 if (ICmpInst::isEquality(P: Pred)) {
1906 // For each part on the LHS and RHS, keep track of the result of XOR-ing
1907 // them together. For each equal part, the result should be all 0s. For
1908 // each non-equal part, we'll get at least one 1.
1909 auto Zero = MIRBuilder.buildConstant(Res: NarrowTy, Val: 0);
1910 SmallVector<Register, 4> Xors;
1911 for (auto LHSAndRHS : zip(t&: LHSPartRegs, u&: RHSPartRegs)) {
1912 auto LHS = std::get<0>(t&: LHSAndRHS);
1913 auto RHS = std::get<1>(t&: LHSAndRHS);
1914 auto Xor = MIRBuilder.buildXor(Dst: NarrowTy, Src0: LHS, Src1: RHS).getReg(Idx: 0);
1915 Xors.push_back(Elt: Xor);
1916 }
1917
1918 // Build a G_XOR for each leftover register. Each G_XOR must be widened
1919 // to the desired narrow type so that we can OR them together later.
1920 SmallVector<Register, 4> WidenedXors;
1921 for (auto LHSAndRHS : zip(t&: LHSLeftoverRegs, u&: RHSLeftoverRegs)) {
1922 auto LHS = std::get<0>(t&: LHSAndRHS);
1923 auto RHS = std::get<1>(t&: LHSAndRHS);
1924 auto Xor = MIRBuilder.buildXor(Dst: LeftoverTy, Src0: LHS, Src1: RHS).getReg(Idx: 0);
1925 LLT GCDTy = extractGCDType(Parts&: WidenedXors, DstTy: NarrowTy, NarrowTy: LeftoverTy, SrcReg: Xor);
1926 buildLCMMergePieces(DstTy: LeftoverTy, NarrowTy, GCDTy, VRegs&: WidenedXors,
1927 /* PadStrategy = */ TargetOpcode::G_ZEXT);
1928 llvm::append_range(C&: Xors, R&: WidenedXors);
1929 }
1930
1931 // Now, for each part we broke up, we know if they are equal/not equal
1932 // based off the G_XOR. We can OR these all together and compare against
1933 // 0 to get the result.
1934 assert(Xors.size() >= 2 && "Should have gotten at least two Xors?");
1935 auto Or = MIRBuilder.buildOr(Dst: NarrowTy, Src0: Xors[0], Src1: Xors[1]);
1936 for (unsigned I = 2, E = Xors.size(); I < E; ++I)
1937 Or = MIRBuilder.buildOr(Dst: NarrowTy, Src0: Or, Src1: Xors[I]);
1938 MIRBuilder.buildICmp(Pred, Res: Dst, Op0: Or, Op1: Zero);
1939 } else {
1940 Register CmpIn;
1941 for (unsigned I = 0, E = LHSPartRegs.size(); I != E; ++I) {
1942 Register CmpOut;
1943 CmpInst::Predicate PartPred;
1944
1945 if (I == E - 1 && LHSLeftoverRegs.empty()) {
1946 PartPred = Pred;
1947 CmpOut = Dst;
1948 } else {
1949 PartPred = ICmpInst::getUnsignedPredicate(Pred);
1950 CmpOut = MRI.createGenericVirtualRegister(Ty: ResTy);
1951 }
1952
1953 if (!CmpIn) {
1954 MIRBuilder.buildICmp(Pred: PartPred, Res: CmpOut, Op0: LHSPartRegs[I],
1955 Op1: RHSPartRegs[I]);
1956 } else {
1957 auto Cmp = MIRBuilder.buildICmp(Pred: PartPred, Res: ResTy, Op0: LHSPartRegs[I],
1958 Op1: RHSPartRegs[I]);
1959 auto CmpEq = MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: ResTy,
1960 Op0: LHSPartRegs[I], Op1: RHSPartRegs[I]);
1961 MIRBuilder.buildSelect(Res: CmpOut, Tst: CmpEq, Op0: CmpIn, Op1: Cmp);
1962 }
1963
1964 CmpIn = CmpOut;
1965 }
1966
1967 for (unsigned I = 0, E = LHSLeftoverRegs.size(); I != E; ++I) {
1968 Register CmpOut;
1969 CmpInst::Predicate PartPred;
1970
1971 if (I == E - 1) {
1972 PartPred = Pred;
1973 CmpOut = Dst;
1974 } else {
1975 PartPred = ICmpInst::getUnsignedPredicate(Pred);
1976 CmpOut = MRI.createGenericVirtualRegister(Ty: ResTy);
1977 }
1978
1979 if (!CmpIn) {
1980 MIRBuilder.buildICmp(Pred: PartPred, Res: CmpOut, Op0: LHSLeftoverRegs[I],
1981 Op1: RHSLeftoverRegs[I]);
1982 } else {
1983 auto Cmp = MIRBuilder.buildICmp(Pred: PartPred, Res: ResTy, Op0: LHSLeftoverRegs[I],
1984 Op1: RHSLeftoverRegs[I]);
1985 auto CmpEq =
1986 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: ResTy,
1987 Op0: LHSLeftoverRegs[I], Op1: RHSLeftoverRegs[I]);
1988 MIRBuilder.buildSelect(Res: CmpOut, Tst: CmpEq, Op0: CmpIn, Op1: Cmp);
1989 }
1990
1991 CmpIn = CmpOut;
1992 }
1993 }
1994 MI.eraseFromParent();
1995 return Legalized;
1996 }
1997 case TargetOpcode::G_FCMP:
1998 if (TypeIdx != 0)
1999 return UnableToLegalize;
2000
2001 Observer.changingInstr(MI);
2002 narrowScalarDst(MI, NarrowTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_ZEXT);
2003 Observer.changedInstr(MI);
2004 return Legalized;
2005
2006 case TargetOpcode::G_SEXT_INREG: {
2007 if (TypeIdx != 0)
2008 return UnableToLegalize;
2009
2010 int64_t SizeInBits = MI.getOperand(i: 2).getImm();
2011
2012 // So long as the new type has more bits than the bits we're extending we
2013 // don't need to break it apart.
2014 if (NarrowTy.getScalarSizeInBits() > SizeInBits) {
2015 Observer.changingInstr(MI);
2016 // We don't lose any non-extension bits by truncating the src and
2017 // sign-extending the dst.
2018 MachineOperand &MO1 = MI.getOperand(i: 1);
2019 auto TruncMIB = MIRBuilder.buildTrunc(Res: NarrowTy, Op: MO1);
2020 MO1.setReg(TruncMIB.getReg(Idx: 0));
2021
2022 MachineOperand &MO2 = MI.getOperand(i: 0);
2023 Register DstExt = MRI.createGenericVirtualRegister(Ty: NarrowTy);
2024 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2025 MIRBuilder.buildSExt(Res: MO2, Op: DstExt);
2026 MO2.setReg(DstExt);
2027 Observer.changedInstr(MI);
2028 return Legalized;
2029 }
2030
2031 // Break it apart. Components below the extension point are unmodified. The
2032 // component containing the extension point becomes a narrower SEXT_INREG.
2033 // Components above it are ashr'd from the component containing the
2034 // extension point.
2035 if (SizeOp0 % NarrowSize != 0)
2036 return UnableToLegalize;
2037 int NumParts = SizeOp0 / NarrowSize;
2038
2039 // List the registers where the destination will be scattered.
2040 SmallVector<Register, 2> DstRegs;
2041 // List the registers where the source will be split.
2042 SmallVector<Register, 2> SrcRegs;
2043
2044 // Create all the temporary registers.
2045 for (int i = 0; i < NumParts; ++i) {
2046 Register SrcReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
2047
2048 SrcRegs.push_back(Elt: SrcReg);
2049 }
2050
2051 // Explode the big arguments into smaller chunks.
2052 MIRBuilder.buildUnmerge(Res: SrcRegs, Op: MI.getOperand(i: 1));
2053
2054 Register AshrCstReg =
2055 MIRBuilder.buildConstant(Res: NarrowTy, Val: NarrowTy.getScalarSizeInBits() - 1)
2056 .getReg(Idx: 0);
2057 Register FullExtensionReg;
2058 Register PartialExtensionReg;
2059
2060 // Do the operation on each small part.
2061 for (int i = 0; i < NumParts; ++i) {
2062 if ((i + 1) * NarrowTy.getScalarSizeInBits() <= SizeInBits) {
2063 DstRegs.push_back(Elt: SrcRegs[i]);
2064 PartialExtensionReg = DstRegs.back();
2065 } else if (i * NarrowTy.getScalarSizeInBits() >= SizeInBits) {
2066 assert(PartialExtensionReg &&
2067 "Expected to visit partial extension before full");
2068 if (FullExtensionReg) {
2069 DstRegs.push_back(Elt: FullExtensionReg);
2070 continue;
2071 }
2072 DstRegs.push_back(
2073 Elt: MIRBuilder.buildAShr(Dst: NarrowTy, Src0: PartialExtensionReg, Src1: AshrCstReg)
2074 .getReg(Idx: 0));
2075 FullExtensionReg = DstRegs.back();
2076 } else {
2077 DstRegs.push_back(
2078 Elt: MIRBuilder
2079 .buildInstr(
2080 Opc: TargetOpcode::G_SEXT_INREG, DstOps: {NarrowTy},
2081 SrcOps: {SrcRegs[i], SizeInBits % NarrowTy.getScalarSizeInBits()})
2082 .getReg(Idx: 0));
2083 PartialExtensionReg = DstRegs.back();
2084 }
2085 }
2086
2087 // Gather the destination registers into the final destination.
2088 Register DstReg = MI.getOperand(i: 0).getReg();
2089 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
2090 MI.eraseFromParent();
2091 return Legalized;
2092 }
2093 case TargetOpcode::G_BSWAP:
2094 case TargetOpcode::G_BITREVERSE: {
2095 if (SizeOp0 % NarrowSize != 0)
2096 return UnableToLegalize;
2097
2098 Observer.changingInstr(MI);
2099 SmallVector<Register, 2> SrcRegs, DstRegs;
2100 unsigned NumParts = SizeOp0 / NarrowSize;
2101 extractParts(Reg: MI.getOperand(i: 1).getReg(), Ty: NarrowTy, NumParts, VRegs&: SrcRegs,
2102 MIRBuilder, MRI);
2103
2104 for (unsigned i = 0; i < NumParts; ++i) {
2105 auto DstPart = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {NarrowTy},
2106 SrcOps: {SrcRegs[NumParts - 1 - i]});
2107 DstRegs.push_back(Elt: DstPart.getReg(Idx: 0));
2108 }
2109
2110 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0), Ops: DstRegs);
2111
2112 Observer.changedInstr(MI);
2113 MI.eraseFromParent();
2114 return Legalized;
2115 }
2116 case TargetOpcode::G_PTR_ADD:
2117 case TargetOpcode::G_PTRMASK: {
2118 if (TypeIdx != 1)
2119 return UnableToLegalize;
2120 Observer.changingInstr(MI);
2121 narrowScalarSrc(MI, NarrowTy, OpIdx: 2);
2122 Observer.changedInstr(MI);
2123 return Legalized;
2124 }
2125 case TargetOpcode::G_FPTOUI:
2126 case TargetOpcode::G_FPTOSI:
2127 case TargetOpcode::G_FPTOUI_SAT:
2128 case TargetOpcode::G_FPTOSI_SAT:
2129 return narrowScalarFPTOI(MI, TypeIdx, Ty: NarrowTy);
2130 case TargetOpcode::G_FPEXT:
2131 if (TypeIdx != 0)
2132 return UnableToLegalize;
2133 Observer.changingInstr(MI);
2134 narrowScalarDst(MI, NarrowTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_FPEXT);
2135 Observer.changedInstr(MI);
2136 return Legalized;
2137 case TargetOpcode::G_FLDEXP:
2138 case TargetOpcode::G_STRICT_FLDEXP:
2139 return narrowScalarFLDEXP(MI, TypeIdx, Ty: NarrowTy);
2140 case TargetOpcode::G_VSCALE: {
2141 Register Dst = MI.getOperand(i: 0).getReg();
2142 LLT Ty = MRI.getType(Reg: Dst);
2143
2144 // Assume VSCALE(1) fits into a legal integer
2145 const APInt One(NarrowTy.getSizeInBits(), 1);
2146 auto VScaleBase = MIRBuilder.buildVScale(Res: NarrowTy, MinElts: One);
2147 auto ZExt = MIRBuilder.buildZExt(Res: Ty, Op: VScaleBase);
2148 auto C = MIRBuilder.buildConstant(Res: Ty, Val: *MI.getOperand(i: 1).getCImm());
2149 MIRBuilder.buildMul(Dst, Src0: ZExt, Src1: C);
2150
2151 MI.eraseFromParent();
2152 return Legalized;
2153 }
2154 }
2155}
2156
2157Register LegalizerHelper::coerceToInteger(Register Val) {
2158 LLT Ty = MRI.getType(Reg: Val);
2159 if (Ty.isScalar() && !Ty.isFloat())
2160 return Val;
2161
2162 const DataLayout &DL = MIRBuilder.getDataLayout();
2163 LLT NewTy = LLT::integer(SizeInBits: Ty.getSizeInBits());
2164
2165 if (Ty.isFloat())
2166 return MIRBuilder.buildBitcast(Dst: NewTy, Src: Val).getReg(Idx: 0);
2167
2168 if (Ty.isPointer()) {
2169 if (DL.isNonIntegralAddressSpace(AddrSpace: Ty.getAddressSpace()))
2170 return Register();
2171 return MIRBuilder.buildPtrToInt(Dst: NewTy, Src: Val).getReg(Idx: 0);
2172 }
2173
2174 Register NewVal = Val;
2175
2176 assert(Ty.isVector());
2177 if (Ty.isPointerVector())
2178 NewVal = MIRBuilder.buildPtrToInt(Dst: NewTy, Src: NewVal).getReg(Idx: 0);
2179 return MIRBuilder.buildBitcast(Dst: NewTy, Src: NewVal).getReg(Idx: 0);
2180}
2181
2182void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy,
2183 unsigned OpIdx, unsigned ExtOpcode) {
2184 MachineOperand &MO = MI.getOperand(i: OpIdx);
2185 auto ExtB = MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {WideTy}, SrcOps: {MO});
2186 MO.setReg(ExtB.getReg(Idx: 0));
2187}
2188
2189void LegalizerHelper::widenScalarSrcUsingFPExt(MachineInstr &MI, LLT WideTy,
2190 unsigned OpIdx) {
2191 MachineOperand &MO = MI.getOperand(i: OpIdx);
2192 auto ExtB = MIRBuilder.buildInstr(Opc: TargetOpcode::G_FPEXT, DstOps: {WideTy}, SrcOps: {MO},
2193 Flags: MI.getFlags());
2194 MO.setReg(ExtB.getReg(Idx: 0));
2195}
2196
2197void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy,
2198 unsigned OpIdx) {
2199 MachineOperand &MO = MI.getOperand(i: OpIdx);
2200 auto ExtB = MIRBuilder.buildTrunc(Res: NarrowTy, Op: MO);
2201 MO.setReg(ExtB.getReg(Idx: 0));
2202}
2203
2204void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy,
2205 unsigned OpIdx, unsigned TruncOpcode) {
2206 MachineOperand &MO = MI.getOperand(i: OpIdx);
2207 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2208 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2209 MIRBuilder.buildInstr(Opc: TruncOpcode, DstOps: {MO}, SrcOps: {DstExt});
2210 MO.setReg(DstExt);
2211}
2212
2213void LegalizerHelper::widenScalarDstUsingFPTrunc(MachineInstr &MI, LLT WideTy,
2214 unsigned OpIdx) {
2215 MachineOperand &MO = MI.getOperand(i: OpIdx);
2216 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2217 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2218 MIRBuilder.buildInstr(Opc: TargetOpcode::G_FPTRUNC, DstOps: {MO}, SrcOps: {DstExt}, Flags: MI.getFlags());
2219 MO.setReg(DstExt);
2220}
2221
2222void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy,
2223 unsigned OpIdx, unsigned ExtOpcode) {
2224 MachineOperand &MO = MI.getOperand(i: OpIdx);
2225 Register DstTrunc = MRI.createGenericVirtualRegister(Ty: NarrowTy);
2226 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2227 MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {MO}, SrcOps: {DstTrunc});
2228 MO.setReg(DstTrunc);
2229}
2230
2231void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy,
2232 unsigned OpIdx) {
2233 MachineOperand &MO = MI.getOperand(i: OpIdx);
2234 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2235 Register Dst = MO.getReg();
2236 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2237 MO.setReg(DstExt);
2238 MIRBuilder.buildDeleteTrailingVectorElements(Res: Dst, Op0: DstExt);
2239}
2240
2241void LegalizerHelper::moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy,
2242 unsigned OpIdx) {
2243 MachineOperand &MO = MI.getOperand(i: OpIdx);
2244 MO.setReg(MIRBuilder.buildPadVectorWithUndefElements(Res: MoreTy, Op0: MO).getReg(Idx: 0));
2245}
2246
2247void LegalizerHelper::bitcastSrc(MachineInstr &MI, LLT CastTy, unsigned OpIdx) {
2248 MachineOperand &Op = MI.getOperand(i: OpIdx);
2249 Op.setReg(MIRBuilder.buildBitcast(Dst: CastTy, Src: Op).getReg(Idx: 0));
2250}
2251
2252void LegalizerHelper::bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx) {
2253 MachineOperand &MO = MI.getOperand(i: OpIdx);
2254 Register CastDst = MRI.createGenericVirtualRegister(Ty: CastTy);
2255 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2256 MIRBuilder.buildBitcast(Dst: MO, Src: CastDst);
2257 MO.setReg(CastDst);
2258}
2259
2260LegalizerHelper::LegalizeResult
2261LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx,
2262 LLT WideTy) {
2263 if (TypeIdx != 1)
2264 return UnableToLegalize;
2265
2266 auto [DstReg, DstTy, Src1Reg, Src1Ty] = MI.getFirst2RegLLTs();
2267 if (DstTy.isVector())
2268 return UnableToLegalize;
2269
2270 LLT SrcTy = MRI.getType(Reg: Src1Reg);
2271 const int DstSize = DstTy.getSizeInBits();
2272 const int SrcSize = SrcTy.getSizeInBits();
2273 const int WideSize = WideTy.getSizeInBits();
2274 const int NumMerge = (DstSize + WideSize - 1) / WideSize;
2275
2276 unsigned NumOps = MI.getNumOperands();
2277 unsigned NumSrc = MI.getNumOperands() - 1;
2278 unsigned PartSize = DstTy.getSizeInBits() / NumSrc;
2279
2280 if (WideSize >= DstSize) {
2281 // Directly pack the bits in the target type.
2282 Register ResultReg = MIRBuilder.buildZExt(Res: WideTy, Op: Src1Reg).getReg(Idx: 0);
2283
2284 for (unsigned I = 2; I != NumOps; ++I) {
2285 const unsigned Offset = (I - 1) * PartSize;
2286
2287 Register SrcReg = MI.getOperand(i: I).getReg();
2288 assert(MRI.getType(SrcReg) == LLT::scalar(PartSize));
2289
2290 auto ZextInput = MIRBuilder.buildZExt(Res: WideTy, Op: SrcReg);
2291
2292 Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg :
2293 MRI.createGenericVirtualRegister(Ty: WideTy);
2294
2295 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: Offset);
2296 auto Shl = MIRBuilder.buildShl(Dst: WideTy, Src0: ZextInput, Src1: ShiftAmt);
2297 MIRBuilder.buildOr(Dst: NextResult, Src0: ResultReg, Src1: Shl);
2298 ResultReg = NextResult;
2299 }
2300
2301 if (WideSize > DstSize)
2302 MIRBuilder.buildTrunc(Res: DstReg, Op: ResultReg);
2303 else if (DstTy.isPointer())
2304 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: ResultReg);
2305 else if (DstTy != WideTy)
2306 MIRBuilder.buildBitcast(Dst: DstReg, Src: ResultReg);
2307
2308 MI.eraseFromParent();
2309 return Legalized;
2310 }
2311
2312 // Unmerge the original values to the GCD type, and recombine to the next
2313 // multiple greater than the original type.
2314 //
2315 // %3:_(s12) = G_MERGE_VALUES %0:_(s4), %1:_(s4), %2:_(s4) -> s6
2316 // %4:_(s2), %5:_(s2) = G_UNMERGE_VALUES %0
2317 // %6:_(s2), %7:_(s2) = G_UNMERGE_VALUES %1
2318 // %8:_(s2), %9:_(s2) = G_UNMERGE_VALUES %2
2319 // %10:_(s6) = G_MERGE_VALUES %4, %5, %6
2320 // %11:_(s6) = G_MERGE_VALUES %7, %8, %9
2321 // %12:_(s12) = G_MERGE_VALUES %10, %11
2322 //
2323 // Padding with undef if necessary:
2324 //
2325 // %2:_(s8) = G_MERGE_VALUES %0:_(s4), %1:_(s4) -> s6
2326 // %3:_(s2), %4:_(s2) = G_UNMERGE_VALUES %0
2327 // %5:_(s2), %6:_(s2) = G_UNMERGE_VALUES %1
2328 // %7:_(s2) = G_IMPLICIT_DEF
2329 // %8:_(s6) = G_MERGE_VALUES %3, %4, %5
2330 // %9:_(s6) = G_MERGE_VALUES %6, %7, %7
2331 // %10:_(s12) = G_MERGE_VALUES %8, %9
2332
2333 const int GCD = std::gcd(m: SrcSize, n: WideSize);
2334 LLT GCDTy = WideTy.changeElementSize(NewEltSize: GCD);
2335
2336 SmallVector<Register, 8> NewMergeRegs;
2337 SmallVector<Register, 8> Unmerges;
2338 LLT WideDstTy = WideTy.changeElementSize(NewEltSize: NumMerge * WideSize);
2339
2340 // Decompose the original operands if they don't evenly divide.
2341 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands())) {
2342 Register SrcReg = MO.getReg();
2343 if (GCD == SrcSize) {
2344 Unmerges.push_back(Elt: SrcReg);
2345 } else {
2346 auto Unmerge = MIRBuilder.buildUnmerge(Res: GCDTy, Op: SrcReg);
2347 for (int J = 0, JE = Unmerge->getNumOperands() - 1; J != JE; ++J)
2348 Unmerges.push_back(Elt: Unmerge.getReg(Idx: J));
2349 }
2350 }
2351
2352 // Pad with undef to the next size that is a multiple of the requested size.
2353 if (static_cast<int>(Unmerges.size()) != NumMerge * WideSize) {
2354 Register UndefReg = MIRBuilder.buildUndef(Res: GCDTy).getReg(Idx: 0);
2355 for (int I = Unmerges.size(); I != NumMerge * WideSize; ++I)
2356 Unmerges.push_back(Elt: UndefReg);
2357 }
2358
2359 const int PartsPerGCD = WideSize / GCD;
2360
2361 // Build merges of each piece.
2362 ArrayRef<Register> Slicer(Unmerges);
2363 for (int I = 0; I != NumMerge; ++I, Slicer = Slicer.drop_front(N: PartsPerGCD)) {
2364 auto Merge =
2365 MIRBuilder.buildMergeLikeInstr(Res: WideTy, Ops: Slicer.take_front(N: PartsPerGCD));
2366 NewMergeRegs.push_back(Elt: Merge.getReg(Idx: 0));
2367 }
2368
2369 // A truncate may be necessary if the requested type doesn't evenly divide the
2370 // original result type.
2371 if (DstTy.getSizeInBits() == WideDstTy.getSizeInBits()) {
2372 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: NewMergeRegs);
2373 } else {
2374 auto FinalMerge = MIRBuilder.buildMergeLikeInstr(Res: WideDstTy, Ops: NewMergeRegs);
2375 MIRBuilder.buildTrunc(Res: DstReg, Op: FinalMerge.getReg(Idx: 0));
2376 }
2377
2378 MI.eraseFromParent();
2379 return Legalized;
2380}
2381
2382LegalizerHelper::LegalizeResult
2383LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx,
2384 LLT WideTy) {
2385 if (TypeIdx != 0)
2386 return UnableToLegalize;
2387
2388 int NumDst = MI.getNumOperands() - 1;
2389 Register SrcReg = MI.getOperand(i: NumDst).getReg();
2390 LLT SrcTy = MRI.getType(Reg: SrcReg);
2391 if (SrcTy.isVector())
2392 return UnableToLegalize;
2393
2394 Register Dst0Reg = MI.getOperand(i: 0).getReg();
2395 LLT DstTy = MRI.getType(Reg: Dst0Reg);
2396 if (!DstTy.isScalar())
2397 return UnableToLegalize;
2398
2399 if (WideTy.getSizeInBits() >= SrcTy.getSizeInBits()) {
2400 if (SrcTy.isPointer()) {
2401 const DataLayout &DL = MIRBuilder.getDataLayout();
2402 if (DL.isNonIntegralAddressSpace(AddrSpace: SrcTy.getAddressSpace())) {
2403 LLVM_DEBUG(
2404 dbgs() << "Not casting non-integral address space integer\n");
2405 return UnableToLegalize;
2406 }
2407
2408 SrcTy = LLT::scalar(SizeInBits: SrcTy.getSizeInBits());
2409 SrcReg = MIRBuilder.buildPtrToInt(Dst: SrcTy, Src: SrcReg).getReg(Idx: 0);
2410 }
2411
2412 // Widen SrcTy to WideTy. This does not affect the result, but since the
2413 // user requested this size, it is probably better handled than SrcTy and
2414 // should reduce the total number of legalization artifacts.
2415 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
2416 SrcTy = WideTy;
2417 SrcReg = MIRBuilder.buildAnyExt(Res: WideTy, Op: SrcReg).getReg(Idx: 0);
2418 }
2419
2420 // Theres no unmerge type to target. Directly extract the bits from the
2421 // source type
2422 unsigned DstSize = DstTy.getSizeInBits();
2423
2424 if (SrcTy.isFloat()) {
2425 SrcReg = coerceToInteger(Val: SrcReg);
2426 SrcTy = MRI.getType(Reg: SrcReg);
2427 }
2428
2429 MIRBuilder.buildTrunc(Res: Dst0Reg, Op: SrcReg);
2430 for (int I = 1; I != NumDst; ++I) {
2431 auto ShiftAmt = MIRBuilder.buildConstant(Res: SrcTy, Val: DstSize * I);
2432 auto Shr = MIRBuilder.buildLShr(Dst: SrcTy, Src0: SrcReg, Src1: ShiftAmt);
2433 MIRBuilder.buildTrunc(Res: MI.getOperand(i: I), Op: Shr);
2434 }
2435
2436 MI.eraseFromParent();
2437 return Legalized;
2438 }
2439
2440 // Extend the source to a wider type.
2441 LLT LCMTy = getLCMType(OrigTy: SrcTy, TargetTy: WideTy);
2442
2443 Register WideSrc = SrcReg;
2444 if (LCMTy.getSizeInBits() != SrcTy.getSizeInBits()) {
2445 // TODO: If this is an integral address space, cast to integer and anyext.
2446 if (SrcTy.isPointer()) {
2447 LLVM_DEBUG(dbgs() << "Widening pointer source types not implemented\n");
2448 return UnableToLegalize;
2449 }
2450
2451 WideSrc = MIRBuilder.buildAnyExt(Res: LCMTy, Op: WideSrc).getReg(Idx: 0);
2452 }
2453
2454 auto Unmerge = MIRBuilder.buildUnmerge(Res: WideTy, Op: WideSrc);
2455
2456 // Create a sequence of unmerges and merges to the original results. Since we
2457 // may have widened the source, we will need to pad the results with dead defs
2458 // to cover the source register.
2459 // e.g. widen s48 to s64:
2460 // %1:_(s48), %2:_(s48) = G_UNMERGE_VALUES %0:_(s96)
2461 //
2462 // =>
2463 // %4:_(s192) = G_ANYEXT %0:_(s96)
2464 // %5:_(s64), %6, %7 = G_UNMERGE_VALUES %4 ; Requested unmerge
2465 // ; unpack to GCD type, with extra dead defs
2466 // %8:_(s16), %9, %10, %11 = G_UNMERGE_VALUES %5:_(s64)
2467 // %12:_(s16), %13, dead %14, dead %15 = G_UNMERGE_VALUES %6:_(s64)
2468 // dead %16:_(s16), dead %17, dead %18, dead %18 = G_UNMERGE_VALUES %7:_(s64)
2469 // %1:_(s48) = G_MERGE_VALUES %8:_(s16), %9, %10 ; Remerge to destination
2470 // %2:_(s48) = G_MERGE_VALUES %11:_(s16), %12, %13 ; Remerge to destination
2471 const LLT GCDTy = getGCDType(OrigTy: WideTy, TargetTy: DstTy);
2472 const int NumUnmerge = Unmerge->getNumOperands() - 1;
2473 const int PartsPerRemerge = DstTy.getSizeInBits() / GCDTy.getSizeInBits();
2474
2475 // Directly unmerge to the destination without going through a GCD type
2476 // if possible
2477 if (PartsPerRemerge == 1) {
2478 const int PartsPerUnmerge = WideTy.getSizeInBits() / DstTy.getSizeInBits();
2479
2480 for (int I = 0; I != NumUnmerge; ++I) {
2481 auto MIB = MIRBuilder.buildInstr(Opcode: TargetOpcode::G_UNMERGE_VALUES);
2482
2483 for (int J = 0; J != PartsPerUnmerge; ++J) {
2484 int Idx = I * PartsPerUnmerge + J;
2485 if (Idx < NumDst)
2486 MIB.addDef(RegNo: MI.getOperand(i: Idx).getReg());
2487 else {
2488 // Create dead def for excess components.
2489 MIB.addDef(RegNo: MRI.createGenericVirtualRegister(Ty: DstTy));
2490 }
2491 }
2492
2493 MIB.addUse(RegNo: Unmerge.getReg(Idx: I));
2494 }
2495 } else {
2496 SmallVector<Register, 16> Parts;
2497 for (int J = 0; J != NumUnmerge; ++J)
2498 extractGCDType(Parts, GCDTy, SrcReg: Unmerge.getReg(Idx: J));
2499
2500 SmallVector<Register, 8> RemergeParts;
2501 for (int I = 0; I != NumDst; ++I) {
2502 for (int J = 0; J < PartsPerRemerge; ++J) {
2503 const int Idx = I * PartsPerRemerge + J;
2504 RemergeParts.emplace_back(Args&: Parts[Idx]);
2505 }
2506
2507 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: I).getReg(), Ops: RemergeParts);
2508 RemergeParts.clear();
2509 }
2510 }
2511
2512 MI.eraseFromParent();
2513 return Legalized;
2514}
2515
2516LegalizerHelper::LegalizeResult
2517LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx,
2518 LLT WideTy) {
2519 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
2520 unsigned Offset = MI.getOperand(i: 2).getImm();
2521
2522 if (TypeIdx == 0) {
2523 if (SrcTy.isVector() || DstTy.isVector())
2524 return UnableToLegalize;
2525
2526 SrcOp Src(SrcReg);
2527 if (SrcTy.isPointer()) {
2528 // Extracts from pointers can be handled only if they are really just
2529 // simple integers.
2530 const DataLayout &DL = MIRBuilder.getDataLayout();
2531 if (DL.isNonIntegralAddressSpace(AddrSpace: SrcTy.getAddressSpace()))
2532 return UnableToLegalize;
2533
2534 LLT SrcAsIntTy = LLT::scalar(SizeInBits: SrcTy.getSizeInBits());
2535 Src = MIRBuilder.buildPtrToInt(Dst: SrcAsIntTy, Src);
2536 SrcTy = SrcAsIntTy;
2537 }
2538
2539 if (DstTy.isPointer())
2540 return UnableToLegalize;
2541
2542 if (Offset == 0) {
2543 // Avoid a shift in the degenerate case.
2544 MIRBuilder.buildTrunc(Res: DstReg,
2545 Op: MIRBuilder.buildAnyExtOrTrunc(Res: WideTy, Op: Src));
2546 MI.eraseFromParent();
2547 return Legalized;
2548 }
2549
2550 // Do a shift in the source type.
2551 LLT ShiftTy = SrcTy;
2552 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
2553 Src = MIRBuilder.buildAnyExt(Res: WideTy, Op: Src);
2554 ShiftTy = WideTy;
2555 }
2556
2557 auto LShr = MIRBuilder.buildLShr(
2558 Dst: ShiftTy, Src0: Src, Src1: MIRBuilder.buildConstant(Res: ShiftTy, Val: Offset));
2559 MIRBuilder.buildTrunc(Res: DstReg, Op: LShr);
2560 MI.eraseFromParent();
2561 return Legalized;
2562 }
2563
2564 if (SrcTy.isScalar()) {
2565 Observer.changingInstr(MI);
2566 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2567 Observer.changedInstr(MI);
2568 return Legalized;
2569 }
2570
2571 if (!SrcTy.isVector())
2572 return UnableToLegalize;
2573
2574 if (DstTy != SrcTy.getElementType())
2575 return UnableToLegalize;
2576
2577 if (Offset % SrcTy.getScalarSizeInBits() != 0)
2578 return UnableToLegalize;
2579
2580 Observer.changingInstr(MI);
2581 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2582
2583 MI.getOperand(i: 2).setImm((WideTy.getSizeInBits() / SrcTy.getSizeInBits()) *
2584 Offset);
2585 widenScalarDst(MI, WideTy: WideTy.getScalarType(), OpIdx: 0);
2586 Observer.changedInstr(MI);
2587 return Legalized;
2588}
2589
2590LegalizerHelper::LegalizeResult
2591LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx,
2592 LLT WideTy) {
2593 if (TypeIdx != 0 || WideTy.isVector())
2594 return UnableToLegalize;
2595 Observer.changingInstr(MI);
2596 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2597 widenScalarDst(MI, WideTy);
2598 Observer.changedInstr(MI);
2599 return Legalized;
2600}
2601
2602LegalizerHelper::LegalizeResult
2603LegalizerHelper::widenScalarAddSubOverflow(MachineInstr &MI, unsigned TypeIdx,
2604 LLT WideTy) {
2605 unsigned Opcode;
2606 unsigned ExtOpcode;
2607 std::optional<Register> CarryIn;
2608 switch (MI.getOpcode()) {
2609 default:
2610 llvm_unreachable("Unexpected opcode!");
2611 case TargetOpcode::G_SADDO:
2612 Opcode = TargetOpcode::G_ADD;
2613 ExtOpcode = TargetOpcode::G_SEXT;
2614 break;
2615 case TargetOpcode::G_SSUBO:
2616 Opcode = TargetOpcode::G_SUB;
2617 ExtOpcode = TargetOpcode::G_SEXT;
2618 break;
2619 case TargetOpcode::G_UADDO:
2620 Opcode = TargetOpcode::G_ADD;
2621 ExtOpcode = TargetOpcode::G_ZEXT;
2622 break;
2623 case TargetOpcode::G_USUBO:
2624 Opcode = TargetOpcode::G_SUB;
2625 ExtOpcode = TargetOpcode::G_ZEXT;
2626 break;
2627 case TargetOpcode::G_SADDE:
2628 Opcode = TargetOpcode::G_UADDE;
2629 ExtOpcode = TargetOpcode::G_SEXT;
2630 CarryIn = MI.getOperand(i: 4).getReg();
2631 break;
2632 case TargetOpcode::G_SSUBE:
2633 Opcode = TargetOpcode::G_USUBE;
2634 ExtOpcode = TargetOpcode::G_SEXT;
2635 CarryIn = MI.getOperand(i: 4).getReg();
2636 break;
2637 case TargetOpcode::G_UADDE:
2638 Opcode = TargetOpcode::G_UADDE;
2639 ExtOpcode = TargetOpcode::G_ZEXT;
2640 CarryIn = MI.getOperand(i: 4).getReg();
2641 break;
2642 case TargetOpcode::G_USUBE:
2643 Opcode = TargetOpcode::G_USUBE;
2644 ExtOpcode = TargetOpcode::G_ZEXT;
2645 CarryIn = MI.getOperand(i: 4).getReg();
2646 break;
2647 }
2648
2649 if (TypeIdx == 1) {
2650 unsigned BoolExtOp = MIRBuilder.getBoolExtOp(IsVec: WideTy.isVector(), IsFP: false);
2651
2652 Observer.changingInstr(MI);
2653 if (CarryIn)
2654 widenScalarSrc(MI, WideTy, OpIdx: 4, ExtOpcode: BoolExtOp);
2655 widenScalarDst(MI, WideTy, OpIdx: 1);
2656
2657 Observer.changedInstr(MI);
2658 return Legalized;
2659 }
2660
2661 auto LHSExt = MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {WideTy}, SrcOps: {MI.getOperand(i: 2)});
2662 auto RHSExt = MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {WideTy}, SrcOps: {MI.getOperand(i: 3)});
2663 // Do the arithmetic in the larger type.
2664 Register NewOp;
2665 if (CarryIn) {
2666 LLT CarryOutTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
2667 NewOp = MIRBuilder
2668 .buildInstr(Opc: Opcode, DstOps: {WideTy, CarryOutTy},
2669 SrcOps: {LHSExt, RHSExt, *CarryIn})
2670 .getReg(Idx: 0);
2671 } else {
2672 NewOp = MIRBuilder.buildInstr(Opc: Opcode, DstOps: {WideTy}, SrcOps: {LHSExt, RHSExt}).getReg(Idx: 0);
2673 }
2674 LLT OrigTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
2675 auto TruncOp = MIRBuilder.buildTrunc(Res: OrigTy, Op: NewOp);
2676 auto ExtOp = MIRBuilder.buildInstr(Opc: ExtOpcode, DstOps: {WideTy}, SrcOps: {TruncOp});
2677 // There is no overflow if the ExtOp is the same as NewOp.
2678 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: MI.getOperand(i: 1), Op0: NewOp, Op1: ExtOp);
2679 // Now trunc the NewOp to the original result.
2680 MIRBuilder.buildTrunc(Res: MI.getOperand(i: 0), Op: NewOp);
2681 MI.eraseFromParent();
2682 return Legalized;
2683}
2684
2685LegalizerHelper::LegalizeResult
2686LegalizerHelper::widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx,
2687 LLT WideTy) {
2688 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SADDSAT ||
2689 MI.getOpcode() == TargetOpcode::G_SSUBSAT ||
2690 MI.getOpcode() == TargetOpcode::G_SSHLSAT;
2691 bool IsShift = MI.getOpcode() == TargetOpcode::G_SSHLSAT ||
2692 MI.getOpcode() == TargetOpcode::G_USHLSAT;
2693 // We can convert this to:
2694 // 1. Any extend iN to iM
2695 // 2. SHL by M-N
2696 // 3. [US][ADD|SUB|SHL]SAT
2697 // 4. L/ASHR by M-N
2698 //
2699 // It may be more efficient to lower this to a min and a max operation in
2700 // the higher precision arithmetic if the promoted operation isn't legal,
2701 // but this decision is up to the target's lowering request.
2702 Register DstReg = MI.getOperand(i: 0).getReg();
2703
2704 unsigned NewBits = WideTy.getScalarSizeInBits();
2705 unsigned SHLAmount = NewBits - MRI.getType(Reg: DstReg).getScalarSizeInBits();
2706
2707 // Shifts must zero-extend the RHS to preserve the unsigned quantity, and
2708 // must not left shift the RHS to preserve the shift amount.
2709 auto LHS = MIRBuilder.buildAnyExt(Res: WideTy, Op: MI.getOperand(i: 1));
2710 auto RHS = IsShift ? MIRBuilder.buildZExt(Res: WideTy, Op: MI.getOperand(i: 2))
2711 : MIRBuilder.buildAnyExt(Res: WideTy, Op: MI.getOperand(i: 2));
2712 auto ShiftK = MIRBuilder.buildConstant(Res: WideTy, Val: SHLAmount);
2713 auto ShiftL = MIRBuilder.buildShl(Dst: WideTy, Src0: LHS, Src1: ShiftK);
2714 auto ShiftR = IsShift ? RHS : MIRBuilder.buildShl(Dst: WideTy, Src0: RHS, Src1: ShiftK);
2715
2716 auto WideInst = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {WideTy},
2717 SrcOps: {ShiftL, ShiftR}, Flags: MI.getFlags());
2718
2719 // Use a shift that will preserve the number of sign bits when the trunc is
2720 // folded away.
2721 auto Result = IsSigned ? MIRBuilder.buildAShr(Dst: WideTy, Src0: WideInst, Src1: ShiftK)
2722 : MIRBuilder.buildLShr(Dst: WideTy, Src0: WideInst, Src1: ShiftK);
2723
2724 MIRBuilder.buildTrunc(Res: DstReg, Op: Result);
2725 MI.eraseFromParent();
2726 return Legalized;
2727}
2728
2729LegalizerHelper::LegalizeResult
2730LegalizerHelper::widenScalarMulo(MachineInstr &MI, unsigned TypeIdx,
2731 LLT WideTy) {
2732 if (TypeIdx == 1) {
2733 Observer.changingInstr(MI);
2734 widenScalarDst(MI, WideTy, OpIdx: 1);
2735 Observer.changedInstr(MI);
2736 return Legalized;
2737 }
2738
2739 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SMULO;
2740 auto [Result, OriginalOverflow, LHS, RHS] = MI.getFirst4Regs();
2741 LLT SrcTy = MRI.getType(Reg: LHS);
2742 LLT OverflowTy = MRI.getType(Reg: OriginalOverflow);
2743 unsigned SrcBitWidth = SrcTy.getScalarSizeInBits();
2744
2745 // To determine if the result overflowed in the larger type, we extend the
2746 // input to the larger type, do the multiply (checking if it overflows),
2747 // then also check the high bits of the result to see if overflow happened
2748 // there.
2749 unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
2750 auto LeftOperand = MIRBuilder.buildInstr(Opc: ExtOp, DstOps: {WideTy}, SrcOps: {LHS});
2751 auto RightOperand = MIRBuilder.buildInstr(Opc: ExtOp, DstOps: {WideTy}, SrcOps: {RHS});
2752
2753 // Multiplication cannot overflow if the WideTy is >= 2 * original width,
2754 // so we don't need to check the overflow result of larger type Mulo.
2755 bool WideMulCanOverflow = WideTy.getScalarSizeInBits() < 2 * SrcBitWidth;
2756
2757 unsigned MulOpc =
2758 WideMulCanOverflow ? MI.getOpcode() : (unsigned)TargetOpcode::G_MUL;
2759
2760 MachineInstrBuilder Mulo;
2761 if (WideMulCanOverflow)
2762 Mulo = MIRBuilder.buildInstr(Opc: MulOpc, DstOps: {WideTy, OverflowTy},
2763 SrcOps: {LeftOperand, RightOperand});
2764 else
2765 Mulo = MIRBuilder.buildInstr(Opc: MulOpc, DstOps: {WideTy}, SrcOps: {LeftOperand, RightOperand});
2766
2767 auto Mul = Mulo->getOperand(i: 0);
2768 MIRBuilder.buildTrunc(Res: Result, Op: Mul);
2769
2770 MachineInstrBuilder ExtResult;
2771 // Overflow occurred if it occurred in the larger type, or if the high part
2772 // of the result does not zero/sign-extend the low part. Check this second
2773 // possibility first.
2774 if (IsSigned) {
2775 // For signed, overflow occurred when the high part does not sign-extend
2776 // the low part.
2777 ExtResult = MIRBuilder.buildSExtInReg(Res: WideTy, Op: Mul, ImmOp: SrcBitWidth);
2778 } else {
2779 // Unsigned overflow occurred when the high part does not zero-extend the
2780 // low part.
2781 ExtResult = MIRBuilder.buildZExtInReg(Res: WideTy, Op: Mul, ImmOp: SrcBitWidth);
2782 }
2783
2784 if (WideMulCanOverflow) {
2785 auto Overflow =
2786 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: OverflowTy, Op0: Mul, Op1: ExtResult);
2787 // Finally check if the multiplication in the larger type itself overflowed.
2788 MIRBuilder.buildOr(Dst: OriginalOverflow, Src0: Mulo->getOperand(i: 1), Src1: Overflow);
2789 } else {
2790 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: OriginalOverflow, Op0: Mul, Op1: ExtResult);
2791 }
2792 MI.eraseFromParent();
2793 return Legalized;
2794}
2795
2796LegalizerHelper::LegalizeResult
2797LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
2798 unsigned Opcode = MI.getOpcode();
2799 switch (Opcode) {
2800 default:
2801 return UnableToLegalize;
2802 case TargetOpcode::G_ATOMICRMW_XCHG:
2803 case TargetOpcode::G_ATOMICRMW_ADD:
2804 case TargetOpcode::G_ATOMICRMW_SUB:
2805 case TargetOpcode::G_ATOMICRMW_AND:
2806 case TargetOpcode::G_ATOMICRMW_OR:
2807 case TargetOpcode::G_ATOMICRMW_XOR:
2808 case TargetOpcode::G_ATOMICRMW_MIN:
2809 case TargetOpcode::G_ATOMICRMW_MAX:
2810 case TargetOpcode::G_ATOMICRMW_UMIN:
2811 case TargetOpcode::G_ATOMICRMW_UMAX:
2812 assert(TypeIdx == 0 && "atomicrmw with second scalar type");
2813 Observer.changingInstr(MI);
2814 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
2815 widenScalarDst(MI, WideTy, OpIdx: 0);
2816 Observer.changedInstr(MI);
2817 return Legalized;
2818 case TargetOpcode::G_ATOMIC_CMPXCHG:
2819 assert(TypeIdx == 0 && "G_ATOMIC_CMPXCHG with second scalar type");
2820 Observer.changingInstr(MI);
2821 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
2822 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ANYEXT);
2823 widenScalarDst(MI, WideTy, OpIdx: 0);
2824 Observer.changedInstr(MI);
2825 return Legalized;
2826 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS:
2827 if (TypeIdx == 0) {
2828 Observer.changingInstr(MI);
2829 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ANYEXT);
2830 widenScalarSrc(MI, WideTy, OpIdx: 4, ExtOpcode: TargetOpcode::G_ANYEXT);
2831 widenScalarDst(MI, WideTy, OpIdx: 0);
2832 Observer.changedInstr(MI);
2833 return Legalized;
2834 }
2835 assert(TypeIdx == 1 &&
2836 "G_ATOMIC_CMPXCHG_WITH_SUCCESS with third scalar type");
2837 Observer.changingInstr(MI);
2838 widenScalarDst(MI, WideTy, OpIdx: 1);
2839 Observer.changedInstr(MI);
2840 return Legalized;
2841 case TargetOpcode::G_EXTRACT:
2842 return widenScalarExtract(MI, TypeIdx, WideTy);
2843 case TargetOpcode::G_INSERT:
2844 return widenScalarInsert(MI, TypeIdx, WideTy);
2845 case TargetOpcode::G_MERGE_VALUES:
2846 return widenScalarMergeValues(MI, TypeIdx, WideTy);
2847 case TargetOpcode::G_UNMERGE_VALUES:
2848 return widenScalarUnmergeValues(MI, TypeIdx, WideTy);
2849 case TargetOpcode::G_SADDO:
2850 case TargetOpcode::G_SSUBO:
2851 case TargetOpcode::G_UADDO:
2852 case TargetOpcode::G_USUBO:
2853 case TargetOpcode::G_SADDE:
2854 case TargetOpcode::G_SSUBE:
2855 case TargetOpcode::G_UADDE:
2856 case TargetOpcode::G_USUBE:
2857 return widenScalarAddSubOverflow(MI, TypeIdx, WideTy);
2858 case TargetOpcode::G_UMULO:
2859 case TargetOpcode::G_SMULO:
2860 return widenScalarMulo(MI, TypeIdx, WideTy);
2861 case TargetOpcode::G_SADDSAT:
2862 case TargetOpcode::G_SSUBSAT:
2863 case TargetOpcode::G_SSHLSAT:
2864 case TargetOpcode::G_UADDSAT:
2865 case TargetOpcode::G_USUBSAT:
2866 case TargetOpcode::G_USHLSAT:
2867 return widenScalarAddSubShlSat(MI, TypeIdx, WideTy);
2868 case TargetOpcode::G_CTTZ:
2869 case TargetOpcode::G_CTTZ_ZERO_POISON:
2870 case TargetOpcode::G_CTLZ:
2871 case TargetOpcode::G_CTLZ_ZERO_POISON:
2872 case TargetOpcode::G_CTLS:
2873 case TargetOpcode::G_CTPOP: {
2874 if (TypeIdx == 0) {
2875 Observer.changingInstr(MI);
2876 widenScalarDst(MI, WideTy, OpIdx: 0);
2877 Observer.changedInstr(MI);
2878 return Legalized;
2879 }
2880
2881 Register SrcReg = MI.getOperand(i: 1).getReg();
2882
2883 // First extend the input.
2884 unsigned ExtOpc;
2885 switch (Opcode) {
2886 case TargetOpcode::G_CTTZ:
2887 case TargetOpcode::G_CTTZ_ZERO_POISON:
2888 case TargetOpcode::G_CTLZ_ZERO_POISON: // poison shifted out below
2889 ExtOpc = TargetOpcode::G_ANYEXT;
2890 break;
2891 case TargetOpcode::G_CTLS:
2892 ExtOpc = TargetOpcode::G_SEXT;
2893 break;
2894 default:
2895 ExtOpc = TargetOpcode::G_ZEXT;
2896 }
2897
2898 auto MIBSrc = MIRBuilder.buildInstr(Opc: ExtOpc, DstOps: {WideTy}, SrcOps: {SrcReg});
2899 LLT CurTy = MRI.getType(Reg: SrcReg);
2900 unsigned NewOpc = Opcode;
2901 if (NewOpc == TargetOpcode::G_CTTZ) {
2902 // The count is the same in the larger type except if the original
2903 // value was zero. This can be handled by setting the bit just off
2904 // the top of the original type.
2905 auto TopBit = APInt::getOneBitSet(numBits: WideTy.getScalarSizeInBits(),
2906 BitNo: CurTy.getScalarSizeInBits());
2907 MIBSrc = MIRBuilder.buildOr(
2908 Dst: WideTy, Src0: MIBSrc, Src1: MIRBuilder.buildConstant(Res: WideTy, Val: TopBit));
2909 // Now we know the operand is non-zero, use the more relaxed opcode.
2910 NewOpc = TargetOpcode::G_CTTZ_ZERO_POISON;
2911 }
2912
2913 unsigned SizeDiff =
2914 WideTy.getScalarSizeInBits() - CurTy.getScalarSizeInBits();
2915
2916 if (Opcode == TargetOpcode::G_CTLZ_ZERO_POISON) {
2917 // An optimization where the result is the CTLZ after the left shift by
2918 // (Difference in widety and current ty), that is,
2919 // MIBSrc = MIBSrc << (sizeinbits(WideTy) - sizeinbits(CurTy))
2920 // Result = ctlz MIBSrc
2921 MIBSrc = MIRBuilder.buildShl(Dst: WideTy, Src0: MIBSrc,
2922 Src1: MIRBuilder.buildConstant(Res: WideTy, Val: SizeDiff));
2923 }
2924
2925 // Perform the operation at the larger size.
2926 auto MIBNewOp = MIRBuilder.buildInstr(Opc: NewOpc, DstOps: {WideTy}, SrcOps: {MIBSrc});
2927 // This is already the correct result for CTPOP and CTTZs
2928 if (Opcode == TargetOpcode::G_CTLZ || Opcode == TargetOpcode::G_CTLS) {
2929 // The correct result is NewOp - (Difference in widety and current ty).
2930 // At this stage SUB is guaranteed to be positive no-wrap,
2931 // that to be used in further KnownBits optimizations for CTLZ.
2932 MIBNewOp = MIRBuilder.buildSub(
2933 Dst: WideTy, Src0: MIBNewOp, Src1: MIRBuilder.buildConstant(Res: WideTy, Val: SizeDiff),
2934 Flags: Opcode == TargetOpcode::G_CTLZ
2935 ? std::optional<unsigned>(MachineInstr::NoUWrap)
2936 : std::nullopt);
2937 }
2938
2939 MIRBuilder.buildZExtOrTrunc(Res: MI.getOperand(i: 0), Op: MIBNewOp);
2940 MI.eraseFromParent();
2941 return Legalized;
2942 }
2943 case TargetOpcode::G_BSWAP: {
2944 Observer.changingInstr(MI);
2945 Register DstReg = MI.getOperand(i: 0).getReg();
2946
2947 Register ShrReg = MRI.createGenericVirtualRegister(Ty: WideTy);
2948 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2949 Register ShiftAmtReg = MRI.createGenericVirtualRegister(Ty: WideTy);
2950 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2951
2952 MI.getOperand(i: 0).setReg(DstExt);
2953
2954 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2955
2956 LLT Ty = MRI.getType(Reg: DstReg);
2957 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
2958 MIRBuilder.buildConstant(Res: ShiftAmtReg, Val: DiffBits);
2959 MIRBuilder.buildLShr(Dst: ShrReg, Src0: DstExt, Src1: ShiftAmtReg);
2960
2961 MIRBuilder.buildTrunc(Res: DstReg, Op: ShrReg);
2962 Observer.changedInstr(MI);
2963 return Legalized;
2964 }
2965 case TargetOpcode::G_BITREVERSE: {
2966 Observer.changingInstr(MI);
2967
2968 Register DstReg = MI.getOperand(i: 0).getReg();
2969 LLT Ty = MRI.getType(Reg: DstReg);
2970 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
2971
2972 Register DstExt = MRI.createGenericVirtualRegister(Ty: WideTy);
2973 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2974 MI.getOperand(i: 0).setReg(DstExt);
2975 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
2976
2977 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: DiffBits);
2978 auto Shift = MIRBuilder.buildLShr(Dst: WideTy, Src0: DstExt, Src1: ShiftAmt);
2979 MIRBuilder.buildTrunc(Res: DstReg, Op: Shift);
2980 Observer.changedInstr(MI);
2981 return Legalized;
2982 }
2983 case TargetOpcode::G_FREEZE:
2984 case TargetOpcode::G_CONSTANT_FOLD_BARRIER:
2985 Observer.changingInstr(MI);
2986 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
2987 widenScalarDst(MI, WideTy);
2988 Observer.changedInstr(MI);
2989 return Legalized;
2990
2991 case TargetOpcode::G_ABS:
2992 Observer.changingInstr(MI);
2993 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_SEXT);
2994 widenScalarDst(MI, WideTy);
2995 Observer.changedInstr(MI);
2996 return Legalized;
2997
2998 case TargetOpcode::G_ADD:
2999 case TargetOpcode::G_AND:
3000 case TargetOpcode::G_MUL:
3001 case TargetOpcode::G_OR:
3002 case TargetOpcode::G_XOR:
3003 case TargetOpcode::G_SUB:
3004 case TargetOpcode::G_SHUFFLE_VECTOR:
3005 // Perform operation at larger width (any extension is fines here, high bits
3006 // don't affect the result) and then truncate the result back to the
3007 // original type.
3008 Observer.changingInstr(MI);
3009 // The G_ANYEXTs below leave the new high bits unconstrained, so no-wrap and
3010 // disjoint claims proved at the narrow width no longer hold. Paths that
3011 // widen with value-preserving G_ZEXT/G_SEXT keep their flags.
3012 MI.clearFlags(flags: MachineInstr::NoUWrap | MachineInstr::NoSWrap |
3013 MachineInstr::Disjoint);
3014 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3015 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
3016 widenScalarDst(MI, WideTy);
3017 Observer.changedInstr(MI);
3018 return Legalized;
3019
3020 case TargetOpcode::G_SBFX:
3021 case TargetOpcode::G_UBFX:
3022 Observer.changingInstr(MI);
3023
3024 if (TypeIdx == 0) {
3025 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3026 widenScalarDst(MI, WideTy);
3027 } else {
3028 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3029 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ZEXT);
3030 }
3031
3032 Observer.changedInstr(MI);
3033 return Legalized;
3034
3035 case TargetOpcode::G_SHL:
3036 Observer.changingInstr(MI);
3037
3038 if (TypeIdx == 0) {
3039 // Widening the result with G_ANYEXT invalidates the no-wrap flags, as in
3040 // the G_ADD/G_SUB/G_MUL case above. TypeIdx 1 widens only the shift
3041 // amount, which is value-preserving, so it keeps them.
3042 MI.clearFlags(flags: MachineInstr::NoUWrap | MachineInstr::NoSWrap);
3043 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3044 widenScalarDst(MI, WideTy);
3045 } else {
3046 assert(TypeIdx == 1);
3047 // The "number of bits to shift" operand must preserve its value as an
3048 // unsigned integer:
3049 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3050 }
3051
3052 Observer.changedInstr(MI);
3053 return Legalized;
3054
3055 case TargetOpcode::G_ROTR:
3056 case TargetOpcode::G_ROTL:
3057 if (TypeIdx != 1)
3058 return UnableToLegalize;
3059
3060 Observer.changingInstr(MI);
3061 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3062 Observer.changedInstr(MI);
3063 return Legalized;
3064
3065 case TargetOpcode::G_SDIV:
3066 case TargetOpcode::G_SREM:
3067 case TargetOpcode::G_SMIN:
3068 case TargetOpcode::G_SMAX:
3069 case TargetOpcode::G_ABDS:
3070 Observer.changingInstr(MI);
3071 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_SEXT);
3072 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_SEXT);
3073 widenScalarDst(MI, WideTy);
3074 Observer.changedInstr(MI);
3075 return Legalized;
3076
3077 case TargetOpcode::G_SDIVREM:
3078 Observer.changingInstr(MI);
3079 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_SEXT);
3080 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_SEXT);
3081 widenScalarDst(MI, WideTy);
3082 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: --MIRBuilder.getInsertPt());
3083 widenScalarDst(MI, WideTy, OpIdx: 1);
3084 Observer.changedInstr(MI);
3085 return Legalized;
3086
3087 case TargetOpcode::G_ASHR:
3088 case TargetOpcode::G_LSHR:
3089 Observer.changingInstr(MI);
3090
3091 if (TypeIdx == 0) {
3092 unsigned CvtOp = Opcode == TargetOpcode::G_ASHR ? TargetOpcode::G_SEXT
3093 : TargetOpcode::G_ZEXT;
3094
3095 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: CvtOp);
3096 widenScalarDst(MI, WideTy);
3097 } else {
3098 assert(TypeIdx == 1);
3099 // The "number of bits to shift" operand must preserve its value as an
3100 // unsigned integer:
3101 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3102 }
3103
3104 Observer.changedInstr(MI);
3105 return Legalized;
3106 case TargetOpcode::G_UDIV:
3107 case TargetOpcode::G_UREM:
3108 case TargetOpcode::G_ABDU:
3109 Observer.changingInstr(MI);
3110 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ZEXT);
3111 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3112 widenScalarDst(MI, WideTy);
3113 Observer.changedInstr(MI);
3114 return Legalized;
3115 case TargetOpcode::G_UDIVREM:
3116 Observer.changingInstr(MI);
3117 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3118 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ZEXT);
3119 widenScalarDst(MI, WideTy);
3120 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: --MIRBuilder.getInsertPt());
3121 widenScalarDst(MI, WideTy, OpIdx: 1);
3122 Observer.changedInstr(MI);
3123 return Legalized;
3124 case TargetOpcode::G_UMIN:
3125 case TargetOpcode::G_UMAX: {
3126 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3127
3128 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
3129 unsigned ExtOpc =
3130 TLI.isSExtCheaperThanZExt(FromTy: getApproximateEVTForLLT(Ty, Ctx),
3131 ToTy: getApproximateEVTForLLT(Ty: WideTy, Ctx))
3132 ? TargetOpcode::G_SEXT
3133 : TargetOpcode::G_ZEXT;
3134
3135 Observer.changingInstr(MI);
3136 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: ExtOpc);
3137 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: ExtOpc);
3138 widenScalarDst(MI, WideTy);
3139 Observer.changedInstr(MI);
3140 return Legalized;
3141 }
3142
3143 case TargetOpcode::G_SELECT:
3144 Observer.changingInstr(MI);
3145 if (TypeIdx == 0) {
3146 // Perform operation at larger width (any extension is fine here, high
3147 // bits don't affect the result) and then truncate the result back to the
3148 // original type.
3149 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
3150 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ANYEXT);
3151 widenScalarDst(MI, WideTy);
3152 } else {
3153 bool IsVec = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).isVector();
3154 // Explicit extension is required here since high bits affect the result.
3155 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: MIRBuilder.getBoolExtOp(IsVec, IsFP: false));
3156 }
3157 Observer.changedInstr(MI);
3158 return Legalized;
3159
3160 case TargetOpcode::G_FPEXT:
3161 if (TypeIdx != 1)
3162 return UnableToLegalize;
3163
3164 Observer.changingInstr(MI);
3165 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3166 Observer.changedInstr(MI);
3167 return Legalized;
3168 case TargetOpcode::G_FPTOSI:
3169 case TargetOpcode::G_FPTOUI:
3170 case TargetOpcode::G_INTRINSIC_LRINT:
3171 case TargetOpcode::G_INTRINSIC_LLRINT:
3172 case TargetOpcode::G_IS_FPCLASS:
3173 Observer.changingInstr(MI);
3174
3175 if (TypeIdx == 0)
3176 widenScalarDst(MI, WideTy);
3177 else
3178 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3179
3180 Observer.changedInstr(MI);
3181 return Legalized;
3182 case TargetOpcode::G_SITOFP:
3183 Observer.changingInstr(MI);
3184
3185 if (TypeIdx == 0)
3186 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3187 else
3188 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_SEXT);
3189
3190 Observer.changedInstr(MI);
3191 return Legalized;
3192 case TargetOpcode::G_UITOFP:
3193 Observer.changingInstr(MI);
3194
3195 if (TypeIdx == 0)
3196 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3197 else
3198 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ZEXT);
3199
3200 Observer.changedInstr(MI);
3201 return Legalized;
3202 case TargetOpcode::G_FPTOSI_SAT:
3203 case TargetOpcode::G_FPTOUI_SAT:
3204 Observer.changingInstr(MI);
3205
3206 if (TypeIdx == 0) {
3207 Register OldDst = MI.getOperand(i: 0).getReg();
3208 LLT Ty = MRI.getType(Reg: OldDst);
3209 Register ExtReg = MRI.createGenericVirtualRegister(Ty: WideTy);
3210 Register NewDst;
3211 MI.getOperand(i: 0).setReg(ExtReg);
3212 uint64_t ShortBits = Ty.getScalarSizeInBits();
3213 uint64_t WideBits = WideTy.getScalarSizeInBits();
3214 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
3215 if (Opcode == TargetOpcode::G_FPTOSI_SAT) {
3216 // z = i16 fptosi_sat(a)
3217 // ->
3218 // x = i32 fptosi_sat(a)
3219 // y = smin(x, 32767)
3220 // z = smax(y, -32768)
3221 auto MaxVal = MIRBuilder.buildConstant(
3222 Res: WideTy, Val: APInt::getSignedMaxValue(numBits: ShortBits).sext(width: WideBits));
3223 auto MinVal = MIRBuilder.buildConstant(
3224 Res: WideTy, Val: APInt::getSignedMinValue(numBits: ShortBits).sext(width: WideBits));
3225 Register MidReg =
3226 MIRBuilder.buildSMin(Dst: WideTy, Src0: ExtReg, Src1: MaxVal).getReg(Idx: 0);
3227 NewDst = MIRBuilder.buildSMax(Dst: WideTy, Src0: MidReg, Src1: MinVal).getReg(Idx: 0);
3228 } else {
3229 // z = i16 fptoui_sat(a)
3230 // ->
3231 // x = i32 fptoui_sat(a)
3232 // y = smin(x, 65535)
3233 auto MaxVal = MIRBuilder.buildConstant(
3234 Res: WideTy, Val: APInt::getAllOnes(numBits: ShortBits).zext(width: WideBits));
3235 NewDst = MIRBuilder.buildUMin(Dst: WideTy, Src0: ExtReg, Src1: MaxVal).getReg(Idx: 0);
3236 }
3237 MIRBuilder.buildTrunc(Res: OldDst, Op: NewDst);
3238 } else
3239 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3240
3241 Observer.changedInstr(MI);
3242 return Legalized;
3243 case TargetOpcode::G_LOAD:
3244 case TargetOpcode::G_SEXTLOAD:
3245 case TargetOpcode::G_ZEXTLOAD:
3246 case TargetOpcode::G_FPEXTLOAD:
3247 Observer.changingInstr(MI);
3248 widenScalarDst(MI, WideTy);
3249 Observer.changedInstr(MI);
3250 return Legalized;
3251
3252 case TargetOpcode::G_STORE: {
3253 if (TypeIdx != 0)
3254 return UnableToLegalize;
3255
3256 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3257 assert(!Ty.isPointerOrPointerVector() && "Can't widen type");
3258 if (!Ty.isScalar()) {
3259 // We need to widen the vector element type.
3260 Observer.changingInstr(MI);
3261 widenScalarSrc(MI, WideTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_ANYEXT);
3262 // We also need to adjust the MMO to turn this into a truncating store.
3263 MachineMemOperand &MMO = **MI.memoperands_begin();
3264 MachineFunction &MF = MIRBuilder.getMF();
3265 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo: MMO.getPointerInfo(), Ty);
3266 MI.setMemRefs(MF, MemRefs: {NewMMO});
3267 Observer.changedInstr(MI);
3268 return Legalized;
3269 }
3270
3271 Observer.changingInstr(MI);
3272
3273 unsigned ExtType = Ty.getScalarSizeInBits() == 1 ?
3274 TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT;
3275 widenScalarSrc(MI, WideTy, OpIdx: 0, ExtOpcode: ExtType);
3276
3277 Observer.changedInstr(MI);
3278 return Legalized;
3279 }
3280 case TargetOpcode::G_FPTRUNCSTORE:
3281 if (TypeIdx != 0)
3282 return UnableToLegalize;
3283 Observer.changingInstr(MI);
3284 widenScalarSrc(MI, WideTy, OpIdx: 0, ExtOpcode: TargetOpcode::G_FPEXT);
3285 Observer.changedInstr(MI);
3286 return Legalized;
3287 case TargetOpcode::G_CONSTANT: {
3288 MachineOperand &SrcMO = MI.getOperand(i: 1);
3289 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
3290 unsigned ExtOpc = LI.getExtOpcodeForWideningConstant(
3291 SmallTy: MRI.getType(Reg: MI.getOperand(i: 0).getReg()));
3292 assert((ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::G_SEXT ||
3293 ExtOpc == TargetOpcode::G_ANYEXT) &&
3294 "Illegal Extend");
3295 const APInt &SrcVal = SrcMO.getCImm()->getValue();
3296 const APInt &Val = (ExtOpc == TargetOpcode::G_SEXT)
3297 ? SrcVal.sext(width: WideTy.getSizeInBits())
3298 : SrcVal.zext(width: WideTy.getSizeInBits());
3299 Observer.changingInstr(MI);
3300 SrcMO.setCImm(ConstantInt::get(Context&: Ctx, V: Val));
3301
3302 widenScalarDst(MI, WideTy);
3303 Observer.changedInstr(MI);
3304 return Legalized;
3305 }
3306 case TargetOpcode::G_FCONSTANT: {
3307 // To avoid changing the bits of the constant due to extension to a larger
3308 // type and then using G_FPTRUNC, we simply convert to a G_CONSTANT.
3309 MachineOperand &SrcMO = MI.getOperand(i: 1);
3310 APInt Val = SrcMO.getFPImm()->getValueAPF().bitcastToAPInt();
3311 MIRBuilder.setInstrAndDebugLoc(MI);
3312 auto IntCst = MIRBuilder.buildConstant(Res: MI.getOperand(i: 0).getReg(), Val);
3313 widenScalarDst(MI&: *IntCst, WideTy, OpIdx: 0, TruncOpcode: TargetOpcode::G_TRUNC);
3314 MI.eraseFromParent();
3315 return Legalized;
3316 }
3317 case TargetOpcode::G_IMPLICIT_DEF: {
3318 Observer.changingInstr(MI);
3319 widenScalarDst(MI, WideTy);
3320 Observer.changedInstr(MI);
3321 return Legalized;
3322 }
3323 case TargetOpcode::G_BRCOND:
3324 Observer.changingInstr(MI);
3325 widenScalarSrc(MI, WideTy, OpIdx: 0, ExtOpcode: MIRBuilder.getBoolExtOp(IsVec: false, IsFP: false));
3326 Observer.changedInstr(MI);
3327 return Legalized;
3328
3329 case TargetOpcode::G_FCMP:
3330 Observer.changingInstr(MI);
3331 if (TypeIdx == 0)
3332 widenScalarDst(MI, WideTy);
3333 else {
3334 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 2);
3335 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 3);
3336 }
3337 Observer.changedInstr(MI);
3338 return Legalized;
3339
3340 case TargetOpcode::G_ICMP:
3341 Observer.changingInstr(MI);
3342 if (TypeIdx == 0)
3343 widenScalarDst(MI, WideTy);
3344 else {
3345 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg());
3346 CmpInst::Predicate Pred =
3347 static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
3348
3349 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
3350 unsigned ExtOpcode =
3351 (CmpInst::isSigned(Pred) ||
3352 TLI.isSExtCheaperThanZExt(FromTy: getApproximateEVTForLLT(Ty: SrcTy, Ctx),
3353 ToTy: getApproximateEVTForLLT(Ty: WideTy, Ctx)))
3354 ? TargetOpcode::G_SEXT
3355 : TargetOpcode::G_ZEXT;
3356 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode);
3357 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode);
3358 }
3359 Observer.changedInstr(MI);
3360 return Legalized;
3361
3362 case TargetOpcode::G_PTR_ADD:
3363 assert(TypeIdx == 1 && "unable to legalize pointer of G_PTR_ADD");
3364 Observer.changingInstr(MI);
3365 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_SEXT);
3366 Observer.changedInstr(MI);
3367 return Legalized;
3368
3369 case TargetOpcode::G_PHI: {
3370 assert(TypeIdx == 0 && "Expecting only Idx 0");
3371
3372 Observer.changingInstr(MI);
3373 for (unsigned I = 1; I < MI.getNumOperands(); I += 2) {
3374 MachineBasicBlock &OpMBB = *MI.getOperand(i: I + 1).getMBB();
3375 MIRBuilder.setInsertPt(MBB&: OpMBB, II: OpMBB.getFirstTerminatorForward());
3376 widenScalarSrc(MI, WideTy, OpIdx: I, ExtOpcode: TargetOpcode::G_ANYEXT);
3377 }
3378
3379 MachineBasicBlock &MBB = *MI.getParent();
3380 MIRBuilder.setInsertPt(MBB, II: --MBB.getFirstNonPHI());
3381 widenScalarDst(MI, WideTy);
3382 Observer.changedInstr(MI);
3383 return Legalized;
3384 }
3385 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
3386 if (TypeIdx == 0) {
3387 Register VecReg = MI.getOperand(i: 1).getReg();
3388 LLT VecTy = MRI.getType(Reg: VecReg);
3389 Observer.changingInstr(MI);
3390
3391 widenScalarSrc(MI, WideTy: LLT::vector(EC: VecTy.getElementCount(), ScalarTy: WideTy), OpIdx: 1,
3392 ExtOpcode: TargetOpcode::G_ANYEXT);
3393
3394 widenScalarDst(MI, WideTy, OpIdx: 0);
3395 Observer.changedInstr(MI);
3396 return Legalized;
3397 }
3398
3399 if (TypeIdx != 2)
3400 return UnableToLegalize;
3401 Observer.changingInstr(MI);
3402 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3403 Observer.changedInstr(MI);
3404 return Legalized;
3405 }
3406 case TargetOpcode::G_INSERT_VECTOR_ELT: {
3407 if (TypeIdx == 0) {
3408 Observer.changingInstr(MI);
3409 const LLT WideEltTy = WideTy.getElementType();
3410
3411 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3412 widenScalarSrc(MI, WideTy: WideEltTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
3413 widenScalarDst(MI, WideTy, OpIdx: 0);
3414 Observer.changedInstr(MI);
3415 return Legalized;
3416 }
3417
3418 if (TypeIdx == 1) {
3419 Observer.changingInstr(MI);
3420
3421 Register VecReg = MI.getOperand(i: 1).getReg();
3422 LLT VecTy = MRI.getType(Reg: VecReg);
3423 LLT WideVecTy = VecTy.changeVectorElementType(NewEltTy: WideTy);
3424
3425 widenScalarSrc(MI, WideTy: WideVecTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3426 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
3427 widenScalarDst(MI, WideTy: WideVecTy, OpIdx: 0);
3428 Observer.changedInstr(MI);
3429 return Legalized;
3430 }
3431
3432 if (TypeIdx == 2) {
3433 Observer.changingInstr(MI);
3434 widenScalarSrc(MI, WideTy, OpIdx: 3, ExtOpcode: TargetOpcode::G_ZEXT);
3435 Observer.changedInstr(MI);
3436 return Legalized;
3437 }
3438
3439 return UnableToLegalize;
3440 }
3441 case TargetOpcode::G_FADD:
3442 case TargetOpcode::G_FMUL:
3443 case TargetOpcode::G_FSUB:
3444 case TargetOpcode::G_FMA:
3445 case TargetOpcode::G_FMAD:
3446 case TargetOpcode::G_FNEG:
3447 case TargetOpcode::G_FABS:
3448 case TargetOpcode::G_FCANONICALIZE:
3449 case TargetOpcode::G_FMINNUM:
3450 case TargetOpcode::G_FMAXNUM:
3451 case TargetOpcode::G_FMINNUM_IEEE:
3452 case TargetOpcode::G_FMAXNUM_IEEE:
3453 case TargetOpcode::G_FMINIMUM:
3454 case TargetOpcode::G_FMAXIMUM:
3455 case TargetOpcode::G_FMINIMUMNUM:
3456 case TargetOpcode::G_FMAXIMUMNUM:
3457 case TargetOpcode::G_FDIV:
3458 case TargetOpcode::G_FREM:
3459 case TargetOpcode::G_FCEIL:
3460 case TargetOpcode::G_FFLOOR:
3461 case TargetOpcode::G_FCOS:
3462 case TargetOpcode::G_FSIN:
3463 case TargetOpcode::G_FTAN:
3464 case TargetOpcode::G_FACOS:
3465 case TargetOpcode::G_FASIN:
3466 case TargetOpcode::G_FATAN:
3467 case TargetOpcode::G_FATAN2:
3468 case TargetOpcode::G_FCOSH:
3469 case TargetOpcode::G_FSINH:
3470 case TargetOpcode::G_FTANH:
3471 case TargetOpcode::G_FLOG10:
3472 case TargetOpcode::G_FLOG:
3473 case TargetOpcode::G_FLOG2:
3474 case TargetOpcode::G_FRINT:
3475 case TargetOpcode::G_FNEARBYINT:
3476 case TargetOpcode::G_FSQRT:
3477 case TargetOpcode::G_FEXP:
3478 case TargetOpcode::G_FEXP2:
3479 case TargetOpcode::G_FEXP10:
3480 case TargetOpcode::G_FPOW:
3481 case TargetOpcode::G_INTRINSIC_TRUNC:
3482 case TargetOpcode::G_INTRINSIC_ROUND:
3483 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
3484 assert(TypeIdx == 0);
3485 Observer.changingInstr(MI);
3486
3487 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I)
3488 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: I);
3489
3490 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3491 Observer.changedInstr(MI);
3492 return Legalized;
3493 case TargetOpcode::G_FMODF: {
3494 Observer.changingInstr(MI);
3495 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 2);
3496
3497 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 1);
3498 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: --MIRBuilder.getInsertPt());
3499 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3500 Observer.changedInstr(MI);
3501 return Legalized;
3502 }
3503 case TargetOpcode::G_FPOWI:
3504 case TargetOpcode::G_FLDEXP:
3505 case TargetOpcode::G_STRICT_FLDEXP: {
3506 if (TypeIdx == 0) {
3507 if (Opcode == TargetOpcode::G_STRICT_FLDEXP)
3508 return UnableToLegalize;
3509
3510 Observer.changingInstr(MI);
3511 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3512 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3513 Observer.changedInstr(MI);
3514 return Legalized;
3515 }
3516
3517 if (TypeIdx == 1) {
3518 // For some reason SelectionDAG tries to promote to a libcall without
3519 // actually changing the integer type for promotion.
3520 Observer.changingInstr(MI);
3521 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_SEXT);
3522 Observer.changedInstr(MI);
3523 return Legalized;
3524 }
3525
3526 return UnableToLegalize;
3527 }
3528 case TargetOpcode::G_FFREXP: {
3529 Observer.changingInstr(MI);
3530
3531 if (TypeIdx == 0) {
3532 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 2);
3533 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3534 } else {
3535 widenScalarDst(MI, WideTy, OpIdx: 1);
3536 }
3537
3538 Observer.changedInstr(MI);
3539 return Legalized;
3540 }
3541 case TargetOpcode::G_LROUND:
3542 case TargetOpcode::G_LLROUND:
3543 Observer.changingInstr(MI);
3544
3545 if (TypeIdx == 0)
3546 widenScalarDst(MI, WideTy);
3547 else
3548 widenScalarSrcUsingFPExt(MI, WideTy, OpIdx: 1);
3549
3550 Observer.changedInstr(MI);
3551 return Legalized;
3552
3553 case TargetOpcode::G_INTTOPTR:
3554 if (TypeIdx != 1)
3555 return UnableToLegalize;
3556
3557 Observer.changingInstr(MI);
3558 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ZEXT);
3559 Observer.changedInstr(MI);
3560 return Legalized;
3561 case TargetOpcode::G_PTRTOINT:
3562 if (TypeIdx != 0)
3563 return UnableToLegalize;
3564
3565 Observer.changingInstr(MI);
3566 widenScalarDst(MI, WideTy, OpIdx: 0);
3567 Observer.changedInstr(MI);
3568 return Legalized;
3569 case TargetOpcode::G_BUILD_VECTOR: {
3570 Observer.changingInstr(MI);
3571
3572 const LLT WideEltTy = TypeIdx == 1 ? WideTy : WideTy.getElementType();
3573 for (int I = 1, E = MI.getNumOperands(); I != E; ++I)
3574 widenScalarSrc(MI, WideTy: WideEltTy, OpIdx: I, ExtOpcode: TargetOpcode::G_ANYEXT);
3575
3576 // Avoid changing the result vector type if the source element type was
3577 // requested.
3578 if (TypeIdx == 1) {
3579 MI.setDesc(MIRBuilder.getTII().get(Opcode: TargetOpcode::G_BUILD_VECTOR_TRUNC));
3580 } else {
3581 widenScalarDst(MI, WideTy, OpIdx: 0);
3582 }
3583
3584 Observer.changedInstr(MI);
3585 return Legalized;
3586 }
3587 case TargetOpcode::G_SEXT_INREG:
3588 if (TypeIdx != 0)
3589 return UnableToLegalize;
3590
3591 Observer.changingInstr(MI);
3592 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3593 widenScalarDst(MI, WideTy, OpIdx: 0, TruncOpcode: TargetOpcode::G_TRUNC);
3594 Observer.changedInstr(MI);
3595 return Legalized;
3596 case TargetOpcode::G_PTRMASK: {
3597 if (TypeIdx != 1)
3598 return UnableToLegalize;
3599 Observer.changingInstr(MI);
3600 widenScalarSrc(MI, WideTy, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
3601 Observer.changedInstr(MI);
3602 return Legalized;
3603 }
3604 case TargetOpcode::G_VECREDUCE_ADD: {
3605 if (TypeIdx != 1)
3606 return UnableToLegalize;
3607 Observer.changingInstr(MI);
3608 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3609 widenScalarDst(MI, WideTy: WideTy.getScalarType(), OpIdx: 0, TruncOpcode: TargetOpcode::G_TRUNC);
3610 Observer.changedInstr(MI);
3611 return Legalized;
3612 }
3613 case TargetOpcode::G_VECREDUCE_FADD:
3614 case TargetOpcode::G_VECREDUCE_FMUL:
3615 case TargetOpcode::G_VECREDUCE_FMIN:
3616 case TargetOpcode::G_VECREDUCE_FMAX:
3617 case TargetOpcode::G_VECREDUCE_FMINIMUM:
3618 case TargetOpcode::G_VECREDUCE_FMAXIMUM: {
3619 if (TypeIdx != 0)
3620 return UnableToLegalize;
3621 Observer.changingInstr(MI);
3622 Register VecReg = MI.getOperand(i: 1).getReg();
3623 LLT VecTy = MRI.getType(Reg: VecReg);
3624 LLT WideVecTy = VecTy.changeElementType(NewEltTy: WideTy);
3625 widenScalarSrcUsingFPExt(MI, WideTy: WideVecTy, OpIdx: 1);
3626 widenScalarDstUsingFPTrunc(MI, WideTy, OpIdx: 0);
3627 Observer.changedInstr(MI);
3628 return Legalized;
3629 }
3630 case TargetOpcode::G_VSCALE: {
3631 MachineOperand &SrcMO = MI.getOperand(i: 1);
3632 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
3633 const APInt &SrcVal = SrcMO.getCImm()->getValue();
3634 // The CImm is always a signed value
3635 const APInt Val = SrcVal.sext(width: WideTy.getSizeInBits());
3636 Observer.changingInstr(MI);
3637 SrcMO.setCImm(ConstantInt::get(Context&: Ctx, V: Val));
3638 widenScalarDst(MI, WideTy);
3639 Observer.changedInstr(MI);
3640 return Legalized;
3641 }
3642 case TargetOpcode::G_SPLAT_VECTOR: {
3643 if (TypeIdx != 1)
3644 return UnableToLegalize;
3645
3646 Observer.changingInstr(MI);
3647 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3648 Observer.changedInstr(MI);
3649 return Legalized;
3650 }
3651 case TargetOpcode::G_INSERT_SUBVECTOR: {
3652 if (TypeIdx != 0)
3653 return UnableToLegalize;
3654
3655 GInsertSubvector &IS = cast<GInsertSubvector>(Val&: MI);
3656 Register BigVec = IS.getBigVec();
3657 Register SubVec = IS.getSubVec();
3658
3659 LLT SubVecTy = MRI.getType(Reg: SubVec);
3660 LLT SubVecWideTy = SubVecTy.changeElementType(NewEltTy: WideTy.getElementType());
3661
3662 // Widen the G_INSERT_SUBVECTOR
3663 auto BigZExt = MIRBuilder.buildZExt(Res: WideTy, Op: BigVec);
3664 auto SubZExt = MIRBuilder.buildZExt(Res: SubVecWideTy, Op: SubVec);
3665 auto WideInsert = MIRBuilder.buildInsertSubvector(Res: WideTy, Src0: BigZExt, Src1: SubZExt,
3666 Index: IS.getIndexImm());
3667
3668 // Truncate back down
3669 auto SplatZero = MIRBuilder.buildSplatVector(
3670 Res: WideTy, Val: MIRBuilder.buildConstant(Res: WideTy.getElementType(), Val: 0));
3671 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_NE, Res: IS.getReg(Idx: 0), Op0: WideInsert,
3672 Op1: SplatZero);
3673
3674 MI.eraseFromParent();
3675
3676 return Legalized;
3677 }
3678 case TargetOpcode::G_BITCAST:
3679 if (WideTy.isVector())
3680 return UnableToLegalize;
3681 Observer.changingInstr(MI);
3682 if (TypeIdx == 0)
3683 widenScalarDst(MI, WideTy, OpIdx: 0, TruncOpcode: TargetOpcode::G_TRUNC);
3684 else
3685 widenScalarSrc(MI, WideTy, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
3686 Observer.changedInstr(MI);
3687
3688 Register Dst = MI.getOperand(i: 0).getReg();
3689 Register Src = MI.getOperand(i: 1).getReg();
3690 if (MRI.getType(Reg: Dst) == MRI.getType(Reg: Src)) {
3691 Observer.changingAllUsesOfReg(MRI, Reg: Dst);
3692 MRI.replaceRegWith(FromReg: Dst, ToReg: Src);
3693 Observer.finishedChangingAllUsesOfReg();
3694 MI.eraseFromParent();
3695 }
3696
3697 return Legalized;
3698 }
3699}
3700
3701static void getUnmergePieces(SmallVectorImpl<Register> &Pieces,
3702 MachineIRBuilder &B, Register Src, LLT Ty) {
3703 auto Unmerge = B.buildUnmerge(Res: Ty, Op: Src);
3704 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
3705 Pieces.push_back(Elt: Unmerge.getReg(Idx: I));
3706}
3707
3708static void emitLoadFromConstantPool(Register DstReg, const Constant *ConstVal,
3709 MachineIRBuilder &MIRBuilder) {
3710 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
3711 MachineFunction &MF = MIRBuilder.getMF();
3712 const DataLayout &DL = MIRBuilder.getDataLayout();
3713 unsigned AddrSpace = DL.getDefaultGlobalsAddressSpace();
3714 LLT AddrPtrTy = LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL.getPointerSizeInBits(AS: AddrSpace));
3715 LLT DstLLT = MRI.getType(Reg: DstReg);
3716
3717 Align Alignment(DL.getABITypeAlign(Ty: ConstVal->getType()));
3718
3719 auto Addr = MIRBuilder.buildConstantPool(
3720 Res: AddrPtrTy,
3721 Idx: MF.getConstantPool()->getConstantPoolIndex(C: ConstVal, Alignment));
3722
3723 MachineMemOperand *MMO =
3724 MF.getMachineMemOperand(PtrInfo: MachinePointerInfo::getConstantPool(MF),
3725 F: MachineMemOperand::MOLoad, MemTy: DstLLT, BaseAlignment: Alignment);
3726
3727 MIRBuilder.buildLoadInstr(Opcode: TargetOpcode::G_LOAD, Res: DstReg, Addr, MMO&: *MMO);
3728}
3729
3730LegalizerHelper::LegalizeResult
3731LegalizerHelper::lowerConstant(MachineInstr &MI) {
3732 const MachineOperand &ConstOperand = MI.getOperand(i: 1);
3733 const Constant *ConstantVal = ConstOperand.getCImm();
3734
3735 emitLoadFromConstantPool(DstReg: MI.getOperand(i: 0).getReg(), ConstVal: ConstantVal, MIRBuilder);
3736 MI.eraseFromParent();
3737
3738 return Legalized;
3739}
3740
3741LegalizerHelper::LegalizeResult
3742LegalizerHelper::lowerFConstant(MachineInstr &MI) {
3743 const MachineOperand &ConstOperand = MI.getOperand(i: 1);
3744 const Constant *ConstantVal = ConstOperand.getFPImm();
3745
3746 emitLoadFromConstantPool(DstReg: MI.getOperand(i: 0).getReg(), ConstVal: ConstantVal, MIRBuilder);
3747 MI.eraseFromParent();
3748
3749 return Legalized;
3750}
3751
3752LegalizerHelper::LegalizeResult
3753LegalizerHelper::lowerBitcast(MachineInstr &MI) {
3754 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
3755 if (SrcTy.isVector()) {
3756 LLT SrcEltTy = SrcTy.getElementType();
3757 SmallVector<Register, 8> SrcRegs;
3758
3759 if (DstTy.isVector()) {
3760 int NumDstElt = DstTy.getNumElements();
3761 int NumSrcElt = SrcTy.getNumElements();
3762
3763 LLT DstEltTy = DstTy.getElementType();
3764 LLT DstCastTy = DstEltTy; // Intermediate bitcast result type
3765 LLT SrcPartTy = SrcEltTy; // Original unmerge result type.
3766
3767 // If there's an element size mismatch, insert intermediate casts to match
3768 // the result element type.
3769 if (NumSrcElt < NumDstElt) { // Source element type is larger.
3770 // %1:_(<4 x s8>) = G_BITCAST %0:_(<2 x s16>)
3771 //
3772 // =>
3773 //
3774 // %2:_(s16), %3:_(s16) = G_UNMERGE_VALUES %0
3775 // %3:_(<2 x s8>) = G_BITCAST %2
3776 // %4:_(<2 x s8>) = G_BITCAST %3
3777 // %1:_(<4 x s16>) = G_CONCAT_VECTORS %3, %4
3778 DstCastTy = DstTy.changeVectorElementCount(
3779 EC: ElementCount::getFixed(MinVal: NumDstElt / NumSrcElt));
3780 SrcPartTy = SrcEltTy;
3781 } else if (NumSrcElt > NumDstElt) { // Source element type is smaller.
3782 //
3783 // %1:_(<2 x s16>) = G_BITCAST %0:_(<4 x s8>)
3784 //
3785 // =>
3786 //
3787 // %2:_(<2 x s8>), %3:_(<2 x s8>) = G_UNMERGE_VALUES %0
3788 // %3:_(s16) = G_BITCAST %2
3789 // %4:_(s16) = G_BITCAST %3
3790 // %1:_(<2 x s16>) = G_BUILD_VECTOR %3, %4
3791 SrcPartTy = SrcTy.changeVectorElementCount(
3792 EC: ElementCount::getFixed(MinVal: NumSrcElt / NumDstElt));
3793 DstCastTy = DstEltTy;
3794 }
3795
3796 getUnmergePieces(Pieces&: SrcRegs, B&: MIRBuilder, Src, Ty: SrcPartTy);
3797 for (Register &SrcReg : SrcRegs)
3798 SrcReg = MIRBuilder.buildBitcast(Dst: DstCastTy, Src: SrcReg).getReg(Idx: 0);
3799 } else
3800 getUnmergePieces(Pieces&: SrcRegs, B&: MIRBuilder, Src, Ty: SrcEltTy);
3801
3802 MIRBuilder.buildMergeLikeInstr(Res: Dst, Ops: SrcRegs);
3803 MI.eraseFromParent();
3804 return Legalized;
3805 }
3806
3807 if (DstTy.isVector()) {
3808 SmallVector<Register, 8> SrcRegs;
3809 getUnmergePieces(Pieces&: SrcRegs, B&: MIRBuilder, Src, Ty: DstTy.getElementType());
3810 MIRBuilder.buildMergeLikeInstr(Res: Dst, Ops: SrcRegs);
3811 MI.eraseFromParent();
3812 return Legalized;
3813 }
3814
3815 return UnableToLegalize;
3816}
3817
3818/// Figure out the bit offset into a register when coercing a vector index for
3819/// the wide element type. This is only for the case when promoting vector to
3820/// one with larger elements.
3821//
3822///
3823/// %offset_idx = G_AND %idx, ~(-1 << Log2(DstEltSize / SrcEltSize))
3824/// %offset_bits = G_SHL %offset_idx, Log2(SrcEltSize)
3825static Register getBitcastWiderVectorElementOffset(MachineIRBuilder &B,
3826 Register Idx,
3827 unsigned NewEltSize,
3828 unsigned OldEltSize) {
3829 const unsigned Log2EltRatio = Log2_32(Value: NewEltSize / OldEltSize);
3830 LLT IdxTy = B.getMRI()->getType(Reg: Idx);
3831
3832 // Now figure out the amount we need to shift to get the target bits.
3833 auto OffsetMask = B.buildConstant(
3834 Res: IdxTy, Val: ~(APInt::getAllOnes(numBits: IdxTy.getSizeInBits()) << Log2EltRatio));
3835 auto OffsetIdx = B.buildAnd(Dst: IdxTy, Src0: Idx, Src1: OffsetMask);
3836 return B.buildShl(Dst: IdxTy, Src0: OffsetIdx,
3837 Src1: B.buildConstant(Res: IdxTy, Val: Log2_32(Value: OldEltSize))).getReg(Idx: 0);
3838}
3839
3840/// Perform a G_EXTRACT_VECTOR_ELT in a different sized vector element. If this
3841/// is casting to a vector with a smaller element size, perform multiple element
3842/// extracts and merge the results. If this is coercing to a vector with larger
3843/// elements, index the bitcasted vector and extract the target element with bit
3844/// operations. This is intended to force the indexing in the native register
3845/// size for architectures that can dynamically index the register file.
3846LegalizerHelper::LegalizeResult
3847LegalizerHelper::bitcastExtractVectorElt(MachineInstr &MI, unsigned TypeIdx,
3848 LLT CastTy) {
3849 if (TypeIdx != 1)
3850 return UnableToLegalize;
3851
3852 auto [Dst, DstTy, SrcVec, SrcVecTy, Idx, IdxTy] = MI.getFirst3RegLLTs();
3853
3854 LLT SrcEltTy = SrcVecTy.getElementType();
3855 unsigned NewNumElts = CastTy.isVector() ? CastTy.getNumElements() : 1;
3856 unsigned OldNumElts = SrcVecTy.getNumElements();
3857
3858 LLT NewEltTy = CastTy.getScalarType();
3859 Register CastVec = MIRBuilder.buildBitcast(Dst: CastTy, Src: SrcVec).getReg(Idx: 0);
3860
3861 const unsigned NewEltSize = NewEltTy.getSizeInBits();
3862 const unsigned OldEltSize = SrcEltTy.getSizeInBits();
3863 if (NewNumElts > OldNumElts) {
3864 // Decreasing the vector element size
3865 //
3866 // e.g. i64 = extract_vector_elt x:v2i64, y:i32
3867 // =>
3868 // v4i32:castx = bitcast x:v2i64
3869 //
3870 // i64 = bitcast
3871 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
3872 // (i32 (extract_vector_elt castx, (2 * y + 1)))
3873 //
3874 if (NewNumElts % OldNumElts != 0)
3875 return UnableToLegalize;
3876
3877 // Type of the intermediate result vector.
3878 const unsigned NewEltsPerOldElt = NewNumElts / OldNumElts;
3879 LLT MidTy =
3880 CastTy.changeElementCount(EC: ElementCount::getFixed(MinVal: NewEltsPerOldElt));
3881
3882 auto NewEltsPerOldEltK = MIRBuilder.buildConstant(Res: IdxTy, Val: NewEltsPerOldElt);
3883
3884 SmallVector<Register, 8> NewOps(NewEltsPerOldElt);
3885 auto NewBaseIdx = MIRBuilder.buildMul(Dst: IdxTy, Src0: Idx, Src1: NewEltsPerOldEltK);
3886
3887 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
3888 auto IdxOffset = MIRBuilder.buildConstant(Res: IdxTy, Val: I);
3889 auto TmpIdx = MIRBuilder.buildAdd(Dst: IdxTy, Src0: NewBaseIdx, Src1: IdxOffset);
3890 auto Elt = MIRBuilder.buildExtractVectorElement(Res: NewEltTy, Val: CastVec, Idx: TmpIdx);
3891 NewOps[I] = Elt.getReg(Idx: 0);
3892 }
3893
3894 auto NewVec = MIRBuilder.buildBuildVector(Res: MidTy, Ops: NewOps);
3895 MIRBuilder.buildBitcast(Dst, Src: NewVec);
3896 MI.eraseFromParent();
3897 return Legalized;
3898 }
3899
3900 if (NewNumElts < OldNumElts) {
3901 if (NewEltSize % OldEltSize != 0)
3902 return UnableToLegalize;
3903
3904 // This only depends on powers of 2 because we use bit tricks to figure out
3905 // the bit offset we need to shift to get the target element. A general
3906 // expansion could emit division/multiply.
3907 if (!isPowerOf2_32(Value: NewEltSize / OldEltSize))
3908 return UnableToLegalize;
3909
3910 // Increasing the vector element size.
3911 // %elt:_(small_elt) = G_EXTRACT_VECTOR_ELT %vec:_(<N x small_elt>), %idx
3912 //
3913 // =>
3914 //
3915 // %cast = G_BITCAST %vec
3916 // %scaled_idx = G_LSHR %idx, Log2(DstEltSize / SrcEltSize)
3917 // %wide_elt = G_EXTRACT_VECTOR_ELT %cast, %scaled_idx
3918 // %offset_idx = G_AND %idx, ~(-1 << Log2(DstEltSize / SrcEltSize))
3919 // %offset_bits = G_SHL %offset_idx, Log2(SrcEltSize)
3920 // %elt_bits = G_LSHR %wide_elt, %offset_bits
3921 // %elt = G_TRUNC %elt_bits
3922
3923 const unsigned Log2EltRatio = Log2_32(Value: NewEltSize / OldEltSize);
3924 auto Log2Ratio = MIRBuilder.buildConstant(Res: IdxTy, Val: Log2EltRatio);
3925
3926 // Divide to get the index in the wider element type.
3927 auto ScaledIdx = MIRBuilder.buildLShr(Dst: IdxTy, Src0: Idx, Src1: Log2Ratio);
3928
3929 Register WideElt = CastVec;
3930 if (CastTy.isVector()) {
3931 WideElt = MIRBuilder.buildExtractVectorElement(Res: NewEltTy, Val: CastVec,
3932 Idx: ScaledIdx).getReg(Idx: 0);
3933 }
3934
3935 // Compute the bit offset into the register of the target element.
3936 Register OffsetBits = getBitcastWiderVectorElementOffset(
3937 B&: MIRBuilder, Idx, NewEltSize, OldEltSize);
3938
3939 // Shift the wide element to get the target element.
3940 auto ExtractedBits = MIRBuilder.buildLShr(Dst: NewEltTy, Src0: WideElt, Src1: OffsetBits);
3941 MIRBuilder.buildTrunc(Res: Dst, Op: ExtractedBits);
3942 MI.eraseFromParent();
3943 return Legalized;
3944 }
3945
3946 return UnableToLegalize;
3947}
3948
3949/// Emit code to insert \p InsertReg into \p TargetRet at \p OffsetBits in \p
3950/// TargetReg, while preserving other bits in \p TargetReg.
3951///
3952/// (InsertReg << Offset) | (TargetReg & ~(-1 >> InsertReg.size()) << Offset)
3953static Register buildBitFieldInsert(MachineIRBuilder &B,
3954 Register TargetReg, Register InsertReg,
3955 Register OffsetBits) {
3956 LLT TargetTy = B.getMRI()->getType(Reg: TargetReg);
3957 LLT InsertTy = B.getMRI()->getType(Reg: InsertReg);
3958 auto ZextVal = B.buildZExt(Res: TargetTy, Op: InsertReg);
3959 auto ShiftedInsertVal = B.buildShl(Dst: TargetTy, Src0: ZextVal, Src1: OffsetBits);
3960
3961 // Produce a bitmask of the value to insert
3962 auto EltMask = B.buildConstant(
3963 Res: TargetTy, Val: APInt::getLowBitsSet(numBits: TargetTy.getSizeInBits(),
3964 loBitsSet: InsertTy.getSizeInBits()));
3965 // Shift it into position
3966 auto ShiftedMask = B.buildShl(Dst: TargetTy, Src0: EltMask, Src1: OffsetBits);
3967 auto InvShiftedMask = B.buildNot(Dst: TargetTy, Src0: ShiftedMask);
3968
3969 // Clear out the bits in the wide element
3970 auto MaskedOldElt = B.buildAnd(Dst: TargetTy, Src0: TargetReg, Src1: InvShiftedMask);
3971
3972 // The value to insert has all zeros already, so stick it into the masked
3973 // wide element.
3974 return B.buildOr(Dst: TargetTy, Src0: MaskedOldElt, Src1: ShiftedInsertVal).getReg(Idx: 0);
3975}
3976
3977/// Perform a G_INSERT_VECTOR_ELT in a different sized vector element. If this
3978/// is increasing the element size, perform the indexing in the target element
3979/// type, and use bit operations to insert at the element position. This is
3980/// intended for architectures that can dynamically index the register file and
3981/// want to force indexing in the native register size.
3982LegalizerHelper::LegalizeResult
3983LegalizerHelper::bitcastInsertVectorElt(MachineInstr &MI, unsigned TypeIdx,
3984 LLT CastTy) {
3985 if (TypeIdx != 0)
3986 return UnableToLegalize;
3987
3988 auto [Dst, DstTy, SrcVec, SrcVecTy, Val, ValTy, Idx, IdxTy] =
3989 MI.getFirst4RegLLTs();
3990 LLT VecTy = DstTy;
3991
3992 LLT VecEltTy = VecTy.getElementType();
3993 LLT NewEltTy = CastTy.isVector() ? CastTy.getElementType() : CastTy;
3994 const unsigned NewEltSize = NewEltTy.getSizeInBits();
3995 const unsigned OldEltSize = VecEltTy.getSizeInBits();
3996
3997 unsigned NewNumElts = CastTy.isVector() ? CastTy.getNumElements() : 1;
3998 unsigned OldNumElts = VecTy.getNumElements();
3999
4000 Register CastVec = MIRBuilder.buildBitcast(Dst: CastTy, Src: SrcVec).getReg(Idx: 0);
4001 if (NewNumElts < OldNumElts) {
4002 if (NewEltSize % OldEltSize != 0)
4003 return UnableToLegalize;
4004
4005 // This only depends on powers of 2 because we use bit tricks to figure out
4006 // the bit offset we need to shift to get the target element. A general
4007 // expansion could emit division/multiply.
4008 if (!isPowerOf2_32(Value: NewEltSize / OldEltSize))
4009 return UnableToLegalize;
4010
4011 const unsigned Log2EltRatio = Log2_32(Value: NewEltSize / OldEltSize);
4012 auto Log2Ratio = MIRBuilder.buildConstant(Res: IdxTy, Val: Log2EltRatio);
4013
4014 // Divide to get the index in the wider element type.
4015 auto ScaledIdx = MIRBuilder.buildLShr(Dst: IdxTy, Src0: Idx, Src1: Log2Ratio);
4016
4017 Register ExtractedElt = CastVec;
4018 if (CastTy.isVector()) {
4019 ExtractedElt = MIRBuilder.buildExtractVectorElement(Res: NewEltTy, Val: CastVec,
4020 Idx: ScaledIdx).getReg(Idx: 0);
4021 }
4022
4023 // Compute the bit offset into the register of the target element.
4024 Register OffsetBits = getBitcastWiderVectorElementOffset(
4025 B&: MIRBuilder, Idx, NewEltSize, OldEltSize);
4026
4027 Register InsertedElt = buildBitFieldInsert(B&: MIRBuilder, TargetReg: ExtractedElt,
4028 InsertReg: Val, OffsetBits);
4029 if (CastTy.isVector()) {
4030 InsertedElt = MIRBuilder.buildInsertVectorElement(
4031 Res: CastTy, Val: CastVec, Elt: InsertedElt, Idx: ScaledIdx).getReg(Idx: 0);
4032 }
4033
4034 MIRBuilder.buildBitcast(Dst, Src: InsertedElt);
4035 MI.eraseFromParent();
4036 return Legalized;
4037 }
4038
4039 return UnableToLegalize;
4040}
4041
4042// This attempts to handle G_CONCAT_VECTORS with illegal operands, particularly
4043// those that have smaller than legal operands.
4044//
4045// <16 x s8> = G_CONCAT_VECTORS <4 x s8>, <4 x s8>, <4 x s8>, <4 x s8>
4046//
4047// ===>
4048//
4049// s32 = G_BITCAST <4 x s8>
4050// s32 = G_BITCAST <4 x s8>
4051// s32 = G_BITCAST <4 x s8>
4052// s32 = G_BITCAST <4 x s8>
4053// <4 x s32> = G_BUILD_VECTOR s32, s32, s32, s32
4054// <16 x s8> = G_BITCAST <4 x s32>
4055LegalizerHelper::LegalizeResult
4056LegalizerHelper::bitcastConcatVector(MachineInstr &MI, unsigned TypeIdx,
4057 LLT CastTy) {
4058 // Convert it to CONCAT instruction
4059 auto ConcatMI = dyn_cast<GConcatVectors>(Val: &MI);
4060 if (!ConcatMI) {
4061 return UnableToLegalize;
4062 }
4063
4064 // Check if bitcast is Legal
4065 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
4066 LLT SrcScalTy = CastTy.getScalarType();
4067
4068 // Check if the build vector is Legal
4069 if (!LI.isLegal(Query: {TargetOpcode::G_BUILD_VECTOR, {CastTy, SrcScalTy}})) {
4070 return UnableToLegalize;
4071 }
4072
4073 // Bitcast the sources
4074 SmallVector<Register> BitcastRegs;
4075 for (unsigned i = 0; i < ConcatMI->getNumSources(); i++) {
4076 BitcastRegs.push_back(
4077 Elt: MIRBuilder.buildBitcast(Dst: SrcScalTy, Src: ConcatMI->getSourceReg(I: i))
4078 .getReg(Idx: 0));
4079 }
4080
4081 // Build the scalar values into a vector
4082 Register BuildReg =
4083 MIRBuilder.buildBuildVector(Res: CastTy, Ops: BitcastRegs).getReg(Idx: 0);
4084 MIRBuilder.buildBitcast(Dst: DstReg, Src: BuildReg);
4085
4086 MI.eraseFromParent();
4087 return Legalized;
4088}
4089
4090// This bitcasts a shuffle vector to a different type currently of the same
4091// element size. Mostly used to legalize ptr vectors, where ptrtoint/inttoptr
4092// will be used instead.
4093//
4094// <16 x p0> = G_CONCAT_VECTORS <4 x p0>, <4 x p0>, mask
4095// ===>
4096// <4 x s64> = G_PTRTOINT <4 x p0>
4097// <4 x s64> = G_PTRTOINT <4 x p0>
4098// <16 x s64> = G_CONCAT_VECTORS <4 x s64>, <4 x s64>, mask
4099// <16 x p0> = G_INTTOPTR <16 x s64>
4100LegalizerHelper::LegalizeResult
4101LegalizerHelper::bitcastShuffleVector(MachineInstr &MI, unsigned TypeIdx,
4102 LLT CastTy) {
4103 auto ShuffleMI = cast<GShuffleVector>(Val: &MI);
4104 LLT DstTy = MRI.getType(Reg: ShuffleMI->getReg(Idx: 0));
4105 LLT SrcTy = MRI.getType(Reg: ShuffleMI->getReg(Idx: 1));
4106
4107 // We currently only handle vectors of the same size.
4108 if (TypeIdx != 0 ||
4109 CastTy.getScalarSizeInBits() != DstTy.getScalarSizeInBits() ||
4110 CastTy.getElementCount() != DstTy.getElementCount())
4111 return UnableToLegalize;
4112
4113 LLT NewSrcTy = SrcTy.changeElementType(NewEltTy: CastTy.getScalarType());
4114
4115 auto Inp1 = MIRBuilder.buildCast(Dst: NewSrcTy, Src: ShuffleMI->getReg(Idx: 1));
4116 auto Inp2 = MIRBuilder.buildCast(Dst: NewSrcTy, Src: ShuffleMI->getReg(Idx: 2));
4117 auto Shuf =
4118 MIRBuilder.buildShuffleVector(Res: CastTy, Src1: Inp1, Src2: Inp2, Mask: ShuffleMI->getMask());
4119 MIRBuilder.buildCast(Dst: ShuffleMI->getReg(Idx: 0), Src: Shuf);
4120
4121 MI.eraseFromParent();
4122 return Legalized;
4123}
4124
4125/// This attempts to bitcast G_EXTRACT_SUBVECTOR to CastTy.
4126///
4127/// <vscale x 8 x i1> = G_EXTRACT_SUBVECTOR <vscale x 16 x i1>, N
4128///
4129/// ===>
4130///
4131/// <vscale x 2 x i1> = G_BITCAST <vscale x 16 x i1>
4132/// <vscale x 1 x i8> = G_EXTRACT_SUBVECTOR <vscale x 2 x i1>, N / 8
4133/// <vscale x 8 x i1> = G_BITCAST <vscale x 1 x i8>
4134LegalizerHelper::LegalizeResult
4135LegalizerHelper::bitcastExtractSubvector(MachineInstr &MI, unsigned TypeIdx,
4136 LLT CastTy) {
4137 auto ES = cast<GExtractSubvector>(Val: &MI);
4138
4139 if (!CastTy.isVector())
4140 return UnableToLegalize;
4141
4142 if (TypeIdx != 0)
4143 return UnableToLegalize;
4144
4145 Register Dst = ES->getReg(Idx: 0);
4146 Register Src = ES->getSrcVec();
4147 uint64_t Idx = ES->getIndexImm();
4148
4149 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4150
4151 LLT DstTy = MRI.getType(Reg: Dst);
4152 LLT SrcTy = MRI.getType(Reg: Src);
4153 ElementCount DstTyEC = DstTy.getElementCount();
4154 ElementCount SrcTyEC = SrcTy.getElementCount();
4155 auto DstTyMinElts = DstTyEC.getKnownMinValue();
4156 auto SrcTyMinElts = SrcTyEC.getKnownMinValue();
4157
4158 if (DstTy == CastTy)
4159 return Legalized;
4160
4161 if (DstTy.getSizeInBits() != CastTy.getSizeInBits())
4162 return UnableToLegalize;
4163
4164 unsigned CastEltSize = CastTy.getElementType().getSizeInBits();
4165 unsigned DstEltSize = DstTy.getElementType().getSizeInBits();
4166 if (CastEltSize < DstEltSize)
4167 return UnableToLegalize;
4168
4169 auto AdjustAmt = CastEltSize / DstEltSize;
4170 if (Idx % AdjustAmt != 0 || DstTyMinElts % AdjustAmt != 0 ||
4171 SrcTyMinElts % AdjustAmt != 0)
4172 return UnableToLegalize;
4173
4174 Idx /= AdjustAmt;
4175 SrcTy = LLT::vector(EC: SrcTyEC.divideCoefficientBy(RHS: AdjustAmt), ScalarSizeInBits: AdjustAmt);
4176 auto CastVec = MIRBuilder.buildBitcast(Dst: SrcTy, Src);
4177 auto PromotedES = MIRBuilder.buildExtractSubvector(Res: CastTy, Src: CastVec, Index: Idx);
4178 MIRBuilder.buildBitcast(Dst, Src: PromotedES);
4179
4180 ES->eraseFromParent();
4181 return Legalized;
4182}
4183
4184/// This attempts to bitcast G_INSERT_SUBVECTOR to CastTy.
4185///
4186/// <vscale x 16 x i1> = G_INSERT_SUBVECTOR <vscale x 16 x i1>,
4187/// <vscale x 8 x i1>,
4188/// N
4189///
4190/// ===>
4191///
4192/// <vscale x 2 x i8> = G_BITCAST <vscale x 16 x i1>
4193/// <vscale x 1 x i8> = G_BITCAST <vscale x 8 x i1>
4194/// <vscale x 2 x i8> = G_INSERT_SUBVECTOR <vscale x 2 x i8>,
4195/// <vscale x 1 x i8>, N / 8
4196/// <vscale x 16 x i1> = G_BITCAST <vscale x 2 x i8>
4197LegalizerHelper::LegalizeResult
4198LegalizerHelper::bitcastInsertSubvector(MachineInstr &MI, unsigned TypeIdx,
4199 LLT CastTy) {
4200 auto ES = cast<GInsertSubvector>(Val: &MI);
4201
4202 if (!CastTy.isVector())
4203 return UnableToLegalize;
4204
4205 if (TypeIdx != 0)
4206 return UnableToLegalize;
4207
4208 Register Dst = ES->getReg(Idx: 0);
4209 Register BigVec = ES->getBigVec();
4210 Register SubVec = ES->getSubVec();
4211 uint64_t Idx = ES->getIndexImm();
4212
4213 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4214
4215 LLT DstTy = MRI.getType(Reg: Dst);
4216 LLT BigVecTy = MRI.getType(Reg: BigVec);
4217 LLT SubVecTy = MRI.getType(Reg: SubVec);
4218
4219 if (DstTy == CastTy)
4220 return Legalized;
4221
4222 if (DstTy.getSizeInBits() != CastTy.getSizeInBits())
4223 return UnableToLegalize;
4224
4225 ElementCount DstTyEC = DstTy.getElementCount();
4226 ElementCount BigVecTyEC = BigVecTy.getElementCount();
4227 ElementCount SubVecTyEC = SubVecTy.getElementCount();
4228 auto DstTyMinElts = DstTyEC.getKnownMinValue();
4229 auto BigVecTyMinElts = BigVecTyEC.getKnownMinValue();
4230 auto SubVecTyMinElts = SubVecTyEC.getKnownMinValue();
4231
4232 unsigned CastEltSize = CastTy.getElementType().getSizeInBits();
4233 unsigned DstEltSize = DstTy.getElementType().getSizeInBits();
4234 if (CastEltSize < DstEltSize)
4235 return UnableToLegalize;
4236
4237 auto AdjustAmt = CastEltSize / DstEltSize;
4238 if (Idx % AdjustAmt != 0 || DstTyMinElts % AdjustAmt != 0 ||
4239 BigVecTyMinElts % AdjustAmt != 0 || SubVecTyMinElts % AdjustAmt != 0)
4240 return UnableToLegalize;
4241
4242 Idx /= AdjustAmt;
4243 BigVecTy = LLT::vector(EC: BigVecTyEC.divideCoefficientBy(RHS: AdjustAmt), ScalarSizeInBits: AdjustAmt);
4244 SubVecTy = LLT::vector(EC: SubVecTyEC.divideCoefficientBy(RHS: AdjustAmt), ScalarSizeInBits: AdjustAmt);
4245 auto CastBigVec = MIRBuilder.buildBitcast(Dst: BigVecTy, Src: BigVec);
4246 auto CastSubVec = MIRBuilder.buildBitcast(Dst: SubVecTy, Src: SubVec);
4247 auto PromotedIS =
4248 MIRBuilder.buildInsertSubvector(Res: CastTy, Src0: CastBigVec, Src1: CastSubVec, Index: Idx);
4249 MIRBuilder.buildBitcast(Dst, Src: PromotedIS);
4250
4251 ES->eraseFromParent();
4252 return Legalized;
4253}
4254
4255LegalizerHelper::LegalizeResult LegalizerHelper::lowerLoad(GAnyLoad &LoadMI) {
4256 // Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT
4257 Register DstReg = LoadMI.getDstReg();
4258 Register PtrReg = LoadMI.getPointerReg();
4259 LLT DstTy = MRI.getType(Reg: DstReg);
4260 MachineMemOperand &MMO = LoadMI.getMMO();
4261 LLT MemTy = MMO.getMemoryType();
4262 MachineFunction &MF = MIRBuilder.getMF();
4263
4264 LLT EltTy = MemTy.getScalarType();
4265
4266 unsigned MemSizeInBits = MemTy.getSizeInBits();
4267 unsigned MemStoreSizeInBits = 8 * MemTy.getSizeInBytes();
4268
4269 if (MemSizeInBits != MemStoreSizeInBits) {
4270 if (MemTy.isVector())
4271 return UnableToLegalize;
4272
4273 // Promote to a byte-sized load if not loading an integral number of
4274 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
4275 LLT WideMemTy = EltTy.changeElementSize(NewEltSize: MemStoreSizeInBits);
4276 MachineMemOperand *NewMMO =
4277 MF.getMachineMemOperand(MMO: &MMO, PtrInfo: MMO.getPointerInfo(), Ty: WideMemTy);
4278
4279 Register LoadReg = DstReg;
4280 LLT LoadTy = DstTy;
4281
4282 // If this wasn't already an extending load, we need to widen the result
4283 // register to avoid creating a load with a narrower result than the source.
4284 if (MemStoreSizeInBits > DstTy.getSizeInBits()) {
4285 LoadTy = WideMemTy;
4286 LoadReg = MRI.createGenericVirtualRegister(Ty: WideMemTy);
4287 }
4288
4289 if (isa<GSExtLoad>(Val: LoadMI)) {
4290 auto NewLoad = MIRBuilder.buildLoad(Res: LoadTy, Addr: PtrReg, MMO&: *NewMMO);
4291 MIRBuilder.buildSExtInReg(Res: LoadReg, Op: NewLoad, ImmOp: MemSizeInBits);
4292 } else if (isa<GZExtLoad>(Val: LoadMI) || WideMemTy == LoadTy) {
4293 auto NewLoad = MIRBuilder.buildLoad(Res: LoadTy, Addr: PtrReg, MMO&: *NewMMO);
4294 // The extra bits are guaranteed to be zero, since we stored them that
4295 // way. A zext load from Wide thus automatically gives zext from MemVT.
4296 MIRBuilder.buildAssertZExt(Res: LoadReg, Op: NewLoad, Size: MemSizeInBits);
4297 } else {
4298 MIRBuilder.buildLoad(Res: LoadReg, Addr: PtrReg, MMO&: *NewMMO);
4299 }
4300
4301 if (DstTy != LoadTy)
4302 MIRBuilder.buildTrunc(Res: DstReg, Op: LoadReg);
4303
4304 LoadMI.eraseFromParent();
4305 return Legalized;
4306 }
4307
4308 // Big endian lowering not implemented.
4309 if (MIRBuilder.getDataLayout().isBigEndian())
4310 return UnableToLegalize;
4311
4312 // This load needs splitting into power of 2 sized loads.
4313 //
4314 // Our strategy here is to generate anyextending loads for the smaller
4315 // types up to next power-2 result type, and then combine the two larger
4316 // result values together, before truncating back down to the non-pow-2
4317 // type.
4318 // E.g. v1 = i24 load =>
4319 // v2 = i32 zextload (2 byte)
4320 // v3 = i32 load (1 byte)
4321 // v4 = i32 shl v3, 16
4322 // v5 = i32 or v4, v2
4323 // v1 = i24 trunc v5
4324 // By doing this we generate the correct truncate which should get
4325 // combined away as an artifact with a matching extend.
4326
4327 uint64_t LargeSplitSize, SmallSplitSize;
4328
4329 if (!isPowerOf2_32(Value: MemSizeInBits)) {
4330 // This load needs splitting into power of 2 sized loads.
4331 LargeSplitSize = llvm::bit_floor(Value: MemSizeInBits);
4332 SmallSplitSize = MemSizeInBits - LargeSplitSize;
4333 } else {
4334 // This is already a power of 2, but we still need to split this in half.
4335 //
4336 // Assume we're being asked to decompose an unaligned load.
4337 // TODO: If this requires multiple splits, handle them all at once.
4338 auto &Ctx = MF.getFunction().getContext();
4339 if (TLI.allowsMemoryAccess(Context&: Ctx, DL: MIRBuilder.getDataLayout(), Ty: MemTy, MMO))
4340 return UnableToLegalize;
4341
4342 SmallSplitSize = LargeSplitSize = MemSizeInBits / 2;
4343 }
4344
4345 if (MemTy.isVector()) {
4346 // TODO: Handle vector extloads
4347 if (MemTy != DstTy)
4348 return UnableToLegalize;
4349
4350 Align Alignment = LoadMI.getAlign();
4351 // Given an alignment larger than the size of the memory, we can increase
4352 // the size of the load without needing to scalarize it.
4353 if (Alignment.value() * 8 > MemSizeInBits &&
4354 isPowerOf2_64(Value: DstTy.getScalarSizeInBits())) {
4355 LLT MoreTy = DstTy.changeVectorElementCount(
4356 EC: ElementCount::getFixed(MinVal: NextPowerOf2(A: DstTy.getNumElements())));
4357 MachineMemOperand *NewMMO = MF.getMachineMemOperand(MMO: &MMO, Offset: 0, Ty: MoreTy);
4358 auto NewLoad = MIRBuilder.buildLoad(Res: MoreTy, Addr: PtrReg, MMO&: *NewMMO);
4359 MIRBuilder.buildDeleteTrailingVectorElements(Res: LoadMI.getReg(Idx: 0),
4360 Op0: NewLoad.getReg(Idx: 0));
4361 LoadMI.eraseFromParent();
4362 return Legalized;
4363 }
4364
4365 // TODO: We can do better than scalarizing the vector and at least split it
4366 // in half.
4367 return reduceLoadStoreWidth(MI&: LoadMI, TypeIdx: 0, NarrowTy: DstTy.getElementType());
4368 }
4369
4370 MachineMemOperand *LargeMMO =
4371 MF.getMachineMemOperand(MMO: &MMO, Offset: 0, Size: LargeSplitSize / 8);
4372 MachineMemOperand *SmallMMO =
4373 MF.getMachineMemOperand(MMO: &MMO, Offset: LargeSplitSize / 8, Size: SmallSplitSize / 8);
4374
4375 LLT PtrTy = MRI.getType(Reg: PtrReg);
4376 unsigned AnyExtSize = PowerOf2Ceil(A: DstTy.getSizeInBits());
4377
4378 LLT AnyExtTy;
4379 LLT OffsetCstRes;
4380 if (EltTy.isPointer()) {
4381 AnyExtTy = LLT::scalar(SizeInBits: AnyExtSize);
4382 OffsetCstRes = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
4383 } else {
4384 AnyExtTy = DstTy.changeElementSize(NewEltSize: AnyExtSize);
4385 OffsetCstRes = DstTy.changeElementSize(NewEltSize: PtrTy.getSizeInBits());
4386 }
4387
4388 auto LargeLoad = MIRBuilder.buildLoadInstr(Opcode: TargetOpcode::G_ZEXTLOAD, Res: AnyExtTy,
4389 Addr: PtrReg, MMO&: *LargeMMO);
4390
4391 auto OffsetCst = MIRBuilder.buildConstant(Res: OffsetCstRes, Val: LargeSplitSize / 8);
4392 Register PtrAddReg = MRI.createGenericVirtualRegister(Ty: PtrTy);
4393 auto SmallPtr = MIRBuilder.buildObjectPtrOffset(Res: PtrAddReg, Op0: PtrReg, Op1: OffsetCst);
4394 auto SmallLoad = MIRBuilder.buildLoadInstr(Opcode: LoadMI.getOpcode(), Res: AnyExtTy,
4395 Addr: SmallPtr, MMO&: *SmallMMO);
4396
4397 auto ShiftAmt = MIRBuilder.buildConstant(Res: AnyExtTy, Val: LargeSplitSize);
4398 auto Shift = MIRBuilder.buildShl(Dst: AnyExtTy, Src0: SmallLoad, Src1: ShiftAmt);
4399
4400 if (AnyExtTy == DstTy)
4401 MIRBuilder.buildOr(Dst: DstReg, Src0: Shift, Src1: LargeLoad);
4402 else if (AnyExtTy.getSizeInBits() != DstTy.getSizeInBits()) {
4403 auto Or = MIRBuilder.buildOr(Dst: AnyExtTy, Src0: Shift, Src1: LargeLoad);
4404 MIRBuilder.buildTrunc(Res: DstReg, Op: {Or});
4405 } else {
4406 assert(DstTy.isPointer() && "expected pointer");
4407 auto Or = MIRBuilder.buildOr(Dst: AnyExtTy, Src0: Shift, Src1: LargeLoad);
4408
4409 // FIXME: We currently consider this to be illegal for non-integral address
4410 // spaces, but we need still need a way to reinterpret the bits.
4411 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: Or);
4412 }
4413
4414 LoadMI.eraseFromParent();
4415 return Legalized;
4416}
4417
4418LegalizerHelper::LegalizeResult LegalizerHelper::lowerStore(GStore &StoreMI) {
4419 // Lower a non-power of 2 store into multiple pow-2 stores.
4420 // E.g. split an i24 store into an i16 store + i8 store.
4421 // We do this by first extending the stored value to the next largest power
4422 // of 2 type, and then using truncating stores to store the components.
4423 // By doing this, likewise with G_LOAD, generate an extend that can be
4424 // artifact-combined away instead of leaving behind extracts.
4425 Register SrcReg = StoreMI.getValueReg();
4426 Register PtrReg = StoreMI.getPointerReg();
4427 LLT SrcTy = MRI.getType(Reg: SrcReg);
4428 MachineFunction &MF = MIRBuilder.getMF();
4429 MachineMemOperand &MMO = **StoreMI.memoperands_begin();
4430 LLT MemTy = MMO.getMemoryType();
4431
4432 unsigned StoreWidth = MemTy.getSizeInBits();
4433 unsigned StoreSizeInBits = 8 * MemTy.getSizeInBytes();
4434
4435 if (StoreWidth != StoreSizeInBits && !SrcTy.isVector()) {
4436 // Promote to a byte-sized store with upper bits zero if not
4437 // storing an integral number of bytes. For example, promote
4438 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
4439 LLT WideTy = LLT::integer(SizeInBits: StoreSizeInBits);
4440
4441 if (StoreSizeInBits > SrcTy.getSizeInBits()) {
4442 // Avoid creating a store with a narrower source than result.
4443 SrcReg = MIRBuilder.buildAnyExt(Res: WideTy, Op: SrcReg).getReg(Idx: 0);
4444 SrcTy = WideTy;
4445 }
4446
4447 auto ZextInReg = MIRBuilder.buildZExtInReg(Res: SrcTy, Op: SrcReg, ImmOp: StoreWidth);
4448
4449 MachineMemOperand *NewMMO =
4450 MF.getMachineMemOperand(MMO: &MMO, PtrInfo: MMO.getPointerInfo(), Ty: WideTy);
4451 MIRBuilder.buildStore(Val: ZextInReg, Addr: PtrReg, MMO&: *NewMMO);
4452 StoreMI.eraseFromParent();
4453 return Legalized;
4454 }
4455
4456 if (MemTy.isVector()) {
4457 if (MemTy != SrcTy)
4458 return scalarizeVectorBooleanStore(MI&: StoreMI);
4459
4460 // TODO: We can do better than scalarizing the vector and at least split it
4461 // in half.
4462 return reduceLoadStoreWidth(MI&: StoreMI, TypeIdx: 0, NarrowTy: SrcTy.getElementType());
4463 }
4464
4465 unsigned MemSizeInBits = MemTy.getSizeInBits();
4466 uint64_t LargeSplitSize, SmallSplitSize;
4467
4468 if (!isPowerOf2_32(Value: MemSizeInBits)) {
4469 LargeSplitSize = llvm::bit_floor<uint64_t>(Value: MemTy.getSizeInBits());
4470 SmallSplitSize = MemTy.getSizeInBits() - LargeSplitSize;
4471 } else {
4472 auto &Ctx = MF.getFunction().getContext();
4473 if (TLI.allowsMemoryAccess(Context&: Ctx, DL: MIRBuilder.getDataLayout(), Ty: MemTy, MMO))
4474 return UnableToLegalize; // Don't know what we're being asked to do.
4475
4476 SmallSplitSize = LargeSplitSize = MemSizeInBits / 2;
4477 }
4478
4479 // Extend to the next pow-2. If this store was itself the result of lowering,
4480 // e.g. an s56 store being broken into s32 + s24, we might have a stored type
4481 // that's wider than the stored size.
4482 unsigned AnyExtSize = PowerOf2Ceil(A: MemTy.getSizeInBits());
4483 const LLT NewSrcTy = LLT::integer(SizeInBits: AnyExtSize);
4484
4485 if (SrcTy.isPointer()) {
4486 const LLT IntPtrTy = LLT::integer(SizeInBits: SrcTy.getSizeInBits());
4487 SrcReg = MIRBuilder.buildPtrToInt(Dst: IntPtrTy, Src: SrcReg).getReg(Idx: 0);
4488 }
4489
4490 auto ExtVal = MIRBuilder.buildAnyExtOrTrunc(Res: NewSrcTy, Op: SrcReg);
4491
4492 // Obtain the smaller value by shifting away the larger value.
4493 auto ShiftAmt = MIRBuilder.buildConstant(Res: NewSrcTy, Val: LargeSplitSize);
4494 auto SmallVal = MIRBuilder.buildLShr(Dst: NewSrcTy, Src0: ExtVal, Src1: ShiftAmt);
4495
4496 // Generate the PtrAdd and truncating stores.
4497 LLT PtrTy = MRI.getType(Reg: PtrReg);
4498 auto OffsetCst = MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: PtrTy.getSizeInBits()),
4499 Val: LargeSplitSize / 8);
4500 auto SmallPtr = MIRBuilder.buildObjectPtrOffset(Res: PtrTy, Op0: PtrReg, Op1: OffsetCst);
4501
4502 MachineMemOperand *LargeMMO =
4503 MF.getMachineMemOperand(MMO: &MMO, Offset: 0, Size: LargeSplitSize / 8);
4504 MachineMemOperand *SmallMMO =
4505 MF.getMachineMemOperand(MMO: &MMO, Offset: LargeSplitSize / 8, Size: SmallSplitSize / 8);
4506 MIRBuilder.buildStore(Val: ExtVal, Addr: PtrReg, MMO&: *LargeMMO);
4507 MIRBuilder.buildStore(Val: SmallVal, Addr: SmallPtr, MMO&: *SmallMMO);
4508 StoreMI.eraseFromParent();
4509 return Legalized;
4510}
4511
4512LegalizerHelper::LegalizeResult
4513LegalizerHelper::scalarizeVectorBooleanStore(GStore &StoreMI) {
4514 Register SrcReg = StoreMI.getValueReg();
4515 Register PtrReg = StoreMI.getPointerReg();
4516 LLT SrcTy = MRI.getType(Reg: SrcReg);
4517 MachineMemOperand &MMO = **StoreMI.memoperands_begin();
4518 LLT MemTy = MMO.getMemoryType();
4519 LLT MemScalarTy = MemTy.getElementType();
4520 MachineFunction &MF = MIRBuilder.getMF();
4521
4522 assert(SrcTy.isVector() && "Expect a vector store type");
4523
4524 if (!MemScalarTy.isByteSized()) {
4525 // We need to build an integer scalar of the vector bit pattern.
4526 // It's not legal for us to add padding when storing a vector.
4527 unsigned NumBits = MemTy.getSizeInBits();
4528 LLT IntTy = LLT::integer(SizeInBits: NumBits);
4529 auto CurrVal = MIRBuilder.buildConstant(Res: IntTy, Val: 0);
4530 LLT IdxTy = TLI.getVectorIdxLLT(DL: MF.getDataLayout());
4531
4532 for (unsigned I = 0, E = MemTy.getNumElements(); I < E; ++I) {
4533 auto Elt = MIRBuilder.buildExtractVectorElement(
4534 Res: SrcTy.getElementType(), Val: SrcReg, Idx: MIRBuilder.buildConstant(Res: IdxTy, Val: I));
4535 auto Trunc = MIRBuilder.buildTrunc(Res: MemScalarTy, Op: Elt);
4536 auto ZExt = MIRBuilder.buildZExt(Res: IntTy, Op: Trunc);
4537 unsigned ShiftIntoIdx = MF.getDataLayout().isBigEndian()
4538 ? (MemTy.getNumElements() - 1) - I
4539 : I;
4540 auto ShiftAmt = MIRBuilder.buildConstant(
4541 Res: IntTy, Val: ShiftIntoIdx * MemScalarTy.getSizeInBits());
4542 auto Shifted = MIRBuilder.buildShl(Dst: IntTy, Src0: ZExt, Src1: ShiftAmt);
4543 CurrVal = MIRBuilder.buildOr(Dst: IntTy, Src0: CurrVal, Src1: Shifted);
4544 }
4545 auto PtrInfo = MMO.getPointerInfo();
4546 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo, Ty: IntTy);
4547 MIRBuilder.buildStore(Val: CurrVal, Addr: PtrReg, MMO&: *NewMMO);
4548 StoreMI.eraseFromParent();
4549 return Legalized;
4550 }
4551
4552 // TODO: implement simple scalarization.
4553 return UnableToLegalize;
4554}
4555
4556LegalizerHelper::LegalizeResult
4557LegalizerHelper::bitcast(MachineInstr &MI, unsigned TypeIdx, LLT CastTy) {
4558 switch (MI.getOpcode()) {
4559 case TargetOpcode::G_LOAD: {
4560 if (TypeIdx != 0)
4561 return UnableToLegalize;
4562 MachineMemOperand &MMO = **MI.memoperands_begin();
4563
4564 // Not sure how to interpret a bitcast of an extending load.
4565 if (MMO.getMemoryType().getSizeInBits() != CastTy.getSizeInBits())
4566 return UnableToLegalize;
4567
4568 Observer.changingInstr(MI);
4569 bitcastDst(MI, CastTy, OpIdx: 0);
4570 MMO.setType(CastTy);
4571 // The range metadata is no longer valid when reinterpreted as a different
4572 // type.
4573 MMO.clearRanges();
4574 Observer.changedInstr(MI);
4575 return Legalized;
4576 }
4577 case TargetOpcode::G_STORE: {
4578 if (TypeIdx != 0)
4579 return UnableToLegalize;
4580
4581 MachineMemOperand &MMO = **MI.memoperands_begin();
4582
4583 // Not sure how to interpret a bitcast of a truncating store.
4584 if (MMO.getMemoryType().getSizeInBits() != CastTy.getSizeInBits())
4585 return UnableToLegalize;
4586
4587 Observer.changingInstr(MI);
4588 bitcastSrc(MI, CastTy, OpIdx: 0);
4589 MMO.setType(CastTy);
4590 Observer.changedInstr(MI);
4591 return Legalized;
4592 }
4593 case TargetOpcode::G_SELECT: {
4594 if (TypeIdx != 0)
4595 return UnableToLegalize;
4596
4597 if (MRI.getType(Reg: MI.getOperand(i: 1).getReg()).isVector()) {
4598 LLVM_DEBUG(
4599 dbgs() << "bitcast action not implemented for vector select\n");
4600 return UnableToLegalize;
4601 }
4602
4603 Observer.changingInstr(MI);
4604 bitcastSrc(MI, CastTy, OpIdx: 2);
4605 bitcastSrc(MI, CastTy, OpIdx: 3);
4606 bitcastDst(MI, CastTy, OpIdx: 0);
4607 Observer.changedInstr(MI);
4608 return Legalized;
4609 }
4610 case TargetOpcode::G_AND:
4611 case TargetOpcode::G_OR:
4612 case TargetOpcode::G_XOR: {
4613 Observer.changingInstr(MI);
4614 bitcastSrc(MI, CastTy, OpIdx: 1);
4615 bitcastSrc(MI, CastTy, OpIdx: 2);
4616 bitcastDst(MI, CastTy, OpIdx: 0);
4617 Observer.changedInstr(MI);
4618 return Legalized;
4619 }
4620 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
4621 return bitcastExtractVectorElt(MI, TypeIdx, CastTy);
4622 case TargetOpcode::G_INSERT_VECTOR_ELT:
4623 return bitcastInsertVectorElt(MI, TypeIdx, CastTy);
4624 case TargetOpcode::G_CONCAT_VECTORS:
4625 return bitcastConcatVector(MI, TypeIdx, CastTy);
4626 case TargetOpcode::G_SHUFFLE_VECTOR:
4627 return bitcastShuffleVector(MI, TypeIdx, CastTy);
4628 case TargetOpcode::G_EXTRACT_SUBVECTOR:
4629 return bitcastExtractSubvector(MI, TypeIdx, CastTy);
4630 case TargetOpcode::G_INSERT_SUBVECTOR:
4631 return bitcastInsertSubvector(MI, TypeIdx, CastTy);
4632 default:
4633 return UnableToLegalize;
4634 }
4635}
4636
4637// Legalize an instruction by changing the opcode in place.
4638void LegalizerHelper::changeOpcode(MachineInstr &MI, unsigned NewOpcode) {
4639 Observer.changingInstr(MI);
4640 MI.setDesc(MIRBuilder.getTII().get(Opcode: NewOpcode));
4641 Observer.changedInstr(MI);
4642}
4643
4644LegalizerHelper::LegalizeResult
4645LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) {
4646 using namespace TargetOpcode;
4647 switch(MI.getOpcode()) {
4648 default:
4649 return UnableToLegalize;
4650 case TargetOpcode::G_FCONSTANT:
4651 return lowerFConstant(MI);
4652 case TargetOpcode::G_BITCAST:
4653 return lowerBitcast(MI);
4654 case TargetOpcode::G_SREM:
4655 case TargetOpcode::G_UREM: {
4656 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4657 auto Quot =
4658 MIRBuilder.buildInstr(Opc: MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV, DstOps: {Ty},
4659 SrcOps: {MI.getOperand(i: 1), MI.getOperand(i: 2)});
4660
4661 auto Prod = MIRBuilder.buildMul(Dst: Ty, Src0: Quot, Src1: MI.getOperand(i: 2));
4662 MIRBuilder.buildSub(Dst: MI.getOperand(i: 0), Src0: MI.getOperand(i: 1), Src1: Prod);
4663 MI.eraseFromParent();
4664 return Legalized;
4665 }
4666 case TargetOpcode::G_SADDO:
4667 case TargetOpcode::G_SSUBO:
4668 return lowerSADDO_SSUBO(MI);
4669 case TargetOpcode::G_SADDE:
4670 return lowerSADDE(MI);
4671 case TargetOpcode::G_SSUBE:
4672 return lowerSSUBE(MI);
4673 case TargetOpcode::G_UMULH:
4674 case TargetOpcode::G_SMULH:
4675 return lowerSMULH_UMULH(MI);
4676 case TargetOpcode::G_SMULO:
4677 case TargetOpcode::G_UMULO: {
4678 // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the
4679 // result.
4680 auto [Res, Overflow, LHS, RHS] = MI.getFirst4Regs();
4681 LLT Ty = MRI.getType(Reg: Res);
4682
4683 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO
4684 ? TargetOpcode::G_SMULH
4685 : TargetOpcode::G_UMULH;
4686
4687 Observer.changingInstr(MI);
4688 const auto &TII = MIRBuilder.getTII();
4689 MI.setDesc(TII.get(Opcode: TargetOpcode::G_MUL));
4690 MI.removeOperand(OpNo: 1);
4691 Observer.changedInstr(MI);
4692
4693 auto HiPart = MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Ty}, SrcOps: {LHS, RHS});
4694 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
4695
4696 // Move insert point forward so we can use the Res register if needed.
4697 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
4698
4699 // For *signed* multiply, overflow is detected by checking:
4700 // (hi != (lo >> bitwidth-1))
4701 if (Opcode == TargetOpcode::G_SMULH) {
4702 auto ShiftAmt = MIRBuilder.buildConstant(Res: Ty, Val: Ty.getSizeInBits() - 1);
4703 auto Shifted = MIRBuilder.buildAShr(Dst: Ty, Src0: Res, Src1: ShiftAmt);
4704 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: Overflow, Op0: HiPart, Op1: Shifted);
4705 } else {
4706 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: Overflow, Op0: HiPart, Op1: Zero);
4707 }
4708 return Legalized;
4709 }
4710 case TargetOpcode::G_FNEG: {
4711 auto [Res, ResTy, SubByReg, SubByRegTy] = MI.getFirst2RegLLTs();
4712 LLT TyInt =
4713 ResTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: ResTy.getScalarSizeInBits()));
4714 Register CastedSubByReg = SubByReg;
4715
4716 if (!SubByRegTy.getScalarType().isAnyScalar() &&
4717 !SubByRegTy.getScalarType().isInteger()) {
4718 auto BitcastDst = SubByRegTy.changeElementType(
4719 NewEltTy: LLT::integer(SizeInBits: SubByRegTy.getScalarSizeInBits()));
4720 CastedSubByReg = MIRBuilder.buildBitcast(Dst: BitcastDst, Src: SubByReg).getReg(Idx: 0);
4721 }
4722
4723 auto SignMask = MIRBuilder.buildConstant(
4724 Res: TyInt, Val: APInt::getSignMask(BitWidth: TyInt.getScalarSizeInBits()));
4725
4726 if (ResTy != TyInt) {
4727 Register NewDst =
4728 MIRBuilder.buildXor(Dst: TyInt, Src0: CastedSubByReg, Src1: SignMask).getReg(Idx: 0);
4729 MIRBuilder.buildBitcast(Dst: Res, Src: NewDst);
4730 } else
4731 MIRBuilder.buildXor(Dst: Res, Src0: CastedSubByReg, Src1: SignMask).getReg(Idx: 0);
4732
4733 MI.eraseFromParent();
4734 return Legalized;
4735 }
4736 case TargetOpcode::G_FSUB:
4737 case TargetOpcode::G_STRICT_FSUB: {
4738 auto [Res, LHS, RHS] = MI.getFirst3Regs();
4739 LLT Ty = MRI.getType(Reg: Res);
4740
4741 // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)).
4742 auto Neg = MIRBuilder.buildFNeg(Dst: Ty, Src0: RHS);
4743
4744 if (MI.getOpcode() == TargetOpcode::G_STRICT_FSUB)
4745 MIRBuilder.buildStrictFAdd(Dst: Res, Src0: LHS, Src1: Neg, Flags: MI.getFlags());
4746 else
4747 MIRBuilder.buildFAdd(Dst: Res, Src0: LHS, Src1: Neg, Flags: MI.getFlags());
4748
4749 MI.eraseFromParent();
4750 return Legalized;
4751 }
4752 case TargetOpcode::G_FMAD:
4753 return lowerFMad(MI);
4754 case TargetOpcode::G_FFLOOR:
4755 return lowerFFloor(MI);
4756 case TargetOpcode::G_LROUND:
4757 case TargetOpcode::G_LLROUND: {
4758 Register DstReg = MI.getOperand(i: 0).getReg();
4759 Register SrcReg = MI.getOperand(i: 1).getReg();
4760 LLT SrcTy = MRI.getType(Reg: SrcReg);
4761 auto Round = MIRBuilder.buildInstr(Opc: TargetOpcode::G_INTRINSIC_ROUND, DstOps: {SrcTy},
4762 SrcOps: {SrcReg});
4763 MIRBuilder.buildFPTOSI(Dst: DstReg, Src0: Round);
4764 MI.eraseFromParent();
4765 return Legalized;
4766 }
4767 case TargetOpcode::G_INTRINSIC_ROUND:
4768 return lowerIntrinsicRound(MI);
4769 case TargetOpcode::G_FRINT: {
4770 // Since round even is the assumed rounding mode for unconstrained FP
4771 // operations, rint and roundeven are the same operation.
4772 changeOpcode(MI, NewOpcode: TargetOpcode::G_INTRINSIC_ROUNDEVEN);
4773 return Legalized;
4774 }
4775 case TargetOpcode::G_INTRINSIC_LRINT:
4776 case TargetOpcode::G_INTRINSIC_LLRINT: {
4777 Register DstReg = MI.getOperand(i: 0).getReg();
4778 Register SrcReg = MI.getOperand(i: 1).getReg();
4779 LLT SrcTy = MRI.getType(Reg: SrcReg);
4780 auto Round =
4781 MIRBuilder.buildInstr(Opc: TargetOpcode::G_FRINT, DstOps: {SrcTy}, SrcOps: {SrcReg});
4782 MIRBuilder.buildFPTOSI(Dst: DstReg, Src0: Round);
4783 MI.eraseFromParent();
4784 return Legalized;
4785 }
4786 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
4787 auto [OldValRes, SuccessRes, Addr, CmpVal, NewVal] = MI.getFirst5Regs();
4788 Register NewOldValRes = MRI.cloneVirtualRegister(VReg: OldValRes);
4789 MIRBuilder.buildAtomicCmpXchg(OldValRes: NewOldValRes, Addr, CmpVal, NewVal,
4790 MMO&: **MI.memoperands_begin());
4791 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: SuccessRes, Op0: NewOldValRes, Op1: CmpVal);
4792 MIRBuilder.buildCopy(Res: OldValRes, Op: NewOldValRes);
4793 MI.eraseFromParent();
4794 return Legalized;
4795 }
4796 case TargetOpcode::G_LOAD:
4797 case TargetOpcode::G_SEXTLOAD:
4798 case TargetOpcode::G_ZEXTLOAD:
4799 return lowerLoad(LoadMI&: cast<GAnyLoad>(Val&: MI));
4800 case TargetOpcode::G_STORE:
4801 return lowerStore(StoreMI&: cast<GStore>(Val&: MI));
4802 case TargetOpcode::G_CTLZ_ZERO_POISON:
4803 case TargetOpcode::G_CTTZ_ZERO_POISON:
4804 case TargetOpcode::G_CTLZ:
4805 case TargetOpcode::G_CTTZ:
4806 case TargetOpcode::G_CTPOP:
4807 case TargetOpcode::G_CTLS:
4808 return lowerBitCount(MI);
4809 case G_UADDO: {
4810 auto [Res, CarryOut, LHS, RHS] = MI.getFirst4Regs();
4811
4812 Register NewRes = MRI.cloneVirtualRegister(VReg: Res);
4813
4814 MIRBuilder.buildAdd(Dst: NewRes, Src0: LHS, Src1: RHS);
4815 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_ULT, Res: CarryOut, Op0: NewRes, Op1: RHS);
4816
4817 MIRBuilder.buildCopy(Res, Op: NewRes);
4818
4819 MI.eraseFromParent();
4820 return Legalized;
4821 }
4822 case G_UADDE: {
4823 auto [Res, CarryOut, LHS, RHS, CarryIn] = MI.getFirst5Regs();
4824 const LLT CondTy = MRI.getType(Reg: CarryOut);
4825 const LLT Ty = MRI.getType(Reg: Res);
4826
4827 Register NewRes = MRI.cloneVirtualRegister(VReg: Res);
4828
4829 // Initial add of the two operands.
4830 auto TmpRes = MIRBuilder.buildAdd(Dst: Ty, Src0: LHS, Src1: RHS);
4831
4832 // Initial check for carry.
4833 auto Carry = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_ULT, Res: CondTy, Op0: TmpRes, Op1: LHS);
4834
4835 // Add the sum and the carry.
4836 auto ZExtCarryIn = MIRBuilder.buildZExt(Res: Ty, Op: CarryIn);
4837 MIRBuilder.buildAdd(Dst: NewRes, Src0: TmpRes, Src1: ZExtCarryIn);
4838
4839 // Second check for carry. We can only carry if the initial sum is all 1s
4840 // and the carry is set, resulting in a new sum of 0.
4841 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
4842 auto ResEqZero =
4843 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CondTy, Op0: NewRes, Op1: Zero);
4844 auto Carry2 = MIRBuilder.buildAnd(Dst: CondTy, Src0: ResEqZero, Src1: CarryIn);
4845 MIRBuilder.buildOr(Dst: CarryOut, Src0: Carry, Src1: Carry2);
4846
4847 MIRBuilder.buildCopy(Res, Op: NewRes);
4848
4849 MI.eraseFromParent();
4850 return Legalized;
4851 }
4852 case G_USUBO: {
4853 auto [Res, BorrowOut, LHS, RHS] = MI.getFirst4Regs();
4854
4855 MIRBuilder.buildSub(Dst: Res, Src0: LHS, Src1: RHS);
4856 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_ULT, Res: BorrowOut, Op0: LHS, Op1: RHS);
4857
4858 MI.eraseFromParent();
4859 return Legalized;
4860 }
4861 case G_USUBE: {
4862 auto [Res, BorrowOut, LHS, RHS, BorrowIn] = MI.getFirst5Regs();
4863 const LLT CondTy = MRI.getType(Reg: BorrowOut);
4864 const LLT Ty = MRI.getType(Reg: Res);
4865
4866 // Initial subtract of the two operands.
4867 auto TmpRes = MIRBuilder.buildSub(Dst: Ty, Src0: LHS, Src1: RHS);
4868
4869 // Initial check for borrow.
4870 auto Borrow = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_UGT, Res: CondTy, Op0: TmpRes, Op1: LHS);
4871
4872 // Subtract the borrow from the first subtract.
4873 auto ZExtBorrowIn = MIRBuilder.buildZExt(Res: Ty, Op: BorrowIn);
4874 MIRBuilder.buildSub(Dst: Res, Src0: TmpRes, Src1: ZExtBorrowIn);
4875
4876 // Second check for borrow. We can only borrow if the initial difference is
4877 // 0 and the borrow is set, resulting in a new difference of all 1s.
4878 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
4879 auto TmpResEqZero =
4880 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CondTy, Op0: TmpRes, Op1: Zero);
4881 auto Borrow2 = MIRBuilder.buildAnd(Dst: CondTy, Src0: TmpResEqZero, Src1: BorrowIn);
4882 MIRBuilder.buildOr(Dst: BorrowOut, Src0: Borrow, Src1: Borrow2);
4883
4884 MI.eraseFromParent();
4885 return Legalized;
4886 }
4887 case G_UITOFP:
4888 return lowerUITOFP(MI);
4889 case G_SITOFP:
4890 return lowerSITOFP(MI);
4891 case G_FPTOUI:
4892 return lowerFPTOUI(MI);
4893 case G_FPTOSI:
4894 return lowerFPTOSI(MI);
4895 case G_FPTOUI_SAT:
4896 case G_FPTOSI_SAT:
4897 return lowerFPTOINT_SAT(MI);
4898 case G_FPEXT:
4899 return lowerFPEXT(MI);
4900 case G_FPTRUNC:
4901 return lowerFPTRUNC(MI);
4902 case G_FPOWI:
4903 return lowerFPOWI(MI);
4904 case G_FMODF:
4905 return lowerFMODF(MI);
4906 case G_SMIN:
4907 case G_SMAX:
4908 case G_UMIN:
4909 case G_UMAX:
4910 return lowerMinMax(MI);
4911 case G_SCMP:
4912 case G_UCMP:
4913 return lowerThreewayCompare(MI);
4914 case G_FCOPYSIGN:
4915 return lowerFCopySign(MI);
4916 case G_FMINNUM:
4917 case G_FMAXNUM:
4918 case G_FMINIMUMNUM:
4919 case G_FMAXIMUMNUM:
4920 return lowerFMinNumMaxNum(MI);
4921 case G_FMINIMUM:
4922 case G_FMAXIMUM:
4923 return lowerFMinimumMaximum(MI);
4924 case G_MERGE_VALUES:
4925 return lowerMergeValues(MI);
4926 case G_UNMERGE_VALUES:
4927 return lowerUnmergeValues(MI);
4928 case TargetOpcode::G_SEXT_INREG: {
4929 assert(MI.getOperand(2).isImm() && "Expected immediate");
4930 int64_t SizeInBits = MI.getOperand(i: 2).getImm();
4931
4932 auto [DstReg, SrcReg] = MI.getFirst2Regs();
4933 LLT DstTy = MRI.getType(Reg: DstReg);
4934 Register TmpRes = MRI.createGenericVirtualRegister(Ty: DstTy);
4935
4936 auto MIBSz = MIRBuilder.buildConstant(Res: DstTy, Val: DstTy.getScalarSizeInBits() - SizeInBits);
4937 MIRBuilder.buildShl(Dst: TmpRes, Src0: SrcReg, Src1: MIBSz->getOperand(i: 0));
4938 MIRBuilder.buildAShr(Dst: DstReg, Src0: TmpRes, Src1: MIBSz->getOperand(i: 0));
4939 MI.eraseFromParent();
4940 return Legalized;
4941 }
4942 case G_EXTRACT_VECTOR_ELT:
4943 case G_INSERT_VECTOR_ELT:
4944 return lowerExtractInsertVectorElt(MI);
4945 case G_SHUFFLE_VECTOR:
4946 return lowerShuffleVector(MI);
4947 case G_VECTOR_COMPRESS:
4948 return lowerVECTOR_COMPRESS(MI);
4949 case G_DYN_STACKALLOC:
4950 return lowerDynStackAlloc(MI);
4951 case G_INSERT_SUBVECTOR: {
4952 if (MRI.getType(Reg: MI.getOperand(i: 1).getReg()).isScalable() ||
4953 MRI.getType(Reg: MI.getOperand(i: 2).getReg()).isScalable())
4954 return UnableToLegalize;
4955
4956 // Check that subvector is half size of main vector
4957 Register Vector = MI.getOperand(i: 1).getReg();
4958 Register Subvector = MI.getOperand(i: 2).getReg();
4959 auto InsertionPointImm = MI.getOperand(i: 3).getImm();
4960
4961 LLT VectorTy = MRI.getType(Reg: Vector);
4962 LLT DstTy = MRI.getType(Reg: Subvector);
4963 // If so, -> concat(subvector, extract(half of vector))
4964 // (Operands can be either way round depending on insertion point
4965 if (VectorTy.getSizeInBits() == DstTy.getSizeInBits() * 2) {
4966 bool InsertInLowHalf = InsertionPointImm == 0;
4967 auto Extract = MIRBuilder.buildExtractSubvector(
4968 Res: DstTy, Src: Vector,
4969 Index: (uint64_t)(InsertInLowHalf ? VectorTy.getNumElements() / 2 : 0));
4970
4971 auto LowHalf = InsertInLowHalf ? Subvector : Extract.getReg(Idx: 0);
4972 auto HighHalf = InsertInLowHalf ? Extract.getReg(Idx: 0) : Subvector;
4973
4974 MIRBuilder.buildInstr(Opc: TargetOpcode::G_CONCAT_VECTORS, DstOps: {MI.getOperand(i: 0)},
4975 SrcOps: {LowHalf, HighHalf});
4976 MI.eraseFromParent();
4977 return Legalized;
4978 }
4979 // Else -> shuffle(vector, extend(subvector, size(vector)), mask)
4980 else {
4981 // Extend subvector to same size as vector
4982 Register ExtendedSubvector = MRI.createGenericVirtualRegister(Ty: VectorTy);
4983 MIRBuilder.buildPadVectorWithUndefElements(Res: ExtendedSubvector, Op0: Subvector);
4984
4985 // Calculate mask required for this shuffle
4986 SmallVector<int> Mask;
4987 for (int i = 0; i < VectorTy.getNumElements(); i++) {
4988 // If this index is within bounds, put subvector's index into mask
4989 if (i >= InsertionPointImm &&
4990 i < InsertionPointImm + DstTy.getNumElements())
4991 Mask.push_back(Elt: VectorTy.getNumElements() + i - InsertionPointImm);
4992 else
4993 Mask.push_back(Elt: i);
4994 }
4995
4996 // Build shuffle
4997 MIRBuilder.buildShuffleVector(Res: MI.getOperand(i: 0), Src1: Vector, Src2: ExtendedSubvector,
4998 Mask);
4999 MI.eraseFromParent();
5000 return Legalized;
5001 }
5002 }
5003 case G_EXTRACT_SUBVECTOR: {
5004 Register DstReg = MI.getOperand(i: 0).getReg();
5005 Register SrcReg = MI.getOperand(i: 1).getReg();
5006 uint64_t ExtractionPointImm = MI.getOperand(i: 2).getImm();
5007
5008 LLT SrcTy = MRI.getType(Reg: SrcReg);
5009 LLT DstTy = MRI.getType(Reg: DstReg);
5010
5011 if (SrcTy.isScalable())
5012 return UnableToLegalize;
5013
5014 if (SrcTy.getScalarType() != DstTy.getScalarType())
5015 return UnableToLegalize;
5016
5017 // extract_subvector = build_vector(extract_element, extract_element, ...)
5018 SmallVector<Register> ExtractedElements;
5019 for (uint64_t i = 0; i < DstTy.getNumElements(); i++) {
5020 ExtractedElements.push_back(
5021 Elt: MIRBuilder
5022 .buildExtractVectorElementConstant(Res: SrcTy.getScalarType(), Val: SrcReg,
5023 Idx: ExtractionPointImm + i)
5024 .getReg(Idx: 0));
5025 }
5026
5027 MIRBuilder.buildBuildVector(Res: DstReg, Ops: ExtractedElements);
5028 MI.eraseFromParent();
5029 return Legalized;
5030 }
5031 case G_STACKSAVE:
5032 return lowerStackSave(MI);
5033 case G_STACKRESTORE:
5034 return lowerStackRestore(MI);
5035 case G_EXTRACT:
5036 return lowerExtract(MI);
5037 case G_INSERT:
5038 return lowerInsert(MI);
5039 case G_BSWAP:
5040 return lowerBswap(MI);
5041 case G_BITREVERSE:
5042 return lowerBitreverse(MI);
5043 case G_READ_REGISTER:
5044 case G_WRITE_REGISTER:
5045 return lowerReadWriteRegister(MI);
5046 case G_UADDSAT:
5047 case G_USUBSAT: {
5048 // Try to make a reasonable guess about which lowering strategy to use. The
5049 // target can override this with custom lowering and calling the
5050 // implementation functions.
5051 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5052 if (LI.isLegalOrCustom(Query: {G_UMIN, Ty}))
5053 return lowerAddSubSatToMinMax(MI);
5054 return lowerAddSubSatToAddoSubo(MI);
5055 }
5056 case G_SADDSAT:
5057 case G_SSUBSAT: {
5058 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5059
5060 // FIXME: It would probably make more sense to see if G_SADDO is preferred,
5061 // since it's a shorter expansion. However, we would need to figure out the
5062 // preferred boolean type for the carry out for the query.
5063 if (LI.isLegalOrCustom(Query: {G_SMIN, Ty}) && LI.isLegalOrCustom(Query: {G_SMAX, Ty}))
5064 return lowerAddSubSatToMinMax(MI);
5065 return lowerAddSubSatToAddoSubo(MI);
5066 }
5067 case G_SSHLSAT:
5068 case G_USHLSAT:
5069 return lowerShlSat(MI);
5070 case G_TRUNC_SSAT_S:
5071 case G_TRUNC_USAT_U:
5072 case G_TRUNC_SSAT_U:
5073 return lowerTruncSat(MI);
5074 case G_ABS:
5075 return lowerAbsToAddXor(MI);
5076 case G_ABDS:
5077 case G_ABDU: {
5078 bool IsSigned = MI.getOpcode() == G_ABDS;
5079 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5080 if ((IsSigned && LI.isLegal(Query: {G_SMIN, Ty}) && LI.isLegal(Query: {G_SMAX, Ty})) ||
5081 (!IsSigned && LI.isLegal(Query: {G_UMIN, Ty}) && LI.isLegal(Query: {G_UMAX, Ty}))) {
5082 return lowerAbsDiffToMinMax(MI);
5083 }
5084 return lowerAbsDiffToSelect(MI);
5085 }
5086 case G_FABS:
5087 return lowerFAbs(MI);
5088 case G_SELECT:
5089 return lowerSelect(MI);
5090 case G_IS_FPCLASS:
5091 return lowerISFPCLASS(MI);
5092 case G_SDIVREM:
5093 case G_UDIVREM:
5094 return lowerDIVREM(MI);
5095 case G_FSHL:
5096 case G_FSHR:
5097 return lowerFunnelShift(MI);
5098 case G_ROTL:
5099 case G_ROTR:
5100 return lowerRotate(MI);
5101 case G_MEMSET:
5102 case G_MEMCPY:
5103 case G_MEMMOVE:
5104 case G_MEMCPY_INLINE:
5105 case G_MEMSET_INLINE:
5106 return lowerMemCpyFamily(MI);
5107 case G_ZEXT:
5108 case G_SEXT:
5109 case G_ANYEXT:
5110 return lowerEXT(MI);
5111 case G_TRUNC:
5112 return lowerTRUNC(MI);
5113 GISEL_VECREDUCE_CASES_NONSEQ
5114 return lowerVectorReduction(MI);
5115 case G_VAARG:
5116 return lowerVAArg(MI);
5117 case G_ATOMICRMW_SUB: {
5118 auto [Ret, Mem, Val] = MI.getFirst3Regs();
5119 const LLT ValTy = MRI.getType(Reg: Val);
5120 MachineMemOperand *MMO = *MI.memoperands_begin();
5121
5122 auto VNeg = MIRBuilder.buildNeg(Dst: ValTy, Src0: Val);
5123 MIRBuilder.buildAtomicRMW(Opcode: G_ATOMICRMW_ADD, OldValRes: Ret, Addr: Mem, Val: VNeg, MMO&: *MMO);
5124 MI.eraseFromParent();
5125 return Legalized;
5126 }
5127 case G_SMULFIX:
5128 case G_UMULFIX:
5129 case G_SMULFIXSAT:
5130 case G_UMULFIXSAT:
5131 return lowerMulfix(MI);
5132 }
5133}
5134
5135Align LegalizerHelper::getStackTemporaryAlignment(LLT Ty,
5136 Align MinAlign) const {
5137 // FIXME: We're missing a way to go back from LLT to llvm::Type to query the
5138 // datalayout for the preferred alignment. Also there should be a target hook
5139 // for this to allow targets to reduce the alignment and ignore the
5140 // datalayout. e.g. AMDGPU should always use a 4-byte alignment, regardless of
5141 // the type.
5142 return std::max(a: Align(PowerOf2Ceil(A: Ty.getSizeInBytes())), b: MinAlign);
5143}
5144
5145MachineInstrBuilder
5146LegalizerHelper::createStackTemporary(TypeSize Bytes, Align Alignment,
5147 MachinePointerInfo &PtrInfo) {
5148 MachineFunction &MF = MIRBuilder.getMF();
5149 const DataLayout &DL = MIRBuilder.getDataLayout();
5150 int FrameIdx = MF.getFrameInfo().CreateStackObject(Size: Bytes, Alignment, isSpillSlot: false);
5151
5152 unsigned AddrSpace = DL.getAllocaAddrSpace();
5153 LLT FramePtrTy = LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL.getPointerSizeInBits(AS: AddrSpace));
5154
5155 PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIdx);
5156 return MIRBuilder.buildFrameIndex(Res: FramePtrTy, Idx: FrameIdx);
5157}
5158
5159MachineInstrBuilder LegalizerHelper::createStackStoreLoad(const DstOp &Res,
5160 const SrcOp &Val) {
5161 LLT SrcTy = Val.getLLTTy(MRI);
5162 Align StackTypeAlign =
5163 std::max(a: getStackTemporaryAlignment(Ty: SrcTy),
5164 b: getStackTemporaryAlignment(Ty: Res.getLLTTy(MRI)));
5165 MachinePointerInfo PtrInfo;
5166 auto StackTemp =
5167 createStackTemporary(Bytes: SrcTy.getSizeInBytes(), Alignment: StackTypeAlign, PtrInfo);
5168
5169 MIRBuilder.buildStore(Val, Addr: StackTemp, PtrInfo, Alignment: StackTypeAlign);
5170 return MIRBuilder.buildLoad(Res, Addr: StackTemp, PtrInfo, Alignment: StackTypeAlign);
5171}
5172
5173static Register clampVectorIndex(MachineIRBuilder &B, Register IdxReg,
5174 LLT VecTy) {
5175 LLT IdxTy = B.getMRI()->getType(Reg: IdxReg);
5176 unsigned NElts = VecTy.getNumElements();
5177
5178 int64_t IdxVal;
5179 if (mi_match(R: IdxReg, MRI: *B.getMRI(), P: m_ICst(Cst&: IdxVal))) {
5180 if (IdxVal < VecTy.getNumElements())
5181 return IdxReg;
5182 // If a constant index would be out of bounds, clamp it as well.
5183 }
5184
5185 if (isPowerOf2_32(Value: NElts)) {
5186 APInt Imm = APInt::getLowBitsSet(numBits: IdxTy.getSizeInBits(), loBitsSet: Log2_32(Value: NElts));
5187 return B.buildAnd(Dst: IdxTy, Src0: IdxReg, Src1: B.buildConstant(Res: IdxTy, Val: Imm)).getReg(Idx: 0);
5188 }
5189
5190 return B.buildUMin(Dst: IdxTy, Src0: IdxReg, Src1: B.buildConstant(Res: IdxTy, Val: NElts - 1))
5191 .getReg(Idx: 0);
5192}
5193
5194Register LegalizerHelper::getVectorElementPointer(Register VecPtr, LLT VecTy,
5195 Register Index) {
5196 LLT EltTy = VecTy.getElementType();
5197
5198 // Calculate the element offset and add it to the pointer.
5199 unsigned EltSize = EltTy.getSizeInBits() / 8; // FIXME: should be ABI size.
5200 assert(EltSize * 8 == EltTy.getSizeInBits() &&
5201 "Converting bits to bytes lost precision");
5202
5203 Index = clampVectorIndex(B&: MIRBuilder, IdxReg: Index, VecTy);
5204
5205 // Convert index to the correct size for the address space.
5206 const DataLayout &DL = MIRBuilder.getDataLayout();
5207 unsigned AS = MRI.getType(Reg: VecPtr).getAddressSpace();
5208 unsigned IndexSizeInBits = DL.getIndexSize(AS) * 8;
5209 LLT IdxTy = MRI.getType(Reg: Index).changeElementSize(NewEltSize: IndexSizeInBits);
5210 if (IdxTy != MRI.getType(Reg: Index))
5211 Index = MIRBuilder.buildSExtOrTrunc(Res: IdxTy, Op: Index).getReg(Idx: 0);
5212
5213 auto Mul = MIRBuilder.buildMul(Dst: IdxTy, Src0: Index,
5214 Src1: MIRBuilder.buildConstant(Res: IdxTy, Val: EltSize));
5215
5216 LLT PtrTy = MRI.getType(Reg: VecPtr);
5217 return MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: VecPtr, Op1: Mul).getReg(Idx: 0);
5218}
5219
5220#ifndef NDEBUG
5221/// Check that all vector operands have same number of elements. Other operands
5222/// should be listed in NonVecOp.
5223static bool hasSameNumEltsOnAllVectorOperands(
5224 GenericMachineInstr &MI, MachineRegisterInfo &MRI,
5225 std::initializer_list<unsigned> NonVecOpIndices) {
5226 if (MI.getNumMemOperands() != 0)
5227 return false;
5228
5229 LLT VecTy = MRI.getType(MI.getReg(0));
5230 if (!VecTy.isVector())
5231 return false;
5232 unsigned NumElts = VecTy.getNumElements();
5233
5234 for (unsigned OpIdx = 1; OpIdx < MI.getNumOperands(); ++OpIdx) {
5235 MachineOperand &Op = MI.getOperand(OpIdx);
5236 if (!Op.isReg()) {
5237 if (!is_contained(NonVecOpIndices, OpIdx))
5238 return false;
5239 continue;
5240 }
5241
5242 LLT Ty = MRI.getType(Op.getReg());
5243 if (!Ty.isVector()) {
5244 if (!is_contained(NonVecOpIndices, OpIdx))
5245 return false;
5246 continue;
5247 }
5248
5249 if (Ty.getNumElements() != NumElts)
5250 return false;
5251 }
5252
5253 return true;
5254}
5255#endif
5256
5257/// Fill \p DstOps with DstOps that have same number of elements combined as
5258/// the Ty. These DstOps have either scalar type when \p NumElts = 1 or are
5259/// vectors with \p NumElts elements. When Ty.getNumElements() is not multiple
5260/// of \p NumElts last DstOp (leftover) has fewer then \p NumElts elements.
5261static void makeDstOps(SmallVectorImpl<DstOp> &DstOps, LLT Ty,
5262 unsigned NumElts) {
5263 LLT LeftoverTy;
5264 assert(Ty.isVector() && "Expected vector type");
5265 LLT NarrowTy = Ty.changeElementCount(EC: ElementCount::getFixed(MinVal: NumElts));
5266 int NumParts, NumLeftover;
5267 std::tie(args&: NumParts, args&: NumLeftover) =
5268 getNarrowTypeBreakDown(OrigTy: Ty, NarrowTy, LeftoverTy);
5269
5270 assert(NumParts > 0 && "Error in getNarrowTypeBreakDown");
5271 for (int i = 0; i < NumParts; ++i) {
5272 DstOps.push_back(Elt: NarrowTy);
5273 }
5274
5275 if (LeftoverTy.isValid()) {
5276 assert(NumLeftover == 1 && "expected exactly one leftover");
5277 DstOps.push_back(Elt: LeftoverTy);
5278 }
5279}
5280
5281/// Operand \p Op is used on \p N sub-instructions. Fill \p Ops with \p N SrcOps
5282/// made from \p Op depending on operand type.
5283static void broadcastSrcOp(SmallVectorImpl<SrcOp> &Ops, unsigned N,
5284 MachineOperand &Op) {
5285 for (unsigned i = 0; i < N; ++i) {
5286 if (Op.isReg())
5287 Ops.push_back(Elt: Op.getReg());
5288 else if (Op.isImm())
5289 Ops.push_back(Elt: Op.getImm());
5290 else if (Op.isPredicate())
5291 Ops.push_back(Elt: static_cast<CmpInst::Predicate>(Op.getPredicate()));
5292 else
5293 llvm_unreachable("Unsupported type");
5294 }
5295}
5296
5297// Handle splitting vector operations which need to have the same number of
5298// elements in each type index, but each type index may have a different element
5299// type.
5300//
5301// e.g. <4 x s64> = G_SHL <4 x s64>, <4 x s32> ->
5302// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
5303// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
5304//
5305// Also handles some irregular breakdown cases, e.g.
5306// e.g. <3 x s64> = G_SHL <3 x s64>, <3 x s32> ->
5307// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
5308// s64 = G_SHL s64, s32
5309LegalizerHelper::LegalizeResult
5310LegalizerHelper::fewerElementsVectorMultiEltType(
5311 GenericMachineInstr &MI, unsigned NumElts,
5312 std::initializer_list<unsigned> NonVecOpIndices) {
5313 assert(hasSameNumEltsOnAllVectorOperands(MI, MRI, NonVecOpIndices) &&
5314 "Non-compatible opcode or not specified non-vector operands");
5315 unsigned OrigNumElts = MRI.getType(Reg: MI.getReg(Idx: 0)).getNumElements();
5316
5317 unsigned NumInputs = MI.getNumOperands() - MI.getNumDefs();
5318 unsigned NumDefs = MI.getNumDefs();
5319
5320 // Create DstOps (sub-vectors with NumElts elts + Leftover) for each output.
5321 // Build instructions with DstOps to use instruction found by CSE directly.
5322 // CSE copies found instruction into given vreg when building with vreg dest.
5323 SmallVector<SmallVector<DstOp, 8>, 2> OutputOpsPieces(NumDefs);
5324 // Output registers will be taken from created instructions.
5325 SmallVector<SmallVector<Register, 8>, 2> OutputRegs(NumDefs);
5326 for (unsigned i = 0; i < NumDefs; ++i) {
5327 makeDstOps(DstOps&: OutputOpsPieces[i], Ty: MRI.getType(Reg: MI.getReg(Idx: i)), NumElts);
5328 }
5329
5330 // Split vector input operands into sub-vectors with NumElts elts + Leftover.
5331 // Operands listed in NonVecOpIndices will be used as is without splitting;
5332 // examples: compare predicate in icmp and fcmp (op 1), vector select with i1
5333 // scalar condition (op 1), immediate in sext_inreg (op 2).
5334 SmallVector<SmallVector<SrcOp, 8>, 3> InputOpsPieces(NumInputs);
5335 for (unsigned UseIdx = NumDefs, UseNo = 0; UseIdx < MI.getNumOperands();
5336 ++UseIdx, ++UseNo) {
5337 if (is_contained(Set: NonVecOpIndices, Element: UseIdx)) {
5338 broadcastSrcOp(Ops&: InputOpsPieces[UseNo], N: OutputOpsPieces[0].size(),
5339 Op&: MI.getOperand(i: UseIdx));
5340 } else {
5341 SmallVector<Register, 8> SplitPieces;
5342 extractVectorParts(Reg: MI.getReg(Idx: UseIdx), NumElts, VRegs&: SplitPieces, MIRBuilder,
5343 MRI);
5344 llvm::append_range(C&: InputOpsPieces[UseNo], R&: SplitPieces);
5345 }
5346 }
5347
5348 unsigned NumLeftovers = OrigNumElts % NumElts ? 1 : 0;
5349
5350 // Take i-th piece of each input operand split and build sub-vector/scalar
5351 // instruction. Set i-th DstOp(s) from OutputOpsPieces as destination(s).
5352 for (unsigned i = 0; i < OrigNumElts / NumElts + NumLeftovers; ++i) {
5353 SmallVector<DstOp, 2> Defs;
5354 for (unsigned DstNo = 0; DstNo < NumDefs; ++DstNo)
5355 Defs.push_back(Elt: OutputOpsPieces[DstNo][i]);
5356
5357 SmallVector<SrcOp, 3> Uses;
5358 for (unsigned InputNo = 0; InputNo < NumInputs; ++InputNo)
5359 Uses.push_back(Elt: InputOpsPieces[InputNo][i]);
5360
5361 auto I = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: Defs, SrcOps: Uses, Flags: MI.getFlags());
5362 for (unsigned DstNo = 0; DstNo < NumDefs; ++DstNo)
5363 OutputRegs[DstNo].push_back(Elt: I.getReg(Idx: DstNo));
5364 }
5365
5366 // Merge small outputs into MI's output for each def operand.
5367 if (NumLeftovers) {
5368 for (unsigned i = 0; i < NumDefs; ++i)
5369 mergeMixedSubvectors(DstReg: MI.getReg(Idx: i), PartRegs: OutputRegs[i]);
5370 } else {
5371 for (unsigned i = 0; i < NumDefs; ++i)
5372 MIRBuilder.buildMergeLikeInstr(Res: MI.getReg(Idx: i), Ops: OutputRegs[i]);
5373 }
5374
5375 MI.eraseFromParent();
5376 return Legalized;
5377}
5378
5379LegalizerHelper::LegalizeResult
5380LegalizerHelper::fewerElementsVectorPhi(GenericMachineInstr &MI,
5381 unsigned NumElts) {
5382 unsigned OrigNumElts = MRI.getType(Reg: MI.getReg(Idx: 0)).getNumElements();
5383
5384 unsigned NumInputs = MI.getNumOperands() - MI.getNumDefs();
5385 unsigned NumDefs = MI.getNumDefs();
5386
5387 SmallVector<DstOp, 8> OutputOpsPieces;
5388 SmallVector<Register, 8> OutputRegs;
5389 makeDstOps(DstOps&: OutputOpsPieces, Ty: MRI.getType(Reg: MI.getReg(Idx: 0)), NumElts);
5390
5391 // Instructions that perform register split will be inserted in basic block
5392 // where register is defined (basic block is in the next operand).
5393 SmallVector<SmallVector<Register, 8>, 3> InputOpsPieces(NumInputs / 2);
5394 for (unsigned UseIdx = NumDefs, UseNo = 0; UseIdx < MI.getNumOperands();
5395 UseIdx += 2, ++UseNo) {
5396 MachineBasicBlock &OpMBB = *MI.getOperand(i: UseIdx + 1).getMBB();
5397 MIRBuilder.setInsertPt(MBB&: OpMBB, II: OpMBB.getFirstTerminatorForward());
5398 extractVectorParts(Reg: MI.getReg(Idx: UseIdx), NumElts, VRegs&: InputOpsPieces[UseNo],
5399 MIRBuilder, MRI);
5400 }
5401
5402 // Build PHIs with fewer elements.
5403 unsigned NumLeftovers = OrigNumElts % NumElts ? 1 : 0;
5404 MIRBuilder.setInsertPt(MBB&: *MI.getParent(), II: MI);
5405 for (unsigned i = 0; i < OrigNumElts / NumElts + NumLeftovers; ++i) {
5406 auto Phi = MIRBuilder.buildInstr(Opcode: TargetOpcode::G_PHI);
5407 Phi.addDef(
5408 RegNo: MRI.createGenericVirtualRegister(Ty: OutputOpsPieces[i].getLLTTy(MRI)));
5409 OutputRegs.push_back(Elt: Phi.getReg(Idx: 0));
5410
5411 for (unsigned j = 0; j < NumInputs / 2; ++j) {
5412 Phi.addUse(RegNo: InputOpsPieces[j][i]);
5413 Phi.add(MO: MI.getOperand(i: 1 + j * 2 + 1));
5414 }
5415 }
5416
5417 // Set the insert point after the existing PHIs
5418 MachineBasicBlock &MBB = *MI.getParent();
5419 MIRBuilder.setInsertPt(MBB, II: MBB.getFirstNonPHI());
5420
5421 // Merge small outputs into MI's def.
5422 if (NumLeftovers) {
5423 mergeMixedSubvectors(DstReg: MI.getReg(Idx: 0), PartRegs: OutputRegs);
5424 } else {
5425 MIRBuilder.buildMergeLikeInstr(Res: MI.getReg(Idx: 0), Ops: OutputRegs);
5426 }
5427
5428 MI.eraseFromParent();
5429 return Legalized;
5430}
5431
5432LegalizerHelper::LegalizeResult
5433LegalizerHelper::fewerElementsVectorUnmergeValues(MachineInstr &MI,
5434 unsigned TypeIdx,
5435 LLT NarrowTy) {
5436 const int NumDst = MI.getNumOperands() - 1;
5437 const Register SrcReg = MI.getOperand(i: NumDst).getReg();
5438 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5439 LLT SrcTy = MRI.getType(Reg: SrcReg);
5440
5441 if (TypeIdx != 1 || NarrowTy == DstTy)
5442 return UnableToLegalize;
5443
5444 // Requires compatible types. Otherwise SrcReg should have been defined by
5445 // merge-like instruction that would get artifact combined. Most likely
5446 // instruction that defines SrcReg has to perform more/fewer elements
5447 // legalization compatible with NarrowTy.
5448 assert(SrcTy.isVector() && NarrowTy.isVector() && "Expected vector types");
5449 assert((SrcTy.getScalarType() == NarrowTy.getScalarType()) && "bad type");
5450
5451 if ((SrcTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0) ||
5452 (NarrowTy.getSizeInBits() % DstTy.getSizeInBits() != 0))
5453 return UnableToLegalize;
5454
5455 // This is most likely DstTy (smaller then register size) packed in SrcTy
5456 // (larger then register size) and since unmerge was not combined it will be
5457 // lowered to bit sequence extracts from register. Unpack SrcTy to NarrowTy
5458 // (register size) pieces first. Then unpack each of NarrowTy pieces to DstTy.
5459
5460 // %1:_(DstTy), %2, %3, %4 = G_UNMERGE_VALUES %0:_(SrcTy)
5461 //
5462 // %5:_(NarrowTy), %6 = G_UNMERGE_VALUES %0:_(SrcTy) - reg sequence
5463 // %1:_(DstTy), %2 = G_UNMERGE_VALUES %5:_(NarrowTy) - sequence of bits in reg
5464 // %3:_(DstTy), %4 = G_UNMERGE_VALUES %6:_(NarrowTy)
5465 auto Unmerge = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: SrcReg);
5466 const int NumUnmerge = Unmerge->getNumOperands() - 1;
5467 const int PartsPerUnmerge = NumDst / NumUnmerge;
5468
5469 for (int I = 0; I != NumUnmerge; ++I) {
5470 auto MIB = MIRBuilder.buildInstr(Opcode: TargetOpcode::G_UNMERGE_VALUES);
5471
5472 for (int J = 0; J != PartsPerUnmerge; ++J)
5473 MIB.addDef(RegNo: MI.getOperand(i: I * PartsPerUnmerge + J).getReg());
5474 MIB.addUse(RegNo: Unmerge.getReg(Idx: I));
5475 }
5476
5477 MI.eraseFromParent();
5478 return Legalized;
5479}
5480
5481LegalizerHelper::LegalizeResult
5482LegalizerHelper::fewerElementsVectorMerge(MachineInstr &MI, unsigned TypeIdx,
5483 LLT NarrowTy) {
5484 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
5485 // Requires compatible types. Otherwise user of DstReg did not perform unmerge
5486 // that should have been artifact combined. Most likely instruction that uses
5487 // DstReg has to do more/fewer elements legalization compatible with NarrowTy.
5488 assert(DstTy.isVector() && NarrowTy.isVector() && "Expected vector types");
5489 assert((DstTy.getScalarType() == NarrowTy.getScalarType()) && "bad type");
5490 if (NarrowTy == SrcTy)
5491 return UnableToLegalize;
5492
5493 // This attempts to lower part of LCMTy merge/unmerge sequence. Intended use
5494 // is for old mir tests. Since the changes to more/fewer elements it should no
5495 // longer be possible to generate MIR like this when starting from llvm-ir
5496 // because LCMTy approach was replaced with merge/unmerge to vector elements.
5497 if (TypeIdx == 1) {
5498 assert(SrcTy.isVector() && "Expected vector types");
5499 assert((SrcTy.getScalarType() == NarrowTy.getScalarType()) && "bad type");
5500 if ((DstTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0) ||
5501 (NarrowTy.getNumElements() >= SrcTy.getNumElements()))
5502 return UnableToLegalize;
5503 // %2:_(DstTy) = G_CONCAT_VECTORS %0:_(SrcTy), %1:_(SrcTy)
5504 //
5505 // %3:_(EltTy), %4, %5 = G_UNMERGE_VALUES %0:_(SrcTy)
5506 // %6:_(EltTy), %7, %8 = G_UNMERGE_VALUES %1:_(SrcTy)
5507 // %9:_(NarrowTy) = G_BUILD_VECTOR %3:_(EltTy), %4
5508 // %10:_(NarrowTy) = G_BUILD_VECTOR %5:_(EltTy), %6
5509 // %11:_(NarrowTy) = G_BUILD_VECTOR %7:_(EltTy), %8
5510 // %2:_(DstTy) = G_CONCAT_VECTORS %9:_(NarrowTy), %10, %11
5511
5512 SmallVector<Register, 8> Elts;
5513 LLT EltTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getScalarType();
5514 for (unsigned i = 1; i < MI.getNumOperands(); ++i) {
5515 auto Unmerge = MIRBuilder.buildUnmerge(Res: EltTy, Op: MI.getOperand(i).getReg());
5516 for (unsigned j = 0; j < Unmerge->getNumDefs(); ++j)
5517 Elts.push_back(Elt: Unmerge.getReg(Idx: j));
5518 }
5519
5520 SmallVector<Register, 8> NarrowTyElts;
5521 unsigned NumNarrowTyElts = NarrowTy.getNumElements();
5522 unsigned NumNarrowTyPieces = DstTy.getNumElements() / NumNarrowTyElts;
5523 for (unsigned i = 0, Offset = 0; i < NumNarrowTyPieces;
5524 ++i, Offset += NumNarrowTyElts) {
5525 ArrayRef<Register> Pieces(&Elts[Offset], NumNarrowTyElts);
5526 NarrowTyElts.push_back(
5527 Elt: MIRBuilder.buildMergeLikeInstr(Res: NarrowTy, Ops: Pieces).getReg(Idx: 0));
5528 }
5529
5530 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: NarrowTyElts);
5531 MI.eraseFromParent();
5532 return Legalized;
5533 }
5534
5535 assert(TypeIdx == 0 && "Bad type index");
5536 if ((NarrowTy.getSizeInBits() % SrcTy.getSizeInBits() != 0) ||
5537 (DstTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0))
5538 return UnableToLegalize;
5539
5540 // This is most likely SrcTy (smaller then register size) packed in DstTy
5541 // (larger then register size) and since merge was not combined it will be
5542 // lowered to bit sequence packing into register. Merge SrcTy to NarrowTy
5543 // (register size) pieces first. Then merge each of NarrowTy pieces to DstTy.
5544
5545 // %0:_(DstTy) = G_MERGE_VALUES %1:_(SrcTy), %2, %3, %4
5546 //
5547 // %5:_(NarrowTy) = G_MERGE_VALUES %1:_(SrcTy), %2 - sequence of bits in reg
5548 // %6:_(NarrowTy) = G_MERGE_VALUES %3:_(SrcTy), %4
5549 // %0:_(DstTy) = G_MERGE_VALUES %5:_(NarrowTy), %6 - reg sequence
5550 SmallVector<Register, 8> NarrowTyElts;
5551 unsigned NumParts = DstTy.getNumElements() / NarrowTy.getNumElements();
5552 unsigned NumSrcElts = SrcTy.isVector() ? SrcTy.getNumElements() : 1;
5553 unsigned NumElts = NarrowTy.getNumElements() / NumSrcElts;
5554 for (unsigned i = 0; i < NumParts; ++i) {
5555 SmallVector<Register, 8> Sources;
5556 for (unsigned j = 0; j < NumElts; ++j)
5557 Sources.push_back(Elt: MI.getOperand(i: 1 + i * NumElts + j).getReg());
5558 NarrowTyElts.push_back(
5559 Elt: MIRBuilder.buildMergeLikeInstr(Res: NarrowTy, Ops: Sources).getReg(Idx: 0));
5560 }
5561
5562 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: NarrowTyElts);
5563 MI.eraseFromParent();
5564 return Legalized;
5565}
5566
5567LegalizerHelper::LegalizeResult
5568LegalizerHelper::fewerElementsVectorExtractInsertVectorElt(MachineInstr &MI,
5569 unsigned TypeIdx,
5570 LLT NarrowVecTy) {
5571 auto [DstReg, SrcVec] = MI.getFirst2Regs();
5572 Register InsertVal;
5573 bool IsInsert = MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT;
5574
5575 assert((IsInsert ? TypeIdx == 0 : TypeIdx == 1) && "not a vector type index");
5576 if (IsInsert)
5577 InsertVal = MI.getOperand(i: 2).getReg();
5578
5579 Register Idx = MI.getOperand(i: MI.getNumOperands() - 1).getReg();
5580 LLT VecTy = MRI.getType(Reg: SrcVec);
5581
5582 // If the index is a constant, we can really break this down as you would
5583 // expect, and index into the target size pieces.
5584 auto MaybeCst = getIConstantVRegValWithLookThrough(VReg: Idx, MRI);
5585 if (MaybeCst) {
5586 uint64_t IdxVal = MaybeCst->Value.getZExtValue();
5587 // Avoid out of bounds indexing the pieces.
5588 if (IdxVal >= VecTy.getNumElements()) {
5589 MIRBuilder.buildUndef(Res: DstReg);
5590 MI.eraseFromParent();
5591 return Legalized;
5592 }
5593
5594 if (!NarrowVecTy.isVector()) {
5595 SmallVector<Register, 8> SplitPieces;
5596 extractParts(Reg: MI.getOperand(i: 1).getReg(), Ty: NarrowVecTy,
5597 NumParts: VecTy.getNumElements(), VRegs&: SplitPieces, MIRBuilder, MRI);
5598 if (IsInsert) {
5599 SplitPieces[IdxVal] = InsertVal;
5600 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0).getReg(), Ops: SplitPieces);
5601 } else {
5602 MIRBuilder.buildCopy(Res: MI.getOperand(i: 0).getReg(), Op: SplitPieces[IdxVal]);
5603 }
5604 } else {
5605 SmallVector<Register, 8> VecParts;
5606 LLT GCDTy = extractGCDType(Parts&: VecParts, DstTy: VecTy, NarrowTy: NarrowVecTy, SrcReg: SrcVec);
5607
5608 // Build a sequence of NarrowTy pieces in VecParts for this operand.
5609 LLT LCMTy = buildLCMMergePieces(DstTy: VecTy, NarrowTy: NarrowVecTy, GCDTy, VRegs&: VecParts,
5610 PadStrategy: TargetOpcode::G_ANYEXT);
5611
5612 unsigned NewNumElts = NarrowVecTy.getNumElements();
5613
5614 LLT IdxTy = MRI.getType(Reg: Idx);
5615 int64_t PartIdx = IdxVal / NewNumElts;
5616 auto NewIdx =
5617 MIRBuilder.buildConstant(Res: IdxTy, Val: IdxVal - NewNumElts * PartIdx);
5618
5619 if (IsInsert) {
5620 LLT PartTy = MRI.getType(Reg: VecParts[PartIdx]);
5621
5622 // Use the adjusted index to insert into one of the subvectors.
5623 auto InsertPart = MIRBuilder.buildInsertVectorElement(
5624 Res: PartTy, Val: VecParts[PartIdx], Elt: InsertVal, Idx: NewIdx);
5625 VecParts[PartIdx] = InsertPart.getReg(Idx: 0);
5626
5627 // Recombine the inserted subvector with the others to reform the result
5628 // vector.
5629 buildWidenedRemergeToDst(DstReg, LCMTy, RemergeRegs: VecParts);
5630 } else {
5631 MIRBuilder.buildExtractVectorElement(Res: DstReg, Val: VecParts[PartIdx], Idx: NewIdx);
5632 }
5633 }
5634
5635 MI.eraseFromParent();
5636 return Legalized;
5637 }
5638
5639 // With a variable index, we can't perform the operation in a smaller type, so
5640 // we're forced to expand this.
5641 //
5642 // TODO: We could emit a chain of compare/select to figure out which piece to
5643 // index.
5644 return lowerExtractInsertVectorElt(MI);
5645}
5646
5647LegalizerHelper::LegalizeResult
5648LegalizerHelper::reduceLoadStoreWidth(GLoadStore &LdStMI, unsigned TypeIdx,
5649 LLT NarrowTy) {
5650 // FIXME: Don't know how to handle secondary types yet.
5651 if (TypeIdx != 0)
5652 return UnableToLegalize;
5653
5654 if (!NarrowTy.isByteSized()) {
5655 LLVM_DEBUG(dbgs() << "Can't narrow load/store to non-byte-sized type\n");
5656 return UnableToLegalize;
5657 }
5658
5659 // This implementation doesn't work for atomics. Give up instead of doing
5660 // something invalid.
5661 if (LdStMI.isAtomic())
5662 return UnableToLegalize;
5663
5664 bool IsLoad = isa<GLoad>(Val: LdStMI);
5665 Register ValReg = LdStMI.getReg(Idx: 0);
5666 Register AddrReg = LdStMI.getPointerReg();
5667 LLT ValTy = MRI.getType(Reg: ValReg);
5668
5669 // FIXME: Do we need a distinct NarrowMemory legalize action?
5670 if (ValTy.getSizeInBits() != 8 * LdStMI.getMemSize().getValue()) {
5671 LLVM_DEBUG(dbgs() << "Can't narrow extload/truncstore\n");
5672 return UnableToLegalize;
5673 }
5674
5675 int NumParts = -1;
5676 int NumLeftover = -1;
5677 LLT LeftoverTy;
5678 SmallVector<Register, 8> NarrowRegs, NarrowLeftoverRegs;
5679 if (IsLoad) {
5680 std::tie(args&: NumParts, args&: NumLeftover) = getNarrowTypeBreakDown(OrigTy: ValTy, NarrowTy, LeftoverTy);
5681 } else {
5682 if (extractParts(Reg: ValReg, RegTy: ValTy, MainTy: NarrowTy, LeftoverTy, VRegs&: NarrowRegs,
5683 LeftoverVRegs&: NarrowLeftoverRegs, MIRBuilder, MRI)) {
5684 NumParts = NarrowRegs.size();
5685 NumLeftover = NarrowLeftoverRegs.size();
5686 }
5687 }
5688
5689 if (NumParts == -1)
5690 return UnableToLegalize;
5691
5692 LLT PtrTy = MRI.getType(Reg: AddrReg);
5693 const LLT OffsetTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
5694
5695 unsigned TotalSize = ValTy.getSizeInBits();
5696
5697 // Split the load/store into PartTy sized pieces starting at Offset. If this
5698 // is a load, return the new registers in ValRegs. For a store, each elements
5699 // of ValRegs should be PartTy. Returns the next offset that needs to be
5700 // handled.
5701 bool isBigEndian = MIRBuilder.getDataLayout().isBigEndian();
5702 auto MMO = LdStMI.getMMO();
5703 auto splitTypePieces = [=](LLT PartTy, SmallVectorImpl<Register> &ValRegs,
5704 unsigned NumParts, unsigned Offset) -> unsigned {
5705 MachineFunction &MF = MIRBuilder.getMF();
5706 unsigned PartSize = PartTy.getSizeInBits();
5707 for (unsigned Idx = 0, E = NumParts; Idx != E && Offset < TotalSize;
5708 ++Idx) {
5709 unsigned ByteOffset = Offset / 8;
5710 Register NewAddrReg;
5711
5712 MIRBuilder.materializeObjectPtrOffset(Res&: NewAddrReg, Op0: AddrReg, ValueTy: OffsetTy,
5713 Value: ByteOffset);
5714
5715 MachineMemOperand *NewMMO =
5716 MF.getMachineMemOperand(MMO: &MMO, Offset: ByteOffset, Ty: PartTy);
5717
5718 if (IsLoad) {
5719 Register Dst = MRI.createGenericVirtualRegister(Ty: PartTy);
5720 ValRegs.push_back(Elt: Dst);
5721 MIRBuilder.buildLoad(Res: Dst, Addr: NewAddrReg, MMO&: *NewMMO);
5722 } else {
5723 MIRBuilder.buildStore(Val: ValRegs[Idx], Addr: NewAddrReg, MMO&: *NewMMO);
5724 }
5725 Offset = isBigEndian ? Offset - PartSize : Offset + PartSize;
5726 }
5727
5728 return Offset;
5729 };
5730
5731 unsigned Offset = isBigEndian ? TotalSize - NarrowTy.getSizeInBits() : 0;
5732 unsigned HandledOffset =
5733 splitTypePieces(NarrowTy, NarrowRegs, NumParts, Offset);
5734
5735 // Handle the rest of the register if this isn't an even type breakdown.
5736 if (LeftoverTy.isValid())
5737 splitTypePieces(LeftoverTy, NarrowLeftoverRegs, NumLeftover, HandledOffset);
5738
5739 if (IsLoad) {
5740 insertParts(DstReg: ValReg, ResultTy: ValTy, PartTy: NarrowTy, PartRegs: NarrowRegs,
5741 LeftoverTy, LeftoverRegs: NarrowLeftoverRegs);
5742 }
5743
5744 LdStMI.eraseFromParent();
5745 return Legalized;
5746}
5747
5748LegalizerHelper::LegalizeResult
5749LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
5750 LLT NarrowTy) {
5751 using namespace TargetOpcode;
5752 GenericMachineInstr &GMI = cast<GenericMachineInstr>(Val&: MI);
5753 unsigned NumElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
5754
5755 switch (MI.getOpcode()) {
5756 case G_IMPLICIT_DEF:
5757 case G_TRUNC:
5758 case G_AND:
5759 case G_OR:
5760 case G_XOR:
5761 case G_ADD:
5762 case G_SUB:
5763 case G_MUL:
5764 case G_PTR_ADD:
5765 case G_SMULH:
5766 case G_UMULH:
5767 case G_FADD:
5768 case G_FMUL:
5769 case G_FSUB:
5770 case G_FNEG:
5771 case G_FABS:
5772 case G_FCANONICALIZE:
5773 case G_FDIV:
5774 case G_FREM:
5775 case G_FMA:
5776 case G_FMAD:
5777 case G_FPOW:
5778 case G_FEXP:
5779 case G_FEXP2:
5780 case G_FEXP10:
5781 case G_FLOG:
5782 case G_FLOG2:
5783 case G_FLOG10:
5784 case G_FLDEXP:
5785 case G_FNEARBYINT:
5786 case G_FCEIL:
5787 case G_FFLOOR:
5788 case G_FRINT:
5789 case G_INTRINSIC_LRINT:
5790 case G_INTRINSIC_LLRINT:
5791 case G_INTRINSIC_ROUND:
5792 case G_INTRINSIC_ROUNDEVEN:
5793 case G_LROUND:
5794 case G_LLROUND:
5795 case G_INTRINSIC_TRUNC:
5796 case G_FMODF:
5797 case G_FCOS:
5798 case G_FSIN:
5799 case G_FTAN:
5800 case G_FACOS:
5801 case G_FASIN:
5802 case G_FATAN:
5803 case G_FATAN2:
5804 case G_FCOSH:
5805 case G_FSINH:
5806 case G_FTANH:
5807 case G_FSQRT:
5808 case G_BSWAP:
5809 case G_BITREVERSE:
5810 case G_SDIV:
5811 case G_UDIV:
5812 case G_SREM:
5813 case G_UREM:
5814 case G_SDIVREM:
5815 case G_UDIVREM:
5816 case G_SMIN:
5817 case G_SMAX:
5818 case G_UMIN:
5819 case G_UMAX:
5820 case G_ABS:
5821 case G_FMINNUM:
5822 case G_FMAXNUM:
5823 case G_FMINNUM_IEEE:
5824 case G_FMAXNUM_IEEE:
5825 case G_FMINIMUM:
5826 case G_FMAXIMUM:
5827 case G_FMINIMUMNUM:
5828 case G_FMAXIMUMNUM:
5829 case G_FSHL:
5830 case G_FSHR:
5831 case G_ROTL:
5832 case G_ROTR:
5833 case G_FREEZE:
5834 case G_SADDSAT:
5835 case G_SSUBSAT:
5836 case G_UADDSAT:
5837 case G_USUBSAT:
5838 case G_UMULO:
5839 case G_SMULO:
5840 case G_SHL:
5841 case G_LSHR:
5842 case G_ASHR:
5843 case G_SSHLSAT:
5844 case G_USHLSAT:
5845 case G_CTLZ:
5846 case G_CTLZ_ZERO_POISON:
5847 case G_CTTZ:
5848 case G_CTTZ_ZERO_POISON:
5849 case G_CTPOP:
5850 case G_CTLS:
5851 case G_FCOPYSIGN:
5852 case G_ZEXT:
5853 case G_SEXT:
5854 case G_ANYEXT:
5855 case G_FPEXT:
5856 case G_FPTRUNC:
5857 case G_SITOFP:
5858 case G_UITOFP:
5859 case G_FPTOSI:
5860 case G_FPTOUI:
5861 case G_FPTOSI_SAT:
5862 case G_FPTOUI_SAT:
5863 case G_INTTOPTR:
5864 case G_PTRTOINT:
5865 case G_ADDRSPACE_CAST:
5866 case G_UADDO:
5867 case G_USUBO:
5868 case G_UADDE:
5869 case G_USUBE:
5870 case G_SADDO:
5871 case G_SSUBO:
5872 case G_SADDE:
5873 case G_SSUBE:
5874 case G_STRICT_FADD:
5875 case G_STRICT_FSUB:
5876 case G_STRICT_FMUL:
5877 case G_STRICT_FMA:
5878 case G_STRICT_FLDEXP:
5879 case G_FFREXP:
5880 case G_TRUNC_SSAT_S:
5881 case G_TRUNC_SSAT_U:
5882 case G_TRUNC_USAT_U:
5883 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts);
5884 case G_ICMP:
5885 case G_FCMP:
5886 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {1 /*cpm predicate*/});
5887 case G_IS_FPCLASS:
5888 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {2, 3 /*mask,fpsem*/});
5889 case G_SELECT:
5890 if (MRI.getType(Reg: MI.getOperand(i: 1).getReg()).isVector())
5891 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts);
5892 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {1 /*scalar cond*/});
5893 case G_PHI:
5894 return fewerElementsVectorPhi(MI&: GMI, NumElts);
5895 case G_UNMERGE_VALUES:
5896 return fewerElementsVectorUnmergeValues(MI, TypeIdx, NarrowTy);
5897 case G_BUILD_VECTOR:
5898 assert(TypeIdx == 0 && "not a vector type index");
5899 return fewerElementsVectorMerge(MI, TypeIdx, NarrowTy);
5900 case G_CONCAT_VECTORS:
5901 if (TypeIdx != 1) // TODO: This probably does work as expected already.
5902 return UnableToLegalize;
5903 return fewerElementsVectorMerge(MI, TypeIdx, NarrowTy);
5904 case G_EXTRACT_SUBVECTOR: {
5905 Register DstReg = MI.getOperand(i: 0).getReg();
5906 LLT DstTy = MRI.getType(Reg: DstReg);
5907 Register SrcReg = MI.getOperand(i: 1).getReg();
5908 uint64_t InsertionPointImm = MI.getOperand(i: 2).getImm();
5909
5910 // If Dst > NarrowTy bits, then cannot legalize
5911 if (DstTy.getSizeInBits() > NarrowTy.getSizeInBits())
5912 return UnableToLegalize;
5913
5914 // If DstTy's size is not a multiple of NarrowTy's, then cannot legalize
5915 if (!DstTy.getElementCount().isKnownMultipleOf(RHS: NarrowTy.getElementCount()))
5916 return UnableToLegalize;
5917
5918 auto Unmerge = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: SrcReg);
5919 uint64_t RequiredSubvectorIndex =
5920 InsertionPointImm / NarrowTy.getNumElements();
5921 // If Dst and Narrow are both same size, convert to a copy
5922 if (DstTy.getNumElements() == NarrowTy.getNumElements())
5923 MIRBuilder.buildCopy(Res: DstReg, Op: Unmerge.getReg(Idx: RequiredSubvectorIndex));
5924 else
5925 MIRBuilder.buildExtractSubvector(
5926 Res: DstReg, Src: Unmerge.getReg(Idx: RequiredSubvectorIndex),
5927 Index: InsertionPointImm % NarrowTy.getNumElements());
5928
5929 MI.eraseFromParent();
5930 return Legalized;
5931 }
5932 case G_EXTRACT_VECTOR_ELT:
5933 case G_INSERT_VECTOR_ELT:
5934 return fewerElementsVectorExtractInsertVectorElt(MI, TypeIdx, NarrowVecTy: NarrowTy);
5935 case G_LOAD:
5936 case G_STORE:
5937 return reduceLoadStoreWidth(LdStMI&: cast<GLoadStore>(Val&: MI), TypeIdx, NarrowTy);
5938 case G_SEXT_INREG:
5939 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {2 /*imm*/});
5940 GISEL_VECREDUCE_CASES_NONSEQ
5941 return fewerElementsVectorReductions(MI, TypeIdx, NarrowTy);
5942 case TargetOpcode::G_VECREDUCE_SEQ_FADD:
5943 case TargetOpcode::G_VECREDUCE_SEQ_FMUL:
5944 return fewerElementsVectorSeqReductions(MI, TypeIdx, NarrowTy);
5945 case G_SHUFFLE_VECTOR:
5946 return fewerElementsVectorShuffle(MI, TypeIdx, NarrowTy);
5947 case G_FPOWI:
5948 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {2 /*pow*/});
5949 case G_BITCAST:
5950 return fewerElementsBitcast(MI, TypeIdx, NarrowTy);
5951 case G_INTRINSIC_FPTRUNC_ROUND:
5952 return fewerElementsVectorMultiEltType(MI&: GMI, NumElts, NonVecOpIndices: {2});
5953 default:
5954 return UnableToLegalize;
5955 }
5956}
5957
5958LegalizerHelper::LegalizeResult
5959LegalizerHelper::fewerElementsBitcast(MachineInstr &MI, unsigned int TypeIdx,
5960 LLT NarrowTy) {
5961 assert(MI.getOpcode() == TargetOpcode::G_BITCAST &&
5962 "Not a bitcast operation");
5963
5964 if (TypeIdx != 0)
5965 return UnableToLegalize;
5966
5967 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
5968
5969 unsigned NewElemCount =
5970 NarrowTy.getSizeInBits() / SrcTy.getScalarSizeInBits();
5971 SmallVector<Register> SrcVRegs, BitcastVRegs;
5972 if (NewElemCount == 1) {
5973 LLT SrcNarrowTy = SrcTy.getElementType();
5974
5975 auto Unmerge = MIRBuilder.buildUnmerge(Res: SrcNarrowTy, Op: SrcReg);
5976 getUnmergeResults(Regs&: SrcVRegs, MI: *Unmerge);
5977 } else {
5978 LLT SrcNarrowTy =
5979 SrcTy.changeVectorElementCount(EC: ElementCount::getFixed(MinVal: NewElemCount));
5980
5981 // Split the Src and Dst Reg into smaller registers
5982 if (extractGCDType(Parts&: SrcVRegs, DstTy, NarrowTy: SrcNarrowTy, SrcReg) != SrcNarrowTy)
5983 return UnableToLegalize;
5984 }
5985
5986 // Build new smaller bitcast instructions
5987 // Not supporting Leftover types for now but will have to
5988 for (Register Reg : SrcVRegs)
5989 BitcastVRegs.push_back(Elt: MIRBuilder.buildBitcast(Dst: NarrowTy, Src: Reg).getReg(Idx: 0));
5990
5991 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: BitcastVRegs);
5992 MI.eraseFromParent();
5993 return Legalized;
5994}
5995
5996LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorShuffle(
5997 MachineInstr &MI, unsigned int TypeIdx, LLT NarrowTy) {
5998 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
5999 if (TypeIdx != 0)
6000 return UnableToLegalize;
6001
6002 auto [DstReg, DstTy, Src1Reg, Src1Ty, Src2Reg, Src2Ty] =
6003 MI.getFirst3RegLLTs();
6004 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
6005 // The shuffle should be canonicalized by now.
6006 if (DstTy != Src1Ty)
6007 return UnableToLegalize;
6008 if (DstTy != Src2Ty)
6009 return UnableToLegalize;
6010
6011 if (!isPowerOf2_32(Value: DstTy.getNumElements()))
6012 return UnableToLegalize;
6013
6014 // We only support splitting a shuffle into 2, so adjust NarrowTy accordingly.
6015 // Further legalization attempts will be needed to do split further.
6016 NarrowTy =
6017 DstTy.changeElementCount(EC: DstTy.getElementCount().divideCoefficientBy(RHS: 2));
6018 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
6019
6020 SmallVector<Register> SplitSrc1Regs, SplitSrc2Regs;
6021 extractParts(Reg: Src1Reg, Ty: NarrowTy, NumParts: 2, VRegs&: SplitSrc1Regs, MIRBuilder, MRI);
6022 extractParts(Reg: Src2Reg, Ty: NarrowTy, NumParts: 2, VRegs&: SplitSrc2Regs, MIRBuilder, MRI);
6023 Register Inputs[4] = {SplitSrc1Regs[0], SplitSrc1Regs[1], SplitSrc2Regs[0],
6024 SplitSrc2Regs[1]};
6025
6026 Register Hi, Lo;
6027
6028 // If Lo or Hi uses elements from at most two of the four input vectors, then
6029 // express it as a vector shuffle of those two inputs. Otherwise extract the
6030 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
6031 SmallVector<int, 16> Ops;
6032 for (unsigned High = 0; High < 2; ++High) {
6033 Register &Output = High ? Hi : Lo;
6034
6035 // Build a shuffle mask for the output, discovering on the fly which
6036 // input vectors to use as shuffle operands (recorded in InputUsed).
6037 // If building a suitable shuffle vector proves too hard, then bail
6038 // out with useBuildVector set.
6039 unsigned InputUsed[2] = {-1U, -1U}; // Not yet discovered.
6040 unsigned FirstMaskIdx = High * NewElts;
6041 bool UseBuildVector = false;
6042 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
6043 // The mask element. This indexes into the input.
6044 int Idx = Mask[FirstMaskIdx + MaskOffset];
6045
6046 // The input vector this mask element indexes into.
6047 unsigned Input = (unsigned)Idx / NewElts;
6048
6049 if (Input >= std::size(Inputs)) {
6050 // The mask element does not index into any input vector.
6051 Ops.push_back(Elt: -1);
6052 continue;
6053 }
6054
6055 // Turn the index into an offset from the start of the input vector.
6056 Idx -= Input * NewElts;
6057
6058 // Find or create a shuffle vector operand to hold this input.
6059 unsigned OpNo;
6060 for (OpNo = 0; OpNo < std::size(InputUsed); ++OpNo) {
6061 if (InputUsed[OpNo] == Input) {
6062 // This input vector is already an operand.
6063 break;
6064 } else if (InputUsed[OpNo] == -1U) {
6065 // Create a new operand for this input vector.
6066 InputUsed[OpNo] = Input;
6067 break;
6068 }
6069 }
6070
6071 if (OpNo >= std::size(InputUsed)) {
6072 // More than two input vectors used! Give up on trying to create a
6073 // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
6074 UseBuildVector = true;
6075 break;
6076 }
6077
6078 // Add the mask index for the new shuffle vector.
6079 Ops.push_back(Elt: Idx + OpNo * NewElts);
6080 }
6081
6082 if (UseBuildVector) {
6083 LLT EltTy = NarrowTy.getElementType();
6084 SmallVector<Register, 16> SVOps;
6085
6086 // Extract the input elements by hand.
6087 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
6088 // The mask element. This indexes into the input.
6089 int Idx = Mask[FirstMaskIdx + MaskOffset];
6090
6091 // The input vector this mask element indexes into.
6092 unsigned Input = (unsigned)Idx / NewElts;
6093
6094 if (Input >= std::size(Inputs)) {
6095 // The mask element is "undef" or indexes off the end of the input.
6096 SVOps.push_back(Elt: MIRBuilder.buildUndef(Res: EltTy).getReg(Idx: 0));
6097 continue;
6098 }
6099
6100 // Turn the index into an offset from the start of the input vector.
6101 Idx -= Input * NewElts;
6102
6103 // Extract the vector element by hand.
6104 SVOps.push_back(Elt: MIRBuilder
6105 .buildExtractVectorElement(
6106 Res: EltTy, Val: Inputs[Input],
6107 Idx: MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: 32), Val: Idx))
6108 .getReg(Idx: 0));
6109 }
6110
6111 // Construct the Lo/Hi output using a G_BUILD_VECTOR.
6112 Output = MIRBuilder.buildBuildVector(Res: NarrowTy, Ops: SVOps).getReg(Idx: 0);
6113 } else if (InputUsed[0] == -1U) {
6114 // No input vectors were used! The result is undefined.
6115 Output = MIRBuilder.buildUndef(Res: NarrowTy).getReg(Idx: 0);
6116 } else if (NewElts == 1) {
6117 Output = MIRBuilder.buildCopy(Res: NarrowTy, Op: Inputs[InputUsed[0]]).getReg(Idx: 0);
6118 } else {
6119 Register Op0 = Inputs[InputUsed[0]];
6120 // If only one input was used, use an undefined vector for the other.
6121 Register Op1 = InputUsed[1] == -1U
6122 ? MIRBuilder.buildUndef(Res: NarrowTy).getReg(Idx: 0)
6123 : Inputs[InputUsed[1]];
6124 // At least one input vector was used. Create a new shuffle vector.
6125 Output = MIRBuilder.buildShuffleVector(Res: NarrowTy, Src1: Op0, Src2: Op1, Mask: Ops).getReg(Idx: 0);
6126 }
6127
6128 Ops.clear();
6129 }
6130
6131 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: {Lo, Hi});
6132 MI.eraseFromParent();
6133 return Legalized;
6134}
6135
6136LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorReductions(
6137 MachineInstr &MI, unsigned int TypeIdx, LLT NarrowTy) {
6138 auto &RdxMI = cast<GVecReduce>(Val&: MI);
6139
6140 if (TypeIdx != 1)
6141 return UnableToLegalize;
6142
6143 // The semantics of the normal non-sequential reductions allow us to freely
6144 // re-associate the operation.
6145 auto [DstReg, DstTy, SrcReg, SrcTy] = RdxMI.getFirst2RegLLTs();
6146
6147 if (NarrowTy.isVector() &&
6148 (SrcTy.getNumElements() % NarrowTy.getNumElements() != 0))
6149 return UnableToLegalize;
6150
6151 unsigned ScalarOpc = RdxMI.getScalarOpcForReduction();
6152 SmallVector<Register> SplitSrcs;
6153 // If NarrowTy is a scalar then we're being asked to scalarize.
6154 const unsigned NumParts =
6155 NarrowTy.isVector() ? SrcTy.getNumElements() / NarrowTy.getNumElements()
6156 : SrcTy.getNumElements();
6157
6158 extractParts(Reg: SrcReg, Ty: NarrowTy, NumParts, VRegs&: SplitSrcs, MIRBuilder, MRI);
6159 if (NarrowTy.isScalar()) {
6160 if (DstTy != NarrowTy)
6161 return UnableToLegalize; // FIXME: handle implicit extensions.
6162
6163 if (isPowerOf2_32(Value: NumParts)) {
6164 // Generate a tree of scalar operations to reduce the critical path.
6165 SmallVector<Register> PartialResults;
6166 unsigned NumPartsLeft = NumParts;
6167 while (NumPartsLeft > 1) {
6168 for (unsigned Idx = 0; Idx < NumPartsLeft - 1; Idx += 2) {
6169 PartialResults.emplace_back(
6170 Args: MIRBuilder
6171 .buildInstr(Opc: ScalarOpc, DstOps: {NarrowTy},
6172 SrcOps: {SplitSrcs[Idx], SplitSrcs[Idx + 1]},
6173 Flags: MI.getFlags())
6174 .getReg(Idx: 0));
6175 }
6176 SplitSrcs = PartialResults;
6177 PartialResults.clear();
6178 NumPartsLeft = SplitSrcs.size();
6179 }
6180 assert(SplitSrcs.size() == 1);
6181 MIRBuilder.buildCopy(Res: DstReg, Op: SplitSrcs[0]);
6182 MI.eraseFromParent();
6183 return Legalized;
6184 }
6185 // If we can't generate a tree, then just do sequential operations.
6186 Register Acc = SplitSrcs[0];
6187 for (unsigned Idx = 1; Idx < NumParts; ++Idx)
6188 Acc = MIRBuilder
6189 .buildInstr(Opc: ScalarOpc, DstOps: {NarrowTy}, SrcOps: {Acc, SplitSrcs[Idx]},
6190 Flags: MI.getFlags())
6191 .getReg(Idx: 0);
6192 MIRBuilder.buildCopy(Res: DstReg, Op: Acc);
6193 MI.eraseFromParent();
6194 return Legalized;
6195 }
6196 SmallVector<Register> PartialReductions;
6197 for (unsigned Part = 0; Part < NumParts; ++Part) {
6198 PartialReductions.push_back(Elt: MIRBuilder
6199 .buildInstr(Opc: RdxMI.getOpcode(), DstOps: {DstTy},
6200 SrcOps: {SplitSrcs[Part]},
6201 Flags: MI.getFlags())
6202 .getReg(Idx: 0));
6203 }
6204
6205 // If the types involved are powers of 2, we can generate intermediate vector
6206 // ops, before generating a final reduction operation.
6207 if (isPowerOf2_32(Value: SrcTy.getNumElements()) &&
6208 isPowerOf2_32(Value: NarrowTy.getNumElements())) {
6209 return tryNarrowPow2Reduction(MI, SrcReg, SrcTy, NarrowTy, ScalarOpc);
6210 }
6211
6212 Register Acc = PartialReductions[0];
6213 for (unsigned Part = 1; Part < NumParts; ++Part) {
6214 if (Part == NumParts - 1) {
6215 MIRBuilder.buildInstr(Opc: ScalarOpc, DstOps: {DstReg}, SrcOps: {Acc, PartialReductions[Part]},
6216 Flags: MI.getFlags());
6217 } else {
6218 Acc = MIRBuilder
6219 .buildInstr(Opc: ScalarOpc, DstOps: {DstTy}, SrcOps: {Acc, PartialReductions[Part]},
6220 Flags: MI.getFlags())
6221 .getReg(Idx: 0);
6222 }
6223 }
6224 MI.eraseFromParent();
6225 return Legalized;
6226}
6227
6228LegalizerHelper::LegalizeResult
6229LegalizerHelper::fewerElementsVectorSeqReductions(MachineInstr &MI,
6230 unsigned int TypeIdx,
6231 LLT NarrowTy) {
6232 auto [DstReg, DstTy, ScalarReg, ScalarTy, SrcReg, SrcTy] =
6233 MI.getFirst3RegLLTs();
6234 if (!NarrowTy.isScalar() || TypeIdx != 2 || DstTy != ScalarTy ||
6235 DstTy != NarrowTy)
6236 return UnableToLegalize;
6237
6238 assert((MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FADD ||
6239 MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FMUL) &&
6240 "Unexpected vecreduce opcode");
6241 unsigned ScalarOpc = MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FADD
6242 ? TargetOpcode::G_FADD
6243 : TargetOpcode::G_FMUL;
6244
6245 SmallVector<Register> SplitSrcs;
6246 unsigned NumParts = SrcTy.getNumElements();
6247 extractParts(Reg: SrcReg, Ty: NarrowTy, NumParts, VRegs&: SplitSrcs, MIRBuilder, MRI);
6248 Register Acc = ScalarReg;
6249 for (unsigned i = 0; i < NumParts; i++)
6250 Acc = MIRBuilder
6251 .buildInstr(Opc: ScalarOpc, DstOps: {NarrowTy}, SrcOps: {Acc, SplitSrcs[i]},
6252 Flags: MI.getFlags())
6253 .getReg(Idx: 0);
6254
6255 MIRBuilder.buildCopy(Res: DstReg, Op: Acc);
6256 MI.eraseFromParent();
6257 return Legalized;
6258}
6259
6260LegalizerHelper::LegalizeResult
6261LegalizerHelper::tryNarrowPow2Reduction(MachineInstr &MI, Register SrcReg,
6262 LLT SrcTy, LLT NarrowTy,
6263 unsigned ScalarOpc) {
6264 SmallVector<Register> SplitSrcs;
6265 // Split the sources into NarrowTy size pieces.
6266 extractParts(Reg: SrcReg, Ty: NarrowTy,
6267 NumParts: SrcTy.getNumElements() / NarrowTy.getNumElements(), VRegs&: SplitSrcs,
6268 MIRBuilder, MRI);
6269 // We're going to do a tree reduction using vector operations until we have
6270 // one NarrowTy size value left.
6271 while (SplitSrcs.size() > 1) {
6272 SmallVector<Register> PartialRdxs;
6273 for (unsigned Idx = 0; Idx < SplitSrcs.size()-1; Idx += 2) {
6274 Register LHS = SplitSrcs[Idx];
6275 Register RHS = SplitSrcs[Idx + 1];
6276 // Create the intermediate vector op.
6277 Register Res =
6278 MIRBuilder
6279 .buildInstr(Opc: ScalarOpc, DstOps: {NarrowTy}, SrcOps: {LHS, RHS}, Flags: MI.getFlags())
6280 .getReg(Idx: 0);
6281 PartialRdxs.push_back(Elt: Res);
6282 }
6283 SplitSrcs = std::move(PartialRdxs);
6284 }
6285 // Finally generate the requested NarrowTy based reduction.
6286 Observer.changingInstr(MI);
6287 MI.getOperand(i: 1).setReg(SplitSrcs[0]);
6288 Observer.changedInstr(MI);
6289 return Legalized;
6290}
6291
6292LegalizerHelper::LegalizeResult
6293LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt,
6294 const LLT HalfTy, const LLT AmtTy) {
6295
6296 Register InL = MRI.createGenericVirtualRegister(Ty: HalfTy);
6297 Register InH = MRI.createGenericVirtualRegister(Ty: HalfTy);
6298 MIRBuilder.buildUnmerge(Res: {InL, InH}, Op: MI.getOperand(i: 1));
6299
6300 if (Amt.isZero()) {
6301 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0), Ops: {InL, InH});
6302 MI.eraseFromParent();
6303 return Legalized;
6304 }
6305
6306 LLT NVT = HalfTy;
6307 unsigned NVTBits = HalfTy.getSizeInBits();
6308 unsigned VTBits = 2 * NVTBits;
6309
6310 SrcOp Lo(Register(0)), Hi(Register(0));
6311 if (MI.getOpcode() == TargetOpcode::G_SHL) {
6312 if (Amt.ugt(RHS: VTBits)) {
6313 Lo = Hi = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6314 } else if (Amt.ugt(RHS: NVTBits)) {
6315 Lo = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6316 Hi = MIRBuilder.buildShl(Dst: NVT, Src0: InL,
6317 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt - NVTBits));
6318 } else if (Amt == NVTBits) {
6319 Lo = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6320 Hi = InL;
6321 } else {
6322 Lo = MIRBuilder.buildShl(Dst: NVT, Src0: InL, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt));
6323 auto OrLHS =
6324 MIRBuilder.buildShl(Dst: NVT, Src0: InH, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt));
6325 auto OrRHS = MIRBuilder.buildLShr(
6326 Dst: NVT, Src0: InL, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: -Amt + NVTBits));
6327 Hi = MIRBuilder.buildOr(Dst: NVT, Src0: OrLHS, Src1: OrRHS);
6328 }
6329 } else if (MI.getOpcode() == TargetOpcode::G_LSHR) {
6330 if (Amt.ugt(RHS: VTBits)) {
6331 Lo = Hi = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6332 } else if (Amt.ugt(RHS: NVTBits)) {
6333 Lo = MIRBuilder.buildLShr(Dst: NVT, Src0: InH,
6334 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt - NVTBits));
6335 Hi = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6336 } else if (Amt == NVTBits) {
6337 Lo = InH;
6338 Hi = MIRBuilder.buildConstant(Res: NVT, Val: 0);
6339 } else {
6340 auto ShiftAmtConst = MIRBuilder.buildConstant(Res: AmtTy, Val: Amt);
6341
6342 auto OrLHS = MIRBuilder.buildLShr(Dst: NVT, Src0: InL, Src1: ShiftAmtConst);
6343 auto OrRHS = MIRBuilder.buildShl(
6344 Dst: NVT, Src0: InH, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: -Amt + NVTBits));
6345
6346 Lo = MIRBuilder.buildOr(Dst: NVT, Src0: OrLHS, Src1: OrRHS);
6347 Hi = MIRBuilder.buildLShr(Dst: NVT, Src0: InH, Src1: ShiftAmtConst);
6348 }
6349 } else {
6350 if (Amt.ugt(RHS: VTBits)) {
6351 Hi = Lo = MIRBuilder.buildAShr(
6352 Dst: NVT, Src0: InH, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: NVTBits - 1));
6353 } else if (Amt.ugt(RHS: NVTBits)) {
6354 Lo = MIRBuilder.buildAShr(Dst: NVT, Src0: InH,
6355 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: Amt - NVTBits));
6356 Hi = MIRBuilder.buildAShr(Dst: NVT, Src0: InH,
6357 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: NVTBits - 1));
6358 } else if (Amt == NVTBits) {
6359 Lo = InH;
6360 Hi = MIRBuilder.buildAShr(Dst: NVT, Src0: InH,
6361 Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: NVTBits - 1));
6362 } else {
6363 auto ShiftAmtConst = MIRBuilder.buildConstant(Res: AmtTy, Val: Amt);
6364
6365 auto OrLHS = MIRBuilder.buildLShr(Dst: NVT, Src0: InL, Src1: ShiftAmtConst);
6366 auto OrRHS = MIRBuilder.buildShl(
6367 Dst: NVT, Src0: InH, Src1: MIRBuilder.buildConstant(Res: AmtTy, Val: -Amt + NVTBits));
6368
6369 Lo = MIRBuilder.buildOr(Dst: NVT, Src0: OrLHS, Src1: OrRHS);
6370 Hi = MIRBuilder.buildAShr(Dst: NVT, Src0: InH, Src1: ShiftAmtConst);
6371 }
6372 }
6373
6374 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0), Ops: {Lo, Hi});
6375 MI.eraseFromParent();
6376
6377 return Legalized;
6378}
6379
6380LegalizerHelper::LegalizeResult
6381LegalizerHelper::narrowScalarShift(MachineInstr &MI, unsigned TypeIdx,
6382 LLT RequestedTy) {
6383 if (TypeIdx == 1) {
6384 Observer.changingInstr(MI);
6385 narrowScalarSrc(MI, NarrowTy: RequestedTy, OpIdx: 2);
6386 Observer.changedInstr(MI);
6387 return Legalized;
6388 }
6389
6390 Register DstReg = MI.getOperand(i: 0).getReg();
6391 LLT DstTy = MRI.getType(Reg: DstReg);
6392 if (DstTy.isVector())
6393 return UnableToLegalize;
6394
6395 Register Amt = MI.getOperand(i: 2).getReg();
6396 LLT ShiftAmtTy = MRI.getType(Reg: Amt);
6397 const unsigned DstEltSize = DstTy.getScalarSizeInBits();
6398 if (DstEltSize % 2 != 0)
6399 return UnableToLegalize;
6400
6401 // Check if we should use multi-way splitting instead of recursive binary
6402 // splitting.
6403 //
6404 // Multi-way splitting directly decomposes wide shifts (e.g., 128-bit ->
6405 // 4×32-bit) in a single legalization step, avoiding the recursive overhead
6406 // and dependency chains created by usual binary splitting approach
6407 // (128->64->32).
6408 //
6409 // The >= 8 parts threshold ensures we only use this optimization when binary
6410 // splitting would require multiple recursive passes, avoiding overhead for
6411 // simple 2-way splits where binary approach is sufficient.
6412 if (RequestedTy.isValid() && RequestedTy.isScalar() &&
6413 DstEltSize % RequestedTy.getSizeInBits() == 0) {
6414 const unsigned NumParts = DstEltSize / RequestedTy.getSizeInBits();
6415 // Use multiway if we have 8 or more parts (i.e., would need 3+ recursive
6416 // steps).
6417 if (NumParts >= 8)
6418 return narrowScalarShiftMultiway(MI, TargetTy: RequestedTy);
6419 }
6420
6421 // Fall back to binary splitting:
6422 // Ignore the input type. We can only go to exactly half the size of the
6423 // input. If that isn't small enough, the resulting pieces will be further
6424 // legalized.
6425 const unsigned NewBitSize = DstEltSize / 2;
6426 const LLT HalfTy = DstTy.getScalarType().changeElementSize(NewEltSize: NewBitSize);
6427 const LLT CondTy = LLT::integer(SizeInBits: 1);
6428
6429 if (auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: Amt, MRI)) {
6430 return narrowScalarShiftByConstant(MI, Amt: VRegAndVal->Value, HalfTy,
6431 AmtTy: ShiftAmtTy);
6432 }
6433
6434 // TODO: Expand with known bits.
6435
6436 // Handle the fully general expansion by an unknown amount.
6437 auto NewBits = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: NewBitSize);
6438
6439 Register InL = MRI.createGenericVirtualRegister(Ty: HalfTy);
6440 Register InH = MRI.createGenericVirtualRegister(Ty: HalfTy);
6441 MIRBuilder.buildUnmerge(Res: {InL, InH}, Op: MI.getOperand(i: 1));
6442
6443 auto AmtExcess = MIRBuilder.buildSub(Dst: ShiftAmtTy, Src0: Amt, Src1: NewBits);
6444 auto AmtLack = MIRBuilder.buildSub(Dst: ShiftAmtTy, Src0: NewBits, Src1: Amt);
6445
6446 auto Zero = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: 0);
6447 auto IsShort = MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_ULT, Res: CondTy, Op0: Amt, Op1: NewBits);
6448 auto IsZero = MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: CondTy, Op0: Amt, Op1: Zero);
6449
6450 Register ResultRegs[2];
6451 switch (MI.getOpcode()) {
6452 case TargetOpcode::G_SHL: {
6453 // Short: ShAmt < NewBitSize
6454 auto LoS = MIRBuilder.buildShl(Dst: HalfTy, Src0: InL, Src1: Amt);
6455
6456 auto LoOr = MIRBuilder.buildLShr(Dst: HalfTy, Src0: InL, Src1: AmtLack);
6457 auto HiOr = MIRBuilder.buildShl(Dst: HalfTy, Src0: InH, Src1: Amt);
6458 auto HiS = MIRBuilder.buildOr(Dst: HalfTy, Src0: LoOr, Src1: HiOr);
6459
6460 // Long: ShAmt >= NewBitSize
6461 auto LoL = MIRBuilder.buildConstant(Res: HalfTy, Val: 0); // Lo part is zero.
6462 auto HiL = MIRBuilder.buildShl(Dst: HalfTy, Src0: InL, Src1: AmtExcess); // Hi from Lo part.
6463
6464 auto Lo = MIRBuilder.buildSelect(Res: HalfTy, Tst: IsShort, Op0: LoS, Op1: LoL);
6465 auto Hi = MIRBuilder.buildSelect(
6466 Res: HalfTy, Tst: IsZero, Op0: InH, Op1: MIRBuilder.buildSelect(Res: HalfTy, Tst: IsShort, Op0: HiS, Op1: HiL));
6467
6468 ResultRegs[0] = Lo.getReg(Idx: 0);
6469 ResultRegs[1] = Hi.getReg(Idx: 0);
6470 break;
6471 }
6472 case TargetOpcode::G_LSHR:
6473 case TargetOpcode::G_ASHR: {
6474 // Short: ShAmt < NewBitSize
6475 auto HiS = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {HalfTy}, SrcOps: {InH, Amt});
6476
6477 auto LoOr = MIRBuilder.buildLShr(Dst: HalfTy, Src0: InL, Src1: Amt);
6478 auto HiOr = MIRBuilder.buildShl(Dst: HalfTy, Src0: InH, Src1: AmtLack);
6479 auto LoS = MIRBuilder.buildOr(Dst: HalfTy, Src0: LoOr, Src1: HiOr);
6480
6481 // Long: ShAmt >= NewBitSize
6482 MachineInstrBuilder HiL;
6483 if (MI.getOpcode() == TargetOpcode::G_LSHR) {
6484 HiL = MIRBuilder.buildConstant(Res: HalfTy, Val: 0); // Hi part is zero.
6485 } else {
6486 auto ShiftAmt = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: NewBitSize - 1);
6487 HiL = MIRBuilder.buildAShr(Dst: HalfTy, Src0: InH, Src1: ShiftAmt); // Sign of Hi part.
6488 }
6489 auto LoL = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {HalfTy},
6490 SrcOps: {InH, AmtExcess}); // Lo from Hi part.
6491
6492 auto Lo = MIRBuilder.buildSelect(
6493 Res: HalfTy, Tst: IsZero, Op0: InL, Op1: MIRBuilder.buildSelect(Res: HalfTy, Tst: IsShort, Op0: LoS, Op1: LoL));
6494
6495 auto Hi = MIRBuilder.buildSelect(Res: HalfTy, Tst: IsShort, Op0: HiS, Op1: HiL);
6496
6497 ResultRegs[0] = Lo.getReg(Idx: 0);
6498 ResultRegs[1] = Hi.getReg(Idx: 0);
6499 break;
6500 }
6501 default:
6502 llvm_unreachable("not a shift");
6503 }
6504
6505 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: ResultRegs);
6506 MI.eraseFromParent();
6507 return Legalized;
6508}
6509
6510Register LegalizerHelper::buildConstantShiftPart(unsigned Opcode,
6511 unsigned PartIdx,
6512 unsigned NumParts,
6513 ArrayRef<Register> SrcParts,
6514 const ShiftParams &Params,
6515 LLT TargetTy, LLT ShiftAmtTy) {
6516 auto WordShiftConst = getIConstantVRegVal(VReg: Params.WordShift, MRI);
6517 auto BitShiftConst = getIConstantVRegVal(VReg: Params.BitShift, MRI);
6518 assert(WordShiftConst && BitShiftConst && "Expected constants");
6519
6520 const unsigned ShiftWords = WordShiftConst->getZExtValue();
6521 const unsigned ShiftBits = BitShiftConst->getZExtValue();
6522 const bool NeedsInterWordShift = ShiftBits != 0;
6523
6524 switch (Opcode) {
6525 case TargetOpcode::G_SHL: {
6526 // Data moves from lower indices to higher indices
6527 // If this part would come from a source beyond our range, it's zero
6528 if (PartIdx < ShiftWords)
6529 return Params.Zero;
6530
6531 unsigned SrcIdx = PartIdx - ShiftWords;
6532 if (!NeedsInterWordShift)
6533 return SrcParts[SrcIdx];
6534
6535 // Combine shifted main part with carry from previous part
6536 auto Hi = MIRBuilder.buildShl(Dst: TargetTy, Src0: SrcParts[SrcIdx], Src1: Params.BitShift);
6537 if (SrcIdx > 0) {
6538 auto Lo = MIRBuilder.buildLShr(Dst: TargetTy, Src0: SrcParts[SrcIdx - 1],
6539 Src1: Params.InvBitShift);
6540 return MIRBuilder.buildOr(Dst: TargetTy, Src0: Hi, Src1: Lo).getReg(Idx: 0);
6541 }
6542 return Hi.getReg(Idx: 0);
6543 }
6544
6545 case TargetOpcode::G_LSHR: {
6546 unsigned SrcIdx = PartIdx + ShiftWords;
6547 if (SrcIdx >= NumParts)
6548 return Params.Zero;
6549 if (!NeedsInterWordShift)
6550 return SrcParts[SrcIdx];
6551
6552 // Combine shifted main part with carry from next part
6553 auto Lo = MIRBuilder.buildLShr(Dst: TargetTy, Src0: SrcParts[SrcIdx], Src1: Params.BitShift);
6554 if (SrcIdx + 1 < NumParts) {
6555 auto Hi = MIRBuilder.buildShl(Dst: TargetTy, Src0: SrcParts[SrcIdx + 1],
6556 Src1: Params.InvBitShift);
6557 return MIRBuilder.buildOr(Dst: TargetTy, Src0: Lo, Src1: Hi).getReg(Idx: 0);
6558 }
6559 return Lo.getReg(Idx: 0);
6560 }
6561
6562 case TargetOpcode::G_ASHR: {
6563 // Like LSHR but preserves sign bit
6564 unsigned SrcIdx = PartIdx + ShiftWords;
6565 if (SrcIdx >= NumParts)
6566 return Params.SignBit;
6567 if (!NeedsInterWordShift)
6568 return SrcParts[SrcIdx];
6569
6570 // Only the original MSB part uses arithmetic shift to preserve sign. All
6571 // other parts use logical shift since they're just moving data bits.
6572 auto Lo =
6573 (SrcIdx == NumParts - 1)
6574 ? MIRBuilder.buildAShr(Dst: TargetTy, Src0: SrcParts[SrcIdx], Src1: Params.BitShift)
6575 : MIRBuilder.buildLShr(Dst: TargetTy, Src0: SrcParts[SrcIdx], Src1: Params.BitShift);
6576 Register HiSrc =
6577 (SrcIdx + 1 < NumParts) ? SrcParts[SrcIdx + 1] : Params.SignBit;
6578 auto Hi = MIRBuilder.buildShl(Dst: TargetTy, Src0: HiSrc, Src1: Params.InvBitShift);
6579 return MIRBuilder.buildOr(Dst: TargetTy, Src0: Lo, Src1: Hi).getReg(Idx: 0);
6580 }
6581
6582 default:
6583 llvm_unreachable("not a shift");
6584 }
6585}
6586
6587Register LegalizerHelper::buildVariableShiftPart(unsigned Opcode,
6588 Register MainOperand,
6589 Register ShiftAmt,
6590 LLT TargetTy,
6591 Register CarryOperand) {
6592 // This helper generates a single output part for variable shifts by combining
6593 // the main operand (shifted by BitShift) with carry bits from an adjacent
6594 // part.
6595
6596 // For G_ASHR, individual parts don't have their own sign bit, only the
6597 // complete value does. So we use LSHR for the main operand shift in ASHR
6598 // context.
6599 unsigned MainOpcode = (Opcode == TargetOpcode::G_ASHR)
6600 ? static_cast<unsigned>(TargetOpcode::G_LSHR)
6601 : Opcode;
6602
6603 // Perform the primary shift on the main operand
6604 Register MainShifted =
6605 MIRBuilder.buildInstr(Opc: MainOpcode, DstOps: {TargetTy}, SrcOps: {MainOperand, ShiftAmt})
6606 .getReg(Idx: 0);
6607
6608 // No carry operand available
6609 if (!CarryOperand.isValid())
6610 return MainShifted;
6611
6612 // If BitShift is 0 (word-aligned shift), no inter-word bit movement occurs,
6613 // so carry bits aren't needed.
6614 LLT ShiftAmtTy = MRI.getType(Reg: ShiftAmt);
6615 auto ZeroConst = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: 0);
6616 LLT BoolTy = LLT::integer(SizeInBits: 1);
6617 auto IsZeroBitShift =
6618 MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: BoolTy, Op0: ShiftAmt, Op1: ZeroConst);
6619
6620 // Extract bits from the adjacent part that will "carry over" into this part.
6621 // The carry direction is opposite to the main shift direction, so we can
6622 // align the two shifted values before combining them with OR.
6623
6624 // Determine the carry shift opcode (opposite direction)
6625 unsigned CarryOpcode = (Opcode == TargetOpcode::G_SHL) ? TargetOpcode::G_LSHR
6626 : TargetOpcode::G_SHL;
6627
6628 // Calculate inverse shift amount: BitWidth - ShiftAmt
6629 auto TargetBitsConst =
6630 MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetTy.getScalarSizeInBits());
6631 auto InvShiftAmt = MIRBuilder.buildSub(Dst: ShiftAmtTy, Src0: TargetBitsConst, Src1: ShiftAmt);
6632
6633 // Shift the carry operand
6634 Register CarryBits =
6635 MIRBuilder
6636 .buildInstr(Opc: CarryOpcode, DstOps: {TargetTy}, SrcOps: {CarryOperand, InvShiftAmt})
6637 .getReg(Idx: 0);
6638
6639 // If BitShift is 0, don't include carry bits (InvShiftAmt would equal
6640 // TargetBits which would be poison for the individual carry shift operation).
6641 auto ZeroReg = MIRBuilder.buildConstant(Res: TargetTy, Val: 0);
6642 Register SafeCarryBits =
6643 MIRBuilder.buildSelect(Res: TargetTy, Tst: IsZeroBitShift, Op0: ZeroReg, Op1: CarryBits)
6644 .getReg(Idx: 0);
6645
6646 // Combine the main shifted part with the carry bits
6647 return MIRBuilder.buildOr(Dst: TargetTy, Src0: MainShifted, Src1: SafeCarryBits).getReg(Idx: 0);
6648}
6649
6650LegalizerHelper::LegalizeResult
6651LegalizerHelper::narrowScalarShiftByConstantMultiway(MachineInstr &MI,
6652 const APInt &Amt,
6653 LLT TargetTy,
6654 LLT ShiftAmtTy) {
6655 // Any wide shift can be decomposed into WordShift + BitShift components.
6656 // When shift amount is known constant, directly compute the decomposition
6657 // values and generate constant registers.
6658 Register DstReg = MI.getOperand(i: 0).getReg();
6659 Register SrcReg = MI.getOperand(i: 1).getReg();
6660 LLT DstTy = MRI.getType(Reg: DstReg);
6661
6662 const unsigned DstBits = DstTy.getScalarSizeInBits();
6663 const unsigned TargetBits = TargetTy.getScalarSizeInBits();
6664 const unsigned NumParts = DstBits / TargetBits;
6665
6666 assert(DstBits % TargetBits == 0 && "Target type must evenly divide source");
6667
6668 // When the shift amount is known at compile time, we just calculate which
6669 // source parts contribute to each output part.
6670
6671 SmallVector<Register, 8> SrcParts;
6672 extractParts(Reg: SrcReg, Ty: TargetTy, NumParts, VRegs&: SrcParts, MIRBuilder, MRI);
6673
6674 if (Amt.isZero()) {
6675 // No shift needed, just copy
6676 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: SrcParts);
6677 MI.eraseFromParent();
6678 return Legalized;
6679 }
6680
6681 ShiftParams Params;
6682 const unsigned ShiftWords = Amt.getZExtValue() / TargetBits;
6683 const unsigned ShiftBits = Amt.getZExtValue() % TargetBits;
6684
6685 // Generate constants and values needed by all shift types
6686 Params.WordShift = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: ShiftWords).getReg(Idx: 0);
6687 Params.BitShift = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: ShiftBits).getReg(Idx: 0);
6688 Params.InvBitShift =
6689 MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBits - ShiftBits).getReg(Idx: 0);
6690 Params.Zero = MIRBuilder.buildConstant(Res: TargetTy, Val: 0).getReg(Idx: 0);
6691
6692 // For ASHR, we need the sign-extended value to fill shifted-out positions
6693 if (MI.getOpcode() == TargetOpcode::G_ASHR)
6694 Params.SignBit =
6695 MIRBuilder
6696 .buildAShr(Dst: TargetTy, Src0: SrcParts[SrcParts.size() - 1],
6697 Src1: MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBits - 1))
6698 .getReg(Idx: 0);
6699
6700 SmallVector<Register, 8> DstParts(NumParts);
6701 for (unsigned I = 0; I < NumParts; ++I)
6702 DstParts[I] = buildConstantShiftPart(Opcode: MI.getOpcode(), PartIdx: I, NumParts, SrcParts,
6703 Params, TargetTy, ShiftAmtTy);
6704
6705 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstParts);
6706 MI.eraseFromParent();
6707 return Legalized;
6708}
6709
6710LegalizerHelper::LegalizeResult
6711LegalizerHelper::narrowScalarShiftMultiway(MachineInstr &MI, LLT TargetTy) {
6712 Register DstReg = MI.getOperand(i: 0).getReg();
6713 Register SrcReg = MI.getOperand(i: 1).getReg();
6714 Register AmtReg = MI.getOperand(i: 2).getReg();
6715 LLT DstTy = MRI.getType(Reg: DstReg);
6716 LLT ShiftAmtTy = MRI.getType(Reg: AmtReg);
6717
6718 const unsigned DstBits = DstTy.getScalarSizeInBits();
6719 const unsigned TargetBits = TargetTy.getScalarSizeInBits();
6720 const unsigned NumParts = DstBits / TargetBits;
6721
6722 assert(DstBits % TargetBits == 0 && "Target type must evenly divide source");
6723 assert(isPowerOf2_32(TargetBits) && "Target bit width must be power of 2");
6724
6725 // If the shift amount is known at compile time, we can use direct indexing
6726 // instead of generating select chains in the general case.
6727 if (auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: AmtReg, MRI))
6728 return narrowScalarShiftByConstantMultiway(MI, Amt: VRegAndVal->Value, TargetTy,
6729 ShiftAmtTy);
6730
6731 // For runtime-variable shift amounts, we must generate a more complex
6732 // sequence that handles all possible shift values using select chains.
6733
6734 // Split the input into target-sized pieces
6735 SmallVector<Register, 8> SrcParts;
6736 extractParts(Reg: SrcReg, Ty: TargetTy, NumParts, VRegs&: SrcParts, MIRBuilder, MRI);
6737
6738 // Shifting by zero should be a no-op.
6739 auto ZeroAmtConst = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: 0);
6740 LLT BoolTy = LLT::integer(SizeInBits: 1);
6741 auto IsZeroShift =
6742 MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: BoolTy, Op0: AmtReg, Op1: ZeroAmtConst);
6743
6744 // Any wide shift can be decomposed into two components:
6745 // 1. WordShift: number of complete target-sized words to shift
6746 // 2. BitShift: number of bits to shift within each word
6747 //
6748 // Example: 128-bit >> 50 with 32-bit target:
6749 // WordShift = 50 / 32 = 1 (shift right by 1 complete word)
6750 // BitShift = 50 % 32 = 18 (shift each word right by 18 bits)
6751 unsigned TargetBitsLog2 = Log2_32(Value: TargetBits);
6752 auto TargetBitsLog2Const =
6753 MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBitsLog2);
6754 auto TargetBitsMask = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBits - 1);
6755
6756 Register WordShift =
6757 MIRBuilder.buildLShr(Dst: ShiftAmtTy, Src0: AmtReg, Src1: TargetBitsLog2Const).getReg(Idx: 0);
6758 Register BitShift =
6759 MIRBuilder.buildAnd(Dst: ShiftAmtTy, Src0: AmtReg, Src1: TargetBitsMask).getReg(Idx: 0);
6760
6761 // Fill values:
6762 // - SHL/LSHR: fill with zeros
6763 // - ASHR: fill with sign-extended MSB
6764 Register ZeroReg = MIRBuilder.buildConstant(Res: TargetTy, Val: 0).getReg(Idx: 0);
6765
6766 Register FillValue;
6767 if (MI.getOpcode() == TargetOpcode::G_ASHR) {
6768 auto TargetBitsMinusOneConst =
6769 MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: TargetBits - 1);
6770 FillValue = MIRBuilder
6771 .buildAShr(Dst: TargetTy, Src0: SrcParts[NumParts - 1],
6772 Src1: TargetBitsMinusOneConst)
6773 .getReg(Idx: 0);
6774 } else {
6775 FillValue = ZeroReg;
6776 }
6777
6778 SmallVector<Register, 8> DstParts(NumParts);
6779
6780 // For each output part, generate a select chain that chooses the correct
6781 // result based on the runtime WordShift value. This handles all possible
6782 // word shift amounts by pre-calculating what each would produce.
6783 for (unsigned I = 0; I < NumParts; ++I) {
6784 // Initialize with appropriate default value for this shift type
6785 Register InBoundsResult = FillValue;
6786
6787 // clang-format off
6788 // Build a branchless select chain by pre-computing results for all possible
6789 // WordShift values (0 to NumParts-1). Each iteration nests a new select:
6790 //
6791 // K=0: select(WordShift==0, result0, FillValue)
6792 // K=1: select(WordShift==1, result1, select(WordShift==0, result0, FillValue))
6793 // K=2: select(WordShift==2, result2, select(WordShift==1, result1, select(...)))
6794 // clang-format on
6795 for (unsigned K = 0; K < NumParts; ++K) {
6796 auto WordShiftKConst = MIRBuilder.buildConstant(Res: ShiftAmtTy, Val: K);
6797 auto IsWordShiftK = MIRBuilder.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: BoolTy,
6798 Op0: WordShift, Op1: WordShiftKConst);
6799
6800 // Calculate source indices for this word shift
6801 //
6802 // For 4-part 128-bit value with K=1 word shift:
6803 // SHL: [3][2][1][0] << K => [2][1][0][Z]
6804 // -> (MainIdx = I-K, CarryIdx = I-K-1)
6805 // LSHR: [3][2][1][0] >> K => [Z][3][2][1]
6806 // -> (MainIdx = I+K, CarryIdx = I+K+1)
6807 int MainSrcIdx;
6808 int CarrySrcIdx; // Index for the word that provides the carried-in bits.
6809
6810 switch (MI.getOpcode()) {
6811 case TargetOpcode::G_SHL:
6812 MainSrcIdx = (int)I - (int)K;
6813 CarrySrcIdx = MainSrcIdx - 1;
6814 break;
6815 case TargetOpcode::G_LSHR:
6816 case TargetOpcode::G_ASHR:
6817 MainSrcIdx = (int)I + (int)K;
6818 CarrySrcIdx = MainSrcIdx + 1;
6819 break;
6820 default:
6821 llvm_unreachable("Not a shift");
6822 }
6823
6824 // Check bounds and build the result for this word shift
6825 Register ResultForK;
6826 if (MainSrcIdx >= 0 && MainSrcIdx < (int)NumParts) {
6827 Register MainOp = SrcParts[MainSrcIdx];
6828 Register CarryOp;
6829
6830 // Determine carry operand with bounds checking
6831 if (CarrySrcIdx >= 0 && CarrySrcIdx < (int)NumParts)
6832 CarryOp = SrcParts[CarrySrcIdx];
6833 else if (MI.getOpcode() == TargetOpcode::G_ASHR &&
6834 CarrySrcIdx >= (int)NumParts)
6835 CarryOp = FillValue; // Use sign extension
6836
6837 ResultForK = buildVariableShiftPart(Opcode: MI.getOpcode(), MainOperand: MainOp, ShiftAmt: BitShift,
6838 TargetTy, CarryOperand: CarryOp);
6839 } else {
6840 // Out of bounds - use fill value for this k
6841 ResultForK = FillValue;
6842 }
6843
6844 // Select this result if WordShift equals k
6845 InBoundsResult =
6846 MIRBuilder
6847 .buildSelect(Res: TargetTy, Tst: IsWordShiftK, Op0: ResultForK, Op1: InBoundsResult)
6848 .getReg(Idx: 0);
6849 }
6850
6851 // Handle zero-shift special case: if shift is 0, use original input
6852 DstParts[I] =
6853 MIRBuilder
6854 .buildSelect(Res: TargetTy, Tst: IsZeroShift, Op0: SrcParts[I], Op1: InBoundsResult)
6855 .getReg(Idx: 0);
6856 }
6857
6858 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstParts);
6859 MI.eraseFromParent();
6860 return Legalized;
6861}
6862
6863LegalizerHelper::LegalizeResult
6864LegalizerHelper::moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
6865 LLT MoreTy) {
6866 assert(TypeIdx == 0 && "Expecting only Idx 0");
6867
6868 Observer.changingInstr(MI);
6869 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
6870 MachineBasicBlock &OpMBB = *MI.getOperand(i: I + 1).getMBB();
6871 MIRBuilder.setInsertPt(MBB&: OpMBB, II: OpMBB.getFirstTerminator());
6872 moreElementsVectorSrc(MI, MoreTy, OpIdx: I);
6873 }
6874
6875 MachineBasicBlock &MBB = *MI.getParent();
6876 MIRBuilder.setInsertPt(MBB, II: --MBB.getFirstNonPHI());
6877 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
6878 Observer.changedInstr(MI);
6879 return Legalized;
6880}
6881
6882MachineInstrBuilder LegalizerHelper::getNeutralElementForVecReduce(
6883 unsigned Opcode, MachineIRBuilder &MIRBuilder, LLT Ty) {
6884 assert(Ty.isScalar() && "Expected scalar type to make neutral element for");
6885
6886 switch (Opcode) {
6887 default:
6888 llvm_unreachable(
6889 "getNeutralElementForVecReduce called with invalid opcode!");
6890 case TargetOpcode::G_VECREDUCE_ADD:
6891 case TargetOpcode::G_VECREDUCE_OR:
6892 case TargetOpcode::G_VECREDUCE_XOR:
6893 case TargetOpcode::G_VECREDUCE_UMAX:
6894 return MIRBuilder.buildConstant(Res: Ty, Val: 0);
6895 case TargetOpcode::G_VECREDUCE_MUL:
6896 return MIRBuilder.buildConstant(Res: Ty, Val: 1);
6897 case TargetOpcode::G_VECREDUCE_AND:
6898 case TargetOpcode::G_VECREDUCE_UMIN:
6899 return MIRBuilder.buildConstant(
6900 Res: Ty, Val: APInt::getAllOnes(numBits: Ty.getScalarSizeInBits()));
6901 case TargetOpcode::G_VECREDUCE_SMAX:
6902 return MIRBuilder.buildConstant(
6903 Res: Ty, Val: APInt::getSignedMinValue(numBits: Ty.getSizeInBits()));
6904 case TargetOpcode::G_VECREDUCE_SMIN:
6905 return MIRBuilder.buildConstant(
6906 Res: Ty, Val: APInt::getSignedMaxValue(numBits: Ty.getSizeInBits()));
6907 case TargetOpcode::G_VECREDUCE_FADD:
6908 return MIRBuilder.buildFConstant(Res: Ty, Val: -0.0);
6909 case TargetOpcode::G_VECREDUCE_FMUL:
6910 return MIRBuilder.buildFConstant(Res: Ty, Val: 1.0);
6911 case TargetOpcode::G_VECREDUCE_FMINIMUM:
6912 case TargetOpcode::G_VECREDUCE_FMAXIMUM:
6913 assert(false && "getNeutralElementForVecReduce unimplemented for "
6914 "G_VECREDUCE_FMINIMUM and G_VECREDUCE_FMAXIMUM!");
6915 }
6916 llvm_unreachable("switch expected to return!");
6917}
6918
6919LegalizerHelper::LegalizeResult
6920LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
6921 LLT MoreTy) {
6922 unsigned Opc = MI.getOpcode();
6923 switch (Opc) {
6924 case TargetOpcode::G_IMPLICIT_DEF:
6925 case TargetOpcode::G_LOAD: {
6926 if (TypeIdx != 0)
6927 return UnableToLegalize;
6928 Observer.changingInstr(MI);
6929 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
6930 Observer.changedInstr(MI);
6931 return Legalized;
6932 }
6933 case TargetOpcode::G_STORE:
6934 if (TypeIdx != 0)
6935 return UnableToLegalize;
6936 Observer.changingInstr(MI);
6937 moreElementsVectorSrc(MI, MoreTy, OpIdx: 0);
6938 Observer.changedInstr(MI);
6939 return Legalized;
6940 case TargetOpcode::G_AND:
6941 case TargetOpcode::G_OR:
6942 case TargetOpcode::G_XOR:
6943 case TargetOpcode::G_ADD:
6944 case TargetOpcode::G_SUB:
6945 case TargetOpcode::G_MUL:
6946 case TargetOpcode::G_FADD:
6947 case TargetOpcode::G_FSUB:
6948 case TargetOpcode::G_FMUL:
6949 case TargetOpcode::G_FDIV:
6950 case TargetOpcode::G_FCOPYSIGN:
6951 case TargetOpcode::G_UADDSAT:
6952 case TargetOpcode::G_USUBSAT:
6953 case TargetOpcode::G_SADDSAT:
6954 case TargetOpcode::G_SSUBSAT:
6955 case TargetOpcode::G_SMIN:
6956 case TargetOpcode::G_SMAX:
6957 case TargetOpcode::G_UMIN:
6958 case TargetOpcode::G_UMAX:
6959 case TargetOpcode::G_FMINNUM:
6960 case TargetOpcode::G_FMAXNUM:
6961 case TargetOpcode::G_FMINNUM_IEEE:
6962 case TargetOpcode::G_FMAXNUM_IEEE:
6963 case TargetOpcode::G_FMINIMUM:
6964 case TargetOpcode::G_FMAXIMUM:
6965 case TargetOpcode::G_FMINIMUMNUM:
6966 case TargetOpcode::G_FMAXIMUMNUM:
6967 case TargetOpcode::G_STRICT_FADD:
6968 case TargetOpcode::G_STRICT_FSUB:
6969 case TargetOpcode::G_STRICT_FMUL: {
6970 Observer.changingInstr(MI);
6971 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
6972 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
6973 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
6974 Observer.changedInstr(MI);
6975 return Legalized;
6976 }
6977 case TargetOpcode::G_SHL:
6978 case TargetOpcode::G_ASHR:
6979 case TargetOpcode::G_LSHR: {
6980 Observer.changingInstr(MI);
6981 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
6982 // The shift operand may have a different scalar type from the source and
6983 // destination operands.
6984 LLT ShiftMoreTy = MoreTy.changeElementType(
6985 NewEltTy: MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getElementType());
6986 moreElementsVectorSrc(MI, MoreTy: ShiftMoreTy, OpIdx: 2);
6987 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
6988 Observer.changedInstr(MI);
6989 return Legalized;
6990 }
6991 case TargetOpcode::G_FMA:
6992 case TargetOpcode::G_STRICT_FMA:
6993 case TargetOpcode::G_FSHR:
6994 case TargetOpcode::G_FSHL: {
6995 Observer.changingInstr(MI);
6996 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
6997 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
6998 moreElementsVectorSrc(MI, MoreTy, OpIdx: 3);
6999 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7000 Observer.changedInstr(MI);
7001 return Legalized;
7002 }
7003 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
7004 case TargetOpcode::G_EXTRACT:
7005 if (TypeIdx != 1)
7006 return UnableToLegalize;
7007 Observer.changingInstr(MI);
7008 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
7009 Observer.changedInstr(MI);
7010 return Legalized;
7011 case TargetOpcode::G_INSERT:
7012 case TargetOpcode::G_INSERT_VECTOR_ELT:
7013 case TargetOpcode::G_FREEZE:
7014 case TargetOpcode::G_FNEG:
7015 case TargetOpcode::G_FABS:
7016 case TargetOpcode::G_FSQRT:
7017 case TargetOpcode::G_FCEIL:
7018 case TargetOpcode::G_FFLOOR:
7019 case TargetOpcode::G_FNEARBYINT:
7020 case TargetOpcode::G_FRINT:
7021 case TargetOpcode::G_INTRINSIC_ROUND:
7022 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
7023 case TargetOpcode::G_INTRINSIC_TRUNC:
7024 case TargetOpcode::G_BITREVERSE:
7025 case TargetOpcode::G_BSWAP:
7026 case TargetOpcode::G_FCANONICALIZE:
7027 case TargetOpcode::G_SEXT_INREG:
7028 case TargetOpcode::G_ABS:
7029 case TargetOpcode::G_CTLZ:
7030 case TargetOpcode::G_CTPOP:
7031 if (TypeIdx != 0)
7032 return UnableToLegalize;
7033 Observer.changingInstr(MI);
7034 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
7035 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7036 Observer.changedInstr(MI);
7037 return Legalized;
7038 case TargetOpcode::G_SELECT: {
7039 auto [DstReg, DstTy, CondReg, CondTy] = MI.getFirst2RegLLTs();
7040 if (TypeIdx == 1) {
7041 if (!CondTy.isScalar() ||
7042 DstTy.getElementCount() != MoreTy.getElementCount())
7043 return UnableToLegalize;
7044
7045 // This is turning a scalar select of vectors into a vector
7046 // select. Broadcast the select condition.
7047 auto ShufSplat = MIRBuilder.buildShuffleSplat(Res: MoreTy, Src: CondReg);
7048 Observer.changingInstr(MI);
7049 MI.getOperand(i: 1).setReg(ShufSplat.getReg(Idx: 0));
7050 Observer.changedInstr(MI);
7051 return Legalized;
7052 }
7053
7054 if (CondTy.isVector())
7055 return UnableToLegalize;
7056
7057 Observer.changingInstr(MI);
7058 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
7059 moreElementsVectorSrc(MI, MoreTy, OpIdx: 3);
7060 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7061 Observer.changedInstr(MI);
7062 return Legalized;
7063 }
7064 case TargetOpcode::G_UNMERGE_VALUES:
7065 return UnableToLegalize;
7066 case TargetOpcode::G_PHI:
7067 return moreElementsVectorPhi(MI, TypeIdx, MoreTy);
7068 case TargetOpcode::G_SHUFFLE_VECTOR:
7069 return moreElementsVectorShuffle(MI, TypeIdx, MoreTy);
7070 case TargetOpcode::G_BUILD_VECTOR: {
7071 SmallVector<SrcOp, 8> Elts;
7072 for (auto Op : MI.uses()) {
7073 Elts.push_back(Elt: Op.getReg());
7074 }
7075
7076 for (unsigned i = Elts.size(); i < MoreTy.getNumElements(); ++i) {
7077 Elts.push_back(Elt: MIRBuilder.buildUndef(Res: MoreTy.getScalarType()));
7078 }
7079
7080 MIRBuilder.buildDeleteTrailingVectorElements(
7081 Res: MI.getOperand(i: 0).getReg(), Op0: MIRBuilder.buildInstr(Opc, DstOps: {MoreTy}, SrcOps: Elts));
7082 MI.eraseFromParent();
7083 return Legalized;
7084 }
7085 case TargetOpcode::G_SEXT:
7086 case TargetOpcode::G_ZEXT:
7087 case TargetOpcode::G_ANYEXT:
7088 case TargetOpcode::G_TRUNC:
7089 case TargetOpcode::G_FPTRUNC:
7090 case TargetOpcode::G_FPEXT:
7091 case TargetOpcode::G_FPTOSI:
7092 case TargetOpcode::G_FPTOUI:
7093 case TargetOpcode::G_FPTOSI_SAT:
7094 case TargetOpcode::G_FPTOUI_SAT:
7095 case TargetOpcode::G_SITOFP:
7096 case TargetOpcode::G_UITOFP:
7097 case TargetOpcode::G_TRUNC_SSAT_S:
7098 case TargetOpcode::G_TRUNC_SSAT_U:
7099 case TargetOpcode::G_TRUNC_USAT_U: {
7100 Observer.changingInstr(MI);
7101 LLT SrcExtTy;
7102 LLT DstExtTy;
7103 if (TypeIdx == 0) {
7104 DstExtTy = MoreTy;
7105 SrcExtTy = MoreTy.changeElementType(
7106 NewEltTy: MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getElementType());
7107 } else {
7108 DstExtTy = MoreTy.changeElementType(
7109 NewEltTy: MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getElementType());
7110 SrcExtTy = MoreTy;
7111 }
7112 moreElementsVectorSrc(MI, MoreTy: SrcExtTy, OpIdx: 1);
7113 moreElementsVectorDst(MI, WideTy: DstExtTy, OpIdx: 0);
7114 Observer.changedInstr(MI);
7115 return Legalized;
7116 }
7117 case TargetOpcode::G_ICMP:
7118 case TargetOpcode::G_FCMP: {
7119 if (TypeIdx != 1)
7120 return UnableToLegalize;
7121
7122 Observer.changingInstr(MI);
7123 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
7124 moreElementsVectorSrc(MI, MoreTy, OpIdx: 3);
7125 LLT CondTy = MoreTy.changeVectorElementType(
7126 NewEltTy: MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getElementType());
7127 moreElementsVectorDst(MI, WideTy: CondTy, OpIdx: 0);
7128 Observer.changedInstr(MI);
7129 return Legalized;
7130 }
7131 case TargetOpcode::G_BITCAST: {
7132 if (TypeIdx != 0)
7133 return UnableToLegalize;
7134
7135 LLT SrcTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
7136 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7137
7138 unsigned coefficient = SrcTy.getNumElements() * MoreTy.getNumElements();
7139 if (coefficient % DstTy.getNumElements() != 0)
7140 return UnableToLegalize;
7141
7142 coefficient = coefficient / DstTy.getNumElements();
7143
7144 LLT NewTy = SrcTy.changeElementCount(
7145 EC: ElementCount::get(MinVal: coefficient, Scalable: MoreTy.isScalable()));
7146 Observer.changingInstr(MI);
7147 moreElementsVectorSrc(MI, MoreTy: NewTy, OpIdx: 1);
7148 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7149 Observer.changedInstr(MI);
7150 return Legalized;
7151 }
7152 case TargetOpcode::G_VECREDUCE_FADD:
7153 case TargetOpcode::G_VECREDUCE_FMUL:
7154 case TargetOpcode::G_VECREDUCE_ADD:
7155 case TargetOpcode::G_VECREDUCE_MUL:
7156 case TargetOpcode::G_VECREDUCE_AND:
7157 case TargetOpcode::G_VECREDUCE_OR:
7158 case TargetOpcode::G_VECREDUCE_XOR:
7159 case TargetOpcode::G_VECREDUCE_SMAX:
7160 case TargetOpcode::G_VECREDUCE_SMIN:
7161 case TargetOpcode::G_VECREDUCE_UMAX:
7162 case TargetOpcode::G_VECREDUCE_UMIN: {
7163 LLT OrigTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
7164 MachineOperand &MO = MI.getOperand(i: 1);
7165 auto NewVec = MIRBuilder.buildPadVectorWithUndefElements(Res: MoreTy, Op0: MO);
7166 auto NeutralElement = getNeutralElementForVecReduce(
7167 Opcode: MI.getOpcode(), MIRBuilder, Ty: MoreTy.getElementType());
7168
7169 LLT IdxTy(TLI.getVectorIdxLLT(DL: MIRBuilder.getDataLayout()));
7170 for (size_t i = OrigTy.getNumElements(), e = MoreTy.getNumElements();
7171 i != e; i++) {
7172 auto Idx = MIRBuilder.buildConstant(Res: IdxTy, Val: i);
7173 NewVec = MIRBuilder.buildInsertVectorElement(Res: MoreTy, Val: NewVec,
7174 Elt: NeutralElement, Idx);
7175 }
7176
7177 Observer.changingInstr(MI);
7178 MO.setReg(NewVec.getReg(Idx: 0));
7179 Observer.changedInstr(MI);
7180 return Legalized;
7181 }
7182
7183 default:
7184 return UnableToLegalize;
7185 }
7186}
7187
7188LegalizerHelper::LegalizeResult
7189LegalizerHelper::equalizeVectorShuffleLengths(MachineInstr &MI) {
7190 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7191 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
7192 unsigned MaskNumElts = Mask.size();
7193 unsigned SrcNumElts = SrcTy.getNumElements();
7194 LLT DestEltTy = DstTy.getElementType();
7195
7196 if (MaskNumElts == SrcNumElts)
7197 return Legalized;
7198
7199 if (MaskNumElts < SrcNumElts) {
7200 // Extend mask to match new destination vector size with
7201 // undef values.
7202 SmallVector<int, 16> NewMask(SrcNumElts, -1);
7203 llvm::copy(Range&: Mask, Out: NewMask.begin());
7204
7205 moreElementsVectorDst(MI, WideTy: SrcTy, OpIdx: 0);
7206 MIRBuilder.setInstrAndDebugLoc(MI);
7207 MIRBuilder.buildShuffleVector(Res: MI.getOperand(i: 0).getReg(),
7208 Src1: MI.getOperand(i: 1).getReg(),
7209 Src2: MI.getOperand(i: 2).getReg(), Mask: NewMask);
7210 MI.eraseFromParent();
7211
7212 return Legalized;
7213 }
7214
7215 unsigned PaddedMaskNumElts = alignTo(Value: MaskNumElts, Align: SrcNumElts);
7216 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
7217 LLT PaddedTy =
7218 DstTy.changeVectorElementCount(EC: ElementCount::getFixed(MinVal: PaddedMaskNumElts));
7219
7220 // Create new source vectors by concatenating the initial
7221 // source vectors with undefined vectors of the same size.
7222 auto Undef = MIRBuilder.buildUndef(Res: SrcTy);
7223 SmallVector<Register, 8> MOps1(NumConcat, Undef.getReg(Idx: 0));
7224 SmallVector<Register, 8> MOps2(NumConcat, Undef.getReg(Idx: 0));
7225 MOps1[0] = MI.getOperand(i: 1).getReg();
7226 MOps2[0] = MI.getOperand(i: 2).getReg();
7227
7228 auto Src1 = MIRBuilder.buildConcatVectors(Res: PaddedTy, Ops: MOps1);
7229 auto Src2 = MIRBuilder.buildConcatVectors(Res: PaddedTy, Ops: MOps2);
7230
7231 // Readjust mask for new input vector length.
7232 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
7233 for (unsigned I = 0; I != MaskNumElts; ++I) {
7234 int Idx = Mask[I];
7235 if (Idx >= static_cast<int>(SrcNumElts))
7236 Idx += PaddedMaskNumElts - SrcNumElts;
7237 MappedOps[I] = Idx;
7238 }
7239
7240 // If we got more elements than required, extract subvector.
7241 if (MaskNumElts != PaddedMaskNumElts) {
7242 auto Shuffle =
7243 MIRBuilder.buildShuffleVector(Res: PaddedTy, Src1, Src2, Mask: MappedOps);
7244
7245 SmallVector<Register, 16> Elts(MaskNumElts);
7246 for (unsigned I = 0; I < MaskNumElts; ++I) {
7247 Elts[I] =
7248 MIRBuilder.buildExtractVectorElementConstant(Res: DestEltTy, Val: Shuffle, Idx: I)
7249 .getReg(Idx: 0);
7250 }
7251 MIRBuilder.buildBuildVector(Res: DstReg, Ops: Elts);
7252 } else {
7253 MIRBuilder.buildShuffleVector(Res: DstReg, Src1, Src2, Mask: MappedOps);
7254 }
7255
7256 MI.eraseFromParent();
7257 return LegalizerHelper::LegalizeResult::Legalized;
7258}
7259
7260LegalizerHelper::LegalizeResult
7261LegalizerHelper::moreElementsVectorShuffle(MachineInstr &MI,
7262 unsigned int TypeIdx, LLT MoreTy) {
7263 auto [DstTy, Src1Ty, Src2Ty] = MI.getFirst3LLTs();
7264 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
7265 unsigned NumElts = DstTy.getNumElements();
7266 unsigned WidenNumElts = MoreTy.getNumElements();
7267
7268 if (DstTy.isVector() && Src1Ty.isVector() &&
7269 DstTy.getNumElements() != Src1Ty.getNumElements()) {
7270 return equalizeVectorShuffleLengths(MI);
7271 }
7272
7273 if (TypeIdx != 0)
7274 return UnableToLegalize;
7275
7276 // Expect a canonicalized shuffle.
7277 if (DstTy != Src1Ty || DstTy != Src2Ty)
7278 return UnableToLegalize;
7279
7280 moreElementsVectorSrc(MI, MoreTy, OpIdx: 1);
7281 moreElementsVectorSrc(MI, MoreTy, OpIdx: 2);
7282
7283 // Adjust mask based on new input vector length.
7284 SmallVector<int, 16> NewMask(WidenNumElts, -1);
7285 for (unsigned I = 0; I != NumElts; ++I) {
7286 int Idx = Mask[I];
7287 if (Idx < static_cast<int>(NumElts))
7288 NewMask[I] = Idx;
7289 else
7290 NewMask[I] = Idx - NumElts + WidenNumElts;
7291 }
7292 moreElementsVectorDst(MI, WideTy: MoreTy, OpIdx: 0);
7293 MIRBuilder.setInstrAndDebugLoc(MI);
7294 MIRBuilder.buildShuffleVector(Res: MI.getOperand(i: 0).getReg(),
7295 Src1: MI.getOperand(i: 1).getReg(),
7296 Src2: MI.getOperand(i: 2).getReg(), Mask: NewMask);
7297 MI.eraseFromParent();
7298 return Legalized;
7299}
7300
7301void LegalizerHelper::multiplyRegisters(SmallVectorImpl<Register> &DstRegs,
7302 ArrayRef<Register> Src1Regs,
7303 ArrayRef<Register> Src2Regs,
7304 LLT NarrowTy) {
7305 MachineIRBuilder &B = MIRBuilder;
7306 unsigned SrcParts = Src1Regs.size();
7307 unsigned DstParts = DstRegs.size();
7308
7309 unsigned DstIdx = 0; // Low bits of the result.
7310 Register FactorSum =
7311 B.buildMul(Dst: NarrowTy, Src0: Src1Regs[DstIdx], Src1: Src2Regs[DstIdx]).getReg(Idx: 0);
7312 DstRegs[DstIdx] = FactorSum;
7313
7314 Register CarrySumPrevDstIdx;
7315 SmallVector<Register, 4> Factors;
7316
7317 for (DstIdx = 1; DstIdx < DstParts; DstIdx++) {
7318 // Collect high parts of muls from previous DstIdx.
7319 for (unsigned i = DstIdx < SrcParts ? 0 : DstIdx - SrcParts;
7320 i <= std::min(a: DstIdx - 1, b: SrcParts - 1); ++i) {
7321 MachineInstrBuilder Umulh =
7322 B.buildUMulH(Dst: NarrowTy, Src0: Src1Regs[DstIdx - 1 - i], Src1: Src2Regs[i]);
7323 Factors.push_back(Elt: Umulh.getReg(Idx: 0));
7324 }
7325 // Collect low parts of muls for DstIdx. Visit the diagonal starting with
7326 // the low Src1 part, so multiply-add selectors can use it as the first
7327 // accumulated cross product.
7328 unsigned LowStart = DstIdx + 1 < SrcParts ? 0 : DstIdx - SrcParts + 1;
7329 unsigned LowEnd = std::min(a: DstIdx, b: SrcParts - 1);
7330 for (unsigned RevI = LowEnd + 1; RevI != LowStart; --RevI) {
7331 unsigned i = RevI - 1;
7332 MachineInstrBuilder Mul =
7333 B.buildMul(Dst: NarrowTy, Src0: Src1Regs[DstIdx - i], Src1: Src2Regs[i]);
7334 Factors.push_back(Elt: Mul.getReg(Idx: 0));
7335 }
7336 // Add CarrySum from additions calculated for previous DstIdx.
7337 if (DstIdx != 1) {
7338 Factors.push_back(Elt: CarrySumPrevDstIdx);
7339 }
7340
7341 Register CarrySum;
7342 // Add all factors and accumulate all carries into CarrySum.
7343 if (DstIdx != DstParts - 1) {
7344 MachineInstrBuilder Uaddo =
7345 B.buildUAddo(Res: NarrowTy, CarryOut: LLT::integer(SizeInBits: 1), Op0: Factors[0], Op1: Factors[1]);
7346 FactorSum = Uaddo.getReg(Idx: 0);
7347 CarrySum = B.buildZExt(Res: NarrowTy, Op: Uaddo.getReg(Idx: 1)).getReg(Idx: 0);
7348 for (unsigned i = 2; i < Factors.size(); ++i) {
7349 MachineInstrBuilder Uaddo =
7350 B.buildUAddo(Res: NarrowTy, CarryOut: LLT::integer(SizeInBits: 1), Op0: FactorSum, Op1: Factors[i]);
7351 FactorSum = Uaddo.getReg(Idx: 0);
7352 MachineInstrBuilder Carry = B.buildZExt(Res: NarrowTy, Op: Uaddo.getReg(Idx: 1));
7353 CarrySum = B.buildAdd(Dst: NarrowTy, Src0: CarrySum, Src1: Carry).getReg(Idx: 0);
7354 }
7355 } else {
7356 // Since value for the next index is not calculated, neither is CarrySum.
7357 FactorSum = B.buildAdd(Dst: NarrowTy, Src0: Factors[0], Src1: Factors[1]).getReg(Idx: 0);
7358 for (unsigned i = 2; i < Factors.size(); ++i)
7359 FactorSum = B.buildAdd(Dst: NarrowTy, Src0: FactorSum, Src1: Factors[i]).getReg(Idx: 0);
7360 }
7361
7362 CarrySumPrevDstIdx = CarrySum;
7363 DstRegs[DstIdx] = FactorSum;
7364 Factors.clear();
7365 }
7366}
7367
7368LegalizerHelper::LegalizeResult
7369LegalizerHelper::narrowScalarAddSub(MachineInstr &MI, unsigned TypeIdx,
7370 LLT NarrowTy) {
7371 if (TypeIdx != 0)
7372 return UnableToLegalize;
7373
7374 Register DstReg = MI.getOperand(i: 0).getReg();
7375 LLT DstType = MRI.getType(Reg: DstReg);
7376 // FIXME: add support for vector types
7377 if (DstType.isVector())
7378 return UnableToLegalize;
7379
7380 unsigned Opcode = MI.getOpcode();
7381 unsigned OpO, OpE, OpF;
7382 switch (Opcode) {
7383 case TargetOpcode::G_SADDO:
7384 case TargetOpcode::G_SADDE:
7385 case TargetOpcode::G_UADDO:
7386 case TargetOpcode::G_UADDE:
7387 case TargetOpcode::G_ADD:
7388 OpO = TargetOpcode::G_UADDO;
7389 OpE = TargetOpcode::G_UADDE;
7390 OpF = TargetOpcode::G_UADDE;
7391 if (Opcode == TargetOpcode::G_SADDO || Opcode == TargetOpcode::G_SADDE)
7392 OpF = TargetOpcode::G_SADDE;
7393 break;
7394 case TargetOpcode::G_SSUBO:
7395 case TargetOpcode::G_SSUBE:
7396 case TargetOpcode::G_USUBO:
7397 case TargetOpcode::G_USUBE:
7398 case TargetOpcode::G_SUB:
7399 OpO = TargetOpcode::G_USUBO;
7400 OpE = TargetOpcode::G_USUBE;
7401 OpF = TargetOpcode::G_USUBE;
7402 if (Opcode == TargetOpcode::G_SSUBO || Opcode == TargetOpcode::G_SSUBE)
7403 OpF = TargetOpcode::G_SSUBE;
7404 break;
7405 default:
7406 llvm_unreachable("Unexpected add/sub opcode!");
7407 }
7408
7409 // 1 for a plain add/sub, 2 if this is an operation with a carry-out.
7410 unsigned NumDefs = MI.getNumExplicitDefs();
7411 Register Src1 = MI.getOperand(i: NumDefs).getReg();
7412 Register Src2 = MI.getOperand(i: NumDefs + 1).getReg();
7413 Register CarryDst, CarryIn;
7414 if (NumDefs == 2)
7415 CarryDst = MI.getOperand(i: 1).getReg();
7416 if (MI.getNumOperands() == NumDefs + 3)
7417 CarryIn = MI.getOperand(i: NumDefs + 2).getReg();
7418
7419 LLT RegTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7420 LLT LeftoverTy, DummyTy;
7421 SmallVector<Register, 2> Src1Regs, Src2Regs, Src1Left, Src2Left, DstRegs;
7422 extractParts(Reg: Src1, RegTy, MainTy: NarrowTy, LeftoverTy, VRegs&: Src1Regs, LeftoverVRegs&: Src1Left,
7423 MIRBuilder, MRI);
7424 extractParts(Reg: Src2, RegTy, MainTy: NarrowTy, LeftoverTy&: DummyTy, VRegs&: Src2Regs, LeftoverVRegs&: Src2Left, MIRBuilder,
7425 MRI);
7426
7427 int NarrowParts = Src1Regs.size();
7428 Src1Regs.append(RHS: Src1Left);
7429 Src2Regs.append(RHS: Src2Left);
7430 DstRegs.reserve(N: Src1Regs.size());
7431
7432 for (int i = 0, e = Src1Regs.size(); i != e; ++i) {
7433 Register DstReg =
7434 MRI.createGenericVirtualRegister(Ty: MRI.getType(Reg: Src1Regs[i]));
7435 Register CarryOut;
7436 // Forward the final carry-out to the destination register
7437 if (i == e - 1 && CarryDst)
7438 CarryOut = CarryDst;
7439 else
7440 CarryOut = MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: 1));
7441
7442 if (!CarryIn) {
7443 MIRBuilder.buildInstr(Opc: OpO, DstOps: {DstReg, CarryOut},
7444 SrcOps: {Src1Regs[i], Src2Regs[i]});
7445 } else if (i == e - 1) {
7446 MIRBuilder.buildInstr(Opc: OpF, DstOps: {DstReg, CarryOut},
7447 SrcOps: {Src1Regs[i], Src2Regs[i], CarryIn});
7448 } else {
7449 MIRBuilder.buildInstr(Opc: OpE, DstOps: {DstReg, CarryOut},
7450 SrcOps: {Src1Regs[i], Src2Regs[i], CarryIn});
7451 }
7452
7453 DstRegs.push_back(Elt: DstReg);
7454 CarryIn = CarryOut;
7455 }
7456 insertParts(DstReg: MI.getOperand(i: 0).getReg(), ResultTy: RegTy, PartTy: NarrowTy,
7457 PartRegs: ArrayRef(DstRegs).take_front(N: NarrowParts), LeftoverTy,
7458 LeftoverRegs: ArrayRef(DstRegs).drop_front(N: NarrowParts));
7459
7460 MI.eraseFromParent();
7461 return Legalized;
7462}
7463
7464LegalizerHelper::LegalizeResult
7465LegalizerHelper::narrowScalarMul(MachineInstr &MI, LLT NarrowTy) {
7466 auto [DstReg, Src1, Src2] = MI.getFirst3Regs();
7467
7468 LLT Ty = MRI.getType(Reg: DstReg);
7469 if (Ty.isVector())
7470 return UnableToLegalize;
7471
7472 unsigned Size = Ty.getSizeInBits();
7473 unsigned NarrowSize = NarrowTy.getSizeInBits();
7474 if (Size % NarrowSize != 0)
7475 return UnableToLegalize;
7476
7477 unsigned NumParts = Size / NarrowSize;
7478 bool IsMulHigh = MI.getOpcode() == TargetOpcode::G_UMULH;
7479 unsigned DstTmpParts = NumParts * (IsMulHigh ? 2 : 1);
7480
7481 SmallVector<Register, 2> Src1Parts, Src2Parts;
7482 SmallVector<Register, 2> DstTmpRegs(DstTmpParts);
7483 extractParts(Reg: Src1, Ty: NarrowTy, NumParts, VRegs&: Src1Parts, MIRBuilder, MRI);
7484 extractParts(Reg: Src2, Ty: NarrowTy, NumParts, VRegs&: Src2Parts, MIRBuilder, MRI);
7485 multiplyRegisters(DstRegs&: DstTmpRegs, Src1Regs: Src1Parts, Src2Regs: Src2Parts, NarrowTy);
7486
7487 // Take only high half of registers if this is high mul.
7488 ArrayRef<Register> DstRegs(&DstTmpRegs[DstTmpParts - NumParts], NumParts);
7489 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
7490 MI.eraseFromParent();
7491 return Legalized;
7492}
7493
7494LegalizerHelper::LegalizeResult
7495LegalizerHelper::narrowScalarFPTOI(MachineInstr &MI, unsigned TypeIdx,
7496 LLT NarrowTy) {
7497 if (TypeIdx != 0)
7498 return UnableToLegalize;
7499
7500 bool IsSigned = MI.getOpcode() == TargetOpcode::G_FPTOSI;
7501
7502 Register Src = MI.getOperand(i: 1).getReg();
7503 LLT SrcTy = MRI.getType(Reg: Src);
7504
7505 // If all finite floats fit into the narrowed integer type, we can just swap
7506 // out the result type. This is practically only useful for conversions from
7507 // half to at least 16-bits, so just handle the one case.
7508 if (SrcTy.getScalarType() != LLT::scalar(SizeInBits: 16) ||
7509 NarrowTy.getScalarSizeInBits() < (IsSigned ? 17u : 16u))
7510 return UnableToLegalize;
7511
7512 Observer.changingInstr(MI);
7513 narrowScalarDst(MI, NarrowTy, OpIdx: 0,
7514 ExtOpcode: IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT);
7515 Observer.changedInstr(MI);
7516 return Legalized;
7517}
7518
7519LegalizerHelper::LegalizeResult
7520LegalizerHelper::narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx,
7521 LLT NarrowTy) {
7522 if (TypeIdx != 1)
7523 return UnableToLegalize;
7524
7525 uint64_t NarrowSize = NarrowTy.getSizeInBits();
7526
7527 int64_t SizeOp1 = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
7528 // FIXME: add support for when SizeOp1 isn't an exact multiple of
7529 // NarrowSize.
7530 if (SizeOp1 % NarrowSize != 0)
7531 return UnableToLegalize;
7532 int NumParts = SizeOp1 / NarrowSize;
7533
7534 SmallVector<Register, 2> SrcRegs, DstRegs;
7535 extractParts(Reg: MI.getOperand(i: 1).getReg(), Ty: NarrowTy, NumParts, VRegs&: SrcRegs,
7536 MIRBuilder, MRI);
7537
7538 Register OpReg = MI.getOperand(i: 0).getReg();
7539 uint64_t OpStart = MI.getOperand(i: 2).getImm();
7540 uint64_t OpSize = MRI.getType(Reg: OpReg).getSizeInBits();
7541 for (int i = 0; i < NumParts; ++i) {
7542 unsigned SrcStart = i * NarrowSize;
7543
7544 if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
7545 // No part of the extract uses this subregister, ignore it.
7546 continue;
7547 } else if (SrcStart == OpStart && NarrowTy == MRI.getType(Reg: OpReg)) {
7548 // The entire subregister is extracted, forward the value.
7549 DstRegs.push_back(Elt: SrcRegs[i]);
7550 continue;
7551 }
7552
7553 // OpSegStart is where this destination segment would start in OpReg if it
7554 // extended infinitely in both directions.
7555 int64_t ExtractOffset;
7556 uint64_t SegSize;
7557 if (OpStart < SrcStart) {
7558 ExtractOffset = 0;
7559 SegSize = std::min(a: NarrowSize, b: OpStart + OpSize - SrcStart);
7560 } else {
7561 ExtractOffset = OpStart - SrcStart;
7562 SegSize = std::min(a: SrcStart + NarrowSize - OpStart, b: OpSize);
7563 }
7564
7565 Register SegReg = SrcRegs[i];
7566 if (ExtractOffset != 0 || SegSize != NarrowSize) {
7567 // A genuine extract is needed.
7568 SegReg = MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: SegSize));
7569 MIRBuilder.buildExtract(Res: SegReg, Src: SrcRegs[i], Index: ExtractOffset);
7570 }
7571
7572 DstRegs.push_back(Elt: SegReg);
7573 }
7574
7575 Register DstReg = MI.getOperand(i: 0).getReg();
7576 if (MRI.getType(Reg: DstReg).isVector())
7577 MIRBuilder.buildBuildVector(Res: DstReg, Ops: DstRegs);
7578 else if (DstRegs.size() > 1)
7579 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
7580 else
7581 MIRBuilder.buildCopy(Res: DstReg, Op: DstRegs[0]);
7582 MI.eraseFromParent();
7583 return Legalized;
7584}
7585
7586LegalizerHelper::LegalizeResult
7587LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx,
7588 LLT NarrowTy) {
7589 // FIXME: Don't know how to handle secondary types yet.
7590 if (TypeIdx != 0)
7591 return UnableToLegalize;
7592
7593 SmallVector<Register, 2> SrcRegs, LeftoverRegs, DstRegs;
7594 LLT RegTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
7595 LLT LeftoverTy;
7596 extractParts(Reg: MI.getOperand(i: 1).getReg(), RegTy, MainTy: NarrowTy, LeftoverTy, VRegs&: SrcRegs,
7597 LeftoverVRegs&: LeftoverRegs, MIRBuilder, MRI);
7598
7599 SrcRegs.append(RHS: LeftoverRegs);
7600
7601 uint64_t NarrowSize = NarrowTy.getSizeInBits();
7602 Register OpReg = MI.getOperand(i: 2).getReg();
7603 uint64_t OpStart = MI.getOperand(i: 3).getImm();
7604 uint64_t OpSize = MRI.getType(Reg: OpReg).getSizeInBits();
7605 for (int I = 0, E = SrcRegs.size(); I != E; ++I) {
7606 unsigned DstStart = I * NarrowSize;
7607
7608 if (DstStart == OpStart && NarrowTy == MRI.getType(Reg: OpReg)) {
7609 // The entire subregister is defined by this insert, forward the new
7610 // value.
7611 DstRegs.push_back(Elt: OpReg);
7612 continue;
7613 }
7614
7615 Register SrcReg = SrcRegs[I];
7616 if (MRI.getType(Reg: SrcRegs[I]) == LeftoverTy) {
7617 // The leftover reg is smaller than NarrowTy, so we need to extend it.
7618 SrcReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
7619 MIRBuilder.buildAnyExt(Res: SrcReg, Op: SrcRegs[I]);
7620 }
7621
7622 if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
7623 // No part of the insert affects this subregister, forward the original.
7624 DstRegs.push_back(Elt: SrcReg);
7625 continue;
7626 }
7627
7628 // OpSegStart is where this destination segment would start in OpReg if it
7629 // extended infinitely in both directions.
7630 int64_t ExtractOffset, InsertOffset;
7631 uint64_t SegSize;
7632 if (OpStart < DstStart) {
7633 InsertOffset = 0;
7634 ExtractOffset = DstStart - OpStart;
7635 SegSize = std::min(a: NarrowSize, b: OpStart + OpSize - DstStart);
7636 } else {
7637 InsertOffset = OpStart - DstStart;
7638 ExtractOffset = 0;
7639 SegSize =
7640 std::min(a: NarrowSize - InsertOffset, b: OpStart + OpSize - DstStart);
7641 }
7642
7643 Register SegReg = OpReg;
7644 if (ExtractOffset != 0 || SegSize != OpSize) {
7645 // A genuine extract is needed.
7646 SegReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: SegSize));
7647 MIRBuilder.buildExtract(Res: SegReg, Src: OpReg, Index: ExtractOffset);
7648 }
7649
7650 Register DstReg = MRI.createGenericVirtualRegister(Ty: NarrowTy);
7651 MIRBuilder.buildInsert(Res: DstReg, Src: SrcReg, Op: SegReg, Index: InsertOffset);
7652 DstRegs.push_back(Elt: DstReg);
7653 }
7654
7655 uint64_t WideSize = DstRegs.size() * NarrowSize;
7656 Register DstReg = MI.getOperand(i: 0).getReg();
7657 if (WideSize > RegTy.getSizeInBits()) {
7658 Register MergeReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: WideSize));
7659 MIRBuilder.buildMergeLikeInstr(Res: MergeReg, Ops: DstRegs);
7660 MIRBuilder.buildTrunc(Res: DstReg, Op: MergeReg);
7661 } else
7662 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: DstRegs);
7663
7664 MI.eraseFromParent();
7665 return Legalized;
7666}
7667
7668LegalizerHelper::LegalizeResult
7669LegalizerHelper::narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx,
7670 LLT NarrowTy) {
7671 Register DstReg = MI.getOperand(i: 0).getReg();
7672 LLT DstTy = MRI.getType(Reg: DstReg);
7673
7674 assert(MI.getNumOperands() == 3 && TypeIdx == 0);
7675
7676 SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
7677 SmallVector<Register, 4> Src0Regs, Src0LeftoverRegs;
7678 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
7679 LLT LeftoverTy;
7680 if (!extractParts(Reg: MI.getOperand(i: 1).getReg(), RegTy: DstTy, MainTy: NarrowTy, LeftoverTy,
7681 VRegs&: Src0Regs, LeftoverVRegs&: Src0LeftoverRegs, MIRBuilder, MRI))
7682 return UnableToLegalize;
7683
7684 LLT Unused;
7685 if (!extractParts(Reg: MI.getOperand(i: 2).getReg(), RegTy: DstTy, MainTy: NarrowTy, LeftoverTy&: Unused,
7686 VRegs&: Src1Regs, LeftoverVRegs&: Src1LeftoverRegs, MIRBuilder, MRI))
7687 llvm_unreachable("inconsistent extractParts result");
7688
7689 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
7690 auto Inst = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {NarrowTy},
7691 SrcOps: {Src0Regs[I], Src1Regs[I]});
7692 DstRegs.push_back(Elt: Inst.getReg(Idx: 0));
7693 }
7694
7695 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
7696 auto Inst = MIRBuilder.buildInstr(
7697 Opc: MI.getOpcode(),
7698 DstOps: {LeftoverTy}, SrcOps: {Src0LeftoverRegs[I], Src1LeftoverRegs[I]});
7699 DstLeftoverRegs.push_back(Elt: Inst.getReg(Idx: 0));
7700 }
7701
7702 insertParts(DstReg, ResultTy: DstTy, PartTy: NarrowTy, PartRegs: DstRegs,
7703 LeftoverTy, LeftoverRegs: DstLeftoverRegs);
7704
7705 MI.eraseFromParent();
7706 return Legalized;
7707}
7708
7709LegalizerHelper::LegalizeResult
7710LegalizerHelper::narrowScalarExt(MachineInstr &MI, unsigned TypeIdx,
7711 LLT NarrowTy) {
7712 if (TypeIdx != 0)
7713 return UnableToLegalize;
7714
7715 auto [DstReg, SrcReg] = MI.getFirst2Regs();
7716
7717 LLT DstTy = MRI.getType(Reg: DstReg);
7718 if (DstTy.isVector())
7719 return UnableToLegalize;
7720
7721 SmallVector<Register, 8> Parts;
7722 LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg);
7723 LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, VRegs&: Parts, PadStrategy: MI.getOpcode());
7724 buildWidenedRemergeToDst(DstReg, LCMTy, RemergeRegs: Parts);
7725
7726 MI.eraseFromParent();
7727 return Legalized;
7728}
7729
7730LegalizerHelper::LegalizeResult
7731LegalizerHelper::narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx,
7732 LLT NarrowTy) {
7733 if (TypeIdx != 0)
7734 return UnableToLegalize;
7735
7736 Register CondReg = MI.getOperand(i: 1).getReg();
7737 LLT CondTy = MRI.getType(Reg: CondReg);
7738 if (CondTy.isVector()) // TODO: Handle vselect
7739 return UnableToLegalize;
7740
7741 Register DstReg = MI.getOperand(i: 0).getReg();
7742 LLT DstTy = MRI.getType(Reg: DstReg);
7743
7744 SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
7745 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
7746 SmallVector<Register, 4> Src2Regs, Src2LeftoverRegs;
7747 LLT LeftoverTy;
7748 if (!extractParts(Reg: MI.getOperand(i: 2).getReg(), RegTy: DstTy, MainTy: NarrowTy, LeftoverTy,
7749 VRegs&: Src1Regs, LeftoverVRegs&: Src1LeftoverRegs, MIRBuilder, MRI))
7750 return UnableToLegalize;
7751
7752 LLT Unused;
7753 if (!extractParts(Reg: MI.getOperand(i: 3).getReg(), RegTy: DstTy, MainTy: NarrowTy, LeftoverTy&: Unused,
7754 VRegs&: Src2Regs, LeftoverVRegs&: Src2LeftoverRegs, MIRBuilder, MRI))
7755 llvm_unreachable("inconsistent extractParts result");
7756
7757 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
7758 auto Select = MIRBuilder.buildSelect(Res: NarrowTy,
7759 Tst: CondReg, Op0: Src1Regs[I], Op1: Src2Regs[I]);
7760 DstRegs.push_back(Elt: Select.getReg(Idx: 0));
7761 }
7762
7763 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
7764 auto Select = MIRBuilder.buildSelect(
7765 Res: LeftoverTy, Tst: CondReg, Op0: Src1LeftoverRegs[I], Op1: Src2LeftoverRegs[I]);
7766 DstLeftoverRegs.push_back(Elt: Select.getReg(Idx: 0));
7767 }
7768
7769 insertParts(DstReg, ResultTy: DstTy, PartTy: NarrowTy, PartRegs: DstRegs,
7770 LeftoverTy, LeftoverRegs: DstLeftoverRegs);
7771
7772 MI.eraseFromParent();
7773 return Legalized;
7774}
7775
7776LegalizerHelper::LegalizeResult
7777LegalizerHelper::narrowScalarCTLZ(MachineInstr &MI, unsigned TypeIdx,
7778 LLT NarrowTy) {
7779 if (TypeIdx != 1)
7780 return UnableToLegalize;
7781
7782 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7783 unsigned NarrowSize = NarrowTy.getSizeInBits();
7784
7785 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
7786 const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_POISON;
7787
7788 MachineIRBuilder &B = MIRBuilder;
7789 auto UnmergeSrc = B.buildUnmerge(Res: NarrowTy, Op: SrcReg);
7790 // ctlz(Hi:Lo) -> Hi == 0 ? (NarrowSize + ctlz(Lo)) : ctlz(Hi)
7791 auto C_0 = B.buildConstant(Res: NarrowTy, Val: 0);
7792 auto HiIsZero = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: LLT::integer(SizeInBits: 1),
7793 Op0: UnmergeSrc.getReg(Idx: 1), Op1: C_0);
7794 auto LoCTLZ = IsUndef ? B.buildCTLZ_ZERO_POISON(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 0))
7795 : B.buildCTLZ(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 0));
7796 auto C_NarrowSize = B.buildConstant(Res: DstTy, Val: NarrowSize);
7797 auto HiIsZeroCTLZ = B.buildAdd(Dst: DstTy, Src0: LoCTLZ, Src1: C_NarrowSize);
7798 auto HiCTLZ = B.buildCTLZ_ZERO_POISON(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 1));
7799 B.buildSelect(Res: DstReg, Tst: HiIsZero, Op0: HiIsZeroCTLZ, Op1: HiCTLZ);
7800
7801 MI.eraseFromParent();
7802 return Legalized;
7803 }
7804
7805 return UnableToLegalize;
7806}
7807
7808LegalizerHelper::LegalizeResult
7809LegalizerHelper::narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx,
7810 LLT NarrowTy) {
7811 if (TypeIdx != 1)
7812 return UnableToLegalize;
7813
7814 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7815 unsigned NarrowSize = NarrowTy.getSizeInBits();
7816
7817 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
7818 const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTTZ_ZERO_POISON;
7819
7820 MachineIRBuilder &B = MIRBuilder;
7821 auto UnmergeSrc = B.buildUnmerge(Res: NarrowTy, Op: SrcReg);
7822 // cttz(Hi:Lo) -> Lo == 0 ? (cttz(Hi) + NarrowSize) : cttz(Lo)
7823 auto C_0 = B.buildConstant(Res: NarrowTy, Val: 0);
7824 auto LoIsZero = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: LLT::integer(SizeInBits: 1),
7825 Op0: UnmergeSrc.getReg(Idx: 0), Op1: C_0);
7826 auto HiCTTZ = IsUndef ? B.buildCTTZ_ZERO_POISON(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 1))
7827 : B.buildCTTZ(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 1));
7828 auto C_NarrowSize = B.buildConstant(Res: DstTy, Val: NarrowSize);
7829 auto LoIsZeroCTTZ = B.buildAdd(Dst: DstTy, Src0: HiCTTZ, Src1: C_NarrowSize);
7830 auto LoCTTZ = B.buildCTTZ_ZERO_POISON(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 0));
7831 B.buildSelect(Res: DstReg, Tst: LoIsZero, Op0: LoIsZeroCTTZ, Op1: LoCTTZ);
7832
7833 MI.eraseFromParent();
7834 return Legalized;
7835 }
7836
7837 return UnableToLegalize;
7838}
7839
7840LegalizerHelper::LegalizeResult
7841LegalizerHelper::narrowScalarCTLS(MachineInstr &MI, unsigned TypeIdx,
7842 LLT NarrowTy) {
7843 if (TypeIdx != 1)
7844 return UnableToLegalize;
7845
7846 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7847 unsigned NarrowSize = NarrowTy.getSizeInBits();
7848
7849 if (!SrcTy.isScalar() || SrcTy.getSizeInBits() != 2 * NarrowSize)
7850 return UnableToLegalize;
7851
7852 MachineIRBuilder &B = MIRBuilder;
7853
7854 auto UnmergeSrc = B.buildUnmerge(Res: NarrowTy, Op: SrcReg);
7855 Register Lo = UnmergeSrc.getReg(Idx: 0);
7856 Register Hi = UnmergeSrc.getReg(Idx: 1);
7857
7858 auto ShAmt = B.buildConstant(Res: NarrowTy, Val: NarrowSize - 1);
7859 auto Sign = B.buildAShr(Dst: NarrowTy, Src0: Hi, Src1: ShAmt);
7860
7861 auto HiIsSign = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: LLT::scalar(SizeInBits: 1), Op0: Hi, Op1: Sign);
7862
7863 // Invert Lo if Hi is negative. Then count the leading zeros. If there are no
7864 // leading zeros, then the MSB of Lo is different than the MSB of Hi.
7865 // Otherwise the leading zeros represent additional sign bits of the original
7866 // value.
7867 auto LoInv = B.buildXor(Dst: DstTy, Src0: Lo, Src1: Sign);
7868 auto LoCTLZ = B.buildCTLZ(Dst: DstTy, Src0: LoInv);
7869
7870 // Add NarrowSize-1 to LoCTLZ. This is the full CTLS if Hi is all sign bits.
7871 auto C_NarrowSizeM1 = B.buildConstant(Res: DstTy, Val: NarrowSize - 1);
7872 auto HiIsSignCTLS = B.buildAdd(Dst: DstTy, Src0: LoCTLZ, Src1: C_NarrowSizeM1);
7873
7874 auto HiCTLS = B.buildCTLS(Dst: DstTy, Src0: Hi);
7875
7876 B.buildSelect(Res: DstReg, Tst: HiIsSign, Op0: HiIsSignCTLS, Op1: HiCTLS);
7877
7878 MI.eraseFromParent();
7879 return Legalized;
7880}
7881
7882LegalizerHelper::LegalizeResult
7883LegalizerHelper::narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx,
7884 LLT NarrowTy) {
7885 if (TypeIdx != 1)
7886 return UnableToLegalize;
7887
7888 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7889 unsigned NarrowSize = NarrowTy.getSizeInBits();
7890
7891 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
7892 auto UnmergeSrc = MIRBuilder.buildUnmerge(Res: NarrowTy, Op: MI.getOperand(i: 1));
7893
7894 auto LoCTPOP = MIRBuilder.buildCTPOP(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 0));
7895 auto HiCTPOP = MIRBuilder.buildCTPOP(Dst: DstTy, Src0: UnmergeSrc.getReg(Idx: 1));
7896 MIRBuilder.buildAdd(Dst: DstReg, Src0: HiCTPOP, Src1: LoCTPOP);
7897
7898 MI.eraseFromParent();
7899 return Legalized;
7900 }
7901
7902 return UnableToLegalize;
7903}
7904
7905LegalizerHelper::LegalizeResult
7906LegalizerHelper::narrowScalarFLDEXP(MachineInstr &MI, unsigned TypeIdx,
7907 LLT NarrowTy) {
7908 if (TypeIdx != 1)
7909 return UnableToLegalize;
7910
7911 MachineIRBuilder &B = MIRBuilder;
7912 Register ExpReg = MI.getOperand(i: 2).getReg();
7913 LLT ExpTy = MRI.getType(Reg: ExpReg);
7914
7915 unsigned ClampSize = NarrowTy.getScalarSizeInBits();
7916
7917 // Clamp the exponent to the range of the target type.
7918 auto MinExp = B.buildConstant(Res: ExpTy, Val: minIntN(N: ClampSize));
7919 auto ClampMin = B.buildSMax(Dst: ExpTy, Src0: ExpReg, Src1: MinExp);
7920 auto MaxExp = B.buildConstant(Res: ExpTy, Val: maxIntN(N: ClampSize));
7921 auto Clamp = B.buildSMin(Dst: ExpTy, Src0: ClampMin, Src1: MaxExp);
7922
7923 auto Trunc = B.buildTrunc(Res: NarrowTy, Op: Clamp);
7924 Observer.changingInstr(MI);
7925 MI.getOperand(i: 2).setReg(Trunc.getReg(Idx: 0));
7926 Observer.changedInstr(MI);
7927 return Legalized;
7928}
7929
7930LegalizerHelper::LegalizeResult
7931LegalizerHelper::lowerBitCount(MachineInstr &MI) {
7932 unsigned Opc = MI.getOpcode();
7933 const auto &TII = MIRBuilder.getTII();
7934 auto isSupported = [this](const LegalityQuery &Q) {
7935 auto QAction = LI.getAction(Query: Q).Action;
7936 return QAction == Legal || QAction == Libcall || QAction == Custom;
7937 };
7938 switch (Opc) {
7939 default:
7940 return UnableToLegalize;
7941 case TargetOpcode::G_CTLZ_ZERO_POISON: {
7942 // This trivially expands to CTLZ.
7943 Observer.changingInstr(MI);
7944 MI.setDesc(TII.get(Opcode: TargetOpcode::G_CTLZ));
7945 Observer.changedInstr(MI);
7946 return Legalized;
7947 }
7948 case TargetOpcode::G_CTLZ: {
7949 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7950 unsigned Len = SrcTy.getScalarSizeInBits();
7951
7952 if (isSupported({TargetOpcode::G_CTLZ_ZERO_POISON, {DstTy, SrcTy}})) {
7953 // If CTLZ_ZERO_POISON is supported, emit that and a select for zero.
7954 auto CtlzZU = MIRBuilder.buildCTLZ_ZERO_POISON(Dst: DstTy, Src0: SrcReg);
7955 auto ZeroSrc = MIRBuilder.buildConstant(Res: SrcTy, Val: 0);
7956 auto ICmp = MIRBuilder.buildICmp(
7957 Pred: CmpInst::ICMP_EQ, Res: SrcTy.changeElementSize(NewEltSize: 1), Op0: SrcReg, Op1: ZeroSrc);
7958 auto LenConst = MIRBuilder.buildConstant(Res: DstTy, Val: Len);
7959 MIRBuilder.buildSelect(Res: DstReg, Tst: ICmp, Op0: LenConst, Op1: CtlzZU);
7960 MI.eraseFromParent();
7961 return Legalized;
7962 }
7963 // for now, we do this:
7964 // NewLen = NextPowerOf2(Len);
7965 // x = x | (x >> 1);
7966 // x = x | (x >> 2);
7967 // ...
7968 // x = x | (x >>16);
7969 // x = x | (x >>32); // for 64-bit input
7970 // Upto NewLen/2
7971 // return Len - popcount(x);
7972 //
7973 // Ref: "Hacker's Delight" by Henry Warren
7974 Register Op = SrcReg;
7975 unsigned NewLen = PowerOf2Ceil(A: Len);
7976 for (unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) {
7977 auto MIBShiftAmt = MIRBuilder.buildConstant(Res: SrcTy, Val: 1ULL << i);
7978 auto MIBOp = MIRBuilder.buildOr(
7979 Dst: SrcTy, Src0: Op, Src1: MIRBuilder.buildLShr(Dst: SrcTy, Src0: Op, Src1: MIBShiftAmt));
7980 Op = MIBOp.getReg(Idx: 0);
7981 }
7982 auto MIBPop = MIRBuilder.buildCTPOP(Dst: DstTy, Src0: Op);
7983 MIRBuilder.buildSub(Dst: MI.getOperand(i: 0), Src0: MIRBuilder.buildConstant(Res: DstTy, Val: Len),
7984 Src1: MIBPop);
7985 MI.eraseFromParent();
7986 return Legalized;
7987 }
7988 case TargetOpcode::G_CTTZ_ZERO_POISON: {
7989 // This trivially expands to CTTZ.
7990 Observer.changingInstr(MI);
7991 MI.setDesc(TII.get(Opcode: TargetOpcode::G_CTTZ));
7992 Observer.changedInstr(MI);
7993 return Legalized;
7994 }
7995 case TargetOpcode::G_CTTZ: {
7996 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
7997
7998 unsigned Len = SrcTy.getScalarSizeInBits();
7999 if (isSupported({TargetOpcode::G_CTTZ_ZERO_POISON, {DstTy, SrcTy}})) {
8000 // If CTTZ_ZERO_POISON is legal or custom, emit that and a select with
8001 // zero.
8002 auto CttzZU = MIRBuilder.buildCTTZ_ZERO_POISON(Dst: DstTy, Src0: SrcReg);
8003 auto Zero = MIRBuilder.buildConstant(Res: SrcTy, Val: 0);
8004 auto ICmp = MIRBuilder.buildICmp(
8005 Pred: CmpInst::ICMP_EQ, Res: DstTy.changeElementSize(NewEltSize: 1), Op0: SrcReg, Op1: Zero);
8006 auto LenConst = MIRBuilder.buildConstant(Res: DstTy, Val: Len);
8007 MIRBuilder.buildSelect(Res: DstReg, Tst: ICmp, Op0: LenConst, Op1: CttzZU);
8008 MI.eraseFromParent();
8009 return Legalized;
8010 }
8011 // for now, we use: { return popcount(~x & (x - 1)); }
8012 // unless the target has ctlz but not ctpop, in which case we use:
8013 // { return 32 - nlz(~x & (x-1)); }
8014 // Ref: "Hacker's Delight" by Henry Warren
8015 auto MIBCstNeg1 = MIRBuilder.buildConstant(Res: SrcTy, Val: -1);
8016 auto MIBNot = MIRBuilder.buildXor(Dst: SrcTy, Src0: SrcReg, Src1: MIBCstNeg1);
8017 auto MIBTmp = MIRBuilder.buildAnd(
8018 Dst: SrcTy, Src0: MIBNot, Src1: MIRBuilder.buildAdd(Dst: SrcTy, Src0: SrcReg, Src1: MIBCstNeg1));
8019 if (!isSupported({TargetOpcode::G_CTPOP, {SrcTy, SrcTy}}) &&
8020 isSupported({TargetOpcode::G_CTLZ, {SrcTy, SrcTy}})) {
8021 auto MIBCstLen = MIRBuilder.buildConstant(Res: SrcTy, Val: Len);
8022 MIRBuilder.buildSub(Dst: MI.getOperand(i: 0), Src0: MIBCstLen,
8023 Src1: MIRBuilder.buildCTLZ(Dst: SrcTy, Src0: MIBTmp));
8024 MI.eraseFromParent();
8025 return Legalized;
8026 }
8027 Observer.changingInstr(MI);
8028 MI.setDesc(TII.get(Opcode: TargetOpcode::G_CTPOP));
8029 MI.getOperand(i: 1).setReg(MIBTmp.getReg(Idx: 0));
8030 Observer.changedInstr(MI);
8031 return Legalized;
8032 }
8033 case TargetOpcode::G_CTPOP: {
8034 Register SrcReg = MI.getOperand(i: 1).getReg();
8035 LLT Ty = MRI.getType(Reg: SrcReg);
8036 unsigned Size = Ty.getScalarSizeInBits();
8037 MachineIRBuilder &B = MIRBuilder;
8038
8039 // Bail out on irregular type lengths.
8040 if (Size > 128 || Size % 8 != 0)
8041 return UnableToLegalize;
8042
8043 // Count set bits in blocks of 2 bits. Default approach would be
8044 // B2Count = { val & 0x55555555 } + { (val >> 1) & 0x55555555 }
8045 // We use following formula instead:
8046 // B2Count = val - { (val >> 1) & 0x55555555 }
8047 // since it gives same result in blocks of 2 with one instruction less.
8048 auto C_1 = B.buildConstant(Res: Ty, Val: 1);
8049 auto B2Set1LoTo1Hi = B.buildLShr(Dst: Ty, Src0: SrcReg, Src1: C_1);
8050 APInt B2Mask1HiTo0 = APInt::getSplat(NewLen: Size, V: APInt(8, 0x55));
8051 auto C_B2Mask1HiTo0 = B.buildConstant(Res: Ty, Val: B2Mask1HiTo0);
8052 auto B2Count1Hi = B.buildAnd(Dst: Ty, Src0: B2Set1LoTo1Hi, Src1: C_B2Mask1HiTo0);
8053 auto B2Count = B.buildSub(Dst: Ty, Src0: SrcReg, Src1: B2Count1Hi);
8054
8055 // In order to get count in blocks of 4 add values from adjacent block of 2.
8056 // B4Count = { B2Count & 0x33333333 } + { (B2Count >> 2) & 0x33333333 }
8057 auto C_2 = B.buildConstant(Res: Ty, Val: 2);
8058 auto B4Set2LoTo2Hi = B.buildLShr(Dst: Ty, Src0: B2Count, Src1: C_2);
8059 APInt B4Mask2HiTo0 = APInt::getSplat(NewLen: Size, V: APInt(8, 0x33));
8060 auto C_B4Mask2HiTo0 = B.buildConstant(Res: Ty, Val: B4Mask2HiTo0);
8061 auto B4HiB2Count = B.buildAnd(Dst: Ty, Src0: B4Set2LoTo2Hi, Src1: C_B4Mask2HiTo0);
8062 auto B4LoB2Count = B.buildAnd(Dst: Ty, Src0: B2Count, Src1: C_B4Mask2HiTo0);
8063 auto B4Count = B.buildAdd(Dst: Ty, Src0: B4HiB2Count, Src1: B4LoB2Count);
8064
8065 // For count in blocks of 8 bits we don't have to mask high 4 bits before
8066 // addition since count value sits in range {0,...,8} and 4 bits are enough
8067 // to hold such binary values. After addition high 4 bits still hold count
8068 // of set bits in high 4 bit block, set them to zero and get 8 bit result.
8069 // B8Count = { B4Count + (B4Count >> 4) } & 0x0F0F0F0F
8070 auto C_4 = B.buildConstant(Res: Ty, Val: 4);
8071 auto B8HiB4Count = B.buildLShr(Dst: Ty, Src0: B4Count, Src1: C_4);
8072 auto B8CountDirty4Hi = B.buildAdd(Dst: Ty, Src0: B8HiB4Count, Src1: B4Count);
8073 APInt B8Mask4HiTo0 = APInt::getSplat(NewLen: Size, V: APInt(8, 0x0F));
8074 auto C_B8Mask4HiTo0 = B.buildConstant(Res: Ty, Val: B8Mask4HiTo0);
8075 auto B8Count = B.buildAnd(Dst: Ty, Src0: B8CountDirty4Hi, Src1: C_B8Mask4HiTo0);
8076
8077 assert(Size <= 128 && "Scalar size is too large for CTPOP lower algorithm");
8078
8079 // Avoid the multiply when shift-add is cheaper.
8080 if (Size == 16 && !Ty.isVector()) {
8081 // v = (v + (v >> 8)) & 0xFF;
8082 auto C_8 = B.buildConstant(Res: Ty, Val: 8);
8083 auto HighSum = B.buildLShr(Dst: Ty, Src0: B8Count, Src1: C_8);
8084 auto Res = B.buildAdd(Dst: Ty, Src0: B8Count, Src1: HighSum);
8085 B.buildAnd(Dst: MI.getOperand(i: 0).getReg(), Src0: Res, Src1: B.buildConstant(Res: Ty, Val: 0xFF));
8086 MI.eraseFromParent();
8087 return Legalized;
8088 }
8089
8090 // 8 bits can hold CTPOP result of 128 bit int or smaller. Mul with this
8091 // bitmask will set 8 msb in ResTmp to sum of all B8Counts in 8 bit blocks.
8092 auto MulMask = B.buildConstant(Res: Ty, Val: APInt::getSplat(NewLen: Size, V: APInt(8, 0x01)));
8093
8094 // Shift count result from 8 high bits to low bits.
8095 auto C_SizeM8 = B.buildConstant(Res: Ty, Val: Size - 8);
8096
8097 auto IsMulSupported = [this](const LLT Ty) {
8098 auto Action = LI.getAction(Query: {TargetOpcode::G_MUL, {Ty}}).Action;
8099 return Action == Legal || Action == WidenScalar || Action == Custom;
8100 };
8101 if (IsMulSupported(Ty)) {
8102 auto ResTmp = B.buildMul(Dst: Ty, Src0: B8Count, Src1: MulMask);
8103 B.buildLShr(Dst: MI.getOperand(i: 0).getReg(), Src0: ResTmp, Src1: C_SizeM8);
8104 } else {
8105 auto ResTmp = B8Count;
8106 for (unsigned Shift = 8; Shift < Size; Shift *= 2) {
8107 auto ShiftC = B.buildConstant(Res: Ty, Val: Shift);
8108 auto Shl = B.buildShl(Dst: Ty, Src0: ResTmp, Src1: ShiftC);
8109 ResTmp = B.buildAdd(Dst: Ty, Src0: ResTmp, Src1: Shl);
8110 }
8111 B.buildLShr(Dst: MI.getOperand(i: 0).getReg(), Src0: ResTmp, Src1: C_SizeM8);
8112 }
8113 MI.eraseFromParent();
8114 return Legalized;
8115 }
8116 case TargetOpcode::G_CTLS: {
8117 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
8118
8119 // ctls(x) -> ctlz(x ^ (x >> (N - 1))) - 1
8120 auto SignIdxC =
8121 MIRBuilder.buildConstant(Res: SrcTy, Val: SrcTy.getScalarSizeInBits() - 1);
8122 auto OneC = MIRBuilder.buildConstant(Res: DstTy, Val: 1);
8123
8124 auto Shr = MIRBuilder.buildAShr(Dst: SrcTy, Src0: SrcReg, Src1: SignIdxC);
8125
8126 auto Xor = MIRBuilder.buildXor(Dst: SrcTy, Src0: SrcReg, Src1: Shr);
8127 auto Ctlz = MIRBuilder.buildCTLZ(Dst: DstTy, Src0: Xor);
8128
8129 MIRBuilder.buildSub(Dst: DstReg, Src0: Ctlz, Src1: OneC);
8130 MI.eraseFromParent();
8131 return Legalized;
8132 }
8133 }
8134}
8135
8136// Check that (every element of) Reg is undef or not an exact multiple of BW.
8137static bool isNonZeroModBitWidthOrUndef(const MachineRegisterInfo &MRI,
8138 Register Reg, unsigned BW) {
8139 return matchUnaryPredicate(
8140 MRI, Reg,
8141 Match: [=](const Constant *C) {
8142 // Null constant here means an undef.
8143 const ConstantInt *CI = dyn_cast_or_null<ConstantInt>(Val: C);
8144 return !CI || CI->getValue().urem(RHS: BW) != 0;
8145 },
8146 /*AllowUndefs*/ true);
8147}
8148
8149LegalizerHelper::LegalizeResult
8150LegalizerHelper::lowerFunnelShiftWithInverse(MachineInstr &MI) {
8151 auto [Dst, X, Y, Z] = MI.getFirst4Regs();
8152 LLT Ty = MRI.getType(Reg: Dst);
8153 LLT ShTy = MRI.getType(Reg: Z);
8154
8155 unsigned BW = Ty.getScalarSizeInBits();
8156
8157 if (!isPowerOf2_32(Value: BW))
8158 return UnableToLegalize;
8159
8160 const bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL;
8161 unsigned RevOpcode = IsFSHL ? TargetOpcode::G_FSHR : TargetOpcode::G_FSHL;
8162
8163 if (isNonZeroModBitWidthOrUndef(MRI, Reg: Z, BW)) {
8164 // fshl X, Y, Z -> fshr X, Y, -Z
8165 // fshr X, Y, Z -> fshl X, Y, -Z
8166 auto Zero = MIRBuilder.buildConstant(Res: ShTy, Val: 0);
8167 Z = MIRBuilder.buildSub(Dst: Ty, Src0: Zero, Src1: Z).getReg(Idx: 0);
8168 } else {
8169 // fshl X, Y, Z -> fshr (srl X, 1), (fshr X, Y, 1), ~Z
8170 // fshr X, Y, Z -> fshl (fshl X, Y, 1), (shl Y, 1), ~Z
8171 auto One = MIRBuilder.buildConstant(Res: ShTy, Val: 1);
8172 if (IsFSHL) {
8173 Y = MIRBuilder.buildInstr(Opc: RevOpcode, DstOps: {Ty}, SrcOps: {X, Y, One}).getReg(Idx: 0);
8174 X = MIRBuilder.buildLShr(Dst: Ty, Src0: X, Src1: One).getReg(Idx: 0);
8175 } else {
8176 X = MIRBuilder.buildInstr(Opc: RevOpcode, DstOps: {Ty}, SrcOps: {X, Y, One}).getReg(Idx: 0);
8177 Y = MIRBuilder.buildShl(Dst: Ty, Src0: Y, Src1: One).getReg(Idx: 0);
8178 }
8179
8180 Z = MIRBuilder.buildNot(Dst: ShTy, Src0: Z).getReg(Idx: 0);
8181 }
8182
8183 MIRBuilder.buildInstr(Opc: RevOpcode, DstOps: {Dst}, SrcOps: {X, Y, Z});
8184 MI.eraseFromParent();
8185 return Legalized;
8186}
8187
8188LegalizerHelper::LegalizeResult
8189LegalizerHelper::lowerFunnelShiftAsShifts(MachineInstr &MI) {
8190 auto [Dst, X, Y, Z] = MI.getFirst4Regs();
8191 LLT Ty = MRI.getType(Reg: Dst);
8192 LLT ShTy = MRI.getType(Reg: Z);
8193
8194 const unsigned BW = Ty.getScalarSizeInBits();
8195 const bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL;
8196
8197 Register ShX, ShY;
8198 Register ShAmt, InvShAmt;
8199
8200 // FIXME: Emit optimized urem by constant instead of letting it expand later.
8201 if (isNonZeroModBitWidthOrUndef(MRI, Reg: Z, BW)) {
8202 // fshl: X << C | Y >> (BW - C)
8203 // fshr: X << (BW - C) | Y >> C
8204 // where C = Z % BW is not zero
8205 auto BitWidthC = MIRBuilder.buildConstant(Res: ShTy, Val: BW);
8206 ShAmt = MIRBuilder.buildURem(Dst: ShTy, Src0: Z, Src1: BitWidthC).getReg(Idx: 0);
8207 InvShAmt = MIRBuilder.buildSub(Dst: ShTy, Src0: BitWidthC, Src1: ShAmt).getReg(Idx: 0);
8208 ShX = MIRBuilder.buildShl(Dst: Ty, Src0: X, Src1: IsFSHL ? ShAmt : InvShAmt).getReg(Idx: 0);
8209 ShY = MIRBuilder.buildLShr(Dst: Ty, Src0: Y, Src1: IsFSHL ? InvShAmt : ShAmt).getReg(Idx: 0);
8210 } else {
8211 // fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))
8212 // fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)
8213 auto Mask = MIRBuilder.buildConstant(Res: ShTy, Val: BW - 1);
8214 if (isPowerOf2_32(Value: BW)) {
8215 // Z % BW -> Z & (BW - 1)
8216 ShAmt = MIRBuilder.buildAnd(Dst: ShTy, Src0: Z, Src1: Mask).getReg(Idx: 0);
8217 // (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
8218 auto NotZ = MIRBuilder.buildNot(Dst: ShTy, Src0: Z);
8219 InvShAmt = MIRBuilder.buildAnd(Dst: ShTy, Src0: NotZ, Src1: Mask).getReg(Idx: 0);
8220 } else {
8221 auto BitWidthC = MIRBuilder.buildConstant(Res: ShTy, Val: BW);
8222 ShAmt = MIRBuilder.buildURem(Dst: ShTy, Src0: Z, Src1: BitWidthC).getReg(Idx: 0);
8223 InvShAmt = MIRBuilder.buildSub(Dst: ShTy, Src0: Mask, Src1: ShAmt).getReg(Idx: 0);
8224 }
8225
8226 auto One = MIRBuilder.buildConstant(Res: ShTy, Val: 1);
8227 if (IsFSHL) {
8228 ShX = MIRBuilder.buildShl(Dst: Ty, Src0: X, Src1: ShAmt).getReg(Idx: 0);
8229 auto ShY1 = MIRBuilder.buildLShr(Dst: Ty, Src0: Y, Src1: One);
8230 ShY = MIRBuilder.buildLShr(Dst: Ty, Src0: ShY1, Src1: InvShAmt).getReg(Idx: 0);
8231 } else {
8232 auto ShX1 = MIRBuilder.buildShl(Dst: Ty, Src0: X, Src1: One);
8233 ShX = MIRBuilder.buildShl(Dst: Ty, Src0: ShX1, Src1: InvShAmt).getReg(Idx: 0);
8234 ShY = MIRBuilder.buildLShr(Dst: Ty, Src0: Y, Src1: ShAmt).getReg(Idx: 0);
8235 }
8236 }
8237
8238 MIRBuilder.buildOr(Dst, Src0: ShX, Src1: ShY, Flags: MachineInstr::Disjoint);
8239 MI.eraseFromParent();
8240 return Legalized;
8241}
8242
8243LegalizerHelper::LegalizeResult
8244LegalizerHelper::lowerFunnelShift(MachineInstr &MI) {
8245 // These operations approximately do the following (while avoiding undefined
8246 // shifts by BW):
8247 // G_FSHL: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
8248 // G_FSHR: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
8249 Register Dst = MI.getOperand(i: 0).getReg();
8250 LLT Ty = MRI.getType(Reg: Dst);
8251 LLT ShTy = MRI.getType(Reg: MI.getOperand(i: 3).getReg());
8252
8253 bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL;
8254 unsigned RevOpcode = IsFSHL ? TargetOpcode::G_FSHR : TargetOpcode::G_FSHL;
8255
8256 // TODO: Use smarter heuristic that accounts for vector legalization.
8257 if (LI.getAction(Query: {RevOpcode, {Ty, ShTy}}).Action == Lower)
8258 return lowerFunnelShiftAsShifts(MI);
8259
8260 // This only works for powers of 2, fallback to shifts if it fails.
8261 LegalizerHelper::LegalizeResult Result = lowerFunnelShiftWithInverse(MI);
8262 if (Result == UnableToLegalize)
8263 return lowerFunnelShiftAsShifts(MI);
8264 return Result;
8265}
8266
8267LegalizerHelper::LegalizeResult LegalizerHelper::lowerEXT(MachineInstr &MI) {
8268 auto [Dst, Src] = MI.getFirst2Regs();
8269 LLT DstTy = MRI.getType(Reg: Dst);
8270 LLT SrcTy = MRI.getType(Reg: Src);
8271
8272 uint32_t DstTySize = DstTy.getSizeInBits();
8273 uint32_t DstTyScalarSize = DstTy.getScalarSizeInBits();
8274 uint32_t SrcTyScalarSize = SrcTy.getScalarSizeInBits();
8275
8276 if (!isPowerOf2_32(Value: DstTySize) || !isPowerOf2_32(Value: DstTyScalarSize) ||
8277 !isPowerOf2_32(Value: SrcTyScalarSize))
8278 return UnableToLegalize;
8279
8280 // The step between extend is too large, split it by creating an intermediate
8281 // extend instruction
8282 if (SrcTyScalarSize * 2 < DstTyScalarSize) {
8283 LLT MidTy = SrcTy.changeElementSize(NewEltSize: SrcTyScalarSize * 2);
8284 // If the destination type is illegal, split it into multiple statements
8285 // zext x -> zext(merge(zext(unmerge), zext(unmerge)))
8286 auto NewExt = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {MidTy}, SrcOps: {Src});
8287 // Unmerge the vector
8288 LLT EltTy = MidTy.changeElementCount(
8289 EC: MidTy.getElementCount().divideCoefficientBy(RHS: 2));
8290 auto UnmergeSrc = MIRBuilder.buildUnmerge(Res: EltTy, Op: NewExt);
8291
8292 // ZExt the vectors
8293 LLT ZExtResTy = DstTy.changeElementCount(
8294 EC: DstTy.getElementCount().divideCoefficientBy(RHS: 2));
8295 auto ZExtRes1 = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {ZExtResTy},
8296 SrcOps: {UnmergeSrc.getReg(Idx: 0)});
8297 auto ZExtRes2 = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {ZExtResTy},
8298 SrcOps: {UnmergeSrc.getReg(Idx: 1)});
8299
8300 // Merge the ending vectors
8301 MIRBuilder.buildMergeLikeInstr(Res: Dst, Ops: {ZExtRes1, ZExtRes2});
8302
8303 MI.eraseFromParent();
8304 return Legalized;
8305 }
8306 return UnableToLegalize;
8307}
8308
8309LegalizerHelper::LegalizeResult LegalizerHelper::lowerTRUNC(MachineInstr &MI) {
8310 // MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
8311 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
8312 // Similar to how operand splitting is done in SelectiondDAG, we can handle
8313 // %res(v8s8) = G_TRUNC %in(v8s32) by generating:
8314 // %inlo(<4x s32>), %inhi(<4 x s32>) = G_UNMERGE %in(<8 x s32>)
8315 // %lo16(<4 x s16>) = G_TRUNC %inlo
8316 // %hi16(<4 x s16>) = G_TRUNC %inhi
8317 // %in16(<8 x s16>) = G_CONCAT_VECTORS %lo16, %hi16
8318 // %res(<8 x s8>) = G_TRUNC %in16
8319
8320 assert(MI.getOpcode() == TargetOpcode::G_TRUNC);
8321
8322 Register DstReg = MI.getOperand(i: 0).getReg();
8323 Register SrcReg = MI.getOperand(i: 1).getReg();
8324 LLT DstTy = MRI.getType(Reg: DstReg);
8325 LLT SrcTy = MRI.getType(Reg: SrcReg);
8326
8327 if (DstTy.isVector() && isPowerOf2_32(Value: DstTy.getNumElements()) &&
8328 isPowerOf2_32(Value: DstTy.getScalarSizeInBits()) &&
8329 isPowerOf2_32(Value: SrcTy.getNumElements()) &&
8330 isPowerOf2_32(Value: SrcTy.getScalarSizeInBits())) {
8331 // Split input type.
8332 LLT SplitSrcTy = SrcTy.changeElementCount(
8333 EC: SrcTy.getElementCount().divideCoefficientBy(RHS: 2));
8334
8335 // First, split the source into two smaller vectors.
8336 SmallVector<Register, 2> SplitSrcs;
8337 extractParts(Reg: SrcReg, Ty: SplitSrcTy, NumParts: 2, VRegs&: SplitSrcs, MIRBuilder, MRI);
8338
8339 // Truncate the splits into intermediate narrower elements.
8340 LLT InterTy;
8341 if (DstTy.getScalarSizeInBits() * 2 < SrcTy.getScalarSizeInBits())
8342 InterTy = SplitSrcTy.changeElementSize(NewEltSize: DstTy.getScalarSizeInBits() * 2);
8343 else
8344 InterTy = SplitSrcTy.changeElementSize(NewEltSize: DstTy.getScalarSizeInBits());
8345 for (Register &Src : SplitSrcs)
8346 Src = MIRBuilder.buildTrunc(Res: InterTy, Op: Src).getReg(Idx: 0);
8347
8348 // Combine the new truncates into one vector
8349 auto Merge = MIRBuilder.buildMergeLikeInstr(
8350 Res: DstTy.changeElementSize(NewEltSize: InterTy.getScalarSizeInBits()), Ops: SplitSrcs);
8351
8352 // Truncate the new vector to the final result type
8353 if (DstTy.getScalarSizeInBits() * 2 < SrcTy.getScalarSizeInBits())
8354 MIRBuilder.buildTrunc(Res: MI.getOperand(i: 0).getReg(), Op: Merge.getReg(Idx: 0));
8355 else
8356 MIRBuilder.buildCopy(Res: MI.getOperand(i: 0).getReg(), Op: Merge.getReg(Idx: 0));
8357
8358 MI.eraseFromParent();
8359
8360 return Legalized;
8361 }
8362 return UnableToLegalize;
8363}
8364
8365LegalizerHelper::LegalizeResult
8366LegalizerHelper::lowerRotateWithReverseRotate(MachineInstr &MI) {
8367 auto [Dst, DstTy, Src, SrcTy, Amt, AmtTy] = MI.getFirst3RegLLTs();
8368 auto Zero = MIRBuilder.buildConstant(Res: AmtTy, Val: 0);
8369 bool IsLeft = MI.getOpcode() == TargetOpcode::G_ROTL;
8370 unsigned RevRot = IsLeft ? TargetOpcode::G_ROTR : TargetOpcode::G_ROTL;
8371 auto Neg = MIRBuilder.buildSub(Dst: AmtTy, Src0: Zero, Src1: Amt);
8372 MIRBuilder.buildInstr(Opc: RevRot, DstOps: {Dst}, SrcOps: {Src, Neg});
8373 MI.eraseFromParent();
8374 return Legalized;
8375}
8376
8377LegalizerHelper::LegalizeResult LegalizerHelper::lowerRotate(MachineInstr &MI) {
8378 auto [Dst, DstTy, Src, SrcTy, Amt, AmtTy] = MI.getFirst3RegLLTs();
8379
8380 unsigned EltSizeInBits = DstTy.getScalarSizeInBits();
8381 bool IsLeft = MI.getOpcode() == TargetOpcode::G_ROTL;
8382
8383 MIRBuilder.setInstrAndDebugLoc(MI);
8384
8385 // If a rotate in the other direction is supported, use it.
8386 unsigned RevRot = IsLeft ? TargetOpcode::G_ROTR : TargetOpcode::G_ROTL;
8387 if (LI.isLegalOrCustom(Query: {RevRot, {DstTy, SrcTy}}) &&
8388 isPowerOf2_32(Value: EltSizeInBits))
8389 return lowerRotateWithReverseRotate(MI);
8390
8391 // If a funnel shift is supported, use it.
8392 unsigned FShOpc = IsLeft ? TargetOpcode::G_FSHL : TargetOpcode::G_FSHR;
8393 unsigned RevFsh = !IsLeft ? TargetOpcode::G_FSHL : TargetOpcode::G_FSHR;
8394 bool IsFShLegal = false;
8395 if ((IsFShLegal = LI.isLegalOrCustom(Query: {FShOpc, {DstTy, AmtTy}})) ||
8396 LI.isLegalOrCustom(Query: {RevFsh, {DstTy, AmtTy}})) {
8397 auto buildFunnelShift = [&](unsigned Opc, Register R1, Register R2,
8398 Register R3) {
8399 MIRBuilder.buildInstr(Opc, DstOps: {R1}, SrcOps: {R2, R2, R3});
8400 MI.eraseFromParent();
8401 return Legalized;
8402 };
8403 // If a funnel shift in the other direction is supported, use it.
8404 if (IsFShLegal) {
8405 return buildFunnelShift(FShOpc, Dst, Src, Amt);
8406 } else if (isPowerOf2_32(Value: EltSizeInBits)) {
8407 Amt = MIRBuilder.buildNeg(Dst: DstTy, Src0: Amt).getReg(Idx: 0);
8408 return buildFunnelShift(RevFsh, Dst, Src, Amt);
8409 }
8410 }
8411
8412 auto Zero = MIRBuilder.buildConstant(Res: AmtTy, Val: 0);
8413 unsigned ShOpc = IsLeft ? TargetOpcode::G_SHL : TargetOpcode::G_LSHR;
8414 unsigned RevShiftOpc = IsLeft ? TargetOpcode::G_LSHR : TargetOpcode::G_SHL;
8415 auto BitWidthMinusOneC = MIRBuilder.buildConstant(Res: AmtTy, Val: EltSizeInBits - 1);
8416 Register ShVal;
8417 Register RevShiftVal;
8418 if (isPowerOf2_32(Value: EltSizeInBits)) {
8419 // (rotl x, c) -> x << (c & (w - 1)) | x >> (-c & (w - 1))
8420 // (rotr x, c) -> x >> (c & (w - 1)) | x << (-c & (w - 1))
8421 auto NegAmt = MIRBuilder.buildSub(Dst: AmtTy, Src0: Zero, Src1: Amt);
8422 auto ShAmt = MIRBuilder.buildAnd(Dst: AmtTy, Src0: Amt, Src1: BitWidthMinusOneC);
8423 ShVal = MIRBuilder.buildInstr(Opc: ShOpc, DstOps: {DstTy}, SrcOps: {Src, ShAmt}).getReg(Idx: 0);
8424 auto RevAmt = MIRBuilder.buildAnd(Dst: AmtTy, Src0: NegAmt, Src1: BitWidthMinusOneC);
8425 RevShiftVal =
8426 MIRBuilder.buildInstr(Opc: RevShiftOpc, DstOps: {DstTy}, SrcOps: {Src, RevAmt}).getReg(Idx: 0);
8427 } else {
8428 // (rotl x, c) -> x << (c % w) | x >> 1 >> (w - 1 - (c % w))
8429 // (rotr x, c) -> x >> (c % w) | x << 1 << (w - 1 - (c % w))
8430 auto BitWidthC = MIRBuilder.buildConstant(Res: AmtTy, Val: EltSizeInBits);
8431 auto ShAmt = MIRBuilder.buildURem(Dst: AmtTy, Src0: Amt, Src1: BitWidthC);
8432 ShVal = MIRBuilder.buildInstr(Opc: ShOpc, DstOps: {DstTy}, SrcOps: {Src, ShAmt}).getReg(Idx: 0);
8433 auto RevAmt = MIRBuilder.buildSub(Dst: AmtTy, Src0: BitWidthMinusOneC, Src1: ShAmt);
8434 auto One = MIRBuilder.buildConstant(Res: AmtTy, Val: 1);
8435 auto Inner = MIRBuilder.buildInstr(Opc: RevShiftOpc, DstOps: {DstTy}, SrcOps: {Src, One});
8436 RevShiftVal =
8437 MIRBuilder.buildInstr(Opc: RevShiftOpc, DstOps: {DstTy}, SrcOps: {Inner, RevAmt}).getReg(Idx: 0);
8438 }
8439 MIRBuilder.buildOr(Dst, Src0: ShVal, Src1: RevShiftVal, Flags: MachineInstr::Disjoint);
8440 MI.eraseFromParent();
8441 return Legalized;
8442}
8443
8444// Expand s32 = G_UITOFP s64 to an IEEE float representation using bit
8445// operations and G_SITOFP
8446LegalizerHelper::LegalizeResult
8447LegalizerHelper::lowerU64ToF32WithSITOFP(MachineInstr &MI) {
8448 auto [Dst, Src] = MI.getFirst2Regs();
8449 const LLT S64 = LLT::scalar(SizeInBits: 64);
8450 const LLT S32 = LLT::scalar(SizeInBits: 32);
8451 const LLT S1 = LLT::scalar(SizeInBits: 1);
8452
8453 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S32);
8454
8455 // For i64 < INT_MAX we simply reuse SITOFP.
8456 // Otherwise, divide i64 by 2, round result by ORing with the lowest bit
8457 // saved before division, convert to float by SITOFP, multiply the result
8458 // by 2.
8459 auto One = MIRBuilder.buildConstant(Res: S64, Val: 1);
8460 auto Zero = MIRBuilder.buildConstant(Res: S64, Val: 0);
8461 // Result if Src < INT_MAX
8462 auto SmallResult = MIRBuilder.buildSITOFP(Dst: S32, Src0: Src);
8463 // Result if Src >= INT_MAX
8464 auto Halved = MIRBuilder.buildLShr(Dst: S64, Src0: Src, Src1: One);
8465 auto LowerBit = MIRBuilder.buildAnd(Dst: S64, Src0: Src, Src1: One);
8466 auto RoundedHalved = MIRBuilder.buildOr(Dst: S64, Src0: Halved, Src1: LowerBit);
8467 auto HalvedFP = MIRBuilder.buildSITOFP(Dst: S32, Src0: RoundedHalved);
8468 auto LargeResult = MIRBuilder.buildFAdd(Dst: S32, Src0: HalvedFP, Src1: HalvedFP);
8469 // Check if the original value is larger than INT_MAX by comparing with
8470 // zero to pick one of the two conversions.
8471 auto IsLarge =
8472 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_SLT, Res: S1, Op0: Src, Op1: Zero);
8473 MIRBuilder.buildSelect(Res: Dst, Tst: IsLarge, Op0: LargeResult, Op1: SmallResult);
8474
8475 MI.eraseFromParent();
8476 return Legalized;
8477}
8478
8479// Expand s64 = G_UITOFP s64 using bit and float arithmetic operations to an
8480// IEEE double representation.
8481LegalizerHelper::LegalizeResult
8482LegalizerHelper::lowerU64ToF64BitFloatOps(MachineInstr &MI) {
8483 auto [Dst, Src] = MI.getFirst2Regs();
8484 const LLT S64 = LLT::scalar(SizeInBits: 64);
8485 const LLT S32 = LLT::scalar(SizeInBits: 32);
8486
8487 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S64);
8488
8489 // We create double value from 32 bit parts with 32 exponent difference.
8490 // Note that + and - are float operations that adjust the implicit leading
8491 // one, the bases 2^52 and 2^84 are for illustrative purposes.
8492 //
8493 // X = 2^52 * 1.0...LowBits
8494 // Y = 2^84 * 1.0...HighBits
8495 // Scratch = 2^84 * 1.0...HighBits - 2^84 * 1.0 - 2^52 * 1.0
8496 // = - 2^52 * 1.0...HighBits
8497 // Result = - 2^52 * 1.0...HighBits + 2^52 * 1.0...LowBits
8498 auto TwoP52 = MIRBuilder.buildConstant(Res: S64, UINT64_C(0x4330000000000000));
8499 auto TwoP84 = MIRBuilder.buildConstant(Res: S64, UINT64_C(0x4530000000000000));
8500 auto TwoP52P84 = llvm::bit_cast<double>(UINT64_C(0x4530000000100000));
8501 auto TwoP52P84FP = MIRBuilder.buildFConstant(Res: S64, Val: TwoP52P84);
8502 auto HalfWidth = MIRBuilder.buildConstant(Res: S64, Val: 32);
8503
8504 auto LowBits = MIRBuilder.buildTrunc(Res: S32, Op: Src);
8505 LowBits = MIRBuilder.buildZExt(Res: S64, Op: LowBits);
8506 auto LowBitsFP = MIRBuilder.buildOr(Dst: S64, Src0: TwoP52, Src1: LowBits);
8507 auto HighBits = MIRBuilder.buildLShr(Dst: S64, Src0: Src, Src1: HalfWidth);
8508 auto HighBitsFP = MIRBuilder.buildOr(Dst: S64, Src0: TwoP84, Src1: HighBits);
8509 auto Scratch = MIRBuilder.buildFSub(Dst: S64, Src0: HighBitsFP, Src1: TwoP52P84FP);
8510 MIRBuilder.buildFAdd(Dst, Src0: Scratch, Src1: LowBitsFP);
8511
8512 MI.eraseFromParent();
8513 return Legalized;
8514}
8515
8516/// i64->fp16 itofp can be lowered to i64->f64,f64->f32,f32->f16. We cannot
8517/// convert fpround f64->f16 without double-rounding, so we manually perform the
8518/// lowering here where we know it is valid.
8519static LegalizerHelper::LegalizeResult
8520loweri64tof16ITOFP(MachineInstr &MI, Register Dst, LLT DstTy, Register Src,
8521 LLT SrcTy, MachineIRBuilder &MIRBuilder) {
8522 auto DstFpTy =
8523 SrcTy.changeElementType(NewEltTy: LLT::floatIEEE(SizeInBits: SrcTy.getScalarSizeInBits()));
8524 auto M1 = MI.getOpcode() == TargetOpcode::G_UITOFP
8525 ? MIRBuilder.buildUITOFP(Dst: DstFpTy, Src0: Src)
8526 : MIRBuilder.buildSITOFP(Dst: DstFpTy, Src0: Src);
8527 LLT F32Ty = DstFpTy.changeElementSize(NewEltSize: 32);
8528 auto M2 = MIRBuilder.buildFPTrunc(Res: F32Ty, Op: M1);
8529 MIRBuilder.buildFPTrunc(Res: Dst, Op: M2);
8530 MI.eraseFromParent();
8531 return LegalizerHelper::Legalized;
8532}
8533
8534LegalizerHelper::LegalizeResult LegalizerHelper::lowerUITOFP(MachineInstr &MI) {
8535 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8536
8537 if (SrcTy == LLT::scalar(SizeInBits: 1)) {
8538 auto True = MIRBuilder.buildFConstant(Res: DstTy, Val: 1.0);
8539 auto False = MIRBuilder.buildFConstant(Res: DstTy, Val: 0.0);
8540 MIRBuilder.buildSelect(Res: Dst, Tst: Src, Op0: True, Op1: False);
8541 MI.eraseFromParent();
8542 return Legalized;
8543 }
8544
8545 if (DstTy.getScalarSizeInBits() == 16 && SrcTy.getScalarSizeInBits() == 64)
8546 return loweri64tof16ITOFP(MI, Dst, DstTy, Src, SrcTy, MIRBuilder);
8547
8548 if (SrcTy != LLT::scalar(SizeInBits: 64))
8549 return UnableToLegalize;
8550
8551 if (DstTy == LLT::scalar(SizeInBits: 32))
8552 // TODO: SelectionDAG has several alternative expansions to port which may
8553 // be more reasonable depending on the available instructions. We also need
8554 // a more advanced mechanism to choose an optimal version depending on
8555 // target features such as sitofp or CTLZ availability.
8556 return lowerU64ToF32WithSITOFP(MI);
8557
8558 if (DstTy == LLT::scalar(SizeInBits: 64))
8559 return lowerU64ToF64BitFloatOps(MI);
8560
8561 return UnableToLegalize;
8562}
8563
8564LegalizerHelper::LegalizeResult LegalizerHelper::lowerSITOFP(MachineInstr &MI) {
8565 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8566
8567 const LLT I64 = LLT::integer(SizeInBits: 64);
8568 const LLT I32 = LLT::integer(SizeInBits: 32);
8569 const LLT I1 = LLT::integer(SizeInBits: 1);
8570
8571 if (SrcTy == I1) {
8572 auto True = MIRBuilder.buildFConstant(Res: DstTy, Val: -1.0);
8573 auto False = MIRBuilder.buildFConstant(Res: DstTy, Val: 0.0);
8574 MIRBuilder.buildSelect(Res: Dst, Tst: Src, Op0: True, Op1: False);
8575 MI.eraseFromParent();
8576 return Legalized;
8577 }
8578
8579 if (DstTy.getScalarSizeInBits() == 16 && SrcTy.getScalarSizeInBits() == 64)
8580 return loweri64tof16ITOFP(MI, Dst, DstTy, Src, SrcTy, MIRBuilder);
8581
8582 if (SrcTy != I64)
8583 return UnableToLegalize;
8584
8585 if (DstTy.getScalarSizeInBits() == 32) {
8586 // signed cl2f(long l) {
8587 // long s = l >> 63;
8588 // float r = cul2f((l + s) ^ s);
8589 // return s ? -r : r;
8590 // }
8591 Register L = Src;
8592 auto SignBit = MIRBuilder.buildConstant(Res: I64, Val: 63);
8593 auto S = MIRBuilder.buildAShr(Dst: I64, Src0: L, Src1: SignBit);
8594
8595 auto LPlusS = MIRBuilder.buildAdd(Dst: I64, Src0: L, Src1: S);
8596 auto Xor = MIRBuilder.buildXor(Dst: I64, Src0: LPlusS, Src1: S);
8597 auto R = MIRBuilder.buildUITOFP(Dst: I32, Src0: Xor);
8598
8599 auto RNeg = MIRBuilder.buildFNeg(Dst: I32, Src0: R);
8600 auto SignNotZero = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: I1, Op0: S,
8601 Op1: MIRBuilder.buildConstant(Res: I64, Val: 0));
8602 MIRBuilder.buildSelect(Res: Dst, Tst: SignNotZero, Op0: RNeg, Op1: R);
8603 MI.eraseFromParent();
8604 return Legalized;
8605 }
8606
8607 return UnableToLegalize;
8608}
8609
8610LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOUI(MachineInstr &MI) {
8611 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8612 const LLT S64 = LLT::scalar(SizeInBits: 64);
8613 const LLT S32 = LLT::scalar(SizeInBits: 32);
8614
8615 if (SrcTy != S64 && SrcTy != S32)
8616 return UnableToLegalize;
8617 if (DstTy != S32 && DstTy != S64)
8618 return UnableToLegalize;
8619
8620 // FPTOSI gives same result as FPTOUI for positive signed integers.
8621 // FPTOUI needs to deal with fp values that convert to unsigned integers
8622 // greater or equal to 2^31 for float or 2^63 for double. For brevity 2^Exp.
8623
8624 APInt TwoPExpInt = APInt::getSignMask(BitWidth: DstTy.getSizeInBits());
8625 APFloat TwoPExpFP(SrcTy.getSizeInBits() == 32 ? APFloat::IEEEsingle()
8626 : APFloat::IEEEdouble(),
8627 APInt::getZero(numBits: SrcTy.getSizeInBits()));
8628 TwoPExpFP.convertFromAPInt(Input: TwoPExpInt, IsSigned: false, RM: APFloat::rmNearestTiesToEven);
8629
8630 MachineInstrBuilder FPTOSI = MIRBuilder.buildFPTOSI(Dst: DstTy, Src0: Src);
8631
8632 MachineInstrBuilder Threshold = MIRBuilder.buildFConstant(Res: SrcTy, Val: TwoPExpFP);
8633 // For fp Value greater or equal to Threshold(2^Exp), we use FPTOSI on
8634 // (Value - 2^Exp) and add 2^Exp by setting highest bit in result to 1.
8635 MachineInstrBuilder FSub = MIRBuilder.buildFSub(Dst: SrcTy, Src0: Src, Src1: Threshold);
8636 MachineInstrBuilder ResLowBits = MIRBuilder.buildFPTOSI(Dst: DstTy, Src0: FSub);
8637 MachineInstrBuilder ResHighBit = MIRBuilder.buildConstant(Res: DstTy, Val: TwoPExpInt);
8638 MachineInstrBuilder Res = MIRBuilder.buildXor(Dst: DstTy, Src0: ResLowBits, Src1: ResHighBit);
8639
8640 const LLT S1 = LLT::scalar(SizeInBits: 1);
8641
8642 MachineInstrBuilder FCMP =
8643 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_ULT, Res: S1, Op0: Src, Op1: Threshold);
8644 MIRBuilder.buildSelect(Res: Dst, Tst: FCMP, Op0: FPTOSI, Op1: Res);
8645
8646 MI.eraseFromParent();
8647 return Legalized;
8648}
8649
8650LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOSI(MachineInstr &MI) {
8651 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8652 const LLT S64 = LLT::scalar(SizeInBits: 64);
8653 const LLT S32 = LLT::scalar(SizeInBits: 32);
8654
8655 // FIXME: Only f32 to i64 conversions are supported.
8656 if (SrcTy.getScalarType() != S32 || DstTy.getScalarType() != S64)
8657 return UnableToLegalize;
8658
8659 // Expand f32 -> i64 conversion
8660 // This algorithm comes from compiler-rt's implementation of fixsfdi:
8661 // https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/fixsfdi.c
8662
8663 unsigned SrcEltBits = SrcTy.getScalarSizeInBits();
8664
8665 auto ExponentMask = MIRBuilder.buildConstant(Res: SrcTy, Val: 0x7F800000);
8666 auto ExponentLoBit = MIRBuilder.buildConstant(Res: SrcTy, Val: 23);
8667
8668 auto AndExpMask = MIRBuilder.buildAnd(Dst: SrcTy, Src0: Src, Src1: ExponentMask);
8669 auto ExponentBits = MIRBuilder.buildLShr(Dst: SrcTy, Src0: AndExpMask, Src1: ExponentLoBit);
8670
8671 auto SignMask = MIRBuilder.buildConstant(Res: SrcTy,
8672 Val: APInt::getSignMask(BitWidth: SrcEltBits));
8673 auto AndSignMask = MIRBuilder.buildAnd(Dst: SrcTy, Src0: Src, Src1: SignMask);
8674 auto SignLowBit = MIRBuilder.buildConstant(Res: SrcTy, Val: SrcEltBits - 1);
8675 auto Sign = MIRBuilder.buildAShr(Dst: SrcTy, Src0: AndSignMask, Src1: SignLowBit);
8676 Sign = MIRBuilder.buildSExt(Res: DstTy, Op: Sign);
8677
8678 auto MantissaMask = MIRBuilder.buildConstant(Res: SrcTy, Val: 0x007FFFFF);
8679 auto AndMantissaMask = MIRBuilder.buildAnd(Dst: SrcTy, Src0: Src, Src1: MantissaMask);
8680 auto K = MIRBuilder.buildConstant(Res: SrcTy, Val: 0x00800000);
8681
8682 auto R = MIRBuilder.buildOr(Dst: SrcTy, Src0: AndMantissaMask, Src1: K);
8683 R = MIRBuilder.buildZExt(Res: DstTy, Op: R);
8684
8685 auto Bias = MIRBuilder.buildConstant(Res: SrcTy, Val: 127);
8686 auto Exponent = MIRBuilder.buildSub(Dst: SrcTy, Src0: ExponentBits, Src1: Bias);
8687 auto SubExponent = MIRBuilder.buildSub(Dst: SrcTy, Src0: Exponent, Src1: ExponentLoBit);
8688 auto ExponentSub = MIRBuilder.buildSub(Dst: SrcTy, Src0: ExponentLoBit, Src1: Exponent);
8689
8690 auto Shl = MIRBuilder.buildShl(Dst: DstTy, Src0: R, Src1: SubExponent);
8691 auto Srl = MIRBuilder.buildLShr(Dst: DstTy, Src0: R, Src1: ExponentSub);
8692
8693 const LLT S1 = LLT::scalar(SizeInBits: 1);
8694 auto CmpGt = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SGT,
8695 Res: S1, Op0: Exponent, Op1: ExponentLoBit);
8696
8697 R = MIRBuilder.buildSelect(Res: DstTy, Tst: CmpGt, Op0: Shl, Op1: Srl);
8698
8699 auto XorSign = MIRBuilder.buildXor(Dst: DstTy, Src0: R, Src1: Sign);
8700 auto Ret = MIRBuilder.buildSub(Dst: DstTy, Src0: XorSign, Src1: Sign);
8701
8702 auto ZeroSrcTy = MIRBuilder.buildConstant(Res: SrcTy, Val: 0);
8703
8704 auto ExponentLt0 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT,
8705 Res: S1, Op0: Exponent, Op1: ZeroSrcTy);
8706
8707 auto ZeroDstTy = MIRBuilder.buildConstant(Res: DstTy, Val: 0);
8708 MIRBuilder.buildSelect(Res: Dst, Tst: ExponentLt0, Op0: ZeroDstTy, Op1: Ret);
8709
8710 MI.eraseFromParent();
8711 return Legalized;
8712}
8713
8714LegalizerHelper::LegalizeResult
8715LegalizerHelper::lowerFPTOINT_SAT(MachineInstr &MI) {
8716 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
8717
8718 bool IsSigned = MI.getOpcode() == TargetOpcode::G_FPTOSI_SAT;
8719 unsigned SatWidth = DstTy.getScalarSizeInBits();
8720
8721 // Determine minimum and maximum integer values and their corresponding
8722 // floating-point values.
8723 APInt MinInt, MaxInt;
8724 if (IsSigned) {
8725 MinInt = APInt::getSignedMinValue(numBits: SatWidth);
8726 MaxInt = APInt::getSignedMaxValue(numBits: SatWidth);
8727 } else {
8728 MinInt = APInt::getMinValue(numBits: SatWidth);
8729 MaxInt = APInt::getMaxValue(numBits: SatWidth);
8730 }
8731
8732 const fltSemantics &Semantics = getFltSemanticForLLT(Ty: SrcTy.getScalarType());
8733 APFloat MinFloat(Semantics);
8734 APFloat MaxFloat(Semantics);
8735
8736 APFloat::opStatus MinStatus =
8737 MinFloat.convertFromAPInt(Input: MinInt, IsSigned, RM: APFloat::rmTowardZero);
8738 APFloat::opStatus MaxStatus =
8739 MaxFloat.convertFromAPInt(Input: MaxInt, IsSigned, RM: APFloat::rmTowardZero);
8740 bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact) &&
8741 !(MaxStatus & APFloat::opStatus::opInexact);
8742
8743 // If the integer bounds are exactly representable as floats, emit a
8744 // min+max+fptoi sequence. Otherwise we have to use a sequence of comparisons
8745 // and selects.
8746 if (AreExactFloatBounds) {
8747 // Clamp Src by MinFloat from below. If Src is NaN the result is MinFloat.
8748 auto MaxC = MIRBuilder.buildFConstant(Res: SrcTy, Val: MinFloat);
8749 auto MaxP =
8750 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OGT, Res: LLT::integer(SizeInBits: 1), Op0: Src, Op1: MaxC);
8751 auto Max = MIRBuilder.buildSelect(Res: SrcTy, Tst: MaxP, Op0: Src, Op1: MaxC);
8752 // Clamp by MaxFloat from above. NaN cannot occur.
8753 auto MinC = MIRBuilder.buildFConstant(Res: SrcTy, Val: MaxFloat);
8754 auto MinP = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OLT, Res: LLT::integer(SizeInBits: 1), Op0: Max,
8755 Op1: MinC, Flags: MachineInstr::FmNoNans);
8756 auto Min =
8757 MIRBuilder.buildSelect(Res: SrcTy, Tst: MinP, Op0: Max, Op1: MinC, Flags: MachineInstr::FmNoNans);
8758 // Convert clamped value to integer. In the unsigned case we're done,
8759 // because we mapped NaN to MinFloat, which will cast to zero.
8760 if (!IsSigned) {
8761 MIRBuilder.buildFPTOUI(Dst, Src0: Min);
8762 MI.eraseFromParent();
8763 return Legalized;
8764 }
8765
8766 // Otherwise, select 0 if Src is NaN.
8767 auto FpToInt = MIRBuilder.buildFPTOSI(Dst: DstTy, Src0: Min);
8768 auto IsZero =
8769 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_UNO, Res: LLT::integer(SizeInBits: 1), Op0: Src, Op1: Src);
8770 MIRBuilder.buildSelect(Res: Dst, Tst: IsZero, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: 0),
8771 Op1: FpToInt);
8772 MI.eraseFromParent();
8773 return Legalized;
8774 }
8775
8776 // Result of direct conversion. The assumption here is that the operation is
8777 // non-trapping and it's fine to apply it to an out-of-range value if we
8778 // select it away later.
8779 auto FpToInt = IsSigned ? MIRBuilder.buildFPTOSI(Dst: DstTy, Src0: Src)
8780 : MIRBuilder.buildFPTOUI(Dst: DstTy, Src0: Src);
8781
8782 // If Src ULT MinFloat, select MinInt. In particular, this also selects
8783 // MinInt if Src is NaN.
8784 auto ULT = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_ULT, Res: LLT::integer(SizeInBits: 1), Op0: Src,
8785 Op1: MIRBuilder.buildFConstant(Res: SrcTy, Val: MinFloat));
8786 auto Max = MIRBuilder.buildSelect(
8787 Res: DstTy, Tst: ULT, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: MinInt), Op1: FpToInt);
8788 // If Src OGT MaxFloat, select MaxInt.
8789 auto OGT = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OGT, Res: LLT::integer(SizeInBits: 1), Op0: Src,
8790 Op1: MIRBuilder.buildFConstant(Res: SrcTy, Val: MaxFloat));
8791
8792 // In the unsigned case we are done, because we mapped NaN to MinInt, which
8793 // is already zero.
8794 if (!IsSigned) {
8795 MIRBuilder.buildSelect(Res: Dst, Tst: OGT, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: MaxInt),
8796 Op1: Max);
8797 MI.eraseFromParent();
8798 return Legalized;
8799 }
8800
8801 // Otherwise, select 0 if Src is NaN.
8802 auto Min = MIRBuilder.buildSelect(
8803 Res: DstTy, Tst: OGT, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: MaxInt), Op1: Max);
8804 auto IsZero =
8805 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_UNO, Res: LLT::integer(SizeInBits: 1), Op0: Src, Op1: Src);
8806 MIRBuilder.buildSelect(Res: Dst, Tst: IsZero, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: 0), Op1: Min);
8807 MI.eraseFromParent();
8808 return Legalized;
8809}
8810
8811// Floating-point conversions using truncating and extending loads and stores.
8812LegalizerHelper::LegalizeResult
8813LegalizerHelper::lowerFPExtAndTruncMem(MachineInstr &MI) {
8814 assert((MI.getOpcode() == TargetOpcode::G_FPEXT ||
8815 MI.getOpcode() == TargetOpcode::G_FPTRUNC) &&
8816 "Only G_FPEXT and G_FPTRUNC are expected");
8817
8818 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
8819 MachinePointerInfo PtrInfo;
8820 unsigned StoreOpc;
8821 unsigned LoadOpc;
8822 LLT StackTy;
8823 if (MI.getOpcode() == TargetOpcode::G_FPEXT) {
8824 StackTy = SrcTy;
8825 StoreOpc = TargetOpcode::G_STORE;
8826 LoadOpc = TargetOpcode::G_FPEXTLOAD;
8827 } else {
8828 StackTy = DstTy;
8829 StoreOpc = TargetOpcode::G_FPTRUNCSTORE;
8830 LoadOpc = TargetOpcode::G_LOAD;
8831 }
8832
8833 Align StackTyAlign = getStackTemporaryAlignment(Ty: StackTy);
8834 auto StackTemp =
8835 createStackTemporary(Bytes: StackTy.getSizeInBytes(), Alignment: StackTyAlign, PtrInfo);
8836
8837 MachineFunction &MF = MIRBuilder.getMF();
8838 auto *StoreMMO = MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
8839 MemTy: StackTy, BaseAlignment: StackTyAlign);
8840 MIRBuilder.buildStoreInstr(Opcode: StoreOpc, Val: SrcReg, Addr: StackTemp, MMO&: *StoreMMO);
8841
8842 auto *LoadMMO = MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOLoad,
8843 MemTy: StackTy, BaseAlignment: StackTyAlign);
8844 MIRBuilder.buildLoadInstr(Opcode: LoadOpc, Res: DstReg, Addr: StackTemp, MMO&: *LoadMMO);
8845
8846 MI.eraseFromParent();
8847 return Legalized;
8848}
8849
8850// Expand a bf16 -> f32/f64 fpext with a shift and bitcast. This is based on the
8851// SDAG ISD::BF16_TO_FP lowering.
8852LegalizerHelper::LegalizeResult
8853LegalizerHelper::lowerFPEXT_BF16(MachineInstr &MI) {
8854 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
8855 assert(SrcTy.getScalarType().isBFloat16() &&
8856 "expected a bf16 source for bf16 fpext lowering");
8857
8858 LLT I16Ty = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 16));
8859 LLT I32Ty = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 32));
8860 LLT F32Ty = SrcTy.changeElementType(NewEltTy: LLT::float32());
8861
8862 auto SrcI =
8863 MIRBuilder.buildAnyExt(Res: I32Ty, Op: MIRBuilder.buildBitcast(Dst: I16Ty, Src: SrcReg));
8864 auto Shl =
8865 MIRBuilder.buildShl(Dst: I32Ty, Src0: SrcI, Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 16));
8866
8867 if (DstTy.getScalarType().isFloat32())
8868 MIRBuilder.buildBitcast(Dst: DstReg, Src: Shl);
8869 else
8870 MIRBuilder.buildFPExt(Res: DstReg, Op: MIRBuilder.buildBitcast(Dst: F32Ty, Src: Shl));
8871
8872 MI.eraseFromParent();
8873 return Legalized;
8874}
8875
8876LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPEXT(MachineInstr &MI) {
8877 auto [DstTy, SrcTy] = MI.getFirst2LLTs();
8878 if (SrcTy.getScalarType().isBFloat16() &&
8879 (DstTy.getScalarType().isFloat32() || DstTy.getScalarType().isFloat64()))
8880 return lowerFPEXT_BF16(MI);
8881
8882 return lowerFPExtAndTruncMem(MI);
8883}
8884
8885// f64 -> f16 conversion using round-to-nearest-even rounding mode.
8886LegalizerHelper::LegalizeResult
8887LegalizerHelper::lowerFPTRUNC_F64_TO_F16(MachineInstr &MI) {
8888 const LLT S1 = LLT::scalar(SizeInBits: 1);
8889 const LLT I32 = LLT::integer(SizeInBits: 32);
8890
8891 auto [Dst, Src] = MI.getFirst2Regs();
8892 assert(MRI.getType(Dst).getScalarType() == LLT::float16() &&
8893 MRI.getType(Src).getScalarType() == LLT::float64());
8894
8895 if (MRI.getType(Reg: Src).isVector()) // TODO: Handle vectors directly.
8896 return UnableToLegalize;
8897
8898 if (MI.getFlag(Flag: MachineInstr::FmAfn)) {
8899 unsigned Flags = MI.getFlags();
8900 auto Src32 = MIRBuilder.buildFPTrunc(Res: LLT::float32(), Op: Src, Flags);
8901 MIRBuilder.buildFPTrunc(Res: Dst, Op: Src32, Flags);
8902 MI.eraseFromParent();
8903 return Legalized;
8904 }
8905
8906 const unsigned ExpMask = 0x7ff;
8907 const unsigned ExpBiasf64 = 1023;
8908 const unsigned ExpBiasf16 = 15;
8909
8910 auto Unmerge = MIRBuilder.buildUnmerge(Res: I32, Op: Src);
8911 Register U = Unmerge.getReg(Idx: 0);
8912 Register UH = Unmerge.getReg(Idx: 1);
8913
8914 auto E = MIRBuilder.buildLShr(Dst: I32, Src0: UH, Src1: MIRBuilder.buildConstant(Res: I32, Val: 20));
8915 E = MIRBuilder.buildAnd(Dst: I32, Src0: E, Src1: MIRBuilder.buildConstant(Res: I32, Val: ExpMask));
8916
8917 // Subtract the fp64 exponent bias (1023) to get the real exponent and
8918 // add the f16 bias (15) to get the biased exponent for the f16 format.
8919 E = MIRBuilder.buildAdd(
8920 Dst: I32, Src0: E, Src1: MIRBuilder.buildConstant(Res: I32, Val: -ExpBiasf64 + ExpBiasf16));
8921
8922 auto M = MIRBuilder.buildLShr(Dst: I32, Src0: UH, Src1: MIRBuilder.buildConstant(Res: I32, Val: 8));
8923 M = MIRBuilder.buildAnd(Dst: I32, Src0: M, Src1: MIRBuilder.buildConstant(Res: I32, Val: 0xffe));
8924
8925 auto MaskedSig =
8926 MIRBuilder.buildAnd(Dst: I32, Src0: UH, Src1: MIRBuilder.buildConstant(Res: I32, Val: 0x1ff));
8927 MaskedSig = MIRBuilder.buildOr(Dst: I32, Src0: MaskedSig, Src1: U);
8928
8929 auto Zero = MIRBuilder.buildConstant(Res: I32, Val: 0);
8930 auto SigCmpNE0 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: S1, Op0: MaskedSig, Op1: Zero);
8931 auto Lo40Set = MIRBuilder.buildZExt(Res: I32, Op: SigCmpNE0);
8932 M = MIRBuilder.buildOr(Dst: I32, Src0: M, Src1: Lo40Set);
8933
8934 // (M != 0 ? 0x0200 : 0) | 0x7c00;
8935 auto Bits0x200 = MIRBuilder.buildConstant(Res: I32, Val: 0x0200);
8936 auto CmpM_NE0 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: S1, Op0: M, Op1: Zero);
8937 auto SelectCC = MIRBuilder.buildSelect(Res: I32, Tst: CmpM_NE0, Op0: Bits0x200, Op1: Zero);
8938
8939 auto Bits0x7c00 = MIRBuilder.buildConstant(Res: I32, Val: 0x7c00);
8940 auto I = MIRBuilder.buildOr(Dst: I32, Src0: SelectCC, Src1: Bits0x7c00);
8941
8942 // N = M | (E << 12);
8943 auto EShl12 = MIRBuilder.buildShl(Dst: I32, Src0: E, Src1: MIRBuilder.buildConstant(Res: I32, Val: 12));
8944 auto N = MIRBuilder.buildOr(Dst: I32, Src0: M, Src1: EShl12);
8945
8946 // B = clamp(1-E, 0, 13);
8947 auto One = MIRBuilder.buildConstant(Res: I32, Val: 1);
8948 auto OneSubExp = MIRBuilder.buildSub(Dst: I32, Src0: One, Src1: E);
8949 auto B = MIRBuilder.buildSMax(Dst: I32, Src0: OneSubExp, Src1: Zero);
8950 B = MIRBuilder.buildSMin(Dst: I32, Src0: B, Src1: MIRBuilder.buildConstant(Res: I32, Val: 13));
8951
8952 auto SigSetHigh =
8953 MIRBuilder.buildOr(Dst: I32, Src0: M, Src1: MIRBuilder.buildConstant(Res: I32, Val: 0x1000));
8954
8955 auto D = MIRBuilder.buildLShr(Dst: I32, Src0: SigSetHigh, Src1: B);
8956 auto D0 = MIRBuilder.buildShl(Dst: I32, Src0: D, Src1: B);
8957
8958 auto D0_NE_SigSetHigh = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: S1,
8959 Op0: D0, Op1: SigSetHigh);
8960 auto D1 = MIRBuilder.buildZExt(Res: I32, Op: D0_NE_SigSetHigh);
8961 D = MIRBuilder.buildOr(Dst: I32, Src0: D, Src1: D1);
8962
8963 auto CmpELtOne = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: S1, Op0: E, Op1: One);
8964 auto V = MIRBuilder.buildSelect(Res: I32, Tst: CmpELtOne, Op0: D, Op1: N);
8965
8966 auto VLow3 = MIRBuilder.buildAnd(Dst: I32, Src0: V, Src1: MIRBuilder.buildConstant(Res: I32, Val: 7));
8967 V = MIRBuilder.buildLShr(Dst: I32, Src0: V, Src1: MIRBuilder.buildConstant(Res: I32, Val: 2));
8968
8969 auto VLow3Eq3 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: S1, Op0: VLow3,
8970 Op1: MIRBuilder.buildConstant(Res: I32, Val: 3));
8971 auto V0 = MIRBuilder.buildZExt(Res: I32, Op: VLow3Eq3);
8972
8973 auto VLow3Gt5 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SGT, Res: S1, Op0: VLow3,
8974 Op1: MIRBuilder.buildConstant(Res: I32, Val: 5));
8975 auto V1 = MIRBuilder.buildZExt(Res: I32, Op: VLow3Gt5);
8976
8977 V1 = MIRBuilder.buildOr(Dst: I32, Src0: V0, Src1: V1);
8978 V = MIRBuilder.buildAdd(Dst: I32, Src0: V, Src1: V1);
8979
8980 auto CmpEGt30 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SGT, Res: S1, Op0: E,
8981 Op1: MIRBuilder.buildConstant(Res: I32, Val: 30));
8982 V = MIRBuilder.buildSelect(Res: I32, Tst: CmpEGt30,
8983 Op0: MIRBuilder.buildConstant(Res: I32, Val: 0x7c00), Op1: V);
8984
8985 auto CmpEGt1039 = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: S1, Op0: E,
8986 Op1: MIRBuilder.buildConstant(Res: I32, Val: 1039));
8987 V = MIRBuilder.buildSelect(Res: I32, Tst: CmpEGt1039, Op0: I, Op1: V);
8988
8989 // Extract the sign bit.
8990 auto Sign = MIRBuilder.buildLShr(Dst: I32, Src0: UH, Src1: MIRBuilder.buildConstant(Res: I32, Val: 16));
8991 Sign = MIRBuilder.buildAnd(Dst: I32, Src0: Sign, Src1: MIRBuilder.buildConstant(Res: I32, Val: 0x8000));
8992
8993 // Insert the sign bit
8994 V = MIRBuilder.buildOr(Dst: I32, Src0: Sign, Src1: V);
8995
8996 MIRBuilder.buildTrunc(Res: Dst, Op: V);
8997 MI.eraseFromParent();
8998 return Legalized;
8999}
9000
9001// f32 -> bf16 conversion using round-to-nearest-even rounding mode.
9002LegalizerHelper::LegalizeResult
9003LegalizerHelper::lowerFPTRUNC_F32_TO_BF16(MachineInstr &MI) {
9004 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
9005 assert(DstTy.getScalarType() == LLT::bfloat16() &&
9006 SrcTy.getScalarType() == LLT::float32());
9007
9008 LLT I1Ty = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9009 LLT I16Ty = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 16));
9010 LLT I32Ty = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 32));
9011
9012 auto SrcI = MIRBuilder.buildBitcast(Dst: I32Ty, Src: SrcReg);
9013
9014 // Conversions should set NaN's quiet bit. This also prevents NaNs from
9015 // turning into infinities.
9016 auto NaN = MIRBuilder.buildOr(Dst: I32Ty, Src0: SrcI,
9017 Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 0x400000));
9018
9019 // Factor in the contribution of the low 16 bits.
9020 auto Lsb =
9021 MIRBuilder.buildLShr(Dst: I32Ty, Src0: SrcI, Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 16));
9022 Lsb = MIRBuilder.buildAnd(Dst: I32Ty, Src0: Lsb, Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 1));
9023 auto RoundingBias =
9024 MIRBuilder.buildAdd(Dst: I32Ty, Src0: Lsb, Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 0x7fff));
9025 auto Add = MIRBuilder.buildAdd(Dst: I32Ty, Src0: SrcI, Src1: RoundingBias);
9026
9027 // Don't round if we had a NaN, we don't want to turn 0x7fffffff into
9028 // 0x80000000.
9029 if (!MI.getFlag(Flag: MachineInstr::FmNoNans)) {
9030 auto IsNaN = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_UNO, Res: I1Ty, Op0: SrcReg,
9031 Op1: MIRBuilder.buildFConstant(Res: SrcTy, Val: 0));
9032 Add = MIRBuilder.buildSelect(Res: I32Ty, Tst: IsNaN, Op0: NaN, Op1: Add);
9033 }
9034
9035 // Now that we have rounded, shift the bits into position.
9036 auto Srl =
9037 MIRBuilder.buildLShr(Dst: I32Ty, Src0: Add, Src1: MIRBuilder.buildConstant(Res: I32Ty, Val: 16));
9038 auto Trunc = MIRBuilder.buildTrunc(Res: I16Ty, Op: Srl);
9039 MIRBuilder.buildBitcast(Dst: DstReg, Src: Trunc);
9040 MI.eraseFromParent();
9041 return Legalized;
9042}
9043
9044// Round a wide fp value to ResultTy's element size, forcing inexact
9045// results to the odd value so a subsequent narrowing round is correct. This
9046// avoids double-rounding when narrowing e.g. f64 -> f32 -> bf16. See Boldo &
9047// Melquiond, "When double rounding is odd" (2005).
9048Register LegalizerHelper::lowerRoundInexactToOdd(LLT ResultTy, Register Op) {
9049 LLT OperandTy = MRI.getType(Reg: Op);
9050 if (OperandTy.getScalarType() == ResultTy.getScalarType())
9051 return Op;
9052
9053 LLT ResultIntTy =
9054 ResultTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: ResultTy.getScalarSizeInBits()));
9055 LLT ResultCCTy = ResultTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9056 LLT OperandCCTy = OperandTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9057
9058 auto Narrow = MIRBuilder.buildFPTrunc(Res: ResultTy, Op);
9059 auto NarrowAsWide = MIRBuilder.buildFPExt(Res: OperandTy, Op: Narrow);
9060
9061 auto NarrowBits = MIRBuilder.buildBitcast(Dst: ResultIntTy, Src: Narrow);
9062 auto One = MIRBuilder.buildConstant(Res: ResultIntTy, Val: 1);
9063 auto NegativeOne = MIRBuilder.buildConstant(Res: ResultIntTy, Val: -1);
9064 auto Zero = MIRBuilder.buildConstant(Res: ResultIntTy, Val: 0);
9065 auto And = MIRBuilder.buildAnd(Dst: ResultIntTy, Src0: NarrowBits, Src1: One);
9066 // The result is already odd so we don't need to do anything.
9067 auto AlreadyOdd =
9068 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: ResultCCTy, Op0: And, Op1: Zero);
9069
9070 // We keep results which are exact, odd or NaN.
9071 auto KeepNarrow =
9072 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_UEQ, Res: OperandCCTy, Op0: Op, Op1: NarrowAsWide);
9073 KeepNarrow = MIRBuilder.buildOr(Dst: OperandCCTy, Src0: KeepNarrow, Src1: AlreadyOdd);
9074 // We morally performed a round-down if AbsNarrow is smaller than AbsWide.
9075 auto AbsWide = MIRBuilder.buildFAbs(Dst: OperandTy, Src0: Op);
9076 auto AbsNarrowAsWide = MIRBuilder.buildFAbs(Dst: OperandTy, Src0: NarrowAsWide);
9077 auto NarrowIsRd = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OGT, Res: OperandCCTy,
9078 Op0: AbsWide, Op1: AbsNarrowAsWide);
9079 // If narrow is the rounded-down value, pick the rounded-up value as it will
9080 // be odd; otherwise adjust down.
9081 auto Adjust =
9082 MIRBuilder.buildSelect(Res: ResultIntTy, Tst: NarrowIsRd, Op0: One, Op1: NegativeOne);
9083 auto Adjusted = MIRBuilder.buildAdd(Dst: ResultIntTy, Src0: NarrowBits, Src1: Adjust);
9084 auto Res =
9085 MIRBuilder.buildSelect(Res: ResultIntTy, Tst: KeepNarrow, Op0: NarrowBits, Op1: Adjusted);
9086 return MIRBuilder.buildBitcast(Dst: ResultTy, Src: Res).getReg(Idx: 0);
9087}
9088
9089// f64 -> bf16 conversion, correcting for double rounding.
9090LegalizerHelper::LegalizeResult
9091LegalizerHelper::lowerFPTRUNC_F64_TO_BF16(MachineInstr &MI) {
9092 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
9093 assert(DstTy.getScalarType() == LLT::bfloat16() &&
9094 SrcTy.getScalarType() == LLT::float64());
9095
9096 LLT F32Ty = SrcTy.changeElementType(NewEltTy: LLT::float32());
9097 Register OddF32 = lowerRoundInexactToOdd(ResultTy: F32Ty, Op: SrcReg);
9098 MIRBuilder.buildFPTrunc(Res: DstReg, Op: OddF32, Flags: MI.getFlags());
9099 MI.eraseFromParent();
9100 return Legalized;
9101}
9102
9103LegalizerHelper::LegalizeResult
9104LegalizerHelper::lowerFPTRUNC(MachineInstr &MI) {
9105 auto [DstTy, SrcTy] = MI.getFirst2LLTs();
9106 if (DstTy.getScalarType().isFloat16() && SrcTy.getScalarType().isFloat64())
9107 return lowerFPTRUNC_F64_TO_F16(MI);
9108
9109 if (DstTy.getScalarType().isBFloat16() && SrcTy.getScalarType().isFloat32())
9110 return lowerFPTRUNC_F32_TO_BF16(MI);
9111
9112 if (DstTy.getScalarType().isBFloat16() && SrcTy.getScalarType().isFloat64())
9113 return lowerFPTRUNC_F64_TO_BF16(MI);
9114
9115 return lowerFPExtAndTruncMem(MI);
9116}
9117
9118LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPOWI(MachineInstr &MI) {
9119 auto [Dst, Src0, Src1] = MI.getFirst3Regs();
9120 LLT Ty = MRI.getType(Reg: Dst);
9121
9122 auto CvtSrc1 = MIRBuilder.buildSITOFP(Dst: Ty, Src0: Src1);
9123 MIRBuilder.buildFPow(Dst, Src0, Src1: CvtSrc1, Flags: MI.getFlags());
9124 MI.eraseFromParent();
9125 return Legalized;
9126}
9127
9128LegalizerHelper::LegalizeResult LegalizerHelper::lowerFMODF(MachineInstr &MI) {
9129 auto [DstFrac, DstInt, Src] = MI.getFirst3Regs();
9130 LLT Ty = MRI.getType(Reg: Src);
9131 auto Flags = MI.getFlags();
9132 const LLT CondTy = Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9133
9134 auto IntPart = MIRBuilder.buildIntrinsicTrunc(Dst: Ty, Src0: Src, Flags);
9135 auto FracPart = MIRBuilder.buildFSub(Dst: Ty, Src0: Src, Src1: IntPart, Flags);
9136
9137 Register FracToUse;
9138 if (MI.getFlag(Flag: MachineInstr::FmNoInfs)) {
9139 FracToUse = FracPart.getReg(Idx: 0);
9140 } else {
9141 auto Abs = MIRBuilder.buildFAbs(Dst: Ty, Src0: Src, Flags);
9142 const fltSemantics &Semantics = getFltSemanticForLLT(Ty: Ty.getScalarType());
9143 auto Inf = MIRBuilder.buildFConstant(Res: Ty, Val: APFloat::getInf(Sem: Semantics));
9144 auto IsInf = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OEQ, Res: CondTy, Op0: Abs, Op1: Inf);
9145 auto Zero = MIRBuilder.buildFConstant(Res: Ty, Val: 0.0);
9146 auto Select = MIRBuilder.buildSelect(Res: Ty, Tst: IsInf, Op0: Zero, Op1: FracPart);
9147 FracToUse = Select.getReg(Idx: 0);
9148 }
9149
9150 MIRBuilder.buildFCopysign(Dst: DstFrac, Src0: FracToUse, Src1: Src, Flags);
9151 MIRBuilder.buildCopy(Res: DstInt, Op: IntPart.getReg(Idx: 0));
9152
9153 MI.eraseFromParent();
9154 return Legalized;
9155}
9156
9157static CmpInst::Predicate minMaxToCompare(unsigned Opc) {
9158 switch (Opc) {
9159 case TargetOpcode::G_SMIN:
9160 return CmpInst::ICMP_SLT;
9161 case TargetOpcode::G_SMAX:
9162 return CmpInst::ICMP_SGT;
9163 case TargetOpcode::G_UMIN:
9164 return CmpInst::ICMP_ULT;
9165 case TargetOpcode::G_UMAX:
9166 return CmpInst::ICMP_UGT;
9167 default:
9168 llvm_unreachable("not in integer min/max");
9169 }
9170}
9171
9172LegalizerHelper::LegalizeResult LegalizerHelper::lowerMinMax(MachineInstr &MI) {
9173 auto [Dst, Src0, Src1] = MI.getFirst3Regs();
9174
9175 const CmpInst::Predicate Pred = minMaxToCompare(Opc: MI.getOpcode());
9176 LLT CmpType = MRI.getType(Reg: Dst).changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9177
9178 auto Cmp = MIRBuilder.buildICmp(Pred, Res: CmpType, Op0: Src0, Op1: Src1);
9179 MIRBuilder.buildSelect(Res: Dst, Tst: Cmp, Op0: Src0, Op1: Src1);
9180
9181 MI.eraseFromParent();
9182 return Legalized;
9183}
9184
9185LegalizerHelper::LegalizeResult
9186LegalizerHelper::lowerThreewayCompare(MachineInstr &MI) {
9187 GSUCmp *Cmp = cast<GSUCmp>(Val: &MI);
9188
9189 Register Dst = Cmp->getReg(Idx: 0);
9190 LLT DstTy = MRI.getType(Reg: Dst);
9191 LLT SrcTy = MRI.getType(Reg: Cmp->getReg(Idx: 1));
9192 LLT CmpTy = DstTy.changeElementSize(NewEltSize: 1);
9193
9194 CmpInst::Predicate LTPredicate = Cmp->isSigned()
9195 ? CmpInst::Predicate::ICMP_SLT
9196 : CmpInst::Predicate::ICMP_ULT;
9197 CmpInst::Predicate GTPredicate = Cmp->isSigned()
9198 ? CmpInst::Predicate::ICMP_SGT
9199 : CmpInst::Predicate::ICMP_UGT;
9200
9201 auto Zero = MIRBuilder.buildConstant(Res: DstTy, Val: 0);
9202 auto IsGT = MIRBuilder.buildICmp(Pred: GTPredicate, Res: CmpTy, Op0: Cmp->getLHSReg(),
9203 Op1: Cmp->getRHSReg());
9204 auto IsLT = MIRBuilder.buildICmp(Pred: LTPredicate, Res: CmpTy, Op0: Cmp->getLHSReg(),
9205 Op1: Cmp->getRHSReg());
9206
9207 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
9208 auto BC = TLI.getBooleanContents(isVec: DstTy.isVector(), /*isFP=*/isFloat: false);
9209 if (TLI.preferSelectsOverBooleanArithmetic(
9210 VT: getApproximateEVTForLLT(Ty: SrcTy, Ctx)) ||
9211 BC == TargetLowering::UndefinedBooleanContent) {
9212 auto One = MIRBuilder.buildConstant(Res: DstTy, Val: 1);
9213 auto SelectZeroOrOne = MIRBuilder.buildSelect(Res: DstTy, Tst: IsGT, Op0: One, Op1: Zero);
9214
9215 auto MinusOne = MIRBuilder.buildConstant(Res: DstTy, Val: -1);
9216 MIRBuilder.buildSelect(Res: Dst, Tst: IsLT, Op0: MinusOne, Op1: SelectZeroOrOne);
9217 } else {
9218 if (BC == TargetLowering::ZeroOrNegativeOneBooleanContent)
9219 std::swap(a&: IsGT, b&: IsLT);
9220 // Extend boolean results to DstTy, which is at least i2, before subtracting
9221 // them.
9222 unsigned BoolExtOp =
9223 MIRBuilder.getBoolExtOp(IsVec: DstTy.isVector(), /*isFP=*/IsFP: false);
9224 IsGT = MIRBuilder.buildInstr(Opc: BoolExtOp, DstOps: {DstTy}, SrcOps: {IsGT});
9225 IsLT = MIRBuilder.buildInstr(Opc: BoolExtOp, DstOps: {DstTy}, SrcOps: {IsLT});
9226 MIRBuilder.buildSub(Dst, Src0: IsGT, Src1: IsLT);
9227 }
9228
9229 MI.eraseFromParent();
9230 return Legalized;
9231}
9232
9233LegalizerHelper::LegalizeResult
9234LegalizerHelper::lowerFCopySign(MachineInstr &MI) {
9235 auto [Dst, DstTy, Src0, Src0Ty, Src1, Src1Ty] = MI.getFirst3RegLLTs();
9236 const int Src0Size = Src0Ty.getScalarSizeInBits();
9237 const int Src1Size = Src1Ty.getScalarSizeInBits();
9238
9239 LLT DstIntTy =
9240 DstTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: DstTy.getScalarSizeInBits()));
9241 LLT Src0IntTy = Src0Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: Src0Size));
9242 LLT Src1IntTy = Src1Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: Src1Size));
9243
9244 Register Src0Int = Src0;
9245 Register Src1Int = Src1;
9246
9247 if (!(Src0Ty.getScalarType().isAnyScalar() ||
9248 Src0Ty.getScalarType().isInteger()))
9249 Src0Int = MIRBuilder.buildBitcast(Dst: Src0IntTy, Src: Src0).getReg(Idx: 0);
9250
9251 if (!(Src1Ty.getScalarType().isAnyScalar() ||
9252 Src1Ty.getScalarType().isInteger()))
9253 Src1Int = MIRBuilder.buildBitcast(Dst: Src1IntTy, Src: Src1).getReg(Idx: 0);
9254
9255 auto SignBitMask =
9256 MIRBuilder.buildConstant(Res: Src0IntTy, Val: APInt::getSignMask(BitWidth: Src0Size));
9257
9258 auto NotSignBitMask = MIRBuilder.buildConstant(
9259 Res: Src0IntTy, Val: APInt::getLowBitsSet(numBits: Src0Size, loBitsSet: Src0Size - 1));
9260
9261 Register And0 =
9262 MIRBuilder.buildAnd(Dst: Src0IntTy, Src0: Src0Int, Src1: NotSignBitMask).getReg(Idx: 0);
9263 Register And1;
9264 if (Src0Ty == Src1Ty) {
9265 And1 = MIRBuilder.buildAnd(Dst: Src1IntTy, Src0: Src1Int, Src1: SignBitMask).getReg(Idx: 0);
9266 } else if (Src0Size > Src1Size) {
9267 auto ShiftAmt = MIRBuilder.buildConstant(Res: Src0IntTy, Val: Src0Size - Src1Size);
9268 auto Zext = MIRBuilder.buildZExt(Res: Src0IntTy, Op: Src1Int);
9269 auto Shift = MIRBuilder.buildShl(Dst: Src0IntTy, Src0: Zext, Src1: ShiftAmt);
9270 And1 = MIRBuilder.buildAnd(Dst: Src0IntTy, Src0: Shift, Src1: SignBitMask).getReg(Idx: 0);
9271 } else {
9272 auto ShiftAmt = MIRBuilder.buildConstant(Res: Src1IntTy, Val: Src1Size - Src0Size);
9273 auto Shift = MIRBuilder.buildLShr(Dst: Src1IntTy, Src0: Src1Int, Src1: ShiftAmt);
9274 auto Trunc = MIRBuilder.buildTrunc(Res: Src0IntTy, Op: Shift);
9275 And1 = MIRBuilder.buildAnd(Dst: Src0IntTy, Src0: Trunc, Src1: SignBitMask).getReg(Idx: 0);
9276 }
9277
9278 // Be careful about setting nsz/nnan/ninf on every instruction, since the
9279 // constants are a nan and -0.0, but the final result should preserve
9280 // everything.
9281 unsigned Flags = MI.getFlags();
9282
9283 // We masked the sign bit and the not-sign bit, so these are disjoint.
9284 Flags |= MachineInstr::Disjoint;
9285
9286 if (DstTy == DstIntTy)
9287 MIRBuilder.buildOr(Dst, Src0: And0, Src1: And1, Flags).getReg(Idx: 0);
9288 else {
9289 Register NewDst = MIRBuilder.buildOr(Dst: DstIntTy, Src0: And0, Src1: And1, Flags).getReg(Idx: 0);
9290 MIRBuilder.buildBitcast(Dst, Src: NewDst);
9291 }
9292
9293 MI.eraseFromParent();
9294 return Legalized;
9295}
9296
9297LegalizerHelper::LegalizeResult
9298LegalizerHelper::lowerFMinNumMaxNum(MachineInstr &MI) {
9299 // FIXME: fminnum/fmaxnum and fminimumnum/fmaximumnum should not have
9300 // identical handling. fminimumnum/fmaximumnum also need a path that do not
9301 // depend on fminnum/fmaxnum.
9302
9303 unsigned NewOp;
9304 switch (MI.getOpcode()) {
9305 case TargetOpcode::G_FMINNUM:
9306 NewOp = TargetOpcode::G_FMINNUM_IEEE;
9307 break;
9308 case TargetOpcode::G_FMINIMUMNUM:
9309 NewOp = TargetOpcode::G_FMINNUM;
9310 break;
9311 case TargetOpcode::G_FMAXNUM:
9312 NewOp = TargetOpcode::G_FMAXNUM_IEEE;
9313 break;
9314 case TargetOpcode::G_FMAXIMUMNUM:
9315 NewOp = TargetOpcode::G_FMAXNUM;
9316 break;
9317 default:
9318 llvm_unreachable("unexpected min/max opcode");
9319 }
9320
9321 auto [Dst, Src0, Src1] = MI.getFirst3Regs();
9322 LLT Ty = MRI.getType(Reg: Dst);
9323
9324 if (!MI.getFlag(Flag: MachineInstr::FmNoNans)) {
9325 // Insert canonicalizes if it's possible we need to quiet to get correct
9326 // sNaN behavior.
9327
9328 // Note this must be done here, and not as an optimization combine in the
9329 // absence of a dedicate quiet-snan instruction as we're using an
9330 // omni-purpose G_FCANONICALIZE.
9331 if (!VT->isKnownNeverSNaN(Val: Src0))
9332 Src0 = MIRBuilder.buildFCanonicalize(Dst: Ty, Src0, Flags: MI.getFlags()).getReg(Idx: 0);
9333
9334 if (!VT->isKnownNeverSNaN(Val: Src1))
9335 Src1 = MIRBuilder.buildFCanonicalize(Dst: Ty, Src0: Src1, Flags: MI.getFlags()).getReg(Idx: 0);
9336 }
9337
9338 // If there are no nans, it's safe to simply replace this with the non-IEEE
9339 // version.
9340 MIRBuilder.buildInstr(Opc: NewOp, DstOps: {Dst}, SrcOps: {Src0, Src1}, Flags: MI.getFlags());
9341 MI.eraseFromParent();
9342 return Legalized;
9343}
9344
9345LegalizerHelper::LegalizeResult
9346LegalizerHelper::lowerFMinimumMaximum(MachineInstr &MI) {
9347 unsigned Opc = MI.getOpcode();
9348 auto [Dst, Src0, Src1] = MI.getFirst3Regs();
9349 LLT Ty = MRI.getType(Reg: Dst);
9350 const LLT CmpTy = Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9351
9352 bool IsMax = (Opc == TargetOpcode::G_FMAXIMUM);
9353 unsigned OpcIeee =
9354 IsMax ? TargetOpcode::G_FMAXNUM_IEEE : TargetOpcode::G_FMINNUM_IEEE;
9355 unsigned OpcNonIeee =
9356 IsMax ? TargetOpcode::G_FMAXNUM : TargetOpcode::G_FMINNUM;
9357 bool MinMaxMustRespectOrderedZero = false;
9358 Register Res;
9359
9360 // IEEE variants don't need canonicalization
9361 if (LI.isLegalOrCustom(Query: {OpcIeee, Ty})) {
9362 Res = MIRBuilder.buildInstr(Opc: OpcIeee, DstOps: {Ty}, SrcOps: {Src0, Src1}).getReg(Idx: 0);
9363 MinMaxMustRespectOrderedZero = true;
9364 } else if (LI.isLegalOrCustom(Query: {OpcNonIeee, Ty})) {
9365 Res = MIRBuilder.buildInstr(Opc: OpcNonIeee, DstOps: {Ty}, SrcOps: {Src0, Src1}).getReg(Idx: 0);
9366 } else {
9367 auto Compare = MIRBuilder.buildFCmp(
9368 Pred: IsMax ? CmpInst::FCMP_OGT : CmpInst::FCMP_OLT, Res: CmpTy, Op0: Src0, Op1: Src1);
9369 Res = MIRBuilder.buildSelect(Res: Ty, Tst: Compare, Op0: Src0, Op1: Src1).getReg(Idx: 0);
9370 }
9371
9372 // Propagate any NaN of both operands
9373 if (!MI.getFlag(Flag: MachineInstr::FmNoNans) &&
9374 (!VT->isKnownNeverNaN(Val: Src0) || !VT->isKnownNeverNaN(Val: Src1))) {
9375 auto IsOrdered = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_ORD, Res: CmpTy, Op0: Src0, Op1: Src1);
9376
9377 LLT ElementTy = Ty.isScalar() ? Ty : Ty.getElementType();
9378 APFloat NaNValue = APFloat::getNaN(Sem: getFltSemanticForLLT(Ty: ElementTy));
9379 Register NaN = MIRBuilder.buildFConstant(Res: ElementTy, Val: NaNValue).getReg(Idx: 0);
9380 if (Ty.isVector())
9381 NaN = MIRBuilder.buildSplatBuildVector(Res: Ty, Src: NaN).getReg(Idx: 0);
9382
9383 Res = MIRBuilder.buildSelect(Res: Ty, Tst: IsOrdered, Op0: Res, Op1: NaN).getReg(Idx: 0);
9384 }
9385
9386 // fminimum/fmaximum requires -0.0 less than +0.0
9387 if (!MinMaxMustRespectOrderedZero && !MI.getFlag(Flag: MachineInstr::FmNsz)) {
9388 GISelValueTracking VT(MIRBuilder.getMF());
9389 KnownFPClass Src0Info = VT.computeKnownFPClass(R: Src0, InterestedClasses: fcZero);
9390 KnownFPClass Src1Info = VT.computeKnownFPClass(R: Src1, InterestedClasses: fcZero);
9391
9392 if (!Src0Info.isKnownNeverZero() && !Src1Info.isKnownNeverZero()) {
9393 const unsigned Flags = MI.getFlags();
9394 Register Zero = MIRBuilder.buildFConstant(Res: Ty, Val: 0.0).getReg(Idx: 0);
9395 auto IsZero = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OEQ, Res: CmpTy, Op0: Res, Op1: Zero);
9396
9397 unsigned TestClass = IsMax ? fcPosZero : fcNegZero;
9398
9399 auto LHSTestZero = MIRBuilder.buildIsFPClass(Res: CmpTy, Src: Src0, Mask: TestClass);
9400 auto LHSSelect =
9401 MIRBuilder.buildSelect(Res: Ty, Tst: LHSTestZero, Op0: Src0, Op1: Res, Flags);
9402
9403 auto RHSTestZero = MIRBuilder.buildIsFPClass(Res: CmpTy, Src: Src1, Mask: TestClass);
9404 auto RHSSelect =
9405 MIRBuilder.buildSelect(Res: Ty, Tst: RHSTestZero, Op0: Src1, Op1: LHSSelect, Flags);
9406
9407 Res = MIRBuilder.buildSelect(Res: Ty, Tst: IsZero, Op0: RHSSelect, Op1: Res, Flags).getReg(Idx: 0);
9408 }
9409 }
9410
9411 MIRBuilder.buildCopy(Res: Dst, Op: Res);
9412 MI.eraseFromParent();
9413 return Legalized;
9414}
9415
9416LegalizerHelper::LegalizeResult LegalizerHelper::lowerFMad(MachineInstr &MI) {
9417 // Expand G_FMAD a, b, c -> G_FADD (G_FMUL a, b), c
9418 Register DstReg = MI.getOperand(i: 0).getReg();
9419 LLT Ty = MRI.getType(Reg: DstReg);
9420 unsigned Flags = MI.getFlags();
9421
9422 auto Mul = MIRBuilder.buildFMul(Dst: Ty, Src0: MI.getOperand(i: 1), Src1: MI.getOperand(i: 2),
9423 Flags);
9424 MIRBuilder.buildFAdd(Dst: DstReg, Src0: Mul, Src1: MI.getOperand(i: 3), Flags);
9425 MI.eraseFromParent();
9426 return Legalized;
9427}
9428
9429LegalizerHelper::LegalizeResult
9430LegalizerHelper::lowerIntrinsicRound(MachineInstr &MI) {
9431 auto [DstReg, X] = MI.getFirst2Regs();
9432 const unsigned Flags = MI.getFlags();
9433 const LLT Ty = MRI.getType(Reg: DstReg);
9434 const LLT CondTy = Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9435
9436 // round(x) =>
9437 // t = trunc(x);
9438 // d = fabs(x - t);
9439 // o = copysign(d >= 0.5 ? 1.0 : 0.0, x);
9440 // return t + o;
9441
9442 auto T = MIRBuilder.buildIntrinsicTrunc(Dst: Ty, Src0: X, Flags);
9443
9444 auto Diff = MIRBuilder.buildFSub(Dst: Ty, Src0: X, Src1: T, Flags);
9445 auto AbsDiff = MIRBuilder.buildFAbs(Dst: Ty, Src0: Diff, Flags);
9446
9447 auto Half = MIRBuilder.buildFConstant(Res: Ty, Val: 0.5);
9448 auto Cmp =
9449 MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OGE, Res: CondTy, Op0: AbsDiff, Op1: Half, Flags);
9450
9451 // Could emit G_UITOFP instead
9452 auto One = MIRBuilder.buildFConstant(Res: Ty, Val: 1.0);
9453 auto Zero = MIRBuilder.buildFConstant(Res: Ty, Val: 0.0);
9454 auto BoolFP = MIRBuilder.buildSelect(Res: Ty, Tst: Cmp, Op0: One, Op1: Zero);
9455 auto SignedOffset = MIRBuilder.buildFCopysign(Dst: Ty, Src0: BoolFP, Src1: X);
9456
9457 MIRBuilder.buildFAdd(Dst: DstReg, Src0: T, Src1: SignedOffset, Flags);
9458
9459 MI.eraseFromParent();
9460 return Legalized;
9461}
9462
9463LegalizerHelper::LegalizeResult LegalizerHelper::lowerFFloor(MachineInstr &MI) {
9464 auto [DstReg, SrcReg] = MI.getFirst2Regs();
9465 unsigned Flags = MI.getFlags();
9466 LLT Ty = MRI.getType(Reg: DstReg);
9467 const LLT CondTy = Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: 1));
9468
9469 // result = trunc(src);
9470 // if (src < 0.0 && src != result)
9471 // result += -1.0.
9472
9473 auto Trunc = MIRBuilder.buildIntrinsicTrunc(Dst: Ty, Src0: SrcReg, Flags);
9474 auto Zero = MIRBuilder.buildFConstant(Res: Ty, Val: 0.0);
9475
9476 auto Lt0 = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_OLT, Res: CondTy,
9477 Op0: SrcReg, Op1: Zero, Flags);
9478 auto NeTrunc = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_ONE, Res: CondTy,
9479 Op0: SrcReg, Op1: Trunc, Flags);
9480 auto And = MIRBuilder.buildAnd(Dst: CondTy, Src0: Lt0, Src1: NeTrunc);
9481 auto AddVal = MIRBuilder.buildSITOFP(Dst: Ty, Src0: And);
9482
9483 MIRBuilder.buildFAdd(Dst: DstReg, Src0: Trunc, Src1: AddVal, Flags);
9484 MI.eraseFromParent();
9485 return Legalized;
9486}
9487
9488LegalizerHelper::LegalizeResult
9489LegalizerHelper::lowerMergeValues(MachineInstr &MI) {
9490 const unsigned NumOps = MI.getNumOperands();
9491 auto [DstReg, DstTy, Src0Reg, Src0Ty] = MI.getFirst2RegLLTs();
9492 unsigned PartSize = Src0Ty.getSizeInBits();
9493
9494 LLT WideTy = LLT::integer(SizeInBits: DstTy.getSizeInBits());
9495 Register ResultReg = MIRBuilder.buildZExt(Res: WideTy, Op: Src0Reg).getReg(Idx: 0);
9496
9497 for (unsigned I = 2; I != NumOps; ++I) {
9498 const unsigned Offset = (I - 1) * PartSize;
9499
9500 Register SrcReg = MI.getOperand(i: I).getReg();
9501 auto ZextInput = MIRBuilder.buildZExt(Res: WideTy, Op: SrcReg);
9502
9503 Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg :
9504 MRI.createGenericVirtualRegister(Ty: WideTy);
9505
9506 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: Offset);
9507 auto Shl = MIRBuilder.buildShl(Dst: WideTy, Src0: ZextInput, Src1: ShiftAmt);
9508 MIRBuilder.buildOr(Dst: NextResult, Src0: ResultReg, Src1: Shl);
9509 ResultReg = NextResult;
9510 }
9511
9512 if (DstTy.isPointer()) {
9513 if (MIRBuilder.getDataLayout().isNonIntegralAddressSpace(
9514 AddrSpace: DstTy.getAddressSpace())) {
9515 LLVM_DEBUG(dbgs() << "Not casting nonintegral address space\n");
9516 return UnableToLegalize;
9517 }
9518
9519 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: ResultReg);
9520 } else if (WideTy != DstTy) {
9521 MIRBuilder.buildBitcast(Dst: DstReg, Src: ResultReg);
9522 }
9523
9524 MI.eraseFromParent();
9525 return Legalized;
9526}
9527
9528LegalizerHelper::LegalizeResult
9529LegalizerHelper::lowerUnmergeValues(MachineInstr &MI) {
9530 const unsigned NumDst = MI.getNumOperands() - 1;
9531 Register SrcReg = MI.getOperand(i: NumDst).getReg();
9532 Register Dst0Reg = MI.getOperand(i: 0).getReg();
9533 LLT DstTy = MRI.getType(Reg: Dst0Reg);
9534 if (DstTy.isPointer())
9535 return UnableToLegalize; // TODO
9536
9537 SrcReg = coerceToInteger(Val: SrcReg);
9538 if (!SrcReg)
9539 return UnableToLegalize;
9540
9541 // Expand scalarizing unmerge as bitcast to integer and shift.
9542 LLT IntTy = MRI.getType(Reg: SrcReg);
9543
9544 MIRBuilder.buildTrunc(Res: Dst0Reg, Op: SrcReg);
9545
9546 const unsigned DstSize = DstTy.getSizeInBits();
9547 unsigned Offset = DstSize;
9548 for (unsigned I = 1; I != NumDst; ++I, Offset += DstSize) {
9549 auto ShiftAmt = MIRBuilder.buildConstant(Res: IntTy, Val: Offset);
9550 auto Shift = MIRBuilder.buildLShr(Dst: IntTy, Src0: SrcReg, Src1: ShiftAmt);
9551 MIRBuilder.buildTrunc(Res: MI.getOperand(i: I), Op: Shift);
9552 }
9553
9554 MI.eraseFromParent();
9555 return Legalized;
9556}
9557
9558/// Lower a vector extract or insert by writing the vector to a stack temporary
9559/// and reloading the element or vector.
9560///
9561/// %dst = G_EXTRACT_VECTOR_ELT %vec, %idx
9562/// =>
9563/// %stack_temp = G_FRAME_INDEX
9564/// G_STORE %vec, %stack_temp
9565/// %idx = clamp(%idx, %vec.getNumElements())
9566/// %element_ptr = G_PTR_ADD %stack_temp, %idx
9567/// %dst = G_LOAD %element_ptr
9568LegalizerHelper::LegalizeResult
9569LegalizerHelper::lowerExtractInsertVectorElt(MachineInstr &MI) {
9570 Register DstReg = MI.getOperand(i: 0).getReg();
9571 Register SrcVec = MI.getOperand(i: 1).getReg();
9572 Register InsertVal;
9573 if (MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)
9574 InsertVal = MI.getOperand(i: 2).getReg();
9575
9576 Register Idx = MI.getOperand(i: MI.getNumOperands() - 1).getReg();
9577
9578 LLT VecTy = MRI.getType(Reg: SrcVec);
9579 LLT EltTy = VecTy.getElementType();
9580 unsigned NumElts = VecTy.getNumElements();
9581
9582 int64_t IdxVal;
9583 if (mi_match(R: Idx, MRI, P: m_ICst(Cst&: IdxVal)) && IdxVal <= NumElts) {
9584 SmallVector<Register, 8> SrcRegs;
9585 extractParts(Reg: SrcVec, Ty: EltTy, NumParts: NumElts, VRegs&: SrcRegs, MIRBuilder, MRI);
9586
9587 if (InsertVal) {
9588 SrcRegs[IdxVal] = MI.getOperand(i: 2).getReg();
9589 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: SrcRegs);
9590 } else {
9591 MIRBuilder.buildCopy(Res: DstReg, Op: SrcRegs[IdxVal]);
9592 }
9593
9594 MI.eraseFromParent();
9595 return Legalized;
9596 }
9597
9598 if (!EltTy.isByteSized()) { // Not implemented.
9599 LLVM_DEBUG(dbgs() << "Can't handle non-byte element vectors yet\n");
9600 return UnableToLegalize;
9601 }
9602
9603 unsigned EltBytes = EltTy.getSizeInBytes();
9604 Align VecAlign = getStackTemporaryAlignment(Ty: VecTy);
9605 Align EltAlign;
9606
9607 MachinePointerInfo PtrInfo;
9608 auto StackTemp = createStackTemporary(
9609 Bytes: TypeSize::getFixed(ExactSize: VecTy.getSizeInBytes()), Alignment: VecAlign, PtrInfo);
9610 MIRBuilder.buildStore(Val: SrcVec, Addr: StackTemp, PtrInfo, Alignment: VecAlign);
9611
9612 // Get the pointer to the element, and be sure not to hit undefined behavior
9613 // if the index is out of bounds.
9614 Register EltPtr = getVectorElementPointer(VecPtr: StackTemp.getReg(Idx: 0), VecTy, Index: Idx);
9615
9616 if (mi_match(R: Idx, MRI, P: m_ICst(Cst&: IdxVal))) {
9617 int64_t Offset = IdxVal * EltBytes;
9618 PtrInfo = PtrInfo.getWithOffset(O: Offset);
9619 EltAlign = commonAlignment(A: VecAlign, Offset);
9620 } else {
9621 // We lose information with a variable offset.
9622 EltAlign = getStackTemporaryAlignment(Ty: EltTy);
9623 PtrInfo = MachinePointerInfo(MRI.getType(Reg: EltPtr).getAddressSpace());
9624 }
9625
9626 if (InsertVal) {
9627 // Write the inserted element
9628 MIRBuilder.buildStore(Val: InsertVal, Addr: EltPtr, PtrInfo, Alignment: EltAlign);
9629
9630 // Reload the whole vector.
9631 MIRBuilder.buildLoad(Res: DstReg, Addr: StackTemp, PtrInfo, Alignment: VecAlign);
9632 } else {
9633 MIRBuilder.buildLoad(Res: DstReg, Addr: EltPtr, PtrInfo, Alignment: EltAlign);
9634 }
9635
9636 MI.eraseFromParent();
9637 return Legalized;
9638}
9639
9640LegalizerHelper::LegalizeResult
9641LegalizerHelper::lowerShuffleVector(MachineInstr &MI) {
9642 auto [DstReg, DstTy, Src0Reg, Src0Ty, Src1Reg, Src1Ty] =
9643 MI.getFirst3RegLLTs();
9644 LLT IdxTy = LLT::scalar(SizeInBits: 32);
9645
9646 ArrayRef<int> Mask = MI.getOperand(i: 3).getShuffleMask();
9647 Register Undef;
9648 SmallVector<Register, 32> BuildVec;
9649 LLT EltTy = DstTy.getScalarType();
9650
9651 DenseMap<unsigned, Register> CachedExtract;
9652
9653 for (int Idx : Mask) {
9654 if (Idx < 0) {
9655 if (!Undef.isValid())
9656 Undef = MIRBuilder.buildUndef(Res: EltTy).getReg(Idx: 0);
9657 BuildVec.push_back(Elt: Undef);
9658 continue;
9659 }
9660
9661 assert(!Src0Ty.isScalar() && "Unexpected scalar G_SHUFFLE_VECTOR");
9662
9663 int NumElts = Src0Ty.getNumElements();
9664 Register SrcVec = Idx < NumElts ? Src0Reg : Src1Reg;
9665 int ExtractIdx = Idx < NumElts ? Idx : Idx - NumElts;
9666 auto [It, Inserted] = CachedExtract.try_emplace(Key: Idx);
9667 if (Inserted) {
9668 auto IdxK = MIRBuilder.buildConstant(Res: IdxTy, Val: ExtractIdx);
9669 It->second =
9670 MIRBuilder.buildExtractVectorElement(Res: EltTy, Val: SrcVec, Idx: IdxK).getReg(Idx: 0);
9671 }
9672 BuildVec.push_back(Elt: It->second);
9673 }
9674
9675 assert(DstTy.isVector() && "Unexpected scalar G_SHUFFLE_VECTOR");
9676 MIRBuilder.buildBuildVector(Res: DstReg, Ops: BuildVec);
9677 MI.eraseFromParent();
9678 return Legalized;
9679}
9680
9681LegalizerHelper::LegalizeResult
9682LegalizerHelper::lowerVECTOR_COMPRESS(llvm::MachineInstr &MI) {
9683 auto [Dst, DstTy, Vec, VecTy, Mask, MaskTy, Passthru, PassthruTy] =
9684 MI.getFirst4RegLLTs();
9685
9686 if (VecTy.isScalableVector())
9687 report_fatal_error(reason: "Cannot expand masked_compress for scalable vectors.");
9688
9689 Align VecAlign = getStackTemporaryAlignment(Ty: VecTy);
9690 MachinePointerInfo PtrInfo;
9691 Register StackPtr =
9692 createStackTemporary(Bytes: TypeSize::getFixed(ExactSize: VecTy.getSizeInBytes()), Alignment: VecAlign,
9693 PtrInfo)
9694 .getReg(Idx: 0);
9695 MachinePointerInfo ValPtrInfo =
9696 MachinePointerInfo::getUnknownStack(MF&: *MI.getMF());
9697
9698 LLT IdxTy = LLT::integer(SizeInBits: 32);
9699 LLT ValTy = VecTy.getElementType();
9700 Align ValAlign = getStackTemporaryAlignment(Ty: ValTy);
9701
9702 auto OutPos = MIRBuilder.buildConstant(Res: IdxTy, Val: 0);
9703
9704 bool HasPassthru = !mi_match(R: Passthru, MRI, P: m_GImplicitDef());
9705
9706 if (HasPassthru)
9707 MIRBuilder.buildStore(Val: Passthru, Addr: StackPtr, PtrInfo, Alignment: VecAlign);
9708
9709 Register LastWriteVal;
9710 std::optional<APInt> PassthruSplatVal =
9711 isConstantOrConstantSplatVector(Def: Passthru, MRI);
9712
9713 if (PassthruSplatVal.has_value()) {
9714 LastWriteVal =
9715 MIRBuilder.buildConstant(Res: ValTy, Val: PassthruSplatVal.value()).getReg(Idx: 0);
9716 } else if (HasPassthru) {
9717 auto Popcount = MIRBuilder.buildZExt(Res: MaskTy.changeElementSize(NewEltSize: 32), Op: Mask);
9718 Popcount = MIRBuilder.buildInstr(Opc: TargetOpcode::G_VECREDUCE_ADD,
9719 DstOps: {LLT::integer(SizeInBits: 32)}, SrcOps: {Popcount});
9720
9721 Register LastElmtPtr =
9722 getVectorElementPointer(VecPtr: StackPtr, VecTy, Index: Popcount.getReg(Idx: 0));
9723 LastWriteVal =
9724 MIRBuilder.buildLoad(Res: ValTy, Addr: LastElmtPtr, PtrInfo: ValPtrInfo, Alignment: ValAlign)
9725 .getReg(Idx: 0);
9726 }
9727
9728 unsigned NumElmts = VecTy.getNumElements();
9729 for (unsigned I = 0; I < NumElmts; ++I) {
9730 auto Idx = MIRBuilder.buildConstant(Res: IdxTy, Val: I);
9731 auto Val = MIRBuilder.buildExtractVectorElement(Res: ValTy, Val: Vec, Idx);
9732 Register ElmtPtr =
9733 getVectorElementPointer(VecPtr: StackPtr, VecTy, Index: OutPos.getReg(Idx: 0));
9734 MIRBuilder.buildStore(Val, Addr: ElmtPtr, PtrInfo: ValPtrInfo, Alignment: ValAlign);
9735
9736 LLT MaskITy = MaskTy.getElementType();
9737 auto MaskI = MIRBuilder.buildExtractVectorElement(Res: MaskITy, Val: Mask, Idx);
9738 if (MaskITy.getSizeInBits() > 1)
9739 MaskI = MIRBuilder.buildTrunc(Res: LLT::integer(SizeInBits: 1), Op: MaskI);
9740
9741 MaskI = MIRBuilder.buildZExt(Res: IdxTy, Op: MaskI);
9742 OutPos = MIRBuilder.buildAdd(Dst: IdxTy, Src0: OutPos, Src1: MaskI);
9743
9744 if (HasPassthru && I == NumElmts - 1) {
9745 auto EndOfVector =
9746 MIRBuilder.buildConstant(Res: IdxTy, Val: VecTy.getNumElements() - 1);
9747 auto AllLanesSelected = MIRBuilder.buildICmp(
9748 Pred: CmpInst::ICMP_UGT, Res: LLT::integer(SizeInBits: 1), Op0: OutPos, Op1: EndOfVector);
9749 OutPos = MIRBuilder.buildInstr(Opc: TargetOpcode::G_UMIN, DstOps: {IdxTy},
9750 SrcOps: {OutPos, EndOfVector});
9751 ElmtPtr = getVectorElementPointer(VecPtr: StackPtr, VecTy, Index: OutPos.getReg(Idx: 0));
9752
9753 LastWriteVal =
9754 MIRBuilder.buildSelect(Res: ValTy, Tst: AllLanesSelected, Op0: Val, Op1: LastWriteVal)
9755 .getReg(Idx: 0);
9756 MIRBuilder.buildStore(Val: LastWriteVal, Addr: ElmtPtr, PtrInfo: ValPtrInfo, Alignment: ValAlign);
9757 }
9758 }
9759
9760 // TODO: Use StackPtr's FrameIndex alignment.
9761 MIRBuilder.buildLoad(Res: Dst, Addr: StackPtr, PtrInfo, Alignment: VecAlign);
9762
9763 MI.eraseFromParent();
9764 return Legalized;
9765}
9766
9767Register LegalizerHelper::getDynStackAllocTargetPtr(Register SPReg,
9768 Register AllocSize,
9769 Align Alignment,
9770 LLT PtrTy) {
9771 LLT IntPtrTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
9772
9773 auto SPTmp = MIRBuilder.buildCopy(Res: PtrTy, Op: SPReg);
9774 SPTmp = MIRBuilder.buildCast(Dst: IntPtrTy, Src: SPTmp);
9775
9776 // Subtract the final alloc from the SP. We use G_PTRTOINT here so we don't
9777 // have to generate an extra instruction to negate the alloc and then use
9778 // G_PTR_ADD to add the negative offset.
9779 auto Alloc = MIRBuilder.buildSub(Dst: IntPtrTy, Src0: SPTmp, Src1: AllocSize);
9780 if (Alignment > Align(1)) {
9781 APInt AlignMask(IntPtrTy.getSizeInBits(), Alignment.value(), true);
9782 AlignMask.negate();
9783 auto AlignCst = MIRBuilder.buildConstant(Res: IntPtrTy, Val: AlignMask);
9784 Alloc = MIRBuilder.buildAnd(Dst: IntPtrTy, Src0: Alloc, Src1: AlignCst);
9785 }
9786
9787 return MIRBuilder.buildCast(Dst: PtrTy, Src: Alloc).getReg(Idx: 0);
9788}
9789
9790LegalizerHelper::LegalizeResult
9791LegalizerHelper::lowerDynStackAlloc(MachineInstr &MI) {
9792 const auto &MF = *MI.getMF();
9793 const auto &TFI = *MF.getSubtarget().getFrameLowering();
9794 if (TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp)
9795 return UnableToLegalize;
9796
9797 Register Dst = MI.getOperand(i: 0).getReg();
9798 Register AllocSize = MI.getOperand(i: 1).getReg();
9799 Align Alignment = assumeAligned(Value: MI.getOperand(i: 2).getImm());
9800
9801 LLT PtrTy = MRI.getType(Reg: Dst);
9802 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
9803 Register SPTmp =
9804 getDynStackAllocTargetPtr(SPReg, AllocSize, Alignment, PtrTy);
9805
9806 MIRBuilder.buildCopy(Res: SPReg, Op: SPTmp);
9807 MIRBuilder.buildCopy(Res: Dst, Op: SPTmp);
9808
9809 MI.eraseFromParent();
9810 return Legalized;
9811}
9812
9813LegalizerHelper::LegalizeResult
9814LegalizerHelper::lowerStackSave(MachineInstr &MI) {
9815 Register StackPtr = TLI.getStackPointerRegisterToSaveRestore();
9816 if (!StackPtr)
9817 return UnableToLegalize;
9818
9819 MIRBuilder.buildCopy(Res: MI.getOperand(i: 0), Op: StackPtr);
9820 MI.eraseFromParent();
9821 return Legalized;
9822}
9823
9824LegalizerHelper::LegalizeResult
9825LegalizerHelper::lowerStackRestore(MachineInstr &MI) {
9826 Register StackPtr = TLI.getStackPointerRegisterToSaveRestore();
9827 if (!StackPtr)
9828 return UnableToLegalize;
9829
9830 MIRBuilder.buildCopy(Res: StackPtr, Op: MI.getOperand(i: 0));
9831 MI.eraseFromParent();
9832 return Legalized;
9833}
9834
9835LegalizerHelper::LegalizeResult
9836LegalizerHelper::lowerExtract(MachineInstr &MI) {
9837 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
9838 unsigned Offset = MI.getOperand(i: 2).getImm();
9839
9840 // Extract sub-vector or one element
9841 if (SrcTy.isVector()) {
9842 unsigned SrcEltSize = SrcTy.getElementType().getSizeInBits();
9843 unsigned DstSize = DstTy.getSizeInBits();
9844
9845 if ((Offset % SrcEltSize == 0) && (DstSize % SrcEltSize == 0) &&
9846 (Offset + DstSize <= SrcTy.getSizeInBits())) {
9847 // Unmerge and allow access to each Src element for the artifact combiner.
9848 auto Unmerge = MIRBuilder.buildUnmerge(Res: SrcTy.getElementType(), Op: SrcReg);
9849
9850 // Take element(s) we need to extract and copy it (merge them).
9851 SmallVector<Register, 8> SubVectorElts;
9852 for (unsigned Idx = Offset / SrcEltSize;
9853 Idx < (Offset + DstSize) / SrcEltSize; ++Idx) {
9854 SubVectorElts.push_back(Elt: Unmerge.getReg(Idx));
9855 }
9856 if (SubVectorElts.size() == 1)
9857 MIRBuilder.buildCopy(Res: DstReg, Op: SubVectorElts[0]);
9858 else
9859 MIRBuilder.buildMergeLikeInstr(Res: DstReg, Ops: SubVectorElts);
9860
9861 MI.eraseFromParent();
9862 return Legalized;
9863 }
9864 }
9865
9866 const DataLayout &DL = MIRBuilder.getDataLayout();
9867 if ((SrcTy.isPointer() &&
9868 DL.isNonIntegralAddressSpace(AddrSpace: SrcTy.getAddressSpace())) ||
9869 (DstTy.isPointer() &&
9870 DL.isNonIntegralAddressSpace(AddrSpace: DstTy.getAddressSpace()))) {
9871 LLVM_DEBUG(dbgs() << "Not casting non-integral address space integer\n");
9872 return UnableToLegalize;
9873 }
9874
9875 if ((DstTy.isScalar() || DstTy.isPointer()) &&
9876 (SrcTy.isScalar() || SrcTy.isPointer() ||
9877 (SrcTy.isVector() && DstTy == SrcTy.getElementType()))) {
9878 LLT SrcIntTy = SrcTy;
9879 if (!SrcTy.isScalar()) {
9880 SrcIntTy = LLT::integer(SizeInBits: SrcTy.getSizeInBits());
9881 SrcReg = MIRBuilder.buildCast(Dst: SrcIntTy, Src: SrcReg).getReg(Idx: 0);
9882 }
9883
9884 Register ResultReg = DstReg;
9885 if (DstTy.isPointer())
9886 ResultReg =
9887 MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: DstTy.getSizeInBits()));
9888
9889 if (Offset == 0)
9890 MIRBuilder.buildTrunc(Res: ResultReg, Op: SrcReg);
9891 else {
9892 auto ShiftAmt = MIRBuilder.buildConstant(Res: SrcIntTy, Val: Offset);
9893 auto Shr = MIRBuilder.buildLShr(Dst: SrcIntTy, Src0: SrcReg, Src1: ShiftAmt);
9894 MIRBuilder.buildTrunc(Res: ResultReg, Op: Shr);
9895 }
9896
9897 if (DstTy.isPointer())
9898 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: ResultReg);
9899
9900 MI.eraseFromParent();
9901 return Legalized;
9902 }
9903
9904 return UnableToLegalize;
9905}
9906
9907LegalizerHelper::LegalizeResult LegalizerHelper::lowerInsert(MachineInstr &MI) {
9908 auto [Dst, Src, InsertSrc] = MI.getFirst3Regs();
9909 uint64_t Offset = MI.getOperand(i: 3).getImm();
9910
9911 LLT DstTy = MRI.getType(Reg: Src);
9912 LLT InsertTy = MRI.getType(Reg: InsertSrc);
9913
9914 const DataLayout &DL = MIRBuilder.getDataLayout();
9915 bool IsNonIntegralInsert =
9916 InsertTy.isPointerOrPointerVector() &&
9917 DL.isNonIntegralAddressSpace(AddrSpace: InsertTy.getAddressSpace());
9918 bool IsNonIntegralDst = DstTy.isPointerOrPointerVector() &&
9919 DL.isNonIntegralAddressSpace(AddrSpace: DstTy.getAddressSpace());
9920
9921 // Insert sub-vector or one element
9922 if (DstTy.isVector()) {
9923 LLT EltTy = DstTy.getElementType();
9924
9925 if ((IsNonIntegralInsert || IsNonIntegralDst) && InsertTy != EltTy) {
9926 LLVM_DEBUG(dbgs() << "Not casting non-integral address space integer\n");
9927 return UnableToLegalize;
9928 }
9929
9930 unsigned EltSize = EltTy.getSizeInBits();
9931 unsigned InsertSize = InsertTy.getSizeInBits();
9932
9933 if ((Offset % EltSize == 0) && (InsertSize % EltSize == 0) &&
9934 (Offset + InsertSize <= DstTy.getSizeInBits())) {
9935 auto UnmergeSrc = MIRBuilder.buildUnmerge(Res: EltTy, Op: Src);
9936 SmallVector<Register, 8> DstElts;
9937 unsigned Idx = 0;
9938 // Elements from Src before insert start Offset
9939 for (; Idx < Offset / EltSize; ++Idx) {
9940 DstElts.push_back(Elt: UnmergeSrc.getReg(Idx));
9941 }
9942
9943 // Replace elements in Src with elements from InsertSrc
9944 if (InsertTy.getSizeInBits() > EltSize) {
9945 auto UnmergeInsertSrc = MIRBuilder.buildUnmerge(Res: EltTy, Op: InsertSrc);
9946 for (unsigned i = 0; Idx < (Offset + InsertSize) / EltSize;
9947 ++Idx, ++i) {
9948 DstElts.push_back(Elt: UnmergeInsertSrc.getReg(Idx: i));
9949 }
9950 } else {
9951 if (InsertTy.isPointer() && !EltTy.isPointer())
9952 InsertSrc = MIRBuilder.buildPtrToInt(Dst: EltTy, Src: InsertSrc).getReg(Idx: 0);
9953 else if (!InsertTy.isPointer() && EltTy.isPointer())
9954 InsertSrc = MIRBuilder.buildIntToPtr(Dst: EltTy, Src: InsertSrc).getReg(Idx: 0);
9955 DstElts.push_back(Elt: InsertSrc);
9956 ++Idx;
9957 }
9958
9959 // Remaining elements from Src after insert
9960 for (; Idx < DstTy.getNumElements(); ++Idx) {
9961 DstElts.push_back(Elt: UnmergeSrc.getReg(Idx));
9962 }
9963
9964 MIRBuilder.buildMergeLikeInstr(Res: Dst, Ops: DstElts);
9965 MI.eraseFromParent();
9966 return Legalized;
9967 }
9968 }
9969
9970 if (InsertTy.isVector() ||
9971 (DstTy.isVector() && DstTy.getElementType() != InsertTy))
9972 return UnableToLegalize;
9973
9974 if (IsNonIntegralDst || IsNonIntegralInsert) {
9975 LLVM_DEBUG(dbgs() << "Not casting non-integral address space integer\n");
9976 return UnableToLegalize;
9977 }
9978
9979 LLT IntDstTy = DstTy;
9980
9981 if (!DstTy.isScalar()) {
9982 IntDstTy = LLT::scalar(SizeInBits: DstTy.getSizeInBits());
9983 Src = MIRBuilder.buildCast(Dst: IntDstTy, Src).getReg(Idx: 0);
9984 }
9985
9986 if (!InsertTy.isScalar()) {
9987 const LLT IntInsertTy = LLT::scalar(SizeInBits: InsertTy.getSizeInBits());
9988 InsertSrc = MIRBuilder.buildPtrToInt(Dst: IntInsertTy, Src: InsertSrc).getReg(Idx: 0);
9989 }
9990
9991 Register ExtInsSrc = MIRBuilder.buildZExt(Res: IntDstTy, Op: InsertSrc).getReg(Idx: 0);
9992 if (Offset != 0) {
9993 auto ShiftAmt = MIRBuilder.buildConstant(Res: IntDstTy, Val: Offset);
9994 ExtInsSrc = MIRBuilder.buildShl(Dst: IntDstTy, Src0: ExtInsSrc, Src1: ShiftAmt).getReg(Idx: 0);
9995 }
9996
9997 APInt MaskVal = APInt::getBitsSetWithWrap(
9998 numBits: DstTy.getSizeInBits(), loBit: Offset + InsertTy.getSizeInBits(), hiBit: Offset);
9999
10000 auto Mask = MIRBuilder.buildConstant(Res: IntDstTy, Val: MaskVal);
10001 auto MaskedSrc = MIRBuilder.buildAnd(Dst: IntDstTy, Src0: Src, Src1: Mask);
10002 auto Or = MIRBuilder.buildOr(Dst: IntDstTy, Src0: MaskedSrc, Src1: ExtInsSrc);
10003
10004 MIRBuilder.buildCast(Dst, Src: Or);
10005 MI.eraseFromParent();
10006 return Legalized;
10007}
10008
10009LegalizerHelper::LegalizeResult
10010LegalizerHelper::lowerSADDO_SSUBO(MachineInstr &MI) {
10011 auto [Dst0, Dst0Ty, Dst1, Dst1Ty, LHS, LHSTy, RHS, RHSTy] =
10012 MI.getFirst4RegLLTs();
10013 const bool IsAdd = MI.getOpcode() == TargetOpcode::G_SADDO;
10014
10015 LLT Ty = Dst0Ty;
10016 LLT BoolTy = Dst1Ty;
10017
10018 Register NewDst0 = MRI.cloneVirtualRegister(VReg: Dst0);
10019
10020 if (IsAdd)
10021 MIRBuilder.buildAdd(Dst: NewDst0, Src0: LHS, Src1: RHS);
10022 else
10023 MIRBuilder.buildSub(Dst: NewDst0, Src0: LHS, Src1: RHS);
10024
10025 // TODO: If SADDSAT/SSUBSAT is legal, compare results to detect overflow.
10026
10027 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
10028
10029 if (IsAdd) {
10030 // For an addition, the result should be less than one of the operands (LHS)
10031 // if and only if the other operand (RHS) is negative, otherwise there will
10032 // be overflow.
10033 auto ResultLowerThanLHS =
10034 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: NewDst0, Op1: LHS);
10035 auto RHSNegative =
10036 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: RHS, Op1: Zero);
10037 MIRBuilder.buildXor(Dst: Dst1, Src0: RHSNegative, Src1: ResultLowerThanLHS);
10038 } else {
10039 // For subtraction, overflow occurs when the signed comparison of operands
10040 // doesn't match the sign of the result.
10041 auto LHSLessThanRHS =
10042 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: LHS, Op1: RHS);
10043 auto ResultNegative =
10044 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: NewDst0, Op1: Zero);
10045 MIRBuilder.buildXor(Dst: Dst1, Src0: LHSLessThanRHS, Src1: ResultNegative);
10046 }
10047
10048 MIRBuilder.buildCopy(Res: Dst0, Op: NewDst0);
10049 MI.eraseFromParent();
10050
10051 return Legalized;
10052}
10053
10054LegalizerHelper::LegalizeResult LegalizerHelper::lowerSADDE(MachineInstr &MI) {
10055 auto [Res, OvOut, LHS, RHS, CarryIn] = MI.getFirst5Regs();
10056 const LLT Ty = MRI.getType(Reg: Res);
10057
10058 // sum = LHS + RHS + zext(CarryIn)
10059 auto Tmp = MIRBuilder.buildAdd(Dst: Ty, Src0: LHS, Src1: RHS);
10060 auto CarryZ = MIRBuilder.buildZExt(Res: Ty, Op: CarryIn);
10061 auto Sum = MIRBuilder.buildAdd(Dst: Ty, Src0: Tmp, Src1: CarryZ);
10062 MIRBuilder.buildCopy(Res, Op: Sum);
10063
10064 // OvOut = icmp slt ((sum ^ lhs) & (sum ^ rhs)), 0
10065 auto AX = MIRBuilder.buildXor(Dst: Ty, Src0: Sum, Src1: LHS);
10066 auto BX = MIRBuilder.buildXor(Dst: Ty, Src0: Sum, Src1: RHS);
10067 auto T = MIRBuilder.buildAnd(Dst: Ty, Src0: AX, Src1: BX);
10068
10069 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
10070 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: OvOut, Op0: T, Op1: Zero);
10071
10072 MI.eraseFromParent();
10073 return Legalized;
10074}
10075
10076LegalizerHelper::LegalizeResult LegalizerHelper::lowerSSUBE(MachineInstr &MI) {
10077 auto [Res, OvOut, LHS, RHS, CarryIn] = MI.getFirst5Regs();
10078 const LLT Ty = MRI.getType(Reg: Res);
10079
10080 // Diff = LHS - (RHS + zext(CarryIn))
10081 auto CarryZ = MIRBuilder.buildZExt(Res: Ty, Op: CarryIn);
10082 auto RHSPlusCI = MIRBuilder.buildAdd(Dst: Ty, Src0: RHS, Src1: CarryZ);
10083 auto Diff = MIRBuilder.buildSub(Dst: Ty, Src0: LHS, Src1: RHSPlusCI);
10084 MIRBuilder.buildCopy(Res, Op: Diff);
10085
10086 // ov = msb((LHS ^ RHS) & (LHS ^ Diff))
10087 auto X1 = MIRBuilder.buildXor(Dst: Ty, Src0: LHS, Src1: RHS);
10088 auto X2 = MIRBuilder.buildXor(Dst: Ty, Src0: LHS, Src1: Diff);
10089 auto T = MIRBuilder.buildAnd(Dst: Ty, Src0: X1, Src1: X2);
10090 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
10091 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: OvOut, Op0: T, Op1: Zero);
10092
10093 MI.eraseFromParent();
10094 return Legalized;
10095}
10096
10097LegalizerHelper::LegalizeResult
10098LegalizerHelper::lowerAddSubSatToMinMax(MachineInstr &MI) {
10099 auto [Res, LHS, RHS] = MI.getFirst3Regs();
10100 LLT Ty = MRI.getType(Reg: Res);
10101 bool IsSigned;
10102 bool IsAdd;
10103 unsigned BaseOp;
10104 switch (MI.getOpcode()) {
10105 default:
10106 llvm_unreachable("unexpected addsat/subsat opcode");
10107 case TargetOpcode::G_UADDSAT:
10108 IsSigned = false;
10109 IsAdd = true;
10110 BaseOp = TargetOpcode::G_ADD;
10111 break;
10112 case TargetOpcode::G_SADDSAT:
10113 IsSigned = true;
10114 IsAdd = true;
10115 BaseOp = TargetOpcode::G_ADD;
10116 break;
10117 case TargetOpcode::G_USUBSAT:
10118 IsSigned = false;
10119 IsAdd = false;
10120 BaseOp = TargetOpcode::G_SUB;
10121 break;
10122 case TargetOpcode::G_SSUBSAT:
10123 IsSigned = true;
10124 IsAdd = false;
10125 BaseOp = TargetOpcode::G_SUB;
10126 break;
10127 }
10128
10129 if (IsSigned) {
10130 // sadd.sat(a, b) ->
10131 // hi = 0x7fffffff - smax(a, 0)
10132 // lo = 0x80000000 - smin(a, 0)
10133 // a + smin(smax(lo, b), hi)
10134 // ssub.sat(a, b) ->
10135 // lo = smax(a, -1) - 0x7fffffff
10136 // hi = smin(a, -1) - 0x80000000
10137 // a - smin(smax(lo, b), hi)
10138 // TODO: AMDGPU can use a "median of 3" instruction here:
10139 // a +/- med3(lo, b, hi)
10140 uint64_t NumBits = Ty.getScalarSizeInBits();
10141 auto MaxVal =
10142 MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMaxValue(numBits: NumBits));
10143 auto MinVal =
10144 MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMinValue(numBits: NumBits));
10145 MachineInstrBuilder Hi, Lo;
10146 if (IsAdd) {
10147 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
10148 Hi = MIRBuilder.buildSub(Dst: Ty, Src0: MaxVal, Src1: MIRBuilder.buildSMax(Dst: Ty, Src0: LHS, Src1: Zero));
10149 Lo = MIRBuilder.buildSub(Dst: Ty, Src0: MinVal, Src1: MIRBuilder.buildSMin(Dst: Ty, Src0: LHS, Src1: Zero));
10150 } else {
10151 auto NegOne = MIRBuilder.buildConstant(Res: Ty, Val: -1);
10152 Lo = MIRBuilder.buildSub(Dst: Ty, Src0: MIRBuilder.buildSMax(Dst: Ty, Src0: LHS, Src1: NegOne),
10153 Src1: MaxVal);
10154 Hi = MIRBuilder.buildSub(Dst: Ty, Src0: MIRBuilder.buildSMin(Dst: Ty, Src0: LHS, Src1: NegOne),
10155 Src1: MinVal);
10156 }
10157 auto RHSClamped =
10158 MIRBuilder.buildSMin(Dst: Ty, Src0: MIRBuilder.buildSMax(Dst: Ty, Src0: Lo, Src1: RHS), Src1: Hi);
10159 MIRBuilder.buildInstr(Opc: BaseOp, DstOps: {Res}, SrcOps: {LHS, RHSClamped});
10160 } else {
10161 // uadd.sat(a, b) -> a + umin(~a, b)
10162 // usub.sat(a, b) -> a - umin(a, b)
10163 Register Not = IsAdd ? MIRBuilder.buildNot(Dst: Ty, Src0: LHS).getReg(Idx: 0) : LHS;
10164 auto Min = MIRBuilder.buildUMin(Dst: Ty, Src0: Not, Src1: RHS);
10165 MIRBuilder.buildInstr(Opc: BaseOp, DstOps: {Res}, SrcOps: {LHS, Min});
10166 }
10167
10168 MI.eraseFromParent();
10169 return Legalized;
10170}
10171
10172LegalizerHelper::LegalizeResult
10173LegalizerHelper::lowerAddSubSatToAddoSubo(MachineInstr &MI) {
10174 auto [Res, LHS, RHS] = MI.getFirst3Regs();
10175 LLT Ty = MRI.getType(Reg: Res);
10176 LLT BoolTy = Ty.changeElementSize(NewEltSize: 1);
10177 bool IsSigned;
10178 bool IsAdd;
10179 unsigned OverflowOp;
10180 switch (MI.getOpcode()) {
10181 default:
10182 llvm_unreachable("unexpected addsat/subsat opcode");
10183 case TargetOpcode::G_UADDSAT:
10184 IsSigned = false;
10185 IsAdd = true;
10186 OverflowOp = TargetOpcode::G_UADDO;
10187 break;
10188 case TargetOpcode::G_SADDSAT:
10189 IsSigned = true;
10190 IsAdd = true;
10191 OverflowOp = TargetOpcode::G_SADDO;
10192 break;
10193 case TargetOpcode::G_USUBSAT:
10194 IsSigned = false;
10195 IsAdd = false;
10196 OverflowOp = TargetOpcode::G_USUBO;
10197 break;
10198 case TargetOpcode::G_SSUBSAT:
10199 IsSigned = true;
10200 IsAdd = false;
10201 OverflowOp = TargetOpcode::G_SSUBO;
10202 break;
10203 }
10204
10205 auto OverflowRes =
10206 MIRBuilder.buildInstr(Opc: OverflowOp, DstOps: {Ty, BoolTy}, SrcOps: {LHS, RHS});
10207 Register Tmp = OverflowRes.getReg(Idx: 0);
10208 Register Ov = OverflowRes.getReg(Idx: 1);
10209 MachineInstrBuilder Clamp;
10210 if (IsSigned) {
10211 // sadd.sat(a, b) ->
10212 // {tmp, ov} = saddo(a, b)
10213 // ov ? (tmp >>s 31) + 0x80000000 : r
10214 // ssub.sat(a, b) ->
10215 // {tmp, ov} = ssubo(a, b)
10216 // ov ? (tmp >>s 31) + 0x80000000 : r
10217 uint64_t NumBits = Ty.getScalarSizeInBits();
10218 auto ShiftAmount = MIRBuilder.buildConstant(Res: Ty, Val: NumBits - 1);
10219 auto Sign = MIRBuilder.buildAShr(Dst: Ty, Src0: Tmp, Src1: ShiftAmount);
10220 auto MinVal =
10221 MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMinValue(numBits: NumBits));
10222 Clamp = MIRBuilder.buildAdd(Dst: Ty, Src0: Sign, Src1: MinVal);
10223 } else {
10224 // uadd.sat(a, b) ->
10225 // {tmp, ov} = uaddo(a, b)
10226 // ov ? 0xffffffff : tmp
10227 // usub.sat(a, b) ->
10228 // {tmp, ov} = usubo(a, b)
10229 // ov ? 0 : tmp
10230 Clamp = MIRBuilder.buildConstant(Res: Ty, Val: IsAdd ? -1 : 0);
10231 }
10232 MIRBuilder.buildSelect(Res, Tst: Ov, Op0: Clamp, Op1: Tmp);
10233
10234 MI.eraseFromParent();
10235 return Legalized;
10236}
10237
10238LegalizerHelper::LegalizeResult
10239LegalizerHelper::lowerShlSat(MachineInstr &MI) {
10240 assert((MI.getOpcode() == TargetOpcode::G_SSHLSAT ||
10241 MI.getOpcode() == TargetOpcode::G_USHLSAT) &&
10242 "Expected shlsat opcode!");
10243 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SSHLSAT;
10244 auto [Res, LHS, RHS] = MI.getFirst3Regs();
10245 LLT Ty = MRI.getType(Reg: Res);
10246 LLT BoolTy = Ty.changeElementSize(NewEltSize: 1);
10247
10248 unsigned BW = Ty.getScalarSizeInBits();
10249 auto Result = MIRBuilder.buildShl(Dst: Ty, Src0: LHS, Src1: RHS);
10250 auto Orig = IsSigned ? MIRBuilder.buildAShr(Dst: Ty, Src0: Result, Src1: RHS)
10251 : MIRBuilder.buildLShr(Dst: Ty, Src0: Result, Src1: RHS);
10252
10253 MachineInstrBuilder SatVal;
10254 if (IsSigned) {
10255 auto SatMin = MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMinValue(numBits: BW));
10256 auto SatMax = MIRBuilder.buildConstant(Res: Ty, Val: APInt::getSignedMaxValue(numBits: BW));
10257 auto Cmp = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SLT, Res: BoolTy, Op0: LHS,
10258 Op1: MIRBuilder.buildConstant(Res: Ty, Val: 0));
10259 SatVal = MIRBuilder.buildSelect(Res: Ty, Tst: Cmp, Op0: SatMin, Op1: SatMax);
10260 } else {
10261 SatVal = MIRBuilder.buildConstant(Res: Ty, Val: APInt::getMaxValue(numBits: BW));
10262 }
10263 auto Ov = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_NE, Res: BoolTy, Op0: LHS, Op1: Orig);
10264 MIRBuilder.buildSelect(Res, Tst: Ov, Op0: SatVal, Op1: Result);
10265
10266 MI.eraseFromParent();
10267 return Legalized;
10268}
10269
10270LegalizerHelper::LegalizeResult
10271LegalizerHelper::lowerTruncSat(MachineInstr &MI) {
10272 unsigned Opc = MI.getOpcode();
10273 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
10274 unsigned DstSize = DstTy.getScalarSizeInBits();
10275 unsigned SrcSize = SrcTy.getScalarSizeInBits();
10276
10277 if (Opc == TargetOpcode::G_TRUNC_SSAT_S) {
10278 auto Max = MIRBuilder.buildConstant(
10279 Res: SrcTy, Val: APInt::getSignedMaxValue(numBits: DstSize).sext(width: SrcSize));
10280 Src = MIRBuilder.buildSMin(Dst: SrcTy, Src0: Src, Src1: Max).getReg(Idx: 0);
10281 auto Min = MIRBuilder.buildConstant(
10282 Res: SrcTy, Val: APInt::getSignedMinValue(numBits: DstSize).sext(width: SrcSize));
10283 Src = MIRBuilder.buildSMax(Dst: SrcTy, Src0: Src, Src1: Min).getReg(Idx: 0);
10284 } else if (Opc == TargetOpcode::G_TRUNC_USAT_U) {
10285 auto Max = MIRBuilder.buildConstant(
10286 Res: SrcTy, Val: APInt::getAllOnes(numBits: DstSize).zext(width: SrcSize));
10287 Src = MIRBuilder.buildUMin(Dst: SrcTy, Src0: Src, Src1: Max).getReg(Idx: 0);
10288 } else if (Opc == TargetOpcode::G_TRUNC_SSAT_U) {
10289 auto Max = MIRBuilder.buildConstant(
10290 Res: SrcTy, Val: APInt::getAllOnes(numBits: DstSize).zext(width: SrcSize));
10291 Src = MIRBuilder.buildSMin(Dst: SrcTy, Src0: Src, Src1: Max).getReg(Idx: 0);
10292 auto Min = MIRBuilder.buildConstant(Res: SrcTy, Val: APInt::getZero(numBits: SrcSize));
10293 Src = MIRBuilder.buildSMax(Dst: SrcTy, Src0: Src, Src1: Min).getReg(Idx: 0);
10294 } else {
10295 llvm_unreachable("Expected truncsat opcode!");
10296 }
10297
10298 MIRBuilder.buildTrunc(Res: Dst, Op: Src);
10299 MI.eraseFromParent();
10300 return Legalized;
10301}
10302
10303LegalizerHelper::LegalizeResult LegalizerHelper::lowerBswap(MachineInstr &MI) {
10304 auto [Dst, Src] = MI.getFirst2Regs();
10305 const LLT Ty = MRI.getType(Reg: Src);
10306 unsigned SizeInBytes = (Ty.getScalarSizeInBits() + 7) / 8;
10307 unsigned BaseShiftAmt = (SizeInBytes - 1) * 8;
10308
10309 // Swap most and least significant byte, set remaining bytes in Res to zero.
10310 auto ShiftAmt = MIRBuilder.buildConstant(Res: Ty, Val: BaseShiftAmt);
10311 auto LSByteShiftedLeft = MIRBuilder.buildShl(Dst: Ty, Src0: Src, Src1: ShiftAmt);
10312 auto MSByteShiftedRight = MIRBuilder.buildLShr(Dst: Ty, Src0: Src, Src1: ShiftAmt);
10313 auto Res = MIRBuilder.buildOr(Dst: Ty, Src0: MSByteShiftedRight, Src1: LSByteShiftedLeft);
10314
10315 // Set i-th high/low byte in Res to i-th low/high byte from Src.
10316 for (unsigned i = 1; i < SizeInBytes / 2; ++i) {
10317 // AND with Mask leaves byte i unchanged and sets remaining bytes to 0.
10318 APInt APMask = APInt::getBitsSet(numBits: SizeInBytes * 8, loBit: i * 8, hiBit: i * 8 + 8);
10319 auto Mask = MIRBuilder.buildConstant(Res: Ty, Val: APMask);
10320 auto ShiftAmt = MIRBuilder.buildConstant(Res: Ty, Val: BaseShiftAmt - 16 * i);
10321 // Low byte shifted left to place of high byte: (Src & Mask) << ShiftAmt.
10322 auto LoByte = MIRBuilder.buildAnd(Dst: Ty, Src0: Src, Src1: Mask);
10323 auto LoShiftedLeft = MIRBuilder.buildShl(Dst: Ty, Src0: LoByte, Src1: ShiftAmt);
10324 Res = MIRBuilder.buildOr(Dst: Ty, Src0: Res, Src1: LoShiftedLeft);
10325 // High byte shifted right to place of low byte: (Src >> ShiftAmt) & Mask.
10326 auto SrcShiftedRight = MIRBuilder.buildLShr(Dst: Ty, Src0: Src, Src1: ShiftAmt);
10327 auto HiShiftedRight = MIRBuilder.buildAnd(Dst: Ty, Src0: SrcShiftedRight, Src1: Mask);
10328 Res = MIRBuilder.buildOr(Dst: Ty, Src0: Res, Src1: HiShiftedRight);
10329 }
10330 Res.getInstr()->getOperand(i: 0).setReg(Dst);
10331
10332 MI.eraseFromParent();
10333 return Legalized;
10334}
10335
10336//{ (Src & Mask) >> N } | { (Src << N) & Mask }
10337static MachineInstrBuilder SwapN(unsigned N, DstOp Dst, MachineIRBuilder &B,
10338 MachineInstrBuilder Src, const APInt &Mask) {
10339 const LLT Ty = Dst.getLLTTy(MRI: *B.getMRI());
10340 MachineInstrBuilder C_N = B.buildConstant(Res: Ty, Val: N);
10341 MachineInstrBuilder MaskLoNTo0 = B.buildConstant(Res: Ty, Val: Mask);
10342 auto LHS = B.buildLShr(Dst: Ty, Src0: B.buildAnd(Dst: Ty, Src0: Src, Src1: MaskLoNTo0), Src1: C_N);
10343 auto RHS = B.buildAnd(Dst: Ty, Src0: B.buildShl(Dst: Ty, Src0: Src, Src1: C_N), Src1: MaskLoNTo0);
10344 return B.buildOr(Dst, Src0: LHS, Src1: RHS);
10345}
10346
10347LegalizerHelper::LegalizeResult
10348LegalizerHelper::lowerBitreverse(MachineInstr &MI) {
10349 auto [Dst, Src] = MI.getFirst2Regs();
10350 const LLT SrcTy = MRI.getType(Reg: Src);
10351 unsigned Size = SrcTy.getScalarSizeInBits();
10352 unsigned VSize = SrcTy.getSizeInBits();
10353
10354 if (Size >= 8) {
10355 if (SrcTy.isVector() && (VSize % 8 == 0) &&
10356 (LI.isLegal(Query: {TargetOpcode::G_BITREVERSE,
10357 {LLT::fixed_vector(NumElements: VSize / 8, ScalarTy: LLT::integer(SizeInBits: 8)),
10358 LLT::fixed_vector(NumElements: VSize / 8, ScalarTy: LLT::integer(SizeInBits: 8))}}))) {
10359 // If bitreverse is legal for i8 vector of the same size, then cast
10360 // to i8 vector type.
10361 // e.g. v4s32 -> v16s8
10362 LLT VTy = LLT::fixed_vector(NumElements: VSize / 8, ScalarTy: LLT::integer(SizeInBits: 8));
10363 auto BSWAP = MIRBuilder.buildBSwap(Dst: SrcTy, Src0: Src);
10364 auto Cast = MIRBuilder.buildBitcast(Dst: VTy, Src: BSWAP);
10365 auto RBIT = MIRBuilder.buildBitReverse(Dst: VTy, Src: Cast);
10366 MIRBuilder.buildBitcast(Dst, Src: RBIT);
10367 } else {
10368 MachineInstrBuilder BSWAP =
10369 MIRBuilder.buildInstr(Opc: TargetOpcode::G_BSWAP, DstOps: {SrcTy}, SrcOps: {Src});
10370
10371 // swap high and low 4 bits in 8 bit blocks 7654|3210 -> 3210|7654
10372 // [(val & 0xF0F0F0F0) >> 4] | [(val & 0x0F0F0F0F) << 4]
10373 // -> [(val & 0xF0F0F0F0) >> 4] | [(val << 4) & 0xF0F0F0F0]
10374 MachineInstrBuilder Swap4 = SwapN(N: 4, Dst: SrcTy, B&: MIRBuilder, Src: BSWAP,
10375 Mask: APInt::getSplat(NewLen: Size, V: APInt(8, 0xF0)));
10376
10377 // swap high and low 2 bits in 4 bit blocks 32|10 76|54 -> 10|32 54|76
10378 // [(val & 0xCCCCCCCC) >> 2] & [(val & 0x33333333) << 2]
10379 // -> [(val & 0xCCCCCCCC) >> 2] & [(val << 2) & 0xCCCCCCCC]
10380 MachineInstrBuilder Swap2 = SwapN(N: 2, Dst: SrcTy, B&: MIRBuilder, Src: Swap4,
10381 Mask: APInt::getSplat(NewLen: Size, V: APInt(8, 0xCC)));
10382
10383 // swap high and low 1 bit in 2 bit blocks 1|0 3|2 5|4 7|6 -> 0|1 2|3 4|5
10384 // 6|7
10385 // [(val & 0xAAAAAAAA) >> 1] & [(val & 0x55555555) << 1]
10386 // -> [(val & 0xAAAAAAAA) >> 1] & [(val << 1) & 0xAAAAAAAA]
10387 SwapN(N: 1, Dst, B&: MIRBuilder, Src: Swap2, Mask: APInt::getSplat(NewLen: Size, V: APInt(8, 0xAA)));
10388 }
10389 } else {
10390 // Expand bitreverse for types smaller than 8 bits.
10391 MachineInstrBuilder Tmp;
10392 for (unsigned I = 0, J = Size - 1; I < Size; ++I, --J) {
10393 MachineInstrBuilder Tmp2;
10394 if (I < J) {
10395 auto ShAmt = MIRBuilder.buildConstant(Res: SrcTy, Val: J - I);
10396 Tmp2 = MIRBuilder.buildShl(Dst: SrcTy, Src0: Src, Src1: ShAmt);
10397 } else {
10398 auto ShAmt = MIRBuilder.buildConstant(Res: SrcTy, Val: I - J);
10399 Tmp2 = MIRBuilder.buildLShr(Dst: SrcTy, Src0: Src, Src1: ShAmt);
10400 }
10401
10402 auto Mask = MIRBuilder.buildConstant(Res: SrcTy, Val: 1ULL << J);
10403 Tmp2 = MIRBuilder.buildAnd(Dst: SrcTy, Src0: Tmp2, Src1: Mask);
10404 if (I == 0)
10405 Tmp = Tmp2;
10406 else
10407 Tmp = MIRBuilder.buildOr(Dst: SrcTy, Src0: Tmp, Src1: Tmp2);
10408 }
10409 MIRBuilder.buildCopy(Res: Dst, Op: Tmp);
10410 }
10411
10412 MI.eraseFromParent();
10413 return Legalized;
10414}
10415
10416LegalizerHelper::LegalizeResult
10417LegalizerHelper::lowerReadWriteRegister(MachineInstr &MI) {
10418 MachineFunction &MF = MIRBuilder.getMF();
10419
10420 bool IsRead = MI.getOpcode() == TargetOpcode::G_READ_REGISTER;
10421 int NameOpIdx = IsRead ? 1 : 0;
10422 int ValRegIndex = IsRead ? 0 : 1;
10423
10424 Register ValReg = MI.getOperand(i: ValRegIndex).getReg();
10425 const LLT Ty = MRI.getType(Reg: ValReg);
10426 const MDString *RegStr = cast<MDString>(
10427 Val: cast<MDNode>(Val: MI.getOperand(i: NameOpIdx).getMetadata())->getOperand(I: 0));
10428
10429 Register PhysReg = TLI.getRegisterByName(RegName: RegStr->getString().data(), Ty, MF);
10430 if (!PhysReg) {
10431 const Function &Fn = MF.getFunction();
10432 Fn.getContext().diagnose(DI: DiagnosticInfoGenericWithLoc(
10433 "invalid register \"" + Twine(RegStr->getString().data()) + "\" for " +
10434 (IsRead ? "llvm.read_register" : "llvm.write_register"),
10435 Fn, MI.getDebugLoc()));
10436 if (IsRead)
10437 MIRBuilder.buildUndef(Res: ValReg);
10438
10439 MI.eraseFromParent();
10440 return Legalized;
10441 }
10442
10443 if (IsRead)
10444 MIRBuilder.buildCopy(Res: ValReg, Op: PhysReg);
10445 else
10446 MIRBuilder.buildCopy(Res: PhysReg, Op: ValReg);
10447
10448 MI.eraseFromParent();
10449 return Legalized;
10450}
10451
10452LegalizerHelper::LegalizeResult
10453LegalizerHelper::lowerSMULH_UMULH(MachineInstr &MI) {
10454 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SMULH;
10455 unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
10456 Register Result = MI.getOperand(i: 0).getReg();
10457 LLT OrigTy = MRI.getType(Reg: Result);
10458 auto SizeInBits = OrigTy.getScalarSizeInBits();
10459 LLT WideTy = OrigTy.changeElementSize(NewEltSize: SizeInBits * 2);
10460
10461 auto LHS = MIRBuilder.buildInstr(Opc: ExtOp, DstOps: {WideTy}, SrcOps: {MI.getOperand(i: 1)});
10462 auto RHS = MIRBuilder.buildInstr(Opc: ExtOp, DstOps: {WideTy}, SrcOps: {MI.getOperand(i: 2)});
10463 auto Mul = MIRBuilder.buildMul(Dst: WideTy, Src0: LHS, Src1: RHS);
10464 unsigned ShiftOp = IsSigned ? TargetOpcode::G_ASHR : TargetOpcode::G_LSHR;
10465
10466 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: SizeInBits);
10467 auto Shifted = MIRBuilder.buildInstr(Opc: ShiftOp, DstOps: {WideTy}, SrcOps: {Mul, ShiftAmt});
10468 MIRBuilder.buildTrunc(Res: Result, Op: Shifted);
10469
10470 MI.eraseFromParent();
10471 return Legalized;
10472}
10473
10474LegalizerHelper::LegalizeResult
10475LegalizerHelper::lowerISFPCLASS(MachineInstr &MI) {
10476 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
10477 FPClassTest Mask = static_cast<FPClassTest>(MI.getOperand(i: 2).getImm());
10478
10479 if (Mask == fcNone) {
10480 MIRBuilder.buildConstant(Res: DstReg, Val: 0);
10481 MI.eraseFromParent();
10482 return Legalized;
10483 }
10484 if (Mask == fcAllFlags) {
10485 MIRBuilder.buildConstant(Res: DstReg, Val: 1);
10486 MI.eraseFromParent();
10487 return Legalized;
10488 }
10489
10490 // TODO: Try inverting the test with getInvertedFPClassTest like the DAG
10491 // version
10492
10493 unsigned BitSize = SrcTy.getScalarSizeInBits();
10494 const fltSemantics &Semantics = getFltSemanticForLLT(Ty: SrcTy.getScalarType());
10495
10496 LLT IntTy = SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: BitSize));
10497 auto AsInt = SrcTy == IntTy ? MIRBuilder.buildCopy(Res: IntTy, Op: SrcReg)
10498 : MIRBuilder.buildBitcast(Dst: IntTy, Src: SrcReg);
10499
10500 // Various masks.
10501 APInt SignBit = APInt::getSignMask(BitWidth: BitSize);
10502 APInt ValueMask = APInt::getSignedMaxValue(numBits: BitSize); // All bits but sign.
10503 APInt Inf = APFloat::getInf(Sem: Semantics).bitcastToAPInt(); // Exp and int bit.
10504 APInt ExpMask = Inf;
10505 APInt AllOneMantissa = APFloat::getLargest(Sem: Semantics).bitcastToAPInt() & ~Inf;
10506 APInt QNaNBitMask =
10507 APInt::getOneBitSet(numBits: BitSize, BitNo: AllOneMantissa.getActiveBits() - 1);
10508 APInt InversionMask = APInt::getAllOnes(numBits: DstTy.getScalarSizeInBits());
10509
10510 auto SignBitC = MIRBuilder.buildConstant(Res: IntTy, Val: SignBit);
10511 auto ValueMaskC = MIRBuilder.buildConstant(Res: IntTy, Val: ValueMask);
10512 auto InfC = MIRBuilder.buildConstant(Res: IntTy, Val: Inf);
10513 auto ExpMaskC = MIRBuilder.buildConstant(Res: IntTy, Val: ExpMask);
10514 auto ZeroC = MIRBuilder.buildConstant(Res: IntTy, Val: 0);
10515
10516 auto Abs = MIRBuilder.buildAnd(Dst: IntTy, Src0: AsInt, Src1: ValueMaskC);
10517 auto Sign =
10518 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_NE, Res: DstTy, Op0: AsInt, Op1: Abs);
10519
10520 auto Res = MIRBuilder.buildConstant(Res: DstTy, Val: 0);
10521 // Clang doesn't support capture of structured bindings:
10522 LLT DstTyCopy = DstTy;
10523 const auto appendToRes = [&](MachineInstrBuilder ToAppend) {
10524 Res = MIRBuilder.buildOr(Dst: DstTyCopy, Src0: Res, Src1: ToAppend);
10525 };
10526
10527 // Tests that involve more than one class should be processed first.
10528 if ((Mask & fcFinite) == fcFinite) {
10529 // finite(V) ==> abs(V) u< exp_mask
10530 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: Abs,
10531 Op1: ExpMaskC));
10532 Mask &= ~fcFinite;
10533 } else if ((Mask & fcFinite) == fcPosFinite) {
10534 // finite(V) && V > 0 ==> V u< exp_mask
10535 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: AsInt,
10536 Op1: ExpMaskC));
10537 Mask &= ~fcPosFinite;
10538 } else if ((Mask & fcFinite) == fcNegFinite) {
10539 // finite(V) && V < 0 ==> abs(V) u< exp_mask && signbit == 1
10540 auto Cmp = MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: Abs,
10541 Op1: ExpMaskC);
10542 auto And = MIRBuilder.buildAnd(Dst: DstTy, Src0: Cmp, Src1: Sign);
10543 appendToRes(And);
10544 Mask &= ~fcNegFinite;
10545 }
10546
10547 if (FPClassTest PartialCheck = Mask & (fcZero | fcSubnormal)) {
10548 // fcZero | fcSubnormal => test all exponent bits are 0
10549 // TODO: Handle sign bit specific cases
10550 // TODO: Handle inverted case
10551 if (PartialCheck == (fcZero | fcSubnormal)) {
10552 auto ExpBits = MIRBuilder.buildAnd(Dst: IntTy, Src0: AsInt, Src1: ExpMaskC);
10553 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10554 Op0: ExpBits, Op1: ZeroC));
10555 Mask &= ~PartialCheck;
10556 }
10557 }
10558
10559 // Check for individual classes.
10560 if (FPClassTest PartialCheck = Mask & fcZero) {
10561 if (PartialCheck == fcPosZero)
10562 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10563 Op0: AsInt, Op1: ZeroC));
10564 else if (PartialCheck == fcZero)
10565 appendToRes(
10566 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy, Op0: Abs, Op1: ZeroC));
10567 else // fcNegZero
10568 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10569 Op0: AsInt, Op1: SignBitC));
10570 }
10571
10572 if (FPClassTest PartialCheck = Mask & fcSubnormal) {
10573 // issubnormal(V) ==> unsigned(abs(V) - 1) u< (all mantissa bits set)
10574 // issubnormal(V) && V>0 ==> unsigned(V - 1) u< (all mantissa bits set)
10575 auto V = (PartialCheck == fcPosSubnormal) ? AsInt : Abs;
10576 auto OneC = MIRBuilder.buildConstant(Res: IntTy, Val: 1);
10577 auto VMinusOne = MIRBuilder.buildSub(Dst: IntTy, Src0: V, Src1: OneC);
10578 auto SubnormalRes =
10579 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: VMinusOne,
10580 Op1: MIRBuilder.buildConstant(Res: IntTy, Val: AllOneMantissa));
10581 if (PartialCheck == fcNegSubnormal)
10582 SubnormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: SubnormalRes, Src1: Sign);
10583 appendToRes(SubnormalRes);
10584 }
10585
10586 if (FPClassTest PartialCheck = Mask & fcInf) {
10587 if (PartialCheck == fcPosInf)
10588 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10589 Op0: AsInt, Op1: InfC));
10590 else if (PartialCheck == fcInf)
10591 appendToRes(
10592 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy, Op0: Abs, Op1: InfC));
10593 else { // fcNegInf
10594 APInt NegInf = APFloat::getInf(Sem: Semantics, Negative: true).bitcastToAPInt();
10595 auto NegInfC = MIRBuilder.buildConstant(Res: IntTy, Val: NegInf);
10596 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
10597 Op0: AsInt, Op1: NegInfC));
10598 }
10599 }
10600
10601 if (FPClassTest PartialCheck = Mask & fcNan) {
10602 auto InfWithQnanBitC = MIRBuilder.buildConstant(Res: IntTy, Val: Inf | QNaNBitMask);
10603 if (PartialCheck == fcNan) {
10604 // isnan(V) ==> abs(V) u> int(inf)
10605 appendToRes(
10606 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: DstTy, Op0: Abs, Op1: InfC));
10607 } else if (PartialCheck == fcQNan) {
10608 // isquiet(V) ==> abs(V) u>= (unsigned(Inf) | quiet_bit)
10609 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGE, Res: DstTy, Op0: Abs,
10610 Op1: InfWithQnanBitC));
10611 } else { // fcSNan
10612 // issignaling(V) ==> abs(V) u> unsigned(Inf) &&
10613 // abs(V) u< (unsigned(Inf) | quiet_bit)
10614 auto IsNan =
10615 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: DstTy, Op0: Abs, Op1: InfC);
10616 auto IsNotQnan = MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy,
10617 Op0: Abs, Op1: InfWithQnanBitC);
10618 appendToRes(MIRBuilder.buildAnd(Dst: DstTy, Src0: IsNan, Src1: IsNotQnan));
10619 }
10620 }
10621
10622 if (FPClassTest PartialCheck = Mask & fcNormal) {
10623 // isnormal(V) ==> (0 u< exp u< max_exp) ==> (unsigned(exp-1) u<
10624 // (max_exp-1))
10625 APInt ExpLSB = ExpMask & ~(ExpMask.shl(shiftAmt: 1));
10626 auto ExpMinusOne = MIRBuilder.buildSub(
10627 Dst: IntTy, Src0: Abs, Src1: MIRBuilder.buildConstant(Res: IntTy, Val: ExpLSB));
10628 APInt MaxExpMinusOne = ExpMask - ExpLSB;
10629 auto NormalRes =
10630 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: ExpMinusOne,
10631 Op1: MIRBuilder.buildConstant(Res: IntTy, Val: MaxExpMinusOne));
10632 if (PartialCheck == fcNegNormal)
10633 NormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: NormalRes, Src1: Sign);
10634 else if (PartialCheck == fcPosNormal) {
10635 auto PosSign = MIRBuilder.buildXor(
10636 Dst: DstTy, Src0: Sign, Src1: MIRBuilder.buildConstant(Res: DstTy, Val: InversionMask));
10637 NormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: NormalRes, Src1: PosSign);
10638 }
10639 appendToRes(NormalRes);
10640 }
10641
10642 MIRBuilder.buildCopy(Res: DstReg, Op: Res);
10643 MI.eraseFromParent();
10644 return Legalized;
10645}
10646
10647LegalizerHelper::LegalizeResult LegalizerHelper::lowerSelect(MachineInstr &MI) {
10648 // Implement G_SELECT in terms of XOR, AND, OR.
10649 auto [DstReg, DstTy, MaskReg, MaskTy, Op1Reg, Op1Ty, Op2Reg, Op2Ty] =
10650 MI.getFirst4RegLLTs();
10651
10652 LLT Op1TyInt =
10653 Op1Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: Op1Ty.getScalarSizeInBits()));
10654
10655 bool IsEltPtr = DstTy.isPointerOrPointerVector();
10656 if (IsEltPtr) {
10657 LLT ScalarPtrTy = LLT::integer(SizeInBits: DstTy.getScalarSizeInBits());
10658 LLT NewTy = DstTy.changeElementType(NewEltTy: ScalarPtrTy);
10659 Op1Reg = MIRBuilder.buildPtrToInt(Dst: NewTy, Src: Op1Reg).getReg(Idx: 0);
10660 Op1Ty = MRI.getType(Reg: Op1Reg);
10661 Op2Reg = MIRBuilder.buildPtrToInt(Dst: NewTy, Src: Op2Reg).getReg(Idx: 0);
10662 Op2Ty = MRI.getType(Reg: Op2Reg);
10663 DstTy = NewTy;
10664 }
10665
10666 if (MaskTy.isScalar()) {
10667 // Turn the scalar condition into a vector condition mask if needed.
10668
10669 Register MaskElt = MaskReg;
10670
10671 // The condition was potentially zero extended before, but we want a sign
10672 // extended boolean.
10673 if (MaskTy != LLT::scalar(SizeInBits: 1))
10674 MaskElt = MIRBuilder.buildSExtInReg(Res: MaskTy, Op: MaskElt, ImmOp: 1).getReg(Idx: 0);
10675
10676 // Continue the sign extension (or truncate) to match the data type.
10677 MaskTy = DstTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: DstTy.getScalarSizeInBits()));
10678 MaskElt =
10679 MIRBuilder.buildSExtOrTrunc(Res: MaskTy.getScalarType(), Op: MaskElt).getReg(Idx: 0);
10680
10681 if (DstTy.isVector()) {
10682 // Generate a vector splat idiom.
10683 auto ShufSplat = MIRBuilder.buildShuffleSplat(Res: MaskTy, Src: MaskElt);
10684 MaskReg = ShufSplat.getReg(Idx: 0);
10685 } else {
10686 MaskReg = MaskElt;
10687 }
10688 } else if (!DstTy.isVector()) {
10689 // Cannot handle the case that mask is a vector and dst is a scalar.
10690 return UnableToLegalize;
10691 }
10692
10693 if (MaskTy.getSizeInBits() != DstTy.getSizeInBits()) {
10694 return UnableToLegalize;
10695 }
10696
10697 if (!Op1Ty.getScalarType().isAnyScalar() &&
10698 !Op1Ty.getScalarType().isInteger())
10699 Op1Reg = MIRBuilder.buildBitcast(Dst: Op1TyInt, Src: Op1Reg).getReg(Idx: 0);
10700
10701 if (!Op2Ty.getScalarType().isAnyScalar() &&
10702 !Op2Ty.getScalarType().isInteger()) {
10703 auto Op2TyInt =
10704 Op2Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: Op2Ty.getScalarSizeInBits()));
10705 Op2Reg = MIRBuilder.buildBitcast(Dst: Op2TyInt, Src: Op2Reg).getReg(Idx: 0);
10706 }
10707
10708 auto NotMask = MIRBuilder.buildNot(Dst: MaskTy, Src0: MaskReg);
10709 auto NewOp1 = MIRBuilder.buildAnd(Dst: MaskTy, Src0: Op1Reg, Src1: MaskReg);
10710 auto NewOp2 = MIRBuilder.buildAnd(Dst: MaskTy, Src0: Op2Reg, Src1: NotMask);
10711 if (IsEltPtr) {
10712 auto Or = MIRBuilder.buildOr(Dst: DstTy, Src0: NewOp1, Src1: NewOp2);
10713 MIRBuilder.buildIntToPtr(Dst: DstReg, Src: Or);
10714 } else {
10715 if (DstTy == Op1TyInt)
10716 MIRBuilder.buildOr(Dst: DstReg, Src0: NewOp1, Src1: NewOp2);
10717 else {
10718 auto Or = MIRBuilder.buildOr(Dst: Op1TyInt, Src0: NewOp1, Src1: NewOp2);
10719 MIRBuilder.buildBitcast(Dst: DstReg, Src: Or.getReg(Idx: 0));
10720 }
10721 }
10722 MI.eraseFromParent();
10723 return Legalized;
10724}
10725
10726LegalizerHelper::LegalizeResult LegalizerHelper::lowerDIVREM(MachineInstr &MI) {
10727 // Split DIVREM into individual instructions.
10728 unsigned Opcode = MI.getOpcode();
10729
10730 MIRBuilder.buildInstr(
10731 Opc: Opcode == TargetOpcode::G_SDIVREM ? TargetOpcode::G_SDIV
10732 : TargetOpcode::G_UDIV,
10733 DstOps: {MI.getOperand(i: 0).getReg()}, SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3)});
10734 MIRBuilder.buildInstr(
10735 Opc: Opcode == TargetOpcode::G_SDIVREM ? TargetOpcode::G_SREM
10736 : TargetOpcode::G_UREM,
10737 DstOps: {MI.getOperand(i: 1).getReg()}, SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3)});
10738 MI.eraseFromParent();
10739 return Legalized;
10740}
10741
10742LegalizerHelper::LegalizeResult
10743LegalizerHelper::lowerAbsToAddXor(MachineInstr &MI) {
10744 // Expand %res = G_ABS %a into:
10745 // %v1 = G_ASHR %a, scalar_size-1
10746 // %v2 = G_ADD %a, %v1
10747 // %res = G_XOR %v2, %v1
10748 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
10749 Register OpReg = MI.getOperand(i: 1).getReg();
10750 auto ShiftAmt =
10751 MIRBuilder.buildConstant(Res: DstTy, Val: DstTy.getScalarSizeInBits() - 1);
10752 auto Shift = MIRBuilder.buildAShr(Dst: DstTy, Src0: OpReg, Src1: ShiftAmt);
10753 auto Add = MIRBuilder.buildAdd(Dst: DstTy, Src0: OpReg, Src1: Shift);
10754 MIRBuilder.buildXor(Dst: MI.getOperand(i: 0).getReg(), Src0: Add, Src1: Shift);
10755 MI.eraseFromParent();
10756 return Legalized;
10757}
10758
10759LegalizerHelper::LegalizeResult
10760LegalizerHelper::lowerAbsToMaxNeg(MachineInstr &MI) {
10761 // Expand %res = G_ABS %a into:
10762 // %v1 = G_CONSTANT 0
10763 // %v2 = G_SUB %v1, %a
10764 // %res = G_SMAX %a, %v2
10765 Register SrcReg = MI.getOperand(i: 1).getReg();
10766 LLT Ty = MRI.getType(Reg: SrcReg);
10767 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0);
10768 auto Sub = MIRBuilder.buildSub(Dst: Ty, Src0: Zero, Src1: SrcReg);
10769 MIRBuilder.buildSMax(Dst: MI.getOperand(i: 0), Src0: SrcReg, Src1: Sub);
10770 MI.eraseFromParent();
10771 return Legalized;
10772}
10773
10774LegalizerHelper::LegalizeResult
10775LegalizerHelper::lowerAbsToCNeg(MachineInstr &MI) {
10776 Register SrcReg = MI.getOperand(i: 1).getReg();
10777 Register DestReg = MI.getOperand(i: 0).getReg();
10778 LLT Ty = MRI.getType(Reg: SrcReg), IType = LLT::integer(SizeInBits: 1);
10779 auto Zero = MIRBuilder.buildConstant(Res: Ty, Val: 0).getReg(Idx: 0);
10780 auto Sub = MIRBuilder.buildSub(Dst: Ty, Src0: Zero, Src1: SrcReg).getReg(Idx: 0);
10781 auto ICmp = MIRBuilder.buildICmp(Pred: CmpInst::ICMP_SGT, Res: IType, Op0: SrcReg, Op1: Zero);
10782 MIRBuilder.buildSelect(Res: DestReg, Tst: ICmp, Op0: SrcReg, Op1: Sub);
10783 MI.eraseFromParent();
10784 return Legalized;
10785}
10786
10787LegalizerHelper::LegalizeResult
10788LegalizerHelper::lowerAbsDiffToSelect(MachineInstr &MI) {
10789 assert((MI.getOpcode() == TargetOpcode::G_ABDS ||
10790 MI.getOpcode() == TargetOpcode::G_ABDU) &&
10791 "Expected G_ABDS or G_ABDU instruction");
10792
10793 auto [DstReg, LHS, RHS] = MI.getFirst3Regs();
10794 LLT Ty = MRI.getType(Reg: LHS);
10795
10796 // abds(lhs, rhs) -> select(sgt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
10797 // abdu(lhs, rhs) -> select(ugt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
10798 Register LHSSub = MIRBuilder.buildSub(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10799 Register RHSSub = MIRBuilder.buildSub(Dst: Ty, Src0: RHS, Src1: LHS).getReg(Idx: 0);
10800 CmpInst::Predicate Pred = (MI.getOpcode() == TargetOpcode::G_ABDS)
10801 ? CmpInst::ICMP_SGT
10802 : CmpInst::ICMP_UGT;
10803 auto ICmp = MIRBuilder.buildICmp(Pred, Res: LLT::scalar(SizeInBits: 1), Op0: LHS, Op1: RHS);
10804 MIRBuilder.buildSelect(Res: DstReg, Tst: ICmp, Op0: LHSSub, Op1: RHSSub);
10805
10806 MI.eraseFromParent();
10807 return Legalized;
10808}
10809
10810LegalizerHelper::LegalizeResult
10811LegalizerHelper::lowerAbsDiffToMinMax(MachineInstr &MI) {
10812 assert((MI.getOpcode() == TargetOpcode::G_ABDS ||
10813 MI.getOpcode() == TargetOpcode::G_ABDU) &&
10814 "Expected G_ABDS or G_ABDU instruction");
10815
10816 auto [DstReg, LHS, RHS] = MI.getFirst3Regs();
10817 LLT Ty = MRI.getType(Reg: LHS);
10818
10819 // abds(lhs, rhs) -→ sub(smax(lhs, rhs), smin(lhs, rhs))
10820 // abdu(lhs, rhs) -→ sub(umax(lhs, rhs), umin(lhs, rhs))
10821 Register MaxReg, MinReg;
10822 if (MI.getOpcode() == TargetOpcode::G_ABDS) {
10823 MaxReg = MIRBuilder.buildSMax(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10824 MinReg = MIRBuilder.buildSMin(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10825 } else {
10826 MaxReg = MIRBuilder.buildUMax(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10827 MinReg = MIRBuilder.buildUMin(Dst: Ty, Src0: LHS, Src1: RHS).getReg(Idx: 0);
10828 }
10829 MIRBuilder.buildSub(Dst: DstReg, Src0: MaxReg, Src1: MinReg);
10830
10831 MI.eraseFromParent();
10832 return Legalized;
10833}
10834
10835LegalizerHelper::LegalizeResult LegalizerHelper::lowerFAbs(MachineInstr &MI) {
10836 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
10837 LLT TyInt =
10838 DstTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: DstTy.getScalarSizeInBits()));
10839 Register CastedSrc = SrcReg;
10840
10841 if (!(SrcTy.getScalarType().isAnyScalar() ||
10842 SrcTy.getScalarType().isInteger())) {
10843 auto SrcTyInt =
10844 SrcTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: SrcTy.getScalarSizeInBits()));
10845 CastedSrc = MIRBuilder.buildBitcast(Dst: SrcTyInt, Src: SrcReg).getReg(Idx: 0);
10846 }
10847
10848 if (MRI.getType(Reg: DstReg) != TyInt) {
10849 // Reset sign bit
10850 Register NewDst =
10851 MIRBuilder
10852 .buildAnd(Dst: TyInt, Src0: CastedSrc,
10853 Src1: MIRBuilder.buildConstant(
10854 Res: TyInt, Val: APInt::getSignedMaxValue(
10855 numBits: DstTy.getScalarSizeInBits())))
10856 .getReg(Idx: 0);
10857
10858 MIRBuilder.buildBitcast(Dst: DstReg, Src: NewDst);
10859 } else
10860 MIRBuilder
10861 .buildAnd(
10862 Dst: DstReg, Src0: CastedSrc,
10863 Src1: MIRBuilder.buildConstant(
10864 Res: TyInt, Val: APInt::getSignedMaxValue(numBits: DstTy.getScalarSizeInBits())))
10865 .getReg(Idx: 0);
10866
10867 MI.eraseFromParent();
10868 return Legalized;
10869}
10870
10871LegalizerHelper::LegalizeResult
10872LegalizerHelper::lowerVectorReduction(MachineInstr &MI) {
10873 Register SrcReg = MI.getOperand(i: 1).getReg();
10874 LLT SrcTy = MRI.getType(Reg: SrcReg);
10875 LLT DstTy = MRI.getType(Reg: SrcReg);
10876
10877 // The source could be a scalar if the IR type was <1 x sN>.
10878 if (SrcTy.isScalar()) {
10879 if (DstTy.getSizeInBits() > SrcTy.getSizeInBits())
10880 return UnableToLegalize; // FIXME: handle extension.
10881 // This can be just a plain copy.
10882 Observer.changingInstr(MI);
10883 MI.setDesc(MIRBuilder.getTII().get(Opcode: TargetOpcode::COPY));
10884 Observer.changedInstr(MI);
10885 return Legalized;
10886 }
10887 return UnableToLegalize;
10888}
10889
10890LegalizerHelper::LegalizeResult LegalizerHelper::lowerVAArg(MachineInstr &MI) {
10891 MachineFunction &MF = *MI.getMF();
10892 const DataLayout &DL = MIRBuilder.getDataLayout();
10893 LLVMContext &Ctx = MF.getFunction().getContext();
10894 Register ListPtr = MI.getOperand(i: 1).getReg();
10895 LLT PtrTy = MRI.getType(Reg: ListPtr);
10896
10897 // LstPtr is a pointer to the head of the list. Get the address
10898 // of the head of the list.
10899 Align PtrAlignment = DL.getABITypeAlign(Ty: getTypeForLLT(Ty: PtrTy, C&: Ctx));
10900 MachineMemOperand *PtrLoadMMO = MF.getMachineMemOperand(
10901 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOLoad, MemTy: PtrTy, BaseAlignment: PtrAlignment);
10902 auto VAList = MIRBuilder.buildLoad(Res: PtrTy, Addr: ListPtr, MMO&: *PtrLoadMMO).getReg(Idx: 0);
10903
10904 const Align A(MI.getOperand(i: 2).getImm());
10905 LLT PtrTyAsScalarTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
10906 if (A > TLI.getMinStackArgumentAlignment()) {
10907 Register AlignAmt =
10908 MIRBuilder.buildConstant(Res: PtrTyAsScalarTy, Val: A.value() - 1).getReg(Idx: 0);
10909 auto AddDst = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: VAList, Op1: AlignAmt);
10910 auto AndDst = MIRBuilder.buildMaskLowPtrBits(Res: PtrTy, Op0: AddDst, NumBits: Log2(A));
10911 VAList = AndDst.getReg(Idx: 0);
10912 }
10913
10914 // Increment the pointer, VAList, to the next vaarg
10915 // The list should be bumped by the size of element in the current head of
10916 // list.
10917 Register Dst = MI.getOperand(i: 0).getReg();
10918 LLT LLTTy = MRI.getType(Reg: Dst);
10919 Type *Ty = getTypeForLLT(Ty: LLTTy, C&: Ctx);
10920 auto IncAmt =
10921 MIRBuilder.buildConstant(Res: PtrTyAsScalarTy, Val: DL.getTypeAllocSize(Ty));
10922 auto Succ = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: VAList, Op1: IncAmt);
10923
10924 // Store the increment VAList to the legalized pointer
10925 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
10926 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOStore, MemTy: PtrTy, BaseAlignment: PtrAlignment);
10927 MIRBuilder.buildStore(Val: Succ, Addr: ListPtr, MMO&: *StoreMMO);
10928 // Load the actual argument out of the pointer VAList
10929 Align EltAlignment = DL.getABITypeAlign(Ty);
10930 MachineMemOperand *EltLoadMMO = MF.getMachineMemOperand(
10931 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOLoad, MemTy: LLTTy, BaseAlignment: EltAlignment);
10932 MIRBuilder.buildLoad(Res: Dst, Addr: VAList, MMO&: *EltLoadMMO);
10933
10934 MI.eraseFromParent();
10935 return Legalized;
10936}
10937
10938LegalizerHelper::LegalizeResult LegalizerHelper::lowerMulfix(MachineInstr &MI) {
10939 unsigned OpCode = MI.getOpcode();
10940 assert((OpCode == TargetOpcode::G_SMULFIX ||
10941 OpCode == TargetOpcode::G_UMULFIX ||
10942 OpCode == TargetOpcode::G_SMULFIXSAT ||
10943 OpCode == TargetOpcode::G_UMULFIXSAT) &&
10944 "Operator must be either G_SMULFIX[SAT] or G_UMULFIX[SAT]!");
10945 auto [Dst, LHS, RHS] = MI.getFirst3Regs();
10946 LLT Ty = MRI.getType(Reg: Dst);
10947 unsigned Scale = MI.getOperand(i: 3).getImm();
10948
10949 bool Saturating = (OpCode == TargetOpcode::G_SMULFIXSAT ||
10950 OpCode == TargetOpcode::G_UMULFIXSAT);
10951 bool IsSigned = (OpCode == TargetOpcode::G_SMULFIX ||
10952 OpCode == TargetOpcode::G_SMULFIXSAT);
10953
10954 if (!Saturating && Scale == 0) {
10955 MIRBuilder.buildMul(Dst, Src0: LHS, Src1: RHS);
10956 MI.eraseFromParent();
10957 return Legalized;
10958 }
10959
10960 // TODO: Port other lowering paths from SelectionDAG.
10961 LLT WideTy = Ty.changeElementSize(NewEltSize: Ty.getScalarSizeInBits() * 2);
10962 auto ShiftAmt = MIRBuilder.buildConstant(Res: WideTy, Val: Scale);
10963 MachineInstrBuilder ExtLHS{}, ExtRHS{}, Shift{};
10964 if (IsSigned) {
10965 ExtLHS = MIRBuilder.buildSExt(Res: WideTy, Op: LHS);
10966 ExtRHS = MIRBuilder.buildSExt(Res: WideTy, Op: RHS);
10967 } else {
10968 ExtLHS = MIRBuilder.buildZExt(Res: WideTy, Op: LHS);
10969 ExtRHS = MIRBuilder.buildZExt(Res: WideTy, Op: RHS);
10970 }
10971
10972 auto Mul = MIRBuilder.buildMul(Dst: WideTy, Src0: ExtLHS, Src1: ExtRHS);
10973 if (IsSigned)
10974 Shift = MIRBuilder.buildAShr(Dst: WideTy, Src0: Mul, Src1: ShiftAmt);
10975 else
10976 Shift = MIRBuilder.buildLShr(Dst: WideTy, Src0: Mul, Src1: ShiftAmt);
10977
10978 if (!Saturating)
10979 MIRBuilder.buildTrunc(Res: Dst, Op: Shift);
10980 else if (IsSigned)
10981 MIRBuilder.buildTruncSSatS(Res: Dst, Op: Shift);
10982 else
10983 MIRBuilder.buildTruncUSatU(Res: Dst, Op: Shift);
10984
10985 MI.eraseFromParent();
10986 return Legalized;
10987}
10988
10989// Get a vectorized representation of the memset value operand, GISel edition.
10990static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB) {
10991 MachineRegisterInfo &MRI = *MIB.getMRI();
10992 unsigned NumBits = Ty.getScalarSizeInBits();
10993 auto ValVRegAndVal = getIConstantVRegValWithLookThrough(VReg: Val, MRI);
10994 if (!Ty.isVector() && ValVRegAndVal) {
10995 APInt Scalar = ValVRegAndVal->Value.trunc(width: 8);
10996 APInt SplatVal = APInt::getSplat(NewLen: NumBits, V: Scalar);
10997 return MIB.buildConstant(Res: Ty, Val: SplatVal).getReg(Idx: 0);
10998 }
10999
11000 // Extend the byte value to the larger type, and then multiply by a magic
11001 // value 0x010101... in order to replicate it across every byte.
11002 // Unless it's zero, in which case just emit a larger G_CONSTANT 0.
11003 if (ValVRegAndVal && ValVRegAndVal->Value == 0) {
11004 return MIB.buildConstant(Res: Ty, Val: 0).getReg(Idx: 0);
11005 }
11006
11007 LLT ExtType = Ty.getScalarType();
11008 auto ZExt = MIB.buildZExtOrTrunc(Res: ExtType, Op: Val);
11009 if (NumBits > 8) {
11010 APInt Magic = APInt::getSplat(NewLen: NumBits, V: APInt(8, 0x01));
11011 auto MagicMI = MIB.buildConstant(Res: ExtType, Val: Magic);
11012 Val = MIB.buildMul(Dst: ExtType, Src0: ZExt, Src1: MagicMI).getReg(Idx: 0);
11013 }
11014
11015 // For vector types create a G_BUILD_VECTOR.
11016 if (Ty.isVector())
11017 Val = MIB.buildSplatBuildVector(Res: Ty, Src: Val).getReg(Idx: 0);
11018
11019 return Val;
11020}
11021
11022LegalizerHelper::LegalizeResult
11023LegalizerHelper::lowerMemset(MachineInstr &MI, Register Dst, Register Val,
11024 uint64_t KnownLen, Align Alignment,
11025 bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
11026 auto &MF = *MI.getParent()->getParent();
11027 const auto &TLI = *MF.getSubtarget().getTargetLowering();
11028 auto &DL = MF.getDataLayout();
11029 LLVMContext &C = MF.getFunction().getContext();
11030
11031 assert(KnownLen != 0 && "Have a zero length memset length!");
11032 assert(!MemOps.empty() && "Expected at least one memory op");
11033
11034 MachineFrameInfo &MFI = MF.getFrameInfo();
11035 MachineInstr *FIDef = getOpcodeDef(Opcode: TargetOpcode::G_FRAME_INDEX, Reg: Dst, MRI);
11036 const auto &DstMMO = **MI.memoperands_begin();
11037
11038 if (DstAlignCanChange) {
11039 // Get an estimate of the type from the LLT.
11040 Type *IRTy = getTypeForLLT(Ty: MemOps[0], C);
11041 Align NewAlign = DL.getABITypeAlign(Ty: IRTy);
11042 if (NewAlign > Alignment) {
11043 Alignment = NewAlign;
11044 unsigned FI = FIDef->getOperand(i: 1).getIndex();
11045 // Give the stack frame object a larger alignment if needed.
11046 if (MFI.getObjectAlign(ObjectIdx: FI) < Alignment)
11047 MFI.setObjectAlignment(ObjectIdx: FI, Alignment);
11048 }
11049 }
11050
11051 MachineIRBuilder MIB(MI);
11052 // Find the largest store and generate the bit pattern for it.
11053 LLT LargestTy = MemOps[0];
11054 for (unsigned i = 1; i < MemOps.size(); i++)
11055 if (MemOps[i].getSizeInBits() > LargestTy.getSizeInBits())
11056 LargestTy = MemOps[i];
11057
11058 // The memset stored value is always defined as an s8, so in order to make it
11059 // work with larger store types we need to repeat the bit pattern across the
11060 // wider type.
11061 Register MemSetValue = getMemsetValue(Val, Ty: LargestTy, MIB);
11062
11063 if (!MemSetValue)
11064 return UnableToLegalize;
11065
11066 // Generate the stores. For each store type in the list, we generate the
11067 // matching store of that type to the destination address.
11068 LLT PtrTy = MRI.getType(Reg: Dst);
11069 unsigned DstOff = 0;
11070 unsigned Size = KnownLen;
11071 for (unsigned I = 0; I < MemOps.size(); I++) {
11072 LLT Ty = MemOps[I];
11073 unsigned TySize = Ty.getSizeInBytes();
11074
11075 if (TySize > Size) {
11076 // Issuing a load / store pair that overlaps with the previous pair.
11077 // Adjust the offset accordingly.
11078 assert(I == MemOps.size() - 1 && I != 0);
11079 DstOff -= TySize - Size;
11080 }
11081
11082 // If this store is smaller than the largest store see whether we can get
11083 // the smaller value for free with a truncate.
11084 Register Value = MemSetValue;
11085 if (Ty.getSizeInBits() < LargestTy.getSizeInBits()) {
11086 MVT VT = getMVTForLLT(Ty);
11087 MVT LargestVT = getMVTForLLT(Ty: LargestTy);
11088 if (!LargestTy.isVector() && !Ty.isVector() &&
11089 TLI.isTruncateFree(FromVT: LargestVT, ToVT: VT))
11090 Value = MIB.buildTrunc(Res: Ty, Op: MemSetValue).getReg(Idx: 0);
11091 else
11092 Value = getMemsetValue(Val, Ty, MIB);
11093 if (!Value)
11094 return UnableToLegalize;
11095 }
11096
11097 auto *StoreMMO = MF.getMachineMemOperand(MMO: &DstMMO, Offset: DstOff, Ty);
11098
11099 Register Ptr = Dst;
11100 if (DstOff != 0) {
11101 auto Offset =
11102 MIB.buildConstant(Res: LLT::integer(SizeInBits: PtrTy.getSizeInBits()), Val: DstOff);
11103 Ptr = MIB.buildObjectPtrOffset(Res: PtrTy, Op0: Dst, Op1: Offset).getReg(Idx: 0);
11104 }
11105
11106 MIB.buildStore(Val: Value, Addr: Ptr, MMO&: *StoreMMO);
11107 DstOff += Ty.getSizeInBytes();
11108 Size -= TySize;
11109 }
11110
11111 MI.eraseFromParent();
11112 return Legalized;
11113}
11114
11115LegalizerHelper::LegalizeResult
11116LegalizerHelper::lowerMemcpy(MachineInstr &MI, Register Dst, Register Src,
11117 uint64_t KnownLen, Align Alignment,
11118 bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
11119 auto &MF = *MI.getParent()->getParent();
11120 auto &DL = MF.getDataLayout();
11121 LLVMContext &C = MF.getFunction().getContext();
11122
11123 assert(KnownLen != 0 && "Have a zero length memcpy length!");
11124 assert(!MemOps.empty() && "Expected at least one memory op");
11125
11126 MachineFrameInfo &MFI = MF.getFrameInfo();
11127 MachineInstr *FIDef = getOpcodeDef(Opcode: TargetOpcode::G_FRAME_INDEX, Reg: Dst, MRI);
11128
11129 // FIXME: infer better src pointer alignment like SelectionDAG does here.
11130 // FIXME: also use the equivalent of isMemSrcFromConstant and alwaysinlining
11131 // if the memcpy is in a tail call position.
11132
11133 const auto &DstMMO = **MI.memoperands_begin();
11134 const auto &SrcMMO = **std::next(x: MI.memoperands_begin());
11135
11136 if (DstAlignCanChange) {
11137 // Get an estimate of the type from the LLT.
11138 Type *IRTy = getTypeForLLT(Ty: MemOps[0], C);
11139 Align NewAlign = DL.getABITypeAlign(Ty: IRTy);
11140
11141 // Don't promote to an alignment that would require dynamic stack
11142 // realignment.
11143 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
11144 if (!TRI->hasStackRealignment(MF))
11145 if (MaybeAlign StackAlign = DL.getStackAlignment())
11146 NewAlign = std::min(a: NewAlign, b: *StackAlign);
11147
11148 if (NewAlign > Alignment) {
11149 Alignment = NewAlign;
11150 unsigned FI = FIDef->getOperand(i: 1).getIndex();
11151 // Give the stack frame object a larger alignment if needed.
11152 if (MFI.getObjectAlign(ObjectIdx: FI) < Alignment)
11153 MFI.setObjectAlignment(ObjectIdx: FI, Alignment);
11154 }
11155 }
11156
11157 LLVM_DEBUG(dbgs() << "Inlining memcpy: " << MI << " into loads & stores\n");
11158
11159 MachineIRBuilder MIB(MI);
11160 // Now we need to emit a pair of load and stores for each of the types we've
11161 // collected. I.e. for each type, generate a load from the source pointer of
11162 // that type width, and then generate a corresponding store to the dest buffer
11163 // of that value loaded. This can result in a sequence of loads and stores
11164 // mixed types, depending on what the target specifies as good types to use.
11165 unsigned CurrOffset = 0;
11166 unsigned Size = KnownLen;
11167 for (auto CopyTy : MemOps) {
11168 TypeSize TySize = CopyTy.getSizeInBytes();
11169
11170 // Issuing a load / store pair that overlaps with the previous pair. Adjust
11171 // the offset accordingly.
11172 if (TySize > Size) {
11173 unsigned Overlap = TySize - Size;
11174 assert(Overlap < CurrOffset &&
11175 "overlapping memcpy load/store spans the whole region or more");
11176 CurrOffset -= Overlap;
11177 }
11178
11179 // Construct MMOs for the accesses.
11180 auto *LoadMMO = MF.getMachineMemOperand(MMO: &SrcMMO, Offset: CurrOffset, Size: TySize);
11181 auto *StoreMMO = MF.getMachineMemOperand(MMO: &DstMMO, Offset: CurrOffset, Size: TySize);
11182
11183 // Create the load.
11184 Register LoadPtr = Src;
11185 Register Offset;
11186 if (CurrOffset != 0) {
11187 LLT SrcTy = MRI.getType(Reg: Src);
11188 Offset =
11189 MIB.buildConstant(Res: LLT::integer(SizeInBits: SrcTy.getSizeInBits()), Val: CurrOffset)
11190 .getReg(Idx: 0);
11191 LoadPtr = MIB.buildObjectPtrOffset(Res: SrcTy, Op0: Src, Op1: Offset).getReg(Idx: 0);
11192 }
11193 auto LdVal = MIB.buildLoad(Res: CopyTy, Addr: LoadPtr, MMO&: *LoadMMO);
11194
11195 // Create the store.
11196 Register StorePtr = Dst;
11197 if (CurrOffset != 0) {
11198 LLT DstTy = MRI.getType(Reg: Dst);
11199 StorePtr = MIB.buildObjectPtrOffset(Res: DstTy, Op0: Dst, Op1: Offset).getReg(Idx: 0);
11200 }
11201 MIB.buildStore(Val: LdVal, Addr: StorePtr, MMO&: *StoreMMO);
11202 CurrOffset += TySize;
11203 Size -= TySize;
11204 }
11205
11206 MI.eraseFromParent();
11207 return Legalized;
11208}
11209
11210LegalizerHelper::LegalizeResult
11211LegalizerHelper::lowerMemmove(MachineInstr &MI, Register Dst, Register Src,
11212 uint64_t KnownLen, Align Alignment,
11213 bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
11214 auto &MF = *MI.getParent()->getParent();
11215 auto &DL = MF.getDataLayout();
11216 LLVMContext &C = MF.getFunction().getContext();
11217
11218 assert(KnownLen != 0 && "Have a zero length memmove length!");
11219 assert(!MemOps.empty() && "Expected at least one memory op");
11220
11221 MachineFrameInfo &MFI = MF.getFrameInfo();
11222 MachineInstr *FIDef = getOpcodeDef(Opcode: TargetOpcode::G_FRAME_INDEX, Reg: Dst, MRI);
11223 const auto &DstMMO = **MI.memoperands_begin();
11224 const auto &SrcMMO = **std::next(x: MI.memoperands_begin());
11225
11226 if (DstAlignCanChange) {
11227 // Get an estimate of the type from the LLT.
11228 Type *IRTy = getTypeForLLT(Ty: MemOps[0], C);
11229 Align NewAlign = DL.getABITypeAlign(Ty: IRTy);
11230
11231 // Don't promote to an alignment that would require dynamic stack
11232 // realignment.
11233 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
11234 if (!TRI->hasStackRealignment(MF))
11235 if (MaybeAlign StackAlign = DL.getStackAlignment())
11236 NewAlign = std::min(a: NewAlign, b: *StackAlign);
11237
11238 if (NewAlign > Alignment) {
11239 Alignment = NewAlign;
11240 unsigned FI = FIDef->getOperand(i: 1).getIndex();
11241 // Give the stack frame object a larger alignment if needed.
11242 if (MFI.getObjectAlign(ObjectIdx: FI) < Alignment)
11243 MFI.setObjectAlignment(ObjectIdx: FI, Alignment);
11244 }
11245 }
11246
11247 LLVM_DEBUG(dbgs() << "Inlining memmove: " << MI << " into loads & stores\n");
11248
11249 MachineIRBuilder MIB(MI);
11250 // Memmove requires that we perform the loads first before issuing the stores.
11251 // Apart from that, this loop is pretty much doing the same thing as the
11252 // memcpy codegen function.
11253 unsigned CurrOffset = 0;
11254 unsigned Size = KnownLen;
11255 SmallVector<Register, 16> LoadVals;
11256 for (auto CopyTy : MemOps) {
11257 TypeSize TySize = CopyTy.getSizeInBytes();
11258
11259 // Issuing a load that overlaps with the previous load. Adjust the offset
11260 // accordingly.
11261 if (TySize > Size) {
11262 unsigned Overlap = TySize - Size;
11263 assert(Overlap < CurrOffset &&
11264 "overlapping memmove load spans the whole region or more");
11265 CurrOffset -= Overlap;
11266 }
11267
11268 // Construct MMO for the load.
11269 auto *LoadMMO = MF.getMachineMemOperand(MMO: &SrcMMO, Offset: CurrOffset, Size: TySize);
11270
11271 // Create the load.
11272 Register LoadPtr = Src;
11273 if (CurrOffset != 0) {
11274 LLT SrcTy = MRI.getType(Reg: Src);
11275 auto Offset =
11276 MIB.buildConstant(Res: LLT::integer(SizeInBits: SrcTy.getSizeInBits()), Val: CurrOffset);
11277 LoadPtr = MIB.buildObjectPtrOffset(Res: SrcTy, Op0: Src, Op1: Offset).getReg(Idx: 0);
11278 }
11279 LoadVals.push_back(Elt: MIB.buildLoad(Res: CopyTy, Addr: LoadPtr, MMO&: *LoadMMO).getReg(Idx: 0));
11280 CurrOffset += TySize;
11281 Size -= TySize;
11282 }
11283
11284 CurrOffset = 0;
11285 Size = KnownLen;
11286 for (unsigned I = 0; I < MemOps.size(); ++I) {
11287 LLT CopyTy = MemOps[I];
11288 TypeSize TySize = CopyTy.getSizeInBytes();
11289
11290 // Issuing a store that overlaps with the previous store. Adjust the offset
11291 // accordingly.
11292 if (TySize > Size) {
11293 unsigned Overlap = TySize - Size;
11294 assert(Overlap < CurrOffset &&
11295 "overlapping memmove store spans the whole region or more");
11296 CurrOffset -= Overlap;
11297 }
11298
11299 // Now store the values loaded.
11300 auto *StoreMMO = MF.getMachineMemOperand(MMO: &DstMMO, Offset: CurrOffset, Size: TySize);
11301
11302 Register StorePtr = Dst;
11303 if (CurrOffset != 0) {
11304 LLT DstTy = MRI.getType(Reg: Dst);
11305 auto Offset =
11306 MIB.buildConstant(Res: LLT::integer(SizeInBits: DstTy.getSizeInBits()), Val: CurrOffset);
11307 StorePtr = MIB.buildObjectPtrOffset(Res: DstTy, Op0: Dst, Op1: Offset).getReg(Idx: 0);
11308 }
11309 MIB.buildStore(Val: LoadVals[I], Addr: StorePtr, MMO&: *StoreMMO);
11310 CurrOffset += TySize;
11311 Size -= TySize;
11312 }
11313 MI.eraseFromParent();
11314 return Legalized;
11315}
11316
11317LegalizerHelper::LegalizeResult LegalizerHelper::lowerMemCpyFamily(
11318 MachineInstr &MI, Register Dst, Register Src, uint64_t KnownLen,
11319 Align Alignment, bool DstAlignCanChange, ArrayRef<LLT> MemOps) {
11320 const unsigned Opc = MI.getOpcode();
11321 assert((Opc == TargetOpcode::G_MEMCPY ||
11322 Opc == TargetOpcode::G_MEMCPY_INLINE ||
11323 Opc == TargetOpcode::G_MEMMOVE || Opc == TargetOpcode::G_MEMSET ||
11324 Opc == TargetOpcode::G_MEMSET_INLINE) &&
11325 "Expected memcpy like instruction");
11326
11327 if (KnownLen == 0) {
11328 MI.eraseFromParent();
11329 return Legalized;
11330 }
11331
11332 if (Opc == TargetOpcode::G_MEMCPY || Opc == TargetOpcode::G_MEMCPY_INLINE) {
11333 return lowerMemcpy(MI, Dst, Src, KnownLen, Alignment, DstAlignCanChange,
11334 MemOps);
11335 }
11336 if (Opc == TargetOpcode::G_MEMMOVE)
11337 return lowerMemmove(MI, Dst, Src, KnownLen, Alignment, DstAlignCanChange,
11338 MemOps);
11339 if (Opc == TargetOpcode::G_MEMSET || Opc == TargetOpcode::G_MEMSET_INLINE)
11340 return lowerMemset(MI, Dst, Val: Src, KnownLen, Alignment, DstAlignCanChange,
11341 MemOps);
11342 return UnableToLegalize;
11343}
11344
11345LegalizerHelper::LegalizeResult
11346LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen) {
11347 Register Dst, Src;
11348 uint64_t KnownLen;
11349 Align Alignment;
11350 bool DstAlignCanChange;
11351 std::vector<LLT> MemOps;
11352 if (!canLowerMemCpyFamily(MI, MRI, MaxLen, Dst, Src, KnownLen, Alignment,
11353 DstAlignCanChange, MemOps))
11354 return UnableToLegalize;
11355 return lowerMemCpyFamily(MI, Dst, Src, KnownLen, Alignment, DstAlignCanChange,
11356 MemOps);
11357}
11358