1//===- AMDGPInstCombineIntrinsic.cpp - AMDGPU specific InstCombine pass ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// \file
10// This file implements a TargetTransformInfo analysis pass specific to the
11// AMDGPU target machine. It uses the target's detailed information to provide
12// more precise answers to certain TTI queries, while letting the target
13// independent and default TTI implementations handle the rest.
14//
15//===----------------------------------------------------------------------===//
16
17#include "AMDGPUInstrInfo.h"
18#include "AMDGPUTargetTransformInfo.h"
19#include "GCNSubtarget.h"
20#include "SIDefines.h"
21#include "llvm/ADT/FloatingPointMode.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/Analysis/ConstantFolding.h"
24#include "llvm/Analysis/ValueTracking.h"
25#include "llvm/IR/Constants.h"
26#include "llvm/IR/Dominators.h"
27#include "llvm/IR/IntrinsicsAMDGPU.h"
28#include "llvm/Support/MathExtras.h"
29#include "llvm/Transforms/InstCombine/InstCombiner.h"
30#include <optional>
31
32using namespace llvm;
33using namespace llvm::PatternMatch;
34
35#define DEBUG_TYPE "AMDGPUtti"
36
37namespace {
38
39struct AMDGPUImageDMaskIntrinsic {
40 unsigned Intr;
41};
42
43#define GET_AMDGPUImageDMaskIntrinsicTable_IMPL
44#include "AMDGPUGenSearchableTables.inc"
45
46} // end anonymous namespace
47
48// Constant fold llvm.amdgcn.fmed3 intrinsics for standard inputs.
49//
50// A single NaN input is folded to minnum, so we rely on that folding for
51// handling NaNs.
52static APFloat fmed3AMDGCN(const APFloat &Src0, const APFloat &Src1,
53 const APFloat &Src2) {
54 assert(!Src0.isNaN() && !Src1.isNaN() && !Src2.isNaN() &&
55 "nans handled separately");
56 APFloat Max3 = maxnum(A: maxnum(A: Src0, B: Src1), B: Src2);
57
58 if (Max3.bitwiseIsEqual(RHS: Src0))
59 return maxnum(A: Src1, B: Src2);
60
61 if (Max3.bitwiseIsEqual(RHS: Src1))
62 return maxnum(A: Src0, B: Src2);
63
64 return maxnum(A: Src0, B: Src1);
65}
66
67// Check if a value can be converted to a 16-bit value without losing precision.
68// The value is expected to be either a float (IsFloat = true) or an unsigned
69// integer (IsFloat = false). When AllowI16SExt is set, a sext from i16 is also
70// accepted: for unsigned addresses sext and zext only differ for a negative
71// i16, which is out of bounds anyway (see caller).
72static bool canSafelyConvertTo16Bit(Value &V, bool IsFloat,
73 bool AllowI16SExt = false) {
74 Type *VTy = V.getType();
75 if (VTy->isHalfTy() || VTy->isIntegerTy(BitWidth: 16)) {
76 // The value is already 16-bit, so we don't want to convert to 16-bit again!
77 return false;
78 }
79 if (IsFloat) {
80 if (ConstantFP *ConstFloat = dyn_cast<ConstantFP>(Val: &V)) {
81 // We need to check that if we cast the index down to a half, we do not
82 // lose precision.
83 APFloat FloatValue(ConstFloat->getValueAPF());
84 bool LosesInfo = true;
85 FloatValue.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmTowardZero,
86 losesInfo: &LosesInfo);
87 return !LosesInfo;
88 }
89 } else {
90 if (ConstantInt *ConstInt = dyn_cast<ConstantInt>(Val: &V)) {
91 // We need to check that if we cast the index down to an i16, we do not
92 // lose precision.
93 APInt IntValue(ConstInt->getValue());
94 return IntValue.getActiveBits() <= 16;
95 }
96 }
97
98 // Coordinates may arrive as extractelement((s|z|fp)ext Vec), Idx. The
99 // widening cast has one use per lane, so it is never sunk into the extract;
100 // strip the extract here so the cast check below is common to scalar and
101 // vector coords.
102 Value *CastCandidate;
103 if (!match(V: &V, P: m_ExtractElt(Val: m_Value(V&: CastCandidate), Idx: m_Value())))
104 CastCandidate = &V;
105
106 Value *CastSrc;
107 bool IsExt = IsFloat ? match(V: CastCandidate, P: m_FPExt(Op: m_Value(V&: CastSrc)))
108 : match(V: CastCandidate, P: m_ZExt(Op: m_Value(V&: CastSrc)));
109 if (!IsExt && !IsFloat && AllowI16SExt)
110 IsExt = match(V: CastCandidate, P: m_SExt(Op: m_Value(V&: CastSrc)));
111 if (IsExt) {
112 Type *CastSrcTy = CastSrc->getType()->getScalarType();
113 if (CastSrcTy->isHalfTy() || CastSrcTy->isIntegerTy(BitWidth: 16))
114 return true;
115 }
116
117 return false;
118}
119
120// Convert a value to 16-bit.
121static Value *convertTo16Bit(Value &V, InstCombiner::BuilderTy &Builder) {
122 Type *VTy = V.getType();
123 if (isa<FPExtInst, SExtInst, ZExtInst>(Val: &V))
124 return cast<Instruction>(Val: &V)->getOperand(i: 0);
125 // Vector form: extractelement((s|z|fp)ext Vec), Idx -> extractelement(Vec,
126 // Idx), taking the narrow lane directly so the widening cast can be removed.
127 Instruction *VecCast;
128 Value *Idx;
129 if (match(V: &V, P: m_ExtractElt(Val: m_Instruction(I&: VecCast), Idx: m_Value(V&: Idx))) &&
130 isa<FPExtInst, SExtInst, ZExtInst>(Val: VecCast))
131 return Builder.CreateExtractElement(Vec: VecCast->getOperand(i: 0), Idx);
132 if (VTy->isIntegerTy())
133 return Builder.CreateIntCast(V: &V, DestTy: Type::getInt16Ty(C&: V.getContext()), isSigned: false);
134 if (VTy->isFloatingPointTy())
135 return Builder.CreateFPCast(V: &V, DestTy: Type::getHalfTy(C&: V.getContext()));
136
137 llvm_unreachable("Should never be called!");
138}
139
140/// Applies Func(OldIntr.Args, OldIntr.ArgTys), creates intrinsic call with
141/// modified arguments (based on OldIntr) and replaces InstToReplace with
142/// this newly created intrinsic call.
143static std::optional<Instruction *> modifyIntrinsicCall(
144 IntrinsicInst &OldIntr, Instruction &InstToReplace, unsigned NewIntr,
145 InstCombiner &IC,
146 std::function<void(SmallVectorImpl<Value *> &, SmallVectorImpl<Type *> &)>
147 Func) {
148 SmallVector<Type *, 4> OverloadTys;
149 if (!Intrinsic::isSignatureValid(F: OldIntr.getCalledFunction(), OverloadTys))
150 return std::nullopt;
151
152 SmallVector<Value *, 8> Args(OldIntr.args());
153
154 // Modify arguments and types
155 Func(Args, OverloadTys);
156
157 CallInst *NewCall =
158 IC.Builder.CreateIntrinsicWithoutFolding(ID: NewIntr, OverloadTypes: OverloadTys, Args);
159 NewCall->takeName(V: &OldIntr);
160 NewCall->copyMetadata(SrcInst: OldIntr);
161 if (isa<FPMathOperator>(Val: NewCall))
162 NewCall->copyFastMathFlags(I: &OldIntr);
163 // Copy attributes
164 AttributeList OldAttrList = OldIntr.getAttributes();
165 NewCall->setAttributes(OldAttrList);
166
167 // Erase and replace uses
168 if (!InstToReplace.getType()->isVoidTy())
169 IC.replaceInstUsesWith(I&: InstToReplace, V: NewCall);
170
171 bool RemoveOldIntr = &OldIntr != &InstToReplace;
172
173 auto *RetValue = IC.eraseInstFromFunction(I&: InstToReplace);
174 if (RemoveOldIntr)
175 IC.eraseInstFromFunction(I&: OldIntr);
176
177 return RetValue;
178}
179
180static std::optional<Instruction *>
181simplifyAMDGCNImageIntrinsic(const GCNSubtarget *ST,
182 const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr,
183 IntrinsicInst &II, InstCombiner &IC) {
184 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
185 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: ImageDimIntr->BaseOpcode);
186
187 // Optimize _L to _LZ when _L is zero
188 if (const auto *LZMappingInfo =
189 AMDGPU::getMIMGLZMappingInfo(L: ImageDimIntr->BaseOpcode)) {
190 if (auto *ConstantLod =
191 dyn_cast<ConstantFP>(Val: II.getOperand(i_nocapture: ImageDimIntr->LodIndex))) {
192 if (ConstantLod->isZero() || ConstantLod->isNegative()) {
193 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
194 AMDGPU::getImageDimIntrinsicByBaseOpcode(BaseOpcode: LZMappingInfo->LZ,
195 Dim: ImageDimIntr->Dim);
196 return modifyIntrinsicCall(
197 OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC, Func: [&](auto &Args, auto &ArgTys) {
198 Args.erase(Args.begin() + ImageDimIntr->LodIndex);
199 });
200 }
201 }
202 }
203
204 // Optimize _mip away, when 'lod' is zero
205 if (const auto *MIPMappingInfo =
206 AMDGPU::getMIMGMIPMappingInfo(MIP: ImageDimIntr->BaseOpcode)) {
207 if (auto *ConstantMip =
208 dyn_cast<ConstantInt>(Val: II.getOperand(i_nocapture: ImageDimIntr->MipIndex))) {
209 if (ConstantMip->isZero()) {
210 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
211 AMDGPU::getImageDimIntrinsicByBaseOpcode(BaseOpcode: MIPMappingInfo->NONMIP,
212 Dim: ImageDimIntr->Dim);
213 return modifyIntrinsicCall(
214 OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC, Func: [&](auto &Args, auto &ArgTys) {
215 Args.erase(Args.begin() + ImageDimIntr->MipIndex);
216 });
217 }
218 }
219 }
220
221 // Optimize _bias away when 'bias' is zero
222 if (const auto *BiasMappingInfo =
223 AMDGPU::getMIMGBiasMappingInfo(Bias: ImageDimIntr->BaseOpcode)) {
224 if (auto *ConstantBias =
225 dyn_cast<ConstantFP>(Val: II.getOperand(i_nocapture: ImageDimIntr->BiasIndex))) {
226 if (ConstantBias->isZero()) {
227 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
228 AMDGPU::getImageDimIntrinsicByBaseOpcode(BaseOpcode: BiasMappingInfo->NoBias,
229 Dim: ImageDimIntr->Dim);
230 return modifyIntrinsicCall(
231 OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC, Func: [&](auto &Args, auto &ArgTys) {
232 Args.erase(Args.begin() + ImageDimIntr->BiasIndex);
233 ArgTys.erase(ArgTys.begin() + ImageDimIntr->BiasTyArg);
234 });
235 }
236 }
237 }
238
239 // Optimize _offset away when 'offset' is zero
240 if (const auto *OffsetMappingInfo =
241 AMDGPU::getMIMGOffsetMappingInfo(Offset: ImageDimIntr->BaseOpcode)) {
242 if (auto *ConstantOffset =
243 dyn_cast<ConstantInt>(Val: II.getOperand(i_nocapture: ImageDimIntr->OffsetIndex))) {
244 if (ConstantOffset->isZero()) {
245 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
246 AMDGPU::getImageDimIntrinsicByBaseOpcode(
247 BaseOpcode: OffsetMappingInfo->NoOffset, Dim: ImageDimIntr->Dim);
248 return modifyIntrinsicCall(
249 OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC, Func: [&](auto &Args, auto &ArgTys) {
250 Args.erase(Args.begin() + ImageDimIntr->OffsetIndex);
251 });
252 }
253 }
254 }
255
256 // Optimize the arrayed dim away when the array slice is zero, since slice 0
257 // is the base layer. Restricted to non-atomic, non-sampled image loads and
258 // stores for now.
259 const AMDGPU::MIMGDimInfo *DimInfo =
260 AMDGPU::getMIMGDimInfo(DimEnum: ImageDimIntr->Dim);
261 if (!BaseOpcode->Atomic && !BaseOpcode->Sampler && BaseOpcode->Coordinates &&
262 DimInfo->NonArrayDim != ImageDimIntr->Dim) {
263 // Address is [coords..., slice, (fragid)] plus an optional mip operand.
264 // The slice is the last coordinate, so index it from CoordStart.
265 unsigned SliceIndex = ImageDimIntr->CoordStart + DimInfo->NumCoords - 1 -
266 (DimInfo->MSAA ? 1 : 0);
267 auto *ConstantSlice = dyn_cast<ConstantInt>(Val: II.getOperand(i_nocapture: SliceIndex));
268 if (ConstantSlice && ConstantSlice->isZero()) {
269 if (const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
270 AMDGPU::getImageDimIntrinsicByBaseOpcode(BaseOpcode: ImageDimIntr->BaseOpcode,
271 Dim: DimInfo->NonArrayDim)) {
272 return modifyIntrinsicCall(OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC,
273 Func: [&](auto &Args, auto &ArgTys) {
274 Args.erase(Args.begin() + SliceIndex);
275 });
276 }
277 }
278 }
279
280 // Try to use D16
281 if (ST->hasD16Images()) {
282 if (BaseOpcode->HasD16) {
283
284 // If the only use of image intrinsic is a fptrunc (with conversion to
285 // half) then both fptrunc and image intrinsic will be replaced with image
286 // intrinsic with D16 flag.
287 if (II.hasOneUse()) {
288 Instruction *User = II.user_back();
289
290 if (User->getOpcode() == Instruction::FPTrunc &&
291 User->getType()->getScalarType()->isHalfTy()) {
292
293 return modifyIntrinsicCall(OldIntr&: II, InstToReplace&: *User, NewIntr: ImageDimIntr->Intr, IC,
294 Func: [&](auto &Args, auto &ArgTys) {
295 // Change return type of image intrinsic.
296 // Set it to return type of fptrunc.
297 ArgTys[0] = User->getType();
298 });
299 }
300 }
301
302 // Only perform D16 folding if every user of the image sample is
303 // an ExtractElementInst immediately followed by an FPTrunc to half.
304 SmallVector<std::pair<ExtractElementInst *, FPTruncInst *>, 4>
305 ExtractTruncPairs;
306 bool AllHalfExtracts = true;
307
308 for (User *U : II.users()) {
309 auto *Ext = dyn_cast<ExtractElementInst>(Val: U);
310 if (!Ext || !Ext->hasOneUse()) {
311 AllHalfExtracts = false;
312 break;
313 }
314
315 auto *Tr = dyn_cast<FPTruncInst>(Val: *Ext->user_begin());
316 if (!Tr || !Tr->getType()->isHalfTy()) {
317 AllHalfExtracts = false;
318 break;
319 }
320
321 ExtractTruncPairs.emplace_back(Args&: Ext, Args&: Tr);
322 }
323
324 if (!ExtractTruncPairs.empty() && AllHalfExtracts) {
325 auto *VecTy = cast<VectorType>(Val: II.getType());
326 Type *HalfVecTy =
327 VecTy->getWithNewType(EltTy: Type::getHalfTy(C&: II.getContext()));
328
329 // Obtain the original image sample intrinsic's signature
330 // and replace its return type with the half-vector for D16 folding
331 SmallVector<Type *, 8> OverloadTys;
332 if (!Intrinsic::isSignatureValid(F: II.getCalledFunction(), OverloadTys))
333 return std::nullopt;
334
335 OverloadTys[0] = HalfVecTy;
336 Module *M = II.getModule();
337 Function *HalfDecl = Intrinsic::getOrInsertDeclaration(
338 M, id: ImageDimIntr->Intr, OverloadTys);
339
340 II.mutateType(Ty: HalfVecTy);
341 II.setCalledFunction(HalfDecl);
342
343 IRBuilder<> Builder(*M);
344 for (auto &[Ext, Tr] : ExtractTruncPairs) {
345 Value *Idx = Ext->getIndexOperand();
346
347 Builder.SetInsertPoint(Tr);
348
349 Value *HalfExtract = Builder.CreateExtractElement(Vec: &II, Idx);
350 HalfExtract->takeName(V: Tr);
351
352 Tr->replaceAllUsesWith(V: HalfExtract);
353 }
354
355 for (auto &[Ext, Tr] : ExtractTruncPairs) {
356 IC.eraseInstFromFunction(I&: *Tr);
357 IC.eraseInstFromFunction(I&: *Ext);
358 }
359
360 return &II;
361 }
362 }
363 }
364
365 // Try to use A16 or G16
366 if (!ST->hasA16() && !ST->hasG16())
367 return std::nullopt;
368
369 // Address is interpreted as float if the instruction has a sampler or as
370 // unsigned int if there is no sampler.
371 bool HasSampler = BaseOpcode->Sampler;
372 bool FloatCoord = false;
373 // true means derivatives can be converted to 16 bit, coordinates not
374 bool OnlyDerivatives = false;
375
376 // Sampler-less addresses are unsigned, so a sext from i16 folds to a16 like a
377 // zext: they only disagree for a negative i16 (>= 0x8000), which is out of
378 // bounds while the max image dimension is <= 0x8000.
379 bool AllowI16SExt = !HasSampler;
380
381 for (unsigned OperandIndex = ImageDimIntr->GradientStart;
382 OperandIndex < ImageDimIntr->VAddrEnd; OperandIndex++) {
383 Value *Coord = II.getOperand(i_nocapture: OperandIndex);
384 // If the values are not derived from 16-bit values, we cannot optimize.
385 if (!canSafelyConvertTo16Bit(V&: *Coord, IsFloat: HasSampler, AllowI16SExt)) {
386 if (OperandIndex < ImageDimIntr->CoordStart ||
387 ImageDimIntr->GradientStart == ImageDimIntr->CoordStart) {
388 return std::nullopt;
389 }
390 // All gradients can be converted, so convert only them
391 OnlyDerivatives = true;
392 break;
393 }
394
395 assert(OperandIndex == ImageDimIntr->GradientStart ||
396 FloatCoord == Coord->getType()->isFloatingPointTy());
397 FloatCoord = Coord->getType()->isFloatingPointTy();
398 }
399
400 if (!OnlyDerivatives && !ST->hasA16())
401 OnlyDerivatives = true; // Only supports G16
402
403 // Check if there is a bias parameter and if it can be converted to f16
404 if (!OnlyDerivatives && ImageDimIntr->NumBiasArgs != 0) {
405 Value *Bias = II.getOperand(i_nocapture: ImageDimIntr->BiasIndex);
406 assert(HasSampler &&
407 "Only image instructions with a sampler can have a bias");
408 if (!canSafelyConvertTo16Bit(V&: *Bias, IsFloat: HasSampler))
409 OnlyDerivatives = true;
410 }
411
412 if (OnlyDerivatives && (!ST->hasG16() || ImageDimIntr->GradientStart ==
413 ImageDimIntr->CoordStart))
414 return std::nullopt;
415
416 Type *CoordType = FloatCoord ? Type::getHalfTy(C&: II.getContext())
417 : Type::getInt16Ty(C&: II.getContext());
418
419 return modifyIntrinsicCall(
420 OldIntr&: II, InstToReplace&: II, NewIntr: II.getIntrinsicID(), IC, Func: [&](auto &Args, auto &ArgTys) {
421 ArgTys[ImageDimIntr->GradientTyArg] = CoordType;
422 if (!OnlyDerivatives) {
423 ArgTys[ImageDimIntr->CoordTyArg] = CoordType;
424
425 // Change the bias type
426 if (ImageDimIntr->NumBiasArgs != 0)
427 ArgTys[ImageDimIntr->BiasTyArg] = Type::getHalfTy(C&: II.getContext());
428 }
429
430 unsigned EndIndex =
431 OnlyDerivatives ? ImageDimIntr->CoordStart : ImageDimIntr->VAddrEnd;
432 for (unsigned OperandIndex = ImageDimIntr->GradientStart;
433 OperandIndex < EndIndex; OperandIndex++) {
434 Args[OperandIndex] =
435 convertTo16Bit(V&: *II.getOperand(i_nocapture: OperandIndex), Builder&: IC.Builder);
436 }
437
438 // Convert the bias
439 if (!OnlyDerivatives && ImageDimIntr->NumBiasArgs != 0) {
440 Value *Bias = II.getOperand(i_nocapture: ImageDimIntr->BiasIndex);
441 Args[ImageDimIntr->BiasIndex] = convertTo16Bit(V&: *Bias, Builder&: IC.Builder);
442 }
443 });
444}
445
446bool GCNTTIImpl::canSimplifyLegacyMulToMul(const Instruction &I,
447 const Value *Op0, const Value *Op1,
448 InstCombiner &IC) const {
449 // The legacy behaviour is that multiplying +/-0.0 by anything, even NaN or
450 // infinity, gives +0.0. If we can prove we don't have one of the special
451 // cases then we can use a normal multiply instead.
452 SimplifyQuery SQ = IC.getSimplifyQuery().getWithInstruction(I: &I);
453 KnownFPClass Known0 =
454 computeKnownFPClass(V: Op0, InterestedClasses: fcZero | fcSubnormal | fcInf | fcNan, SQ);
455 DenormalMode Mode = I.getFunction()->getDenormalMode(FPType: APFloat::IEEEsingle());
456
457 // Bail early if Op0 may be zero and nsz is not set -- Op1 cannot help.
458 if (!Known0.isKnownNeverLogicalZero(Mode) && !I.hasNoSignedZeros())
459 return false;
460
461 KnownFPClass Known1 =
462 computeKnownFPClass(V: Op1, InterestedClasses: fcZero | fcSubnormal | fcInf | fcNan, SQ);
463
464 // Simplify if both operands are known non-zero.
465 if (Known0.isKnownNeverLogicalZero(Mode) &&
466 Known1.isKnownNeverLogicalZero(Mode))
467 return true;
468
469 // With nsz, two additional cases allow simplification:
470 // 1. One operand is not zero or infinity or NaN:
471 // Op0 NeverLogicalZero && NeverInfOrNaN, or symmetric for Op1.
472 // 2. Neither operand is infinity or NaN:
473 // Op0 NeverInfOrNaN && Op1 NeverInfOrNaN.
474 // The following condition captures both cases.
475 if (I.hasNoSignedZeros() &&
476 (Known0.isKnownNeverLogicalZero(Mode) || Known1.isKnownNeverInfOrNaN()) &&
477 (Known1.isKnownNeverLogicalZero(Mode) || Known0.isKnownNeverInfOrNaN()))
478 return true;
479
480 return false;
481}
482
483/// Match an fpext from half to float, or a constant we can convert.
484static Value *matchFPExtFromF16(Value *Arg) {
485 Value *Src = nullptr;
486 ConstantFP *CFP = nullptr;
487 if (match(V: Arg, P: m_OneUse(SubPattern: m_FPExt(Op: m_Value(V&: Src))))) {
488 if (Src->getType()->isHalfTy())
489 return Src;
490 } else if (match(V: Arg, P: m_ConstantFP(C&: CFP))) {
491 bool LosesInfo;
492 APFloat Val(CFP->getValueAPF());
493 Val.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmNearestTiesToEven, losesInfo: &LosesInfo);
494 if (!LosesInfo)
495 return ConstantFP::get(Ty: Type::getHalfTy(C&: Arg->getContext()), V: Val);
496 }
497 return nullptr;
498}
499
500// Trim all zero components from the end of the vector \p UseV and return
501// an appropriate bitset with known elements.
502static APInt trimTrailingZerosInVector(InstCombiner &IC, Value *UseV,
503 Instruction *I) {
504 auto *VTy = cast<FixedVectorType>(Val: UseV->getType());
505 unsigned VWidth = VTy->getNumElements();
506 APInt DemandedElts = APInt::getAllOnes(numBits: VWidth);
507
508 for (int i = VWidth - 1; i > 0; --i) {
509 auto *Elt = findScalarElement(V: UseV, EltNo: i);
510 if (!Elt)
511 break;
512
513 if (auto *ConstElt = dyn_cast<Constant>(Val: Elt)) {
514 if (!ConstElt->isNullValue() && !isa<UndefValue>(Val: Elt))
515 break;
516 } else {
517 break;
518 }
519
520 DemandedElts.clearBit(BitPosition: i);
521 }
522
523 return DemandedElts;
524}
525
526// Trim elements of the end of the vector \p V, if they are
527// equal to the first element of the vector.
528static APInt defaultComponentBroadcast(Value *V) {
529 auto *VTy = cast<FixedVectorType>(Val: V->getType());
530 unsigned VWidth = VTy->getNumElements();
531 APInt DemandedElts = APInt::getAllOnes(numBits: VWidth);
532 Value *FirstComponent = findScalarElement(V, EltNo: 0);
533
534 SmallVector<int> ShuffleMask;
535 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: V))
536 SVI->getShuffleMask(Result&: ShuffleMask);
537
538 for (int I = VWidth - 1; I > 0; --I) {
539 if (ShuffleMask.empty()) {
540 auto *Elt = findScalarElement(V, EltNo: I);
541 if (!Elt || (Elt != FirstComponent && !isa<UndefValue>(Val: Elt)))
542 break;
543 } else {
544 // Detect identical elements in the shufflevector result, even though
545 // findScalarElement cannot tell us what that element is.
546 if (ShuffleMask[I] != ShuffleMask[0] && ShuffleMask[I] != PoisonMaskElem)
547 break;
548 }
549 DemandedElts.clearBit(BitPosition: I);
550 }
551
552 return DemandedElts;
553}
554
555static Value *simplifyAMDGCNMemoryIntrinsicDemanded(InstCombiner &IC,
556 IntrinsicInst &II,
557 APInt DemandedElts,
558 int DMaskIdx = -1,
559 bool IsLoad = true);
560
561/// Return true if it's legal to contract llvm.amdgcn.rcp(llvm.sqrt)
562static bool canContractSqrtToRsq(const FPMathOperator *SqrtOp) {
563 return (SqrtOp->getType()->isFloatTy() &&
564 (SqrtOp->hasApproxFunc() || SqrtOp->getFPAccuracy() >= 1.0f)) ||
565 SqrtOp->getType()->isHalfTy();
566}
567
568/// Return true if we can easily prove that use U is uniform.
569static bool isTriviallyUniform(const Use &U) {
570 Value *V = U.get();
571 if (isa<Constant>(Val: V))
572 return true;
573 if (const auto *A = dyn_cast<Argument>(Val: V))
574 return AMDGPU::isArgPassedInSGPR(Arg: A);
575 if (const auto *II = dyn_cast<IntrinsicInst>(Val: V)) {
576 if (!AMDGPU::isIntrinsicAlwaysUniform(IntrID: II->getIntrinsicID()))
577 return false;
578 // If II and U are in different blocks then there is a possibility of
579 // temporal divergence.
580 return II->getParent() == cast<Instruction>(Val: U.getUser())->getParent();
581 }
582 return false;
583}
584
585/// Simplify a lane index operand (e.g. llvm.amdgcn.readlane src1).
586///
587/// The instruction only reads the low 5 bits for wave32, and 6 bits for wave64.
588bool GCNTTIImpl::simplifyDemandedLaneMaskArg(InstCombiner &IC,
589 IntrinsicInst &II,
590 unsigned LaneArgIdx) const {
591 unsigned MaskBits = ST->getWavefrontSizeLog2();
592 APInt DemandedMask(32, maskTrailingOnes<unsigned>(N: MaskBits));
593
594 KnownBits Known(32);
595 if (IC.SimplifyDemandedBits(I: &II, OpNo: LaneArgIdx, DemandedMask, Known))
596 return true;
597
598 if (!Known.isConstant())
599 return false;
600
601 // Out of bounds indexes may appear in wave64 code compiled for wave32.
602 // Unlike the DAG version, SimplifyDemandedBits does not change constants, so
603 // manually fix it up.
604
605 Value *LaneArg = II.getArgOperand(i: LaneArgIdx);
606 Constant *MaskedConst =
607 ConstantInt::get(Ty: LaneArg->getType(), V: Known.getConstant() & DemandedMask);
608 if (MaskedConst != LaneArg) {
609 II.getOperandUse(i: LaneArgIdx).set(MaskedConst);
610 return true;
611 }
612
613 return false;
614}
615
616static CallInst *rewriteCall(IRBuilderBase &B, CallInst &Old,
617 Function &NewCallee, ArrayRef<Value *> Ops) {
618 SmallVector<OperandBundleDef, 2> OpBundles;
619 Old.getOperandBundlesAsDefs(Defs&: OpBundles);
620
621 CallInst *NewCall = B.CreateCall(Callee: &NewCallee, Args: Ops, OpBundles);
622 NewCall->takeName(V: &Old);
623 return NewCall;
624}
625
626// Return true for sequences of instructions that effectively assign
627// each lane to its thread ID
628static bool isThreadID(const GCNSubtarget &ST, Value *V) {
629 // Case 1:
630 // wave32: mbcnt_lo(-1, 0)
631 // wave64: mbcnt_hi(-1, mbcnt_lo(-1, 0))
632 auto W32Pred = m_Intrinsic<Intrinsic::amdgcn_mbcnt_lo>(Ops: m_ConstantInt<-1>(),
633 Ops: m_ConstantInt<0>());
634 auto W64Pred = m_Intrinsic<Intrinsic::amdgcn_mbcnt_hi>(
635 Ops: m_ConstantInt<-1>(), Ops: m_Intrinsic<Intrinsic::amdgcn_mbcnt_lo>(
636 Ops: m_ConstantInt<-1>(), Ops: m_ConstantInt<0>()));
637 if (ST.isWave32() && match(V, P: W32Pred))
638 return true;
639 if (ST.isWave64() && match(V, P: W64Pred))
640 return true;
641
642 return false;
643}
644
645Instruction *
646GCNTTIImpl::hoistLaneIntrinsicThroughOperand(InstCombiner &IC,
647 IntrinsicInst &II) const {
648 const auto IID = II.getIntrinsicID();
649 assert(IID == Intrinsic::amdgcn_readlane ||
650 IID == Intrinsic::amdgcn_readfirstlane ||
651 IID == Intrinsic::amdgcn_permlane64);
652
653 Instruction *OpInst = dyn_cast<Instruction>(Val: II.getOperand(i_nocapture: 0));
654
655 // Only do this if both instructions are in the same block
656 // (so the exec mask won't change) and the readlane is the only user of its
657 // operand.
658 if (!OpInst || !OpInst->hasOneUser() || OpInst->getParent() != II.getParent())
659 return nullptr;
660
661 const bool IsReadLane = (IID == Intrinsic::amdgcn_readlane);
662
663 // If this is a readlane, check that the second operand is a constant, or is
664 // defined before OpInst so we know it's safe to move this intrinsic higher.
665 Value *LaneID = nullptr;
666 if (IsReadLane) {
667 LaneID = II.getOperand(i_nocapture: 1);
668
669 // readlane take an extra operand for the lane ID, so we must check if that
670 // LaneID value can be used at the point where we want to move the
671 // intrinsic.
672 if (auto *LaneIDInst = dyn_cast<Instruction>(Val: LaneID)) {
673 if (!IC.getDominatorTree().dominates(Def: LaneIDInst, User: OpInst))
674 return nullptr;
675 }
676 }
677
678 // Hoist the intrinsic (II) through OpInst.
679 //
680 // (II (OpInst x)) -> (OpInst (II x))
681 const auto DoIt = [&](unsigned OpIdx,
682 Function *NewIntrinsic) -> Instruction * {
683 SmallVector<Value *, 2> Ops{OpInst->getOperand(i: OpIdx)};
684 if (IsReadLane)
685 Ops.push_back(Elt: LaneID);
686
687 // Rewrite the intrinsic call.
688 CallInst *NewII = rewriteCall(B&: IC.Builder, Old&: II, NewCallee&: *NewIntrinsic, Ops);
689
690 // Rewrite OpInst so it takes the result of the intrinsic now.
691 Instruction &NewOp = *OpInst->clone();
692 NewOp.setOperand(i: OpIdx, Val: NewII);
693 return &NewOp;
694 };
695
696 // TODO(?): Should we do more with permlane64?
697 if (IID == Intrinsic::amdgcn_permlane64 && !isa<BitCastInst>(Val: OpInst))
698 return nullptr;
699
700 if (isa<UnaryOperator>(Val: OpInst))
701 return DoIt(0, II.getCalledFunction());
702
703 if (isa<CastInst>(Val: OpInst)) {
704 Value *Src = OpInst->getOperand(i: 0);
705 Type *SrcTy = Src->getType();
706 if (!isTypeLegal(Ty: SrcTy))
707 return nullptr;
708
709 Function *Remangled =
710 Intrinsic::getOrInsertDeclaration(M: II.getModule(), id: IID, OverloadTys: {SrcTy});
711 return DoIt(0, Remangled);
712 }
713
714 // We can also hoist through binary operators if the other operand is uniform.
715 if (isa<BinaryOperator>(Val: OpInst)) {
716 // FIXME: If we had access to UniformityInfo here we could just check
717 // if the operand is uniform.
718 if (isTriviallyUniform(U: OpInst->getOperandUse(i: 0)))
719 return DoIt(1, II.getCalledFunction());
720 if (isTriviallyUniform(U: OpInst->getOperandUse(i: 1)))
721 return DoIt(0, II.getCalledFunction());
722 }
723
724 return nullptr;
725}
726
727/// Evaluate V as a function of the lane ID and return its value on Lane, or
728/// std::nullopt if V is not a closed-form expression of the lane ID.
729static std::optional<uint64_t> evalLaneExpr(Value *V, unsigned Lane,
730 const GCNSubtarget &ST,
731 const DataLayout &DL,
732 unsigned Depth = 0) {
733 if (Depth >= MaxAnalysisRecursionDepth)
734 return std::nullopt;
735
736 // Poison/undef in the index expression: bail and let InstCombine fold the
737 // intrinsic the usual way.
738 if (isa<UndefValue>(Val: V))
739 return std::nullopt;
740
741 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: V))
742 return CI->getZExtValue();
743
744 if (isThreadID(ST, V))
745 return Lane;
746
747 const BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: V);
748 if (!BO)
749 return std::nullopt;
750
751 std::optional<uint64_t> LHS =
752 evalLaneExpr(V: BO->getOperand(i_nocapture: 0), Lane, ST, DL, Depth: Depth + 1);
753 if (!LHS)
754 return std::nullopt;
755 std::optional<uint64_t> RHS =
756 evalLaneExpr(V: BO->getOperand(i_nocapture: 1), Lane, ST, DL, Depth: Depth + 1);
757 if (!RHS)
758 return std::nullopt;
759
760 Type *Ty = BO->getType();
761 Constant *Ops[] = {ConstantInt::get(Ty, V: *LHS), ConstantInt::get(Ty, V: *RHS)};
762 auto *CI =
763 dyn_cast_or_null<ConstantInt>(Val: ConstantFoldInstOperands(I: BO, Ops, DL));
764 return CI ? std::optional<uint64_t>(CI->getZExtValue()) : std::nullopt;
765}
766
767/// Build the per-lane shuffle map by evaluating Index for every lane in the
768/// wave. Returns false if any lane index is non-constant or out of range.
769static bool tryBuildShuffleMap(Value *Index, const GCNSubtarget &ST,
770 SmallVectorImpl<uint8_t> &Ids,
771 const DataLayout &DL) {
772 unsigned WaveSize = ST.getWavefrontSize();
773 Ids.resize(N: WaveSize);
774 for (unsigned Lane : seq(Size: WaveSize)) {
775 std::optional<uint64_t> Val = evalLaneExpr(V: Index, Lane, ST, DL);
776 if (!Val || *Val >= WaveSize)
777 return false;
778 Ids[Lane] = static_cast<uint8_t>(*Val);
779 }
780 return true;
781}
782
783/// Lanes are partitioned into groups of Period; each group is a translated
784/// copy of the first: Ids[I] = Ids[I % Period] + (I & ~(Period - 1)).
785template <unsigned Period>
786static bool hasPeriodicLayout(ArrayRef<uint8_t> Ids) {
787 static_assert(isPowerOf2_32(Value: Period), "Period must be a power of two");
788 for (size_t I = Period, E = Ids.size(); I < E; ++I)
789 if (Ids[I] != Ids[I % Period] + (I & ~(Period - 1)))
790 return false;
791 return true;
792}
793
794/// Match an N-lane row pattern: each lane in [0, N) reads from a source lane
795/// in the same N-lane row, and the pattern repeats periodically across rows.
796template <unsigned N> static bool isRowPattern(ArrayRef<uint8_t> Ids) {
797 for (unsigned I = 0; I < N; ++I)
798 if (Ids[I] >= N)
799 return false;
800 return hasPeriodicLayout<N>(Ids);
801}
802
803static constexpr auto isQuadPattern = isRowPattern<4>;
804static constexpr auto isHalfRowPattern = isRowPattern<8>;
805static constexpr auto isFullRowPattern = isRowPattern<16>;
806
807/// Match a 4-lane (quad) permutation, encoded as the v_mov_b32_dpp
808/// QUAD_PERM control word: bits[1:0]=Ids[0], [3:2]=Ids[1], [5:4]=Ids[2],
809/// [7:6]=Ids[3].
810static std::optional<unsigned> matchQuadPermPattern(ArrayRef<uint8_t> Ids) {
811 if (!isQuadPattern(Ids))
812 return std::nullopt;
813 return Ids[3] << 6 | Ids[2] << 4 | Ids[1] << 2 | Ids[0];
814}
815
816/// Match an N-lane reversal (mirror) pattern.
817template <unsigned N> static bool matchMirrorPattern(ArrayRef<uint8_t> Ids) {
818 if (!isRowPattern<N>(Ids))
819 return false;
820 for (unsigned J = 0; J < N; ++J)
821 if (Ids[J] != (N - 1) - J)
822 return false;
823 return true;
824}
825
826static constexpr auto matchHalfRowMirrorPattern = matchMirrorPattern<8>;
827static constexpr auto matchFullRowMirrorPattern = matchMirrorPattern<16>;
828
829/// Match a 16-lane cyclic rotation; returns the rotation amount in [1, 15].
830static std::optional<unsigned> matchRowRotatePattern(ArrayRef<uint8_t> Ids) {
831 if (Ids[0] == 0 || !isFullRowPattern(Ids))
832 return std::nullopt;
833 for (unsigned J = 1; J < 16; ++J)
834 if (Ids[J] != (Ids[0] + J) % 16)
835 return std::nullopt;
836 return 16u - Ids[0];
837}
838
839/// Match a row-share pattern: all 16 lanes of each row read the same source
840/// lane. Returns the shared source lane index in [0, 16).
841static std::optional<unsigned> matchRowSharePattern(ArrayRef<uint8_t> Ids) {
842 if (!isFullRowPattern(Ids))
843 return std::nullopt;
844 if (!all_equal(Range: Ids.take_front(N: 16)))
845 return std::nullopt;
846 return Ids[0];
847}
848
849/// Match an XOR mask pattern within each 16-lane row: Ids[J] == Mask ^ J,
850/// with Mask in [1, 15].
851static std::optional<unsigned> matchRowXMaskPattern(ArrayRef<uint8_t> Ids) {
852 unsigned Mask = Ids[0];
853 if (Mask == 0 || !isFullRowPattern(Ids))
854 return std::nullopt;
855 for (unsigned J = 0; J < 16; ++J)
856 if (Ids[J] != (Mask ^ J))
857 return std::nullopt;
858 return Mask;
859}
860
861/// Match an 8-lane arbitrary permutation, encoded as the v_mov_b32_dpp8
862/// 24-bit selector (three bits per output lane).
863static std::optional<unsigned> matchHalfRowPermPattern(ArrayRef<uint8_t> Ids) {
864 if (!isHalfRowPattern(Ids))
865 return std::nullopt;
866 unsigned Selector = 0;
867 for (unsigned J = 0; J < 8; ++J)
868 Selector |= Ids[J] << (J * 3);
869 return Selector;
870}
871
872/// Pack a 16-lane permutation into a single 64-bit value: four bits per output
873/// lane, lane J in bits [J*4 + 3 : J*4]. The caller splits it into the low and
874/// high 32-bit selector operands of v_permlane16 / v_permlanex16.
875static uint64_t computePermlane16Masks(ArrayRef<uint8_t> Ids) {
876 uint64_t Sel = 0;
877 for (unsigned J = 0; J < 16; ++J)
878 Sel |= static_cast<uint64_t>(Ids[J] & 0xF) << (J * 4);
879 return Sel;
880}
881
882/// Match a half-wave swap: lane J reads from lane J ^ 32. Only meaningful on
883/// wave64 targets.
884static bool matchHalfWaveSwapPattern(ArrayRef<uint8_t> Ids) {
885 if (Ids.size() != 64)
886 return false;
887 for (unsigned J = 0; J < 64; ++J)
888 if (Ids[J] != (J ^ 32))
889 return false;
890 return true;
891}
892
893/// Match a cross-row permutation suitable for v_permlanex16: every lane in
894/// the low 16-lane half reads from the high half of its own row, and vice
895/// versa.
896static bool isCrossRowPattern(ArrayRef<uint8_t> Ids) {
897 if (!hasPeriodicLayout<32>(Ids))
898 return false;
899 for (unsigned J = 0; J < 16; ++J) {
900 if (Ids[J] < 16 || Ids[J] >= 32)
901 return false;
902 if (Ids[J + 16] != Ids[J] - 16)
903 return false;
904 }
905 return true;
906}
907
908/// Match a DS_SWIZZLE bitmask-mode permutation:
909/// dst_lane = ((src_lane & AND) | OR) ^ XOR
910/// with each mask being five bits. Returns the encoded swizzle immediate.
911/// The hardware applies the formula independently within each 32-lane group,
912/// so on wave64 the high group must replicate the low one (translated by 32).
913static std::optional<unsigned>
914matchDsSwizzleBitmaskPattern(ArrayRef<uint8_t> Ids) {
915 if (!hasPeriodicLayout<32>(Ids))
916 return std::nullopt;
917
918 // The formula is per-bit: output bit B depends only on input bit B. Probe
919 // each bit with src=0 and src=(1<<B); if the output bit flipped, AND[B]=1
920 // and XOR[B] carries the constant offset; otherwise it is a constant bit
921 // encoded in OR (with AND[B]=0, XOR[B]=0).
922 unsigned AndMask = 0, OrMask = 0, XorMask = 0;
923 for (unsigned B = 0; B < 5; ++B) {
924 unsigned Bit0 = (Ids[0] >> B) & 1;
925 unsigned Bit1 = (Ids[1u << B] >> B) & 1;
926 if (Bit0 != Bit1) {
927 AndMask |= 1u << B;
928 XorMask |= Bit0 << B;
929 } else {
930 OrMask |= Bit0 << B;
931 }
932 }
933
934 // The per-bit derivation assumes bit independence; verify the masks
935 // actually reproduce every lane in the 32-lane group.
936 for (unsigned I : seq(Size: 32u)) {
937 unsigned Expected = ((I & AndMask) | OrMask) ^ XorMask;
938 if (Ids[I] != Expected)
939 return std::nullopt;
940 }
941
942 return AMDGPU::Swizzle::BITMASK_PERM_ENC |
943 AndMask << AMDGPU::Swizzle::BITMASK_AND_SHIFT |
944 OrMask << AMDGPU::Swizzle::BITMASK_OR_SHIFT |
945 XorMask << AMDGPU::Swizzle::BITMASK_XOR_SHIFT;
946}
947
948/// Match a GFX9+ DS_SWIZZLE rotate-mode permutation: a cyclic left-rotation
949/// of all 32 lanes within each 32-lane group by a constant N in [0, 31],
950/// i.e. dst_lane = (src_lane + N) % 32. On wave64, hasPeriodicLayout<32>
951/// ensures both 32-lane groups rotate by the same amount.
952static std::optional<unsigned>
953matchDsSwizzleRotatePattern(ArrayRef<uint8_t> Ids) {
954 if (!hasPeriodicLayout<32>(Ids))
955 return std::nullopt;
956
957 // Determine the rotation amount from lane 0: every lane must read from
958 // lane (I + N) % 32 where N = Ids[0] and 0 <= N <= 31.
959 unsigned N = Ids[0];
960 if (N >= 32)
961 return std::nullopt;
962
963 for (unsigned I = 0; I < 32; ++I)
964 if (Ids[I] != (I + N) % 32)
965 return std::nullopt;
966
967 return AMDGPU::Swizzle::ROTATE_MODE_ENC |
968 (N << AMDGPU::Swizzle::ROTATE_SIZE_SHIFT);
969}
970
971/// Emit v_mov_b32_dpp with the given control word, row/bank masks 0xF, and
972/// bound_ctrl=1 so out-of-bounds lanes are well-defined and the DPP mov can
973/// be folded into a consuming VALU op by GCNDPPCombine.
974static Value *createUpdateDpp(IRBuilderBase &B, Value *Val, unsigned Ctrl) {
975 Type *Ty = Val->getType();
976 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_update_dpp, OverloadTypes: {Ty},
977 Args: {PoisonValue::get(T: Ty), Val, B.getInt32(C: Ctrl),
978 B.getInt32(C: 0xF), B.getInt32(C: 0xF), B.getTrue()});
979}
980
981/// Emit v_mov_b32_dpp8 with the given 24-bit lane selector.
982static Value *createMovDpp8(IRBuilderBase &B, Value *Val, unsigned Selector) {
983 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_mov_dpp8, OverloadTypes: {Val->getType()},
984 Args: {Val, B.getInt32(C: Selector)});
985}
986
987/// Emit v_permlane16 with the precomputed lane-select halves.
988static Value *createPermlane16(IRBuilderBase &B, Value *Val, uint32_t Lo,
989 uint32_t Hi) {
990 Type *Ty = Val->getType();
991 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_permlane16, OverloadTypes: {Ty},
992 Args: {PoisonValue::get(T: Ty), Val, B.getInt32(C: Lo),
993 B.getInt32(C: Hi), B.getFalse(), B.getFalse()});
994}
995
996/// Emit v_permlanex16 with the precomputed lane-select halves. Each output
997/// lane reads from the other 16-lane half of the same row.
998static Value *createPermlaneX16(IRBuilderBase &B, Value *Val, uint32_t Lo,
999 uint32_t Hi) {
1000 Type *Ty = Val->getType();
1001 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_permlanex16, OverloadTypes: {Ty},
1002 Args: {PoisonValue::get(T: Ty), Val, B.getInt32(C: Lo),
1003 B.getInt32(C: Hi), B.getFalse(), B.getFalse()});
1004}
1005
1006/// Emit ds_swizzle with the given immediate, bitcasting/converting between
1007/// pointer/float types and i32 as required by the intrinsic signature.
1008static Value *createDsSwizzle(IRBuilderBase &B, Value *Val, unsigned Offset,
1009 const DataLayout &DL) {
1010 Type *OrigTy = Val->getType();
1011 assert(DL.getTypeSizeInBits(OrigTy) == 32 &&
1012 "ds_swizzle only supports 32-bit operands");
1013 IntegerType *I32Ty = B.getInt32Ty();
1014 Value *Src = Val;
1015 if (OrigTy->isPointerTy())
1016 Src = B.CreatePtrToInt(V: Src, DestTy: I32Ty);
1017 else if (OrigTy != I32Ty)
1018 Src = B.CreateBitCast(V: Src, DestTy: I32Ty);
1019 Value *Result = B.CreateIntrinsic(ID: Intrinsic::amdgcn_ds_swizzle, OverloadTypes: {},
1020 Args: {Src, B.getInt32(C: Offset)});
1021 if (OrigTy->isPointerTy())
1022 return B.CreateIntToPtr(V: Result, DestTy: OrigTy);
1023 if (OrigTy != I32Ty)
1024 return B.CreateBitCast(V: Result, DestTy: OrigTy);
1025 return Result;
1026}
1027
1028/// Emit v_permlane64 (swap of the two 32-lane halves of a wave64).
1029static Value *createPermlane64(IRBuilderBase &B, Value *Val) {
1030 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_permlane64, OverloadTypes: {Val->getType()},
1031 Args: {Val});
1032}
1033
1034/// Given a shuffle map, try to emit the best hardware intrinsic.
1035static Value *matchShuffleToHWIntrinsic(IRBuilderBase &B, Value *Src,
1036 ArrayRef<uint8_t> Ids,
1037 const GCNSubtarget &ST,
1038 const DataLayout &DL) {
1039 // Identity shuffle (every lane reads itself) folds to the source value.
1040 if (all_of(Range: enumerate(First&: Ids),
1041 P: [](const auto &E) { return E.value() == E.index(); }))
1042 return Src;
1043
1044 // Uniform shuffle (all lanes read the same value) is handled by cheaper
1045 // broadcast/readlane intrinsics.
1046 if (all_equal(Range&: Ids))
1047 return nullptr;
1048
1049 if (std::optional<unsigned> QP = matchQuadPermPattern(Ids)) {
1050 if (ST.hasDPP())
1051 return createUpdateDpp(B, Val: Src, Ctrl: *QP);
1052 return createDsSwizzle(B, Val: Src, Offset: AMDGPU::Swizzle::QUAD_PERM_ENC | *QP, DL);
1053 }
1054
1055 if (ST.hasDPP()) {
1056 if (matchHalfRowMirrorPattern(Ids))
1057 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_HALF_MIRROR);
1058 if (matchFullRowMirrorPattern(Ids))
1059 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_MIRROR);
1060 if (std::optional<unsigned> Amt = matchRowRotatePattern(Ids))
1061 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_ROR_FIRST + *Amt - 1);
1062 }
1063
1064 // row_share is supported on GFX90A and GFX10+; row_xmask is GFX10+ only.
1065 if (ST.hasDPPRowShare()) {
1066 if (std::optional<unsigned> Lane = matchRowSharePattern(Ids))
1067 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_SHARE_FIRST + *Lane);
1068 }
1069
1070 if (ST.hasDPP() && ST.hasGFX10Insts()) {
1071 if (std::optional<unsigned> Mask = matchRowXMaskPattern(Ids))
1072 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_XMASK_FIRST + *Mask);
1073 }
1074
1075 if (ST.hasDPP8()) {
1076 if (std::optional<unsigned> Sel = matchHalfRowPermPattern(Ids))
1077 return createMovDpp8(B, Val: Src, Selector: *Sel);
1078 }
1079
1080 if (ST.hasPermlane16Insts()) {
1081 if (isFullRowPattern(Ids)) {
1082 uint64_t Sel = computePermlane16Masks(Ids);
1083 return createPermlane16(B, Val: Src, Lo: Lo_32(Value: Sel), Hi: Hi_32(Value: Sel));
1084 }
1085 // Cross-row shuffles (e.g. XOR 16..31) — covered by permlanex16.
1086 if (isCrossRowPattern(Ids)) {
1087 uint64_t Sel = computePermlane16Masks(Ids);
1088 return createPermlaneX16(B, Val: Src, Lo: Lo_32(Value: Sel), Hi: Hi_32(Value: Sel));
1089 }
1090 }
1091
1092 // Generic DS_SWIZZLE bitmask-mode fallback: handles any 32-lane shuffle that
1093 // can be expressed as dst = ((src & AND) | OR) ^ XOR with 5-bit masks. This
1094 // is available on every target that has ds_swizzle.
1095 if (std::optional<unsigned> Imm = matchDsSwizzleBitmaskPattern(Ids))
1096 return createDsSwizzle(B, Val: Src, Offset: *Imm, DL);
1097
1098 // DS_SWIZZLE rotate mode (GFX9+): handles cyclic 32-lane rotations that
1099 // bitmask mode cannot express (e.g. +1 mod 32 requires inter-bit carry).
1100 if (ST.hasDsSwizzleRotateMode()) {
1101 if (std::optional<unsigned> Imm = matchDsSwizzleRotatePattern(Ids))
1102 return createDsSwizzle(B, Val: Src, Offset: *Imm, DL);
1103 }
1104
1105 if (ST.hasPermLane64() && matchHalfWaveSwapPattern(Ids))
1106 return createPermlane64(B, Val: Src);
1107
1108 return nullptr;
1109}
1110
1111/// Try to fold a wave_shuffle/ds_bpermute whose lane index is a constant
1112/// function of the lane ID into a hardware-specific lane permutation intrinsic.
1113static std::optional<Instruction *>
1114tryOptimizeShufflePattern(InstCombiner &IC, IntrinsicInst &II,
1115 const GCNSubtarget &ST) {
1116 const DataLayout &DL = IC.getDataLayout();
1117 if (DL.getTypeSizeInBits(Ty: II.getType()) != 32)
1118 return std::nullopt;
1119
1120 if (!ST.isWaveSizeKnown())
1121 return std::nullopt;
1122
1123 unsigned WaveSize = ST.getWavefrontSize();
1124 bool IsBpermute = II.getIntrinsicID() == Intrinsic::amdgcn_ds_bpermute;
1125 Value *Src = II.getArgOperand(i: IsBpermute ? 1 : 0);
1126 Value *Index = II.getArgOperand(i: IsBpermute ? 0 : 1);
1127
1128 SmallVector<uint8_t, 64> Ids;
1129 if (IsBpermute) {
1130 Ids.resize(N: WaveSize);
1131 for (unsigned Lane : seq(Size: WaveSize)) {
1132 std::optional<uint64_t> Val = evalLaneExpr(V: Index, Lane, ST, DL);
1133 if (!Val || (*Val & 3) || (*Val >> 2) >= WaveSize)
1134 return std::nullopt;
1135 Ids[Lane] = static_cast<uint8_t>(*Val >> 2);
1136 }
1137 } else {
1138 if (!tryBuildShuffleMap(Index, ST, Ids, DL))
1139 return std::nullopt;
1140 }
1141
1142 Value *Result = matchShuffleToHWIntrinsic(B&: IC.Builder, Src, Ids, ST, DL);
1143 if (!Result)
1144 return std::nullopt;
1145
1146 return IC.replaceInstUsesWith(I&: II, V: Result);
1147}
1148
1149/// Try to fold an addition into the accumulator when saturation is disabled.
1150static Instruction *foldAddIntoDotAccumulator(IntrinsicInst &II,
1151 unsigned AccIdx,
1152 unsigned ClampIdx,
1153 InstCombiner &IC) {
1154 // Reassociating across a saturating accumulation is not valid.
1155 if (!match(V: II.getArgOperand(i: ClampIdx), P: m_Zero()) || !II.hasOneUse())
1156 return nullptr;
1157
1158 const APInt *Acc = nullptr;
1159 if (!match(V: II.getArgOperand(i: AccIdx), P: m_APInt(Res&: Acc)))
1160 return nullptr;
1161
1162 auto *AccumUser = dyn_cast<BinaryOperator>(Val: II.user_back());
1163 if (!AccumUser)
1164 return nullptr;
1165
1166 Value *AccumDelta = nullptr;
1167 if (!match(V: AccumUser, P: m_c_Add(L: m_Specific(V: &II), R: m_Value(V&: AccumDelta))))
1168 return nullptr;
1169
1170 Value *NewAcc = nullptr;
1171 const APInt *AccumDeltaC = nullptr;
1172 if (Acc->isZero())
1173 NewAcc = AccumDelta;
1174 else if (match(V: AccumDelta, P: m_APInt(Res&: AccumDeltaC)))
1175 NewAcc = ConstantInt::get(Ty: II.getType(), V: *Acc + *AccumDeltaC);
1176 else
1177 return nullptr;
1178
1179 Instruction *NewII =
1180 IC.InsertNewInstWith(New: II.clone(), Old: AccumUser->getIterator());
1181 NewII->setOperand(i: AccIdx, Val: NewAcc);
1182 NewII->dropPoisonGeneratingAnnotations();
1183 NewII->dropUBImplyingAttrsAndMetadata();
1184 IC.replaceInstUsesWith(I&: *AccumUser, V: NewII);
1185 IC.eraseInstFromFunction(I&: *AccumUser);
1186 return &II;
1187}
1188
1189std::optional<Instruction *>
1190GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
1191 Intrinsic::ID IID = II.getIntrinsicID();
1192 switch (IID) {
1193 case Intrinsic::amdgcn_implicitarg_ptr: {
1194 if (II.getFunction()->hasFnAttribute(Kind: "amdgpu-no-implicitarg-ptr"))
1195 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1196 uint64_t ImplicitArgBytes = ST->getImplicitArgNumBytes(F: *II.getFunction());
1197
1198 uint64_t CurrentOrNullBytes =
1199 II.getAttributes().getRetDereferenceableOrNullBytes();
1200 if (CurrentOrNullBytes != 0) {
1201 // Refine "dereferenceable (A) meets dereferenceable_or_null(B)"
1202 // into dereferenceable(max(A, B))
1203 uint64_t NewBytes = std::max(a: CurrentOrNullBytes, b: ImplicitArgBytes);
1204 II.addRetAttr(
1205 Attr: Attribute::getWithDereferenceableBytes(Context&: II.getContext(), Bytes: NewBytes));
1206 II.removeRetAttr(Kind: Attribute::DereferenceableOrNull);
1207 return &II;
1208 }
1209
1210 uint64_t CurrentBytes = II.getAttributes().getRetDereferenceableBytes();
1211 uint64_t NewBytes = std::max(a: CurrentBytes, b: ImplicitArgBytes);
1212 if (NewBytes != CurrentBytes) {
1213 II.addRetAttr(
1214 Attr: Attribute::getWithDereferenceableBytes(Context&: II.getContext(), Bytes: NewBytes));
1215 return &II;
1216 }
1217
1218 return std::nullopt;
1219 }
1220 case Intrinsic::amdgcn_rcp: {
1221 Value *Src = II.getArgOperand(i: 0);
1222 if (isa<PoisonValue>(Val: Src))
1223 return IC.replaceInstUsesWith(I&: II, V: Src);
1224
1225 // TODO: Move to ConstantFolding/InstSimplify?
1226 if (isa<UndefValue>(Val: Src)) {
1227 Type *Ty = II.getType();
1228 auto *QNaN = ConstantFP::get(Ty, V: APFloat::getQNaN(Sem: Ty->getFltSemantics()));
1229 return IC.replaceInstUsesWith(I&: II, V: QNaN);
1230 }
1231
1232 if (II.isStrictFP())
1233 break;
1234
1235 if (const ConstantFP *C = dyn_cast<ConstantFP>(Val: Src)) {
1236 std::optional<APFloat> Val = AMDGPU::evaluateRcp(Val: C->getValueAPF());
1237 if (!Val)
1238 break;
1239
1240 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::get(Context&: II.getContext(), V: *Val));
1241 }
1242
1243 FastMathFlags FMF = cast<FPMathOperator>(Val&: II).getFastMathFlags();
1244 if (!FMF.allowContract())
1245 break;
1246 auto *SrcCI = dyn_cast<IntrinsicInst>(Val: Src);
1247 if (!SrcCI)
1248 break;
1249
1250 auto IID = SrcCI->getIntrinsicID();
1251 // llvm.amdgcn.rcp(llvm.amdgcn.sqrt(x)) -> llvm.amdgcn.rsq(x) if contractable
1252 //
1253 // llvm.amdgcn.rcp(llvm.sqrt(x)) -> llvm.amdgcn.rsq(x) if contractable and
1254 // relaxed.
1255 if (IID == Intrinsic::amdgcn_sqrt || IID == Intrinsic::sqrt) {
1256 const FPMathOperator *SqrtOp = cast<FPMathOperator>(Val: SrcCI);
1257 FastMathFlags InnerFMF = SqrtOp->getFastMathFlags();
1258 if (!InnerFMF.allowContract() || !SrcCI->hasOneUse())
1259 break;
1260
1261 if (IID == Intrinsic::sqrt && !canContractSqrtToRsq(SqrtOp))
1262 break;
1263
1264 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
1265 M: SrcCI->getModule(), id: Intrinsic::amdgcn_rsq, OverloadTys: {SrcCI->getType()});
1266
1267 InnerFMF |= FMF;
1268 II.setFastMathFlags(InnerFMF);
1269
1270 II.setCalledFunction(NewDecl);
1271 return IC.replaceOperand(I&: II, OpNum: 0, V: SrcCI->getArgOperand(i: 0));
1272 }
1273
1274 break;
1275 }
1276 case Intrinsic::amdgcn_sqrt:
1277 case Intrinsic::amdgcn_rsq:
1278 case Intrinsic::amdgcn_tanh: {
1279 Value *Src = II.getArgOperand(i: 0);
1280 if (isa<PoisonValue>(Val: Src))
1281 return IC.replaceInstUsesWith(I&: II, V: Src);
1282
1283 // TODO: Move to ConstantFolding/InstSimplify?
1284 if (isa<UndefValue>(Val: Src)) {
1285 Type *Ty = II.getType();
1286 auto *QNaN = ConstantFP::get(Ty, V: APFloat::getQNaN(Sem: Ty->getFltSemantics()));
1287 return IC.replaceInstUsesWith(I&: II, V: QNaN);
1288 }
1289
1290 // f16 amdgcn.sqrt is identical to regular sqrt.
1291 if (IID == Intrinsic::amdgcn_sqrt && Src->getType()->isHalfTy()) {
1292 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
1293 M: II.getModule(), id: Intrinsic::sqrt, OverloadTys: {II.getType()});
1294 II.setCalledFunction(NewDecl);
1295 return &II;
1296 }
1297
1298 break;
1299 }
1300 case Intrinsic::amdgcn_log:
1301 case Intrinsic::amdgcn_exp2: {
1302 const bool IsLog = IID == Intrinsic::amdgcn_log;
1303 const bool IsExp = IID == Intrinsic::amdgcn_exp2;
1304 Value *Src = II.getArgOperand(i: 0);
1305 Type *Ty = II.getType();
1306
1307 if (isa<PoisonValue>(Val: Src))
1308 return IC.replaceInstUsesWith(I&: II, V: Src);
1309
1310 if (IC.getSimplifyQuery().isUndefValue(V: Src))
1311 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getNaN(Ty));
1312
1313 if (ConstantFP *C = dyn_cast<ConstantFP>(Val: Src)) {
1314 if (C->isInfinity()) {
1315 // exp2(+inf) -> +inf
1316 // log2(+inf) -> +inf
1317 if (!C->isNegative())
1318 return IC.replaceInstUsesWith(I&: II, V: C);
1319
1320 // exp2(-inf) -> 0
1321 if (IsExp && C->isNegative())
1322 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getZero(Ty));
1323 }
1324
1325 if (II.isStrictFP())
1326 break;
1327
1328 if (C->isNaN()) {
1329 Constant *Quieted = ConstantFP::get(Ty, V: C->getValue().makeQuiet());
1330 return IC.replaceInstUsesWith(I&: II, V: Quieted);
1331 }
1332
1333 // f32 instruction doesn't handle denormals, f16 does.
1334 if (C->isZero() || (C->getValue().isDenormal() && Ty->isFloatTy())) {
1335 Constant *FoldedValue = IsLog ? ConstantFP::getInfinity(Ty, Negative: true)
1336 : ConstantFP::get(Ty, V: 1.0);
1337 return IC.replaceInstUsesWith(I&: II, V: FoldedValue);
1338 }
1339
1340 if (IsLog && C->isNegative())
1341 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getNaN(Ty));
1342
1343 // TODO: Full constant folding matching hardware behavior.
1344 }
1345
1346 break;
1347 }
1348 case Intrinsic::amdgcn_frexp_mant:
1349 case Intrinsic::amdgcn_frexp_exp: {
1350 Value *Src = II.getArgOperand(i: 0);
1351 if (const ConstantFP *C = dyn_cast<ConstantFP>(Val: Src)) {
1352 int Exp;
1353 APFloat Significand =
1354 frexp(X: C->getValueAPF(), Exp, RM: APFloat::rmNearestTiesToEven);
1355
1356 if (IID == Intrinsic::amdgcn_frexp_mant) {
1357 return IC.replaceInstUsesWith(
1358 I&: II, V: ConstantFP::get(Context&: II.getContext(), V: Significand));
1359 }
1360
1361 // Match instruction special case behavior.
1362 if (Exp == APFloat::IEK_NaN || Exp == APFloat::IEK_Inf)
1363 Exp = 0;
1364
1365 return IC.replaceInstUsesWith(I&: II,
1366 V: ConstantInt::getSigned(Ty: II.getType(), V: Exp));
1367 }
1368
1369 if (isa<PoisonValue>(Val: Src))
1370 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1371
1372 if (isa<UndefValue>(Val: Src)) {
1373 return IC.replaceInstUsesWith(I&: II, V: UndefValue::get(T: II.getType()));
1374 }
1375
1376 break;
1377 }
1378 case Intrinsic::amdgcn_class: {
1379 Value *Src0 = II.getArgOperand(i: 0);
1380 Value *Src1 = II.getArgOperand(i: 1);
1381 const ConstantInt *CMask = dyn_cast<ConstantInt>(Val: Src1);
1382 if (CMask) {
1383 II.setCalledOperand(Intrinsic::getOrInsertDeclaration(
1384 M: II.getModule(), id: Intrinsic::is_fpclass, OverloadTys: Src0->getType()));
1385
1386 // Clamp any excess bits, as they're illegal for the generic intrinsic.
1387 II.setArgOperand(i: 1, v: ConstantInt::get(Ty: Src1->getType(),
1388 V: CMask->getZExtValue() & fcAllFlags));
1389 return &II;
1390 }
1391
1392 // Propagate poison.
1393 if (isa<PoisonValue>(Val: Src0) || isa<PoisonValue>(Val: Src1))
1394 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1395
1396 // llvm.amdgcn.class(_, undef) -> false
1397 if (IC.getSimplifyQuery().isUndefValue(V: Src1))
1398 return IC.replaceInstUsesWith(I&: II, V: ConstantInt::get(Ty: II.getType(), V: false));
1399
1400 // llvm.amdgcn.class(undef, mask) -> mask != 0
1401 if (IC.getSimplifyQuery().isUndefValue(V: Src0)) {
1402 Value *CmpMask = IC.Builder.CreateICmpNE(
1403 LHS: Src1, RHS: ConstantInt::getNullValue(Ty: Src1->getType()));
1404 return IC.replaceInstUsesWith(I&: II, V: CmpMask);
1405 }
1406 break;
1407 }
1408 case Intrinsic::amdgcn_cvt_pkrtz: {
1409 auto foldFPTruncToF16RTZ = [](Value *Arg) -> Value * {
1410 Type *HalfTy = Type::getHalfTy(C&: Arg->getContext());
1411
1412 if (isa<PoisonValue>(Val: Arg))
1413 return PoisonValue::get(T: HalfTy);
1414 if (isa<UndefValue>(Val: Arg))
1415 return UndefValue::get(T: HalfTy);
1416
1417 ConstantFP *CFP = nullptr;
1418 if (match(V: Arg, P: m_ConstantFP(C&: CFP))) {
1419 bool LosesInfo;
1420 APFloat Val(CFP->getValueAPF());
1421 Val.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmTowardZero, losesInfo: &LosesInfo);
1422 return ConstantFP::get(Ty: HalfTy, V: Val);
1423 }
1424
1425 Value *Src = nullptr;
1426 if (match(V: Arg, P: m_FPExt(Op: m_Value(V&: Src)))) {
1427 if (Src->getType()->isHalfTy())
1428 return Src;
1429 }
1430
1431 return nullptr;
1432 };
1433
1434 if (Value *Src0 = foldFPTruncToF16RTZ(II.getArgOperand(i: 0))) {
1435 if (Value *Src1 = foldFPTruncToF16RTZ(II.getArgOperand(i: 1))) {
1436 Value *V = PoisonValue::get(T: II.getType());
1437 V = IC.Builder.CreateInsertElement(Vec: V, NewElt: Src0, Idx: (uint64_t)0);
1438 V = IC.Builder.CreateInsertElement(Vec: V, NewElt: Src1, Idx: (uint64_t)1);
1439 return IC.replaceInstUsesWith(I&: II, V);
1440 }
1441 }
1442
1443 break;
1444 }
1445 case Intrinsic::amdgcn_cvt_pknorm_i16:
1446 case Intrinsic::amdgcn_cvt_pknorm_u16:
1447 case Intrinsic::amdgcn_cvt_pk_i16:
1448 case Intrinsic::amdgcn_cvt_pk_u16: {
1449 Value *Src0 = II.getArgOperand(i: 0);
1450 Value *Src1 = II.getArgOperand(i: 1);
1451
1452 // TODO: Replace call with scalar operation if only one element is poison.
1453 if (isa<PoisonValue>(Val: Src0) && isa<PoisonValue>(Val: Src1))
1454 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1455
1456 if (isa<UndefValue>(Val: Src0) && isa<UndefValue>(Val: Src1)) {
1457 return IC.replaceInstUsesWith(I&: II, V: UndefValue::get(T: II.getType()));
1458 }
1459
1460 break;
1461 }
1462 case Intrinsic::amdgcn_cvt_off_f32_i4: {
1463 Value* Arg = II.getArgOperand(i: 0);
1464 Type *Ty = II.getType();
1465
1466 if (isa<PoisonValue>(Val: Arg))
1467 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: Ty));
1468
1469 if(IC.getSimplifyQuery().isUndefValue(V: Arg))
1470 return IC.replaceInstUsesWith(I&: II, V: Constant::getNullValue(Ty));
1471
1472 ConstantInt *CArg = dyn_cast<ConstantInt>(Val: II.getArgOperand(i: 0));
1473 if (!CArg)
1474 break;
1475
1476 // Tabulated 0.0625 * (sext (CArg & 0xf)).
1477 constexpr size_t ResValsSize = 16;
1478 static constexpr float ResVals[ResValsSize] = {
1479 0.0, 0.0625, 0.125, 0.1875, 0.25, 0.3125, 0.375, 0.4375,
1480 -0.5, -0.4375, -0.375, -0.3125, -0.25, -0.1875, -0.125, -0.0625};
1481 Constant *Res =
1482 ConstantFP::get(Ty, V: ResVals[CArg->getZExtValue() & (ResValsSize - 1)]);
1483 return IC.replaceInstUsesWith(I&: II, V: Res);
1484 }
1485 case Intrinsic::amdgcn_ubfe:
1486 case Intrinsic::amdgcn_sbfe: {
1487 // Decompose simple cases into standard shifts.
1488 Value *Src = II.getArgOperand(i: 0);
1489 if (isa<UndefValue>(Val: Src)) {
1490 return IC.replaceInstUsesWith(I&: II, V: Src);
1491 }
1492
1493 uint64_t Width;
1494 Type *Ty = II.getType();
1495 unsigned IntSize = Ty->getIntegerBitWidth();
1496
1497 ConstantInt *CWidth = dyn_cast<ConstantInt>(Val: II.getArgOperand(i: 2));
1498 if (CWidth) {
1499 Width = CWidth->getZExtValue();
1500 if ((Width & (IntSize - 1)) == 0) {
1501 return IC.replaceInstUsesWith(I&: II, V: ConstantInt::getNullValue(Ty));
1502 }
1503
1504 // Hardware ignores high bits, so remove those.
1505 if (Width >= IntSize) {
1506 return IC.replaceOperand(
1507 I&: II, OpNum: 2, V: ConstantInt::get(Ty: CWidth->getType(), V: Width & (IntSize - 1)));
1508 }
1509 }
1510
1511 uint64_t Offset;
1512 ConstantInt *COffset = dyn_cast<ConstantInt>(Val: II.getArgOperand(i: 1));
1513 if (COffset) {
1514 Offset = COffset->getZExtValue();
1515 if (Offset >= IntSize) {
1516 return IC.replaceOperand(
1517 I&: II, OpNum: 1,
1518 V: ConstantInt::get(Ty: COffset->getType(), V: Offset & (IntSize - 1)));
1519 }
1520 }
1521
1522 bool Signed = IID == Intrinsic::amdgcn_sbfe;
1523
1524 if (!CWidth || !COffset)
1525 break;
1526
1527 // The case of Width == 0 is handled above, which makes this transformation
1528 // safe. If Width == 0, then the ashr and lshr instructions become poison
1529 // value since the shift amount would be equal to the bit size.
1530 assert(Width != 0);
1531
1532 // TODO: This allows folding to undef when the hardware has specific
1533 // behavior?
1534 if (Offset + Width < IntSize) {
1535 Value *Shl = IC.Builder.CreateShl(LHS: Src, RHS: IntSize - Offset - Width);
1536 Value *RightShift = Signed ? IC.Builder.CreateAShr(LHS: Shl, RHS: IntSize - Width)
1537 : IC.Builder.CreateLShr(LHS: Shl, RHS: IntSize - Width);
1538 RightShift->takeName(V: &II);
1539 return IC.replaceInstUsesWith(I&: II, V: RightShift);
1540 }
1541
1542 Value *RightShift = Signed ? IC.Builder.CreateAShr(LHS: Src, RHS: Offset)
1543 : IC.Builder.CreateLShr(LHS: Src, RHS: Offset);
1544
1545 RightShift->takeName(V: &II);
1546 return IC.replaceInstUsesWith(I&: II, V: RightShift);
1547 }
1548 case Intrinsic::amdgcn_exp:
1549 case Intrinsic::amdgcn_exp_row:
1550 case Intrinsic::amdgcn_exp_compr: {
1551 ConstantInt *En = cast<ConstantInt>(Val: II.getArgOperand(i: 1));
1552 uint64_t EnBits = En->getZExtValue();
1553 if (EnBits == 0xf)
1554 break; // All inputs enabled.
1555
1556 bool IsCompr = IID == Intrinsic::amdgcn_exp_compr;
1557 bool Changed = false;
1558 for (int I = 0; I < (IsCompr ? 2 : 4); ++I) {
1559 if ((!IsCompr && (EnBits & (1 << I)) == 0) ||
1560 (IsCompr && ((EnBits & (0x3 << (2 * I))) == 0))) {
1561 Value *Src = II.getArgOperand(i: I + 2);
1562 if (!isa<PoisonValue>(Val: Src)) {
1563 IC.replaceOperand(I&: II, OpNum: I + 2, V: PoisonValue::get(T: Src->getType()));
1564 Changed = true;
1565 }
1566 }
1567 }
1568
1569 if (Changed) {
1570 return &II;
1571 }
1572
1573 break;
1574 }
1575 case Intrinsic::amdgcn_fmed3: {
1576 Value *Src0 = II.getArgOperand(i: 0);
1577 Value *Src1 = II.getArgOperand(i: 1);
1578 Value *Src2 = II.getArgOperand(i: 2);
1579
1580 for (Value *Src : {Src0, Src1, Src2}) {
1581 if (isa<PoisonValue>(Val: Src))
1582 return IC.replaceInstUsesWith(I&: II, V: Src);
1583 }
1584
1585 if (II.isStrictFP())
1586 break;
1587
1588 // med3 with a nan input acts like
1589 // v_min_f32(v_min_f32(s0, s1), s2)
1590 //
1591 // Signalingness is ignored with ieee=0, so we fold to
1592 // minimumnum/maximumnum. With ieee=1, the v_min_f32 acts like llvm.minnum
1593 // with signaling nan handling. With ieee=0, like llvm.minimumnum except a
1594 // returned signaling nan will not be quieted.
1595
1596 // ieee=1
1597 // s0 snan: s2
1598 // s1 snan: s2
1599 // s2 snan: qnan
1600
1601 // s0 qnan: min(s1, s2)
1602 // s1 qnan: min(s0, s2)
1603 // s2 qnan: min(s0, s1)
1604
1605 // ieee=0
1606 // s0 _nan: min(s1, s2)
1607 // s1 _nan: min(s0, s2)
1608 // s2 _nan: min(s0, s1)
1609
1610 // med3 behavior with infinity
1611 // s0 +inf: max(s1, s2)
1612 // s1 +inf: max(s0, s2)
1613 // s2 +inf: max(s0, s1)
1614 // s0 -inf: min(s1, s2)
1615 // s1 -inf: min(s0, s2)
1616 // s2 -inf: min(s0, s1)
1617
1618 // Checking for NaN before canonicalization provides better fidelity when
1619 // mapping other operations onto fmed3 since the order of operands is
1620 // unchanged.
1621 Value *V = nullptr;
1622 const APFloat *ConstSrc0 = nullptr;
1623 const APFloat *ConstSrc1 = nullptr;
1624 const APFloat *ConstSrc2 = nullptr;
1625
1626 if ((match(V: Src0, P: m_APFloat(Res&: ConstSrc0)) &&
1627 (ConstSrc0->isNaN() || ConstSrc0->isInfinity())) ||
1628 isa<UndefValue>(Val: Src0)) {
1629 const bool IsPosInfinity = ConstSrc0 && ConstSrc0->isPosInfinity();
1630 switch (fpenvIEEEMode(I: II)) {
1631 case KnownIEEEMode::On:
1632 // TODO: If Src2 is snan, does it need quieting?
1633 if (ConstSrc0 && ConstSrc0->isNaN() && ConstSrc0->isSignaling())
1634 return IC.replaceInstUsesWith(I&: II, V: Src2);
1635
1636 V = IsPosInfinity ? IC.Builder.CreateMaxNum(LHS: Src1, RHS: Src2)
1637 : IC.Builder.CreateMinNum(LHS: Src1, RHS: Src2);
1638 break;
1639 case KnownIEEEMode::Off:
1640 V = IsPosInfinity ? IC.Builder.CreateMaximumNum(LHS: Src1, RHS: Src2)
1641 : IC.Builder.CreateMinimumNum(LHS: Src1, RHS: Src2);
1642 break;
1643 case KnownIEEEMode::Unknown:
1644 break;
1645 }
1646 } else if ((match(V: Src1, P: m_APFloat(Res&: ConstSrc1)) &&
1647 (ConstSrc1->isNaN() || ConstSrc1->isInfinity())) ||
1648 isa<UndefValue>(Val: Src1)) {
1649 const bool IsPosInfinity = ConstSrc1 && ConstSrc1->isPosInfinity();
1650 switch (fpenvIEEEMode(I: II)) {
1651 case KnownIEEEMode::On:
1652 // TODO: If Src2 is snan, does it need quieting?
1653 if (ConstSrc1 && ConstSrc1->isNaN() && ConstSrc1->isSignaling())
1654 return IC.replaceInstUsesWith(I&: II, V: Src2);
1655
1656 V = IsPosInfinity ? IC.Builder.CreateMaxNum(LHS: Src0, RHS: Src2)
1657 : IC.Builder.CreateMinNum(LHS: Src0, RHS: Src2);
1658 break;
1659 case KnownIEEEMode::Off:
1660 V = IsPosInfinity ? IC.Builder.CreateMaximumNum(LHS: Src0, RHS: Src2)
1661 : IC.Builder.CreateMinimumNum(LHS: Src0, RHS: Src2);
1662 break;
1663 case KnownIEEEMode::Unknown:
1664 break;
1665 }
1666 } else if ((match(V: Src2, P: m_APFloat(Res&: ConstSrc2)) &&
1667 (ConstSrc2->isNaN() || ConstSrc2->isInfinity())) ||
1668 isa<UndefValue>(Val: Src2)) {
1669 switch (fpenvIEEEMode(I: II)) {
1670 case KnownIEEEMode::On:
1671 if (ConstSrc2 && ConstSrc2->isNaN() && ConstSrc2->isSignaling()) {
1672 auto *Quieted = ConstantFP::get(Ty: II.getType(), V: ConstSrc2->makeQuiet());
1673 return IC.replaceInstUsesWith(I&: II, V: Quieted);
1674 }
1675
1676 V = (ConstSrc2 && ConstSrc2->isPosInfinity())
1677 ? IC.Builder.CreateMaxNum(LHS: Src0, RHS: Src1)
1678 : IC.Builder.CreateMinNum(LHS: Src0, RHS: Src1);
1679 break;
1680 case KnownIEEEMode::Off:
1681 V = (ConstSrc2 && ConstSrc2->isPosInfinity())
1682 ? IC.Builder.CreateMaximumNum(LHS: Src0, RHS: Src1)
1683 : IC.Builder.CreateMinimumNum(LHS: Src0, RHS: Src1);
1684 break;
1685 case KnownIEEEMode::Unknown:
1686 break;
1687 }
1688 }
1689
1690 if (V) {
1691 if (auto *CI = dyn_cast<CallInst>(Val: V)) {
1692 CI->copyFastMathFlags(I: &II);
1693 CI->takeName(V: &II);
1694 }
1695 return IC.replaceInstUsesWith(I&: II, V);
1696 }
1697
1698 bool Swap = false;
1699 // Canonicalize constants to RHS operands.
1700 //
1701 // fmed3(c0, x, c1) -> fmed3(x, c0, c1)
1702 if (isa<Constant>(Val: Src0) && !isa<Constant>(Val: Src1)) {
1703 std::swap(a&: Src0, b&: Src1);
1704 Swap = true;
1705 }
1706
1707 if (isa<Constant>(Val: Src1) && !isa<Constant>(Val: Src2)) {
1708 std::swap(a&: Src1, b&: Src2);
1709 Swap = true;
1710 }
1711
1712 if (isa<Constant>(Val: Src0) && !isa<Constant>(Val: Src1)) {
1713 std::swap(a&: Src0, b&: Src1);
1714 Swap = true;
1715 }
1716
1717 if (Swap) {
1718 II.setArgOperand(i: 0, v: Src0);
1719 II.setArgOperand(i: 1, v: Src1);
1720 II.setArgOperand(i: 2, v: Src2);
1721 return &II;
1722 }
1723
1724 if (const ConstantFP *C0 = dyn_cast<ConstantFP>(Val: Src0)) {
1725 if (const ConstantFP *C1 = dyn_cast<ConstantFP>(Val: Src1)) {
1726 if (const ConstantFP *C2 = dyn_cast<ConstantFP>(Val: Src2)) {
1727 APFloat Result = fmed3AMDGCN(Src0: C0->getValueAPF(), Src1: C1->getValueAPF(),
1728 Src2: C2->getValueAPF());
1729 return IC.replaceInstUsesWith(I&: II,
1730 V: ConstantFP::get(Ty: II.getType(), V: Result));
1731 }
1732 }
1733 }
1734
1735 if (!ST->hasMed3_16())
1736 break;
1737
1738 // Repeat floating-point width reduction done for minnum/maxnum.
1739 // fmed3((fpext X), (fpext Y), (fpext Z)) -> fpext (fmed3(X, Y, Z))
1740 if (Value *X = matchFPExtFromF16(Arg: Src0)) {
1741 if (Value *Y = matchFPExtFromF16(Arg: Src1)) {
1742 if (Value *Z = matchFPExtFromF16(Arg: Src2)) {
1743 Value *NewCall = IC.Builder.CreateIntrinsic(
1744 ID: IID, OverloadTypes: {X->getType()}, Args: {X, Y, Z}, FMFSource: &II, Name: II.getName());
1745 return new FPExtInst(NewCall, II.getType());
1746 }
1747 }
1748 }
1749
1750 break;
1751 }
1752 case Intrinsic::amdgcn_mbcnt_hi:
1753 // exec_hi is all 0, so this is just a copy.
1754 if (ST->isWave32())
1755 return IC.replaceInstUsesWith(I&: II, V: II.getArgOperand(i: 1));
1756 [[fallthrough]];
1757 case Intrinsic::amdgcn_mbcnt_lo: {
1758 ConstantRange AccRange =
1759 computeConstantRange(V: II.getArgOperand(i: 1),
1760 /*ForSigned=*/false, SQ: IC.getSimplifyQuery());
1761 if (AccRange.isFullSet())
1762 return nullptr;
1763
1764 // TODO: Can raise lower bound by inspecting first argument.
1765 // Wave64 mbcnt_lo adds at most 32. Otherwise these add at most 31.
1766 unsigned MaxCount =
1767 IID == Intrinsic::amdgcn_mbcnt_lo && ST->isWave64() ? 32 : 31;
1768 ConstantRange MbcntRange(APInt(32, 0), APInt(32, MaxCount + 1));
1769 ConstantRange ComputedRange = AccRange.add(Other: MbcntRange);
1770 if (ComputedRange.isFullSet())
1771 return nullptr;
1772
1773 if (std::optional<ConstantRange> ExistingRange = II.getRange()) {
1774 ComputedRange = ComputedRange.intersectWith(CR: *ExistingRange);
1775 if (ComputedRange == *ExistingRange)
1776 return nullptr;
1777 }
1778
1779 II.addRangeRetAttr(CR: ComputedRange);
1780 return nullptr;
1781 }
1782 case Intrinsic::amdgcn_ballot: {
1783 Value *Arg = II.getArgOperand(i: 0);
1784 if (isa<PoisonValue>(Val: Arg))
1785 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1786
1787 if (auto *Src = dyn_cast<ConstantInt>(Val: Arg)) {
1788 if (Src->isZero()) {
1789 // amdgcn.ballot(i1 0) is zero.
1790 return IC.replaceInstUsesWith(I&: II, V: Constant::getNullValue(Ty: II.getType()));
1791 }
1792 }
1793 if (ST->isWave32() && II.getType()->getIntegerBitWidth() == 64) {
1794 // %b64 = call i64 ballot.i64(...)
1795 // =>
1796 // %b32 = call i32 ballot.i32(...)
1797 // %b64 = zext i32 %b32 to i64
1798 Value *Call = IC.Builder.CreateZExt(
1799 V: IC.Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_ballot,
1800 OverloadTypes: {IC.Builder.getInt32Ty()},
1801 Args: {II.getArgOperand(i: 0)}),
1802 DestTy: II.getType());
1803 Call->takeName(V: &II);
1804 return IC.replaceInstUsesWith(I&: II, V: Call);
1805 }
1806 break;
1807 }
1808 case Intrinsic::amdgcn_wavefrontsize: {
1809 if (ST->isWaveSizeKnown())
1810 return IC.replaceInstUsesWith(
1811 I&: II, V: ConstantInt::get(Ty: II.getType(), V: ST->getWavefrontSize()));
1812 break;
1813 }
1814 case Intrinsic::amdgcn_wqm_vote: {
1815 // wqm_vote is identity when the argument is constant.
1816 if (!isa<Constant>(Val: II.getArgOperand(i: 0)))
1817 break;
1818
1819 return IC.replaceInstUsesWith(I&: II, V: II.getArgOperand(i: 0));
1820 }
1821 case Intrinsic::amdgcn_kill: {
1822 const ConstantInt *C = dyn_cast<ConstantInt>(Val: II.getArgOperand(i: 0));
1823 if (!C || !C->getZExtValue())
1824 break;
1825
1826 // amdgcn.kill(i1 1) is a no-op
1827 return IC.eraseInstFromFunction(I&: II);
1828 }
1829 case Intrinsic::amdgcn_s_sendmsg:
1830 case Intrinsic::amdgcn_s_sendmsghalt: {
1831 // The second operand is copied to m0, but is only actually used for
1832 // certain message types. For message types that are known to not use m0,
1833 // fold it to poison.
1834 using namespace AMDGPU::SendMsg;
1835
1836 Value *M0Val = II.getArgOperand(i: 1);
1837 if (isa<PoisonValue>(Val: M0Val))
1838 break;
1839
1840 auto *MsgImm = cast<ConstantInt>(Val: II.getArgOperand(i: 0));
1841 uint16_t MsgId, OpId, StreamId;
1842 decodeMsg(Val: static_cast<unsigned>(MsgImm->getZExtValue()), MsgId, OpId,
1843 StreamId, STI: *ST);
1844
1845 if (!msgDoesNotUseM0(MsgId, STI: *ST))
1846 break;
1847
1848 // Drop UB-implying attributes since we're replacing with poison.
1849 II.dropUBImplyingAttrsAndMetadata();
1850 IC.replaceOperand(I&: II, OpNum: 1, V: PoisonValue::get(T: M0Val->getType()));
1851 return nullptr;
1852 }
1853 case Intrinsic::amdgcn_update_dpp: {
1854 Value *Old = II.getArgOperand(i: 0);
1855
1856 auto *BC = cast<ConstantInt>(Val: II.getArgOperand(i: 5));
1857 auto *RM = cast<ConstantInt>(Val: II.getArgOperand(i: 3));
1858 auto *BM = cast<ConstantInt>(Val: II.getArgOperand(i: 4));
1859 if (BC->isNullValue() || RM->getZExtValue() != 0xF ||
1860 BM->getZExtValue() != 0xF || isa<PoisonValue>(Val: Old))
1861 break;
1862
1863 // If bound_ctrl = 1, row mask = bank mask = 0xf we can omit old value.
1864 return IC.replaceOperand(I&: II, OpNum: 0, V: PoisonValue::get(T: Old->getType()));
1865 }
1866 case Intrinsic::amdgcn_permlane16:
1867 case Intrinsic::amdgcn_permlane16_var:
1868 case Intrinsic::amdgcn_permlanex16:
1869 case Intrinsic::amdgcn_permlanex16_var: {
1870 // Discard vdst_in if it's not going to be read.
1871 Value *VDstIn = II.getArgOperand(i: 0);
1872 if (isa<PoisonValue>(Val: VDstIn))
1873 break;
1874
1875 // FetchInvalid operand idx.
1876 unsigned int FiIdx = (IID == Intrinsic::amdgcn_permlane16 ||
1877 IID == Intrinsic::amdgcn_permlanex16)
1878 ? 4 /* for permlane16 and permlanex16 */
1879 : 3; /* for permlane16_var and permlanex16_var */
1880
1881 // BoundCtrl operand idx.
1882 // For permlane16 and permlanex16 it should be 5
1883 // For Permlane16_var and permlanex16_var it should be 4
1884 unsigned int BcIdx = FiIdx + 1;
1885
1886 ConstantInt *FetchInvalid = cast<ConstantInt>(Val: II.getArgOperand(i: FiIdx));
1887 ConstantInt *BoundCtrl = cast<ConstantInt>(Val: II.getArgOperand(i: BcIdx));
1888 if (!FetchInvalid->getZExtValue() && !BoundCtrl->getZExtValue())
1889 break;
1890
1891 return IC.replaceOperand(I&: II, OpNum: 0, V: PoisonValue::get(T: VDstIn->getType()));
1892 }
1893 case Intrinsic::amdgcn_wave_shuffle:
1894 return tryOptimizeShufflePattern(IC, II, ST: *ST);
1895 case Intrinsic::amdgcn_permlane64:
1896 case Intrinsic::amdgcn_readfirstlane:
1897 case Intrinsic::amdgcn_readlane:
1898 case Intrinsic::amdgcn_ds_bpermute: {
1899 // If the data argument is uniform these intrinsics return it unchanged.
1900 unsigned SrcIdx = IID == Intrinsic::amdgcn_ds_bpermute ? 1 : 0;
1901 const Use &Src = II.getArgOperandUse(i: SrcIdx);
1902 if (isTriviallyUniform(U: Src))
1903 return IC.replaceInstUsesWith(I&: II, V: Src.get());
1904
1905 if (IID == Intrinsic::amdgcn_readlane &&
1906 simplifyDemandedLaneMaskArg(IC, II, LaneArgIdx: 1))
1907 return &II;
1908
1909 // If the lane argument of bpermute is uniform, change it to readlane. This
1910 // generates better code and can enable further optimizations because
1911 // readlane is AlwaysUniform.
1912 if (IID == Intrinsic::amdgcn_ds_bpermute) {
1913 const Use &Lane = II.getArgOperandUse(i: 0);
1914 if (isTriviallyUniform(U: Lane)) {
1915 Value *NewLane = IC.Builder.CreateLShr(LHS: Lane, RHS: 2);
1916 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
1917 M: II.getModule(), id: Intrinsic::amdgcn_readlane, OverloadTys: II.getType());
1918 II.setCalledFunction(NewDecl);
1919 II.setOperand(i_nocapture: 0, Val_nocapture: Src);
1920 II.setOperand(i_nocapture: 1, Val_nocapture: NewLane);
1921 return &II;
1922 }
1923 }
1924
1925 if (IID == Intrinsic::amdgcn_ds_bpermute)
1926 return tryOptimizeShufflePattern(IC, II, ST: *ST);
1927
1928 if (Instruction *Res = hoistLaneIntrinsicThroughOperand(IC, II))
1929 return Res;
1930
1931 return std::nullopt;
1932 }
1933 case Intrinsic::amdgcn_wave_match_b32: {
1934 const Use &Src0 = II.getArgOperandUse(i: 0);
1935 const Use &Src1 = II.getArgOperandUse(i: 1);
1936 if (Src0.get() == Src1.get() && isTriviallyUniform(U: Src0)) {
1937 Function *NewF = Intrinsic::getOrInsertDeclaration(
1938 M: II.getModule(), id: Intrinsic::amdgcn_ballot, OverloadTys: II.getType());
1939 CallInst *NewCall =
1940 IC.Builder.CreateCall(Callee: NewF, Args: {IC.Builder.getInt1(V: true)});
1941 return IC.replaceInstUsesWith(I&: II, V: NewCall);
1942 }
1943 break;
1944 }
1945 case Intrinsic::amdgcn_writelane: {
1946 // TODO: Fold bitcast like readlane.
1947 if (simplifyDemandedLaneMaskArg(IC, II, LaneArgIdx: 1))
1948 return &II;
1949 return std::nullopt;
1950 }
1951 case Intrinsic::amdgcn_trig_preop: {
1952 // The intrinsic is declared with name mangling, but currently the
1953 // instruction only exists for f64
1954 if (!II.getType()->isDoubleTy())
1955 break;
1956
1957 Value *Src = II.getArgOperand(i: 0);
1958 Value *Segment = II.getArgOperand(i: 1);
1959 if (isa<PoisonValue>(Val: Src) || isa<PoisonValue>(Val: Segment))
1960 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1961
1962 if (isa<UndefValue>(Val: Segment))
1963 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getZero(Ty: II.getType()));
1964
1965 // Sign bit is not used.
1966 Value *StrippedSign = InstCombiner::stripSignOnlyFPOps(Val: Src);
1967 if (StrippedSign != Src)
1968 return IC.replaceOperand(I&: II, OpNum: 0, V: StrippedSign);
1969
1970 if (II.isStrictFP())
1971 break;
1972
1973 const ConstantFP *CSrc = dyn_cast<ConstantFP>(Val: Src);
1974 if (!CSrc && !isa<UndefValue>(Val: Src))
1975 break;
1976
1977 // The instruction ignores special cases, and literally just extracts the
1978 // exponents. Fold undef to nan, and index the table as normal.
1979 APInt FSrcInt = CSrc ? CSrc->getValueAPF().bitcastToAPInt()
1980 : APFloat::getQNaN(Sem: II.getType()->getFltSemantics())
1981 .bitcastToAPInt();
1982
1983 const ConstantInt *Cseg = dyn_cast<ConstantInt>(Val: Segment);
1984 if (!Cseg) {
1985 if (isa<UndefValue>(Val: Src))
1986 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getZero(Ty: II.getType()));
1987 break;
1988 }
1989
1990 unsigned Exponent =
1991 static_cast<unsigned>(FSrcInt.extractBitsAsZExtValue(numBits: 11, bitPosition: 52));
1992 unsigned SegmentVal =
1993 static_cast<unsigned>(Cseg->getValue().trunc(width: 5).getZExtValue());
1994 unsigned Shift = SegmentVal * 53;
1995 if (Exponent > 1077)
1996 Shift += Exponent - 1077;
1997
1998 // 2.0/PI table.
1999 static const uint32_t TwoByPi[] = {
2000 0xa2f9836e, 0x4e441529, 0xfc2757d1, 0xf534ddc0, 0xdb629599, 0x3c439041,
2001 0xfe5163ab, 0xdebbc561, 0xb7246e3a, 0x424dd2e0, 0x06492eea, 0x09d1921c,
2002 0xfe1deb1c, 0xb129a73e, 0xe88235f5, 0x2ebb4484, 0xe99c7026, 0xb45f7e41,
2003 0x3991d639, 0x835339f4, 0x9c845f8b, 0xbdf9283b, 0x1ff897ff, 0xde05980f,
2004 0xef2f118b, 0x5a0a6d1f, 0x6d367ecf, 0x27cb09b7, 0x4f463f66, 0x9e5fea2d,
2005 0x7527bac7, 0xebe5f17b, 0x3d0739f7, 0x8a5292ea, 0x6bfb5fb1, 0x1f8d5d08,
2006 0x56033046};
2007
2008 // Return 0 for outbound segment (hardware behavior).
2009 unsigned Idx = Shift >> 5;
2010 if (Idx + 2 >= std::size(TwoByPi)) {
2011 APFloat Zero = APFloat::getZero(Sem: II.getType()->getFltSemantics());
2012 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::get(Ty: II.getType(), V: Zero));
2013 }
2014
2015 unsigned BShift = Shift & 0x1f;
2016 uint64_t Thi = Make_64(High: TwoByPi[Idx], Low: TwoByPi[Idx + 1]);
2017 uint64_t Tlo = Make_64(High: TwoByPi[Idx + 2], Low: 0);
2018 if (BShift)
2019 Thi = (Thi << BShift) | (Tlo >> (64 - BShift));
2020 Thi = Thi >> 11;
2021 APFloat Result = APFloat((double)Thi);
2022
2023 int Scale = -53 - Shift;
2024 if (Exponent >= 1968)
2025 Scale += 128;
2026
2027 Result = scalbn(X: Result, Exp: Scale, RM: RoundingMode::NearestTiesToEven);
2028 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::get(Ty: Src->getType(), V: Result));
2029 }
2030 case Intrinsic::amdgcn_sdot2:
2031 case Intrinsic::amdgcn_udot2:
2032 case Intrinsic::amdgcn_sdot4:
2033 case Intrinsic::amdgcn_udot4:
2034 case Intrinsic::amdgcn_sdot8:
2035 case Intrinsic::amdgcn_udot8: {
2036 Value *Src0 = II.getArgOperand(i: 0);
2037 Value *Src1 = II.getArgOperand(i: 1);
2038
2039 // Canonicalize the constant multiplicand to Src1.
2040 if (isa<Constant>(Val: Src0) && !isa<Constant>(Val: Src1)) {
2041 II.setArgOperand(i: 0, v: Src1);
2042 II.setArgOperand(i: 1, v: Src0);
2043 return &II;
2044 }
2045
2046 if (match(V: Src1, P: m_Zero()))
2047 return IC.replaceInstUsesWith(I&: II, V: II.getArgOperand(i: 2));
2048
2049 if (Instruction *I = foldAddIntoDotAccumulator(II, AccIdx: 2, ClampIdx: 3, IC))
2050 return I;
2051
2052 break;
2053 }
2054 case Intrinsic::amdgcn_sudot4:
2055 case Intrinsic::amdgcn_sudot8: {
2056 Value *Src0 = II.getArgOperand(i: 1);
2057 Value *Src1 = II.getArgOperand(i: 3);
2058
2059 // Canonicalize the constant multiplicand to Src1, moving its sign flag with
2060 // it.
2061 if (isa<Constant>(Val: Src0) && !isa<Constant>(Val: Src1)) {
2062 Value *Sign0 = II.getArgOperand(i: 0);
2063 Value *Sign1 = II.getArgOperand(i: 2);
2064 II.setArgOperand(i: 0, v: Sign1);
2065 II.setArgOperand(i: 1, v: Src1);
2066 II.setArgOperand(i: 2, v: Sign0);
2067 II.setArgOperand(i: 3, v: Src0);
2068 return &II;
2069 }
2070
2071 if (match(V: Src1, P: m_Zero()))
2072 return IC.replaceInstUsesWith(I&: II, V: II.getArgOperand(i: 4));
2073
2074 if (Instruction *I = foldAddIntoDotAccumulator(II, AccIdx: 4, ClampIdx: 5, IC))
2075 return I;
2076
2077 break;
2078 }
2079 case Intrinsic::amdgcn_fmul_legacy: {
2080 Value *Op0 = II.getArgOperand(i: 0);
2081 Value *Op1 = II.getArgOperand(i: 1);
2082
2083 for (Value *Src : {Op0, Op1}) {
2084 if (isa<PoisonValue>(Val: Src))
2085 return IC.replaceInstUsesWith(I&: II, V: Src);
2086 }
2087
2088 // The legacy behaviour is that multiplying +/-0.0 by anything, even NaN or
2089 // infinity, gives +0.0.
2090 // TODO: Move to InstSimplify?
2091 if (match(V: Op0, P: PatternMatch::m_AnyZeroFP()) ||
2092 match(V: Op1, P: PatternMatch::m_AnyZeroFP()))
2093 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getZero(Ty: II.getType()));
2094
2095 // If we can prove we don't have one of the special cases then we can use a
2096 // normal fmul instruction instead.
2097 if (canSimplifyLegacyMulToMul(I: II, Op0, Op1, IC)) {
2098 auto *FMul = IC.Builder.CreateFMulFMF(L: Op0, R: Op1, FMFSource: &II);
2099 FMul->takeName(V: &II);
2100 return IC.replaceInstUsesWith(I&: II, V: FMul);
2101 }
2102 break;
2103 }
2104 case Intrinsic::amdgcn_fma_legacy: {
2105 Value *Op0 = II.getArgOperand(i: 0);
2106 Value *Op1 = II.getArgOperand(i: 1);
2107 Value *Op2 = II.getArgOperand(i: 2);
2108
2109 for (Value *Src : {Op0, Op1, Op2}) {
2110 if (isa<PoisonValue>(Val: Src))
2111 return IC.replaceInstUsesWith(I&: II, V: Src);
2112 }
2113
2114 // The legacy behaviour is that multiplying +/-0.0 by anything, even NaN or
2115 // infinity, gives +0.0.
2116 // TODO: Move to InstSimplify?
2117 if (match(V: Op0, P: PatternMatch::m_AnyZeroFP()) ||
2118 match(V: Op1, P: PatternMatch::m_AnyZeroFP())) {
2119 // It's tempting to just return Op2 here, but that would give the wrong
2120 // result if Op2 was -0.0.
2121 auto *Zero = ConstantFP::getZero(Ty: II.getType());
2122 auto *FAdd = IC.Builder.CreateFAddFMF(L: Zero, R: Op2, FMFSource: &II);
2123 FAdd->takeName(V: &II);
2124 return IC.replaceInstUsesWith(I&: II, V: FAdd);
2125 }
2126
2127 // If we can prove we don't have one of the special cases then we can use a
2128 // normal fma instead.
2129 if (canSimplifyLegacyMulToMul(I: II, Op0, Op1, IC)) {
2130 II.setCalledOperand(Intrinsic::getOrInsertDeclaration(
2131 M: II.getModule(), id: Intrinsic::fma, OverloadTys: II.getType()));
2132 return &II;
2133 }
2134 break;
2135 }
2136 case Intrinsic::amdgcn_is_shared:
2137 case Intrinsic::amdgcn_is_private: {
2138 Value *Src = II.getArgOperand(i: 0);
2139 if (isa<PoisonValue>(Val: Src))
2140 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
2141 if (isa<UndefValue>(Val: Src))
2142 return IC.replaceInstUsesWith(I&: II, V: UndefValue::get(T: II.getType()));
2143
2144 if (isa<ConstantPointerNull>(Val: II.getArgOperand(i: 0)))
2145 return IC.replaceInstUsesWith(I&: II, V: ConstantInt::getFalse(Ty: II.getType()));
2146 break;
2147 }
2148 case Intrinsic::amdgcn_make_buffer_rsrc: {
2149 Value *Src = II.getArgOperand(i: 0);
2150 if (isa<PoisonValue>(Val: Src))
2151 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
2152
2153 // Normalize num_records to the correct width.
2154 std::optional<unsigned> Width = ST->getBufferResourceNumRecordsWidth();
2155 if (!Width)
2156 return std::nullopt;
2157 Type *NumRecordsTy = IC.Builder.getIntNTy(N: *Width);
2158 if (II.getArgOperand(i: 2)->getType() == NumRecordsTy)
2159 return std::nullopt;
2160 SmallVector<Value *, 4> Args(II.args());
2161 Args[2] = IC.Builder.CreateZExtOrTrunc(V: Args[2], DestTy: NumRecordsTy);
2162 CallInst *NewCall = IC.Builder.CreateIntrinsicWithoutFolding(
2163 ID: Intrinsic::amdgcn_make_buffer_rsrc,
2164 OverloadTypes: {II.getType(), Src->getType(), NumRecordsTy}, Args);
2165 NewCall->copyMetadata(SrcInst: II);
2166 NewCall->setTailCallKind(II.getTailCallKind());
2167 // Copy over all attributes except those on num_records, which may no longer
2168 // be valid.
2169 NewCall->setAttributes(
2170 II.getAttributes().removeParamAttributes(C&: II.getContext(), ArgNo: 2));
2171 NewCall->takeName(V: &II);
2172 return IC.replaceInstUsesWith(I&: II, V: NewCall);
2173 }
2174 case Intrinsic::amdgcn_raw_buffer_store_format:
2175 case Intrinsic::amdgcn_raw_ptr_buffer_store_format:
2176 case Intrinsic::amdgcn_struct_buffer_store_format:
2177 case Intrinsic::amdgcn_struct_ptr_buffer_store_format:
2178 case Intrinsic::amdgcn_raw_tbuffer_store:
2179 case Intrinsic::amdgcn_raw_ptr_tbuffer_store:
2180 case Intrinsic::amdgcn_struct_tbuffer_store:
2181 case Intrinsic::amdgcn_struct_ptr_tbuffer_store:
2182 case Intrinsic::amdgcn_image_store_1d:
2183 case Intrinsic::amdgcn_image_store_1darray:
2184 case Intrinsic::amdgcn_image_store_2d:
2185 case Intrinsic::amdgcn_image_store_2darray:
2186 case Intrinsic::amdgcn_image_store_2darraymsaa:
2187 case Intrinsic::amdgcn_image_store_2dmsaa:
2188 case Intrinsic::amdgcn_image_store_3d:
2189 case Intrinsic::amdgcn_image_store_cube:
2190 case Intrinsic::amdgcn_image_store_mip_1d:
2191 case Intrinsic::amdgcn_image_store_mip_1darray:
2192 case Intrinsic::amdgcn_image_store_mip_2d:
2193 case Intrinsic::amdgcn_image_store_mip_2darray:
2194 case Intrinsic::amdgcn_image_store_mip_3d:
2195 case Intrinsic::amdgcn_image_store_mip_cube: {
2196 if (!isa<FixedVectorType>(Val: II.getArgOperand(i: 0)->getType()))
2197 break;
2198
2199 APInt DemandedElts;
2200 if (ST->hasDefaultComponentBroadcast())
2201 DemandedElts = defaultComponentBroadcast(V: II.getArgOperand(i: 0));
2202 else if (ST->hasDefaultComponentZero())
2203 DemandedElts = trimTrailingZerosInVector(IC, UseV: II.getArgOperand(i: 0), I: &II);
2204 else
2205 break;
2206
2207 int DMaskIdx = getAMDGPUImageDMaskIntrinsic(Intr: II.getIntrinsicID()) ? 1 : -1;
2208 if (simplifyAMDGCNMemoryIntrinsicDemanded(IC, II, DemandedElts, DMaskIdx,
2209 IsLoad: false)) {
2210 return IC.eraseInstFromFunction(I&: II);
2211 }
2212
2213 break;
2214 }
2215 case Intrinsic::amdgcn_prng_b32: {
2216 auto *Src = II.getArgOperand(i: 0);
2217 if (isa<UndefValue>(Val: Src)) {
2218 return IC.replaceInstUsesWith(I&: II, V: Src);
2219 }
2220 return std::nullopt;
2221 }
2222 case Intrinsic::amdgcn_mfma_scale_f32_16x16x128_f8f6f4:
2223 case Intrinsic::amdgcn_mfma_scale_f32_32x32x64_f8f6f4: {
2224 Value *Src0 = II.getArgOperand(i: 0);
2225 Value *Src1 = II.getArgOperand(i: 1);
2226 uint64_t CBSZ = cast<ConstantInt>(Val: II.getArgOperand(i: 3))->getZExtValue();
2227 uint64_t BLGP = cast<ConstantInt>(Val: II.getArgOperand(i: 4))->getZExtValue();
2228 auto *Src0Ty = cast<FixedVectorType>(Val: Src0->getType());
2229 auto *Src1Ty = cast<FixedVectorType>(Val: Src1->getType());
2230
2231 auto getFormatNumRegs = [](uint64_t FormatVal) {
2232 switch (FormatVal) {
2233 case AMDGPU::MFMAScaleFormats::FP6_E2M3:
2234 case AMDGPU::MFMAScaleFormats::FP6_E3M2:
2235 return 6u;
2236 case AMDGPU::MFMAScaleFormats::FP4_E2M1:
2237 return 4u;
2238 case AMDGPU::MFMAScaleFormats::FP8_E4M3:
2239 case AMDGPU::MFMAScaleFormats::FP8_E5M2:
2240 return 8u;
2241 default:
2242 llvm_unreachable("invalid format value");
2243 }
2244 };
2245
2246 bool MadeChange = false;
2247 unsigned Src0NumElts = getFormatNumRegs(CBSZ);
2248 unsigned Src1NumElts = getFormatNumRegs(BLGP);
2249
2250 // Depending on the used format, fewer registers are required so shrink the
2251 // vector type.
2252 if (Src0Ty->getNumElements() > Src0NumElts) {
2253 Src0 = IC.Builder.CreateExtractVector(
2254 DstType: FixedVectorType::get(ElementType: Src0Ty->getElementType(), NumElts: Src0NumElts), SrcVec: Src0,
2255 Idx: uint64_t(0));
2256 MadeChange = true;
2257 }
2258
2259 if (Src1Ty->getNumElements() > Src1NumElts) {
2260 Src1 = IC.Builder.CreateExtractVector(
2261 DstType: FixedVectorType::get(ElementType: Src1Ty->getElementType(), NumElts: Src1NumElts), SrcVec: Src1,
2262 Idx: uint64_t(0));
2263 MadeChange = true;
2264 }
2265
2266 if (!MadeChange)
2267 return std::nullopt;
2268
2269 SmallVector<Value *, 10> Args(II.args());
2270 Args[0] = Src0;
2271 Args[1] = Src1;
2272
2273 Value *NewII = IC.Builder.CreateIntrinsic(
2274 ID: IID, OverloadTypes: {Src0->getType(), Src1->getType()}, Args, FMFSource: &II);
2275 NewII->takeName(V: &II);
2276 return IC.replaceInstUsesWith(I&: II, V: NewII);
2277 }
2278 case Intrinsic::amdgcn_wmma_f32_16x16x128_f8f6f4:
2279 case Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
2280 case Intrinsic::amdgcn_wmma_scale16_f32_16x16x128_f8f6f4: {
2281 Value *Src0 = II.getArgOperand(i: 1);
2282 Value *Src1 = II.getArgOperand(i: 3);
2283 unsigned FmtA = cast<ConstantInt>(Val: II.getArgOperand(i: 0))->getZExtValue();
2284 uint64_t FmtB = cast<ConstantInt>(Val: II.getArgOperand(i: 2))->getZExtValue();
2285 auto *Src0Ty = cast<FixedVectorType>(Val: Src0->getType());
2286 auto *Src1Ty = cast<FixedVectorType>(Val: Src1->getType());
2287
2288 bool MadeChange = false;
2289 unsigned Src0NumElts = AMDGPU::wmmaScaleF8F6F4FormatToNumRegs(Fmt: FmtA);
2290 unsigned Src1NumElts = AMDGPU::wmmaScaleF8F6F4FormatToNumRegs(Fmt: FmtB);
2291
2292 // Depending on the used format, fewer registers are required so shrink the
2293 // vector type.
2294 if (Src0Ty->getNumElements() > Src0NumElts) {
2295 Src0 = IC.Builder.CreateExtractVector(
2296 DstType: FixedVectorType::get(ElementType: Src0Ty->getElementType(), NumElts: Src0NumElts), SrcVec: Src0,
2297 Idx: IC.Builder.getInt64(C: 0));
2298 MadeChange = true;
2299 }
2300
2301 if (Src1Ty->getNumElements() > Src1NumElts) {
2302 Src1 = IC.Builder.CreateExtractVector(
2303 DstType: FixedVectorType::get(ElementType: Src1Ty->getElementType(), NumElts: Src1NumElts), SrcVec: Src1,
2304 Idx: IC.Builder.getInt64(C: 0));
2305 MadeChange = true;
2306 }
2307
2308 if (!MadeChange)
2309 return std::nullopt;
2310
2311 SmallVector<Value *, 13> Args(II.args());
2312 Args[1] = Src0;
2313 Args[3] = Src1;
2314
2315 Value *NewII = IC.Builder.CreateIntrinsic(
2316 ID: IID, OverloadTypes: {II.getArgOperand(i: 5)->getType(), Src0->getType(), Src1->getType()},
2317 Args, FMFSource: &II);
2318 NewII->takeName(V: &II);
2319 return IC.replaceInstUsesWith(I&: II, V: NewII);
2320 }
2321 }
2322 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
2323 AMDGPU::getImageDimIntrinsicInfo(Intr: II.getIntrinsicID())) {
2324 return simplifyAMDGCNImageIntrinsic(ST, ImageDimIntr, II, IC);
2325 }
2326 return std::nullopt;
2327}
2328
2329/// Implement SimplifyDemandedVectorElts for amdgcn buffer and image intrinsics.
2330///
2331/// The result of simplifying amdgcn image and buffer store intrinsics is updating
2332/// definitions of the intrinsics vector argument, not Uses of the result like
2333/// image and buffer loads.
2334/// Note: This only supports non-TFE/LWE image intrinsic calls; those have
2335/// struct returns.
2336static Value *simplifyAMDGCNMemoryIntrinsicDemanded(InstCombiner &IC,
2337 IntrinsicInst &II,
2338 APInt DemandedElts,
2339 int DMaskIdx, bool IsLoad) {
2340
2341 auto *IIVTy = cast<FixedVectorType>(Val: IsLoad ? II.getType()
2342 : II.getOperand(i_nocapture: 0)->getType());
2343 unsigned VWidth = IIVTy->getNumElements();
2344 if (VWidth == 1)
2345 return nullptr;
2346 Type *EltTy = IIVTy->getElementType();
2347
2348 IRBuilderBase::InsertPointGuard Guard(IC.Builder);
2349 IC.Builder.SetInsertPoint(&II);
2350
2351 // Assume the arguments are unchanged and later override them, if needed.
2352 SmallVector<Value *, 16> Args(II.args());
2353
2354 if (DMaskIdx < 0) {
2355 // Buffer case.
2356
2357 const unsigned ActiveBits = DemandedElts.getActiveBits();
2358 const unsigned UnusedComponentsAtFront = DemandedElts.countr_zero();
2359
2360 // Start assuming the prefix of elements is demanded, but possibly clear
2361 // some other bits if there are trailing zeros (unused components at front)
2362 // and update offset.
2363 DemandedElts = (1 << ActiveBits) - 1;
2364
2365 if (UnusedComponentsAtFront > 0) {
2366 static const unsigned InvalidOffsetIdx = 0xf;
2367
2368 unsigned OffsetIdx;
2369 switch (II.getIntrinsicID()) {
2370 case Intrinsic::amdgcn_raw_buffer_load:
2371 case Intrinsic::amdgcn_raw_ptr_buffer_load:
2372 OffsetIdx = 1;
2373 break;
2374 case Intrinsic::amdgcn_s_buffer_load:
2375 case Intrinsic::amdgcn_ptr_s_buffer_load:
2376 // If resulting type is vec3, there is no point in trimming the
2377 // load with updated offset, as the vec3 would most likely be widened to
2378 // vec4 anyway during lowering.
2379 if (ActiveBits == 4 && UnusedComponentsAtFront == 1)
2380 OffsetIdx = InvalidOffsetIdx;
2381 else
2382 OffsetIdx = 1;
2383 break;
2384 case Intrinsic::amdgcn_struct_buffer_load:
2385 case Intrinsic::amdgcn_struct_ptr_buffer_load:
2386 OffsetIdx = 2;
2387 break;
2388 default:
2389 // TODO: handle tbuffer* intrinsics.
2390 OffsetIdx = InvalidOffsetIdx;
2391 break;
2392 }
2393
2394 if (OffsetIdx != InvalidOffsetIdx) {
2395 // Clear demanded bits and update the offset.
2396 DemandedElts &= ~((1 << UnusedComponentsAtFront) - 1);
2397 auto *Offset = Args[OffsetIdx];
2398 uint64_t SingleComponentSizeInBits =
2399 IC.getDataLayout().getTypeSizeInBits(Ty: EltTy).getFixedValue();
2400 uint64_t OffsetAdd =
2401 UnusedComponentsAtFront * SingleComponentSizeInBits / 8;
2402 auto *OffsetAddVal = ConstantInt::get(Ty: Offset->getType(), V: OffsetAdd);
2403 Args[OffsetIdx] = IC.Builder.CreateAdd(LHS: Offset, RHS: OffsetAddVal);
2404 }
2405 }
2406 } else {
2407 // Image case.
2408
2409 ConstantInt *DMask = cast<ConstantInt>(Val: Args[DMaskIdx]);
2410 unsigned DMaskVal = DMask->getZExtValue() & 0xf;
2411
2412 // dmask 0 has special semantics, do not simplify.
2413 if (DMaskVal == 0)
2414 return nullptr;
2415
2416 if (!IsLoad && !isMask_32(Value: DMaskVal))
2417 return nullptr;
2418
2419 // Mask off values that are undefined because the dmask doesn't cover them
2420 DemandedElts &= (1 << llvm::popcount(Value: DMaskVal)) - 1;
2421
2422 unsigned NewDMaskVal = 0;
2423 unsigned OrigLdStIdx = 0;
2424 for (unsigned SrcIdx = 0; SrcIdx < 4; ++SrcIdx) {
2425 const unsigned Bit = 1 << SrcIdx;
2426 if (!!(DMaskVal & Bit)) {
2427 if (!!DemandedElts[OrigLdStIdx])
2428 NewDMaskVal |= Bit;
2429 OrigLdStIdx++;
2430 }
2431 }
2432
2433 if (DMaskVal != NewDMaskVal)
2434 Args[DMaskIdx] = ConstantInt::get(Ty: DMask->getType(), V: NewDMaskVal);
2435 }
2436
2437 unsigned NewNumElts = DemandedElts.popcount();
2438 if (!NewNumElts)
2439 return PoisonValue::get(T: IIVTy);
2440
2441 if (NewNumElts >= VWidth && DemandedElts.isMask()) {
2442 if (DMaskIdx >= 0)
2443 II.setArgOperand(i: DMaskIdx, v: Args[DMaskIdx]);
2444 return nullptr;
2445 }
2446
2447 // Validate function argument and return types, extracting overloaded types
2448 // along the way.
2449 SmallVector<Type *, 6> OverloadTys;
2450 if (!Intrinsic::isSignatureValid(F: II.getCalledFunction(), OverloadTys))
2451 return nullptr;
2452
2453 Type *NewTy =
2454 (NewNumElts == 1) ? EltTy : FixedVectorType::get(ElementType: EltTy, NumElts: NewNumElts);
2455 OverloadTys[0] = NewTy;
2456
2457 if (!IsLoad) {
2458 SmallVector<int, 8> EltMask;
2459 for (unsigned OrigStoreIdx = 0; OrigStoreIdx < VWidth; ++OrigStoreIdx)
2460 if (DemandedElts[OrigStoreIdx])
2461 EltMask.push_back(Elt: OrigStoreIdx);
2462
2463 if (NewNumElts == 1)
2464 Args[0] = IC.Builder.CreateExtractElement(Vec: II.getOperand(i_nocapture: 0), Idx: EltMask[0]);
2465 else
2466 Args[0] = IC.Builder.CreateShuffleVector(V: II.getOperand(i_nocapture: 0), Mask: EltMask);
2467 }
2468
2469 CallInst *NewCall = IC.Builder.CreateIntrinsicWithoutFolding(
2470 ID: II.getIntrinsicID(), OverloadTypes: OverloadTys, Args);
2471 NewCall->takeName(V: &II);
2472 NewCall->copyMetadata(SrcInst: II);
2473 AttributeList OldAttrList = II.getAttributes();
2474 NewCall->setAttributes(OldAttrList);
2475
2476 if (IsLoad) {
2477 if (NewNumElts == 1) {
2478 return IC.Builder.CreateInsertElement(Vec: PoisonValue::get(T: IIVTy), NewElt: NewCall,
2479 Idx: DemandedElts.countr_zero());
2480 }
2481
2482 SmallVector<int, 8> EltMask;
2483 unsigned NewLoadIdx = 0;
2484 for (unsigned OrigLoadIdx = 0; OrigLoadIdx < VWidth; ++OrigLoadIdx) {
2485 if (!!DemandedElts[OrigLoadIdx])
2486 EltMask.push_back(Elt: NewLoadIdx++);
2487 else
2488 EltMask.push_back(Elt: NewNumElts);
2489 }
2490
2491 auto *Shuffle = IC.Builder.CreateShuffleVector(V: NewCall, Mask: EltMask);
2492
2493 return Shuffle;
2494 }
2495
2496 return NewCall;
2497}
2498
2499Value *GCNTTIImpl::simplifyAMDGCNLaneIntrinsicDemanded(
2500 InstCombiner &IC, IntrinsicInst &II, const APInt &DemandedElts,
2501 APInt &UndefElts) const {
2502 auto *VT = dyn_cast<FixedVectorType>(Val: II.getType());
2503 if (!VT)
2504 return nullptr;
2505
2506 const unsigned FirstElt = DemandedElts.countr_zero();
2507 const unsigned LastElt = DemandedElts.getActiveBits() - 1;
2508 const unsigned MaskLen = LastElt - FirstElt + 1;
2509
2510 unsigned OldNumElts = VT->getNumElements();
2511 if (MaskLen == OldNumElts && MaskLen != 1)
2512 return nullptr;
2513
2514 Type *EltTy = VT->getElementType();
2515 Type *NewVT = MaskLen == 1 ? EltTy : FixedVectorType::get(ElementType: EltTy, NumElts: MaskLen);
2516
2517 // Theoretically we should support these intrinsics for any legal type. Avoid
2518 // introducing cases that aren't direct register types like v3i16.
2519 if (!isTypeLegal(Ty: NewVT))
2520 return nullptr;
2521
2522 Value *Src = II.getArgOperand(i: 0);
2523
2524 // Make sure convergence tokens are preserved.
2525 // TODO: CreateIntrinsic should allow directly copying bundles
2526 SmallVector<OperandBundleDef, 2> OpBundles;
2527 II.getOperandBundlesAsDefs(Defs&: OpBundles);
2528
2529 Module *M = IC.Builder.getModule();
2530 Function *Remangled =
2531 Intrinsic::getOrInsertDeclaration(M, id: II.getIntrinsicID(), OverloadTys: {NewVT});
2532
2533 if (MaskLen == 1) {
2534 Value *Extract = IC.Builder.CreateExtractElement(Vec: Src, Idx: FirstElt);
2535
2536 // TODO: Preserve callsite attributes?
2537 CallInst *NewCall = IC.Builder.CreateCall(Callee: Remangled, Args: {Extract}, OpBundles);
2538
2539 return IC.Builder.CreateInsertElement(Vec: PoisonValue::get(T: II.getType()),
2540 NewElt: NewCall, Idx: FirstElt);
2541 }
2542
2543 SmallVector<int> ExtractMask(MaskLen, -1);
2544 for (unsigned I = 0; I != MaskLen; ++I) {
2545 if (DemandedElts[FirstElt + I])
2546 ExtractMask[I] = FirstElt + I;
2547 }
2548
2549 Value *Extract = IC.Builder.CreateShuffleVector(V: Src, Mask: ExtractMask);
2550
2551 // TODO: Preserve callsite attributes?
2552 CallInst *NewCall = IC.Builder.CreateCall(Callee: Remangled, Args: {Extract}, OpBundles);
2553
2554 SmallVector<int> InsertMask(OldNumElts, -1);
2555 for (unsigned I = 0; I != MaskLen; ++I) {
2556 if (DemandedElts[FirstElt + I])
2557 InsertMask[FirstElt + I] = I;
2558 }
2559
2560 // FIXME: If the call has a convergence bundle, we end up leaving the dead
2561 // call behind.
2562 return IC.Builder.CreateShuffleVector(V: NewCall, Mask: InsertMask);
2563}
2564
2565std::optional<Value *> GCNTTIImpl::simplifyDemandedVectorEltsIntrinsic(
2566 InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
2567 APInt &UndefElts2, APInt &UndefElts3,
2568 std::function<void(Instruction *, unsigned, APInt, APInt &)>
2569 SimplifyAndSetOp) const {
2570 switch (II.getIntrinsicID()) {
2571 case Intrinsic::amdgcn_readfirstlane:
2572 SimplifyAndSetOp(&II, 0, DemandedElts, UndefElts);
2573 return simplifyAMDGCNLaneIntrinsicDemanded(IC, II, DemandedElts, UndefElts);
2574 case Intrinsic::amdgcn_raw_buffer_load:
2575 case Intrinsic::amdgcn_raw_ptr_buffer_load:
2576 case Intrinsic::amdgcn_raw_buffer_load_format:
2577 case Intrinsic::amdgcn_raw_ptr_buffer_load_format:
2578 case Intrinsic::amdgcn_raw_tbuffer_load:
2579 case Intrinsic::amdgcn_raw_ptr_tbuffer_load:
2580 case Intrinsic::amdgcn_s_buffer_load:
2581 case Intrinsic::amdgcn_ptr_s_buffer_load:
2582 case Intrinsic::amdgcn_struct_buffer_load:
2583 case Intrinsic::amdgcn_struct_ptr_buffer_load:
2584 case Intrinsic::amdgcn_struct_buffer_load_format:
2585 case Intrinsic::amdgcn_struct_ptr_buffer_load_format:
2586 case Intrinsic::amdgcn_struct_tbuffer_load:
2587 case Intrinsic::amdgcn_struct_ptr_tbuffer_load:
2588 return simplifyAMDGCNMemoryIntrinsicDemanded(IC, II, DemandedElts);
2589 default: {
2590 if (getAMDGPUImageDMaskIntrinsic(Intr: II.getIntrinsicID())) {
2591 return simplifyAMDGCNMemoryIntrinsicDemanded(IC, II, DemandedElts, DMaskIdx: 0);
2592 }
2593 break;
2594 }
2595 }
2596 return std::nullopt;
2597}
2598