1//===- AMDGPInstCombineIntrinsic.cpp - AMDGPU specific InstCombine pass ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// \file
10// This file implements a TargetTransformInfo analysis pass specific to the
11// AMDGPU target machine. It uses the target's detailed information to provide
12// more precise answers to certain TTI queries, while letting the target
13// independent and default TTI implementations handle the rest.
14//
15//===----------------------------------------------------------------------===//
16
17#include "AMDGPUInstrInfo.h"
18#include "AMDGPUTargetTransformInfo.h"
19#include "GCNSubtarget.h"
20#include "SIDefines.h"
21#include "llvm/ADT/FloatingPointMode.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/Analysis/ConstantFolding.h"
24#include "llvm/Analysis/ValueTracking.h"
25#include "llvm/IR/Constants.h"
26#include "llvm/IR/Dominators.h"
27#include "llvm/IR/IntrinsicsAMDGPU.h"
28#include "llvm/Support/MathExtras.h"
29#include "llvm/Transforms/InstCombine/InstCombiner.h"
30#include <optional>
31
32using namespace llvm;
33using namespace llvm::PatternMatch;
34
35#define DEBUG_TYPE "AMDGPUtti"
36
37namespace {
38
39struct AMDGPUImageDMaskIntrinsic {
40 unsigned Intr;
41};
42
43#define GET_AMDGPUImageDMaskIntrinsicTable_IMPL
44#include "AMDGPUGenSearchableTables.inc"
45
46} // end anonymous namespace
47
48// Constant fold llvm.amdgcn.fmed3 intrinsics for standard inputs.
49//
50// A single NaN input is folded to minnum, so we rely on that folding for
51// handling NaNs.
52static APFloat fmed3AMDGCN(const APFloat &Src0, const APFloat &Src1,
53 const APFloat &Src2) {
54 assert(!Src0.isNaN() && !Src1.isNaN() && !Src2.isNaN() &&
55 "nans handled separately");
56 APFloat Max3 = maxnum(A: maxnum(A: Src0, B: Src1), B: Src2);
57
58 if (Max3.bitwiseIsEqual(RHS: Src0))
59 return maxnum(A: Src1, B: Src2);
60
61 if (Max3.bitwiseIsEqual(RHS: Src1))
62 return maxnum(A: Src0, B: Src2);
63
64 return maxnum(A: Src0, B: Src1);
65}
66
67// Check if a value can be converted to a 16-bit value without losing precision.
68// The value is expected to be either a float (IsFloat = true) or an unsigned
69// integer (IsFloat = false). When AllowI16SExt is set, a sext from i16 is also
70// accepted: for unsigned addresses sext and zext only differ for a negative
71// i16, which is out of bounds anyway (see caller).
72static bool canSafelyConvertTo16Bit(Value &V, bool IsFloat,
73 bool AllowI16SExt = false) {
74 Type *VTy = V.getType();
75 if (VTy->isHalfTy() || VTy->isIntegerTy(BitWidth: 16)) {
76 // The value is already 16-bit, so we don't want to convert to 16-bit again!
77 return false;
78 }
79 if (IsFloat) {
80 if (ConstantFP *ConstFloat = dyn_cast<ConstantFP>(Val: &V)) {
81 // We need to check that if we cast the index down to a half, we do not
82 // lose precision.
83 APFloat FloatValue(ConstFloat->getValueAPF());
84 bool LosesInfo = true;
85 FloatValue.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmTowardZero,
86 losesInfo: &LosesInfo);
87 return !LosesInfo;
88 }
89 } else {
90 if (ConstantInt *ConstInt = dyn_cast<ConstantInt>(Val: &V)) {
91 // We need to check that if we cast the index down to an i16, we do not
92 // lose precision.
93 APInt IntValue(ConstInt->getValue());
94 return IntValue.getActiveBits() <= 16;
95 }
96 }
97
98 // Coordinates may arrive as extractelement((s|z|fp)ext Vec), Idx. The
99 // widening cast has one use per lane, so it is never sunk into the extract;
100 // strip the extract here so the cast check below is common to scalar and
101 // vector coords.
102 Value *CastCandidate;
103 if (!match(V: &V, P: m_ExtractElt(Val: m_Value(V&: CastCandidate), Idx: m_Value())))
104 CastCandidate = &V;
105
106 Value *CastSrc;
107 bool IsExt = IsFloat ? match(V: CastCandidate, P: m_FPExt(Op: m_Value(V&: CastSrc)))
108 : match(V: CastCandidate, P: m_ZExt(Op: m_Value(V&: CastSrc)));
109 if (!IsExt && !IsFloat && AllowI16SExt)
110 IsExt = match(V: CastCandidate, P: m_SExt(Op: m_Value(V&: CastSrc)));
111 if (IsExt) {
112 Type *CastSrcTy = CastSrc->getType()->getScalarType();
113 if (CastSrcTy->isHalfTy() || CastSrcTy->isIntegerTy(BitWidth: 16))
114 return true;
115 }
116
117 return false;
118}
119
120// Convert a value to 16-bit.
121static Value *convertTo16Bit(Value &V, InstCombiner::BuilderTy &Builder) {
122 Type *VTy = V.getType();
123 if (isa<FPExtInst, SExtInst, ZExtInst>(Val: &V))
124 return cast<Instruction>(Val: &V)->getOperand(i: 0);
125 // Vector form: extractelement((s|z|fp)ext Vec), Idx -> extractelement(Vec,
126 // Idx), taking the narrow lane directly so the widening cast can be removed.
127 Instruction *VecCast;
128 Value *Idx;
129 if (match(V: &V, P: m_ExtractElt(Val: m_Instruction(I&: VecCast), Idx: m_Value(V&: Idx))) &&
130 isa<FPExtInst, SExtInst, ZExtInst>(Val: VecCast))
131 return Builder.CreateExtractElement(Vec: VecCast->getOperand(i: 0), Idx);
132 if (VTy->isIntegerTy())
133 return Builder.CreateIntCast(V: &V, DestTy: Type::getInt16Ty(C&: V.getContext()), isSigned: false);
134 if (VTy->isFloatingPointTy())
135 return Builder.CreateFPCast(V: &V, DestTy: Type::getHalfTy(C&: V.getContext()));
136
137 llvm_unreachable("Should never be called!");
138}
139
140/// Applies Func(OldIntr.Args, OldIntr.ArgTys), creates intrinsic call with
141/// modified arguments (based on OldIntr) and replaces InstToReplace with
142/// this newly created intrinsic call.
143static std::optional<Instruction *> modifyIntrinsicCall(
144 IntrinsicInst &OldIntr, Instruction &InstToReplace, unsigned NewIntr,
145 InstCombiner &IC,
146 std::function<void(SmallVectorImpl<Value *> &, SmallVectorImpl<Type *> &)>
147 Func) {
148 SmallVector<Type *, 4> OverloadTys;
149 if (!Intrinsic::isSignatureValid(F: OldIntr.getCalledFunction(), OverloadTys))
150 return std::nullopt;
151
152 SmallVector<Value *, 8> Args(OldIntr.args());
153
154 // Modify arguments and types
155 Func(Args, OverloadTys);
156
157 CallInst *NewCall =
158 IC.Builder.CreateIntrinsicWithoutFolding(ID: NewIntr, OverloadTypes: OverloadTys, Args);
159 NewCall->takeName(V: &OldIntr);
160 NewCall->copyMetadata(SrcInst: OldIntr);
161 if (isa<FPMathOperator>(Val: NewCall))
162 NewCall->copyFastMathFlags(I: &OldIntr);
163 // Copy attributes
164 AttributeList OldAttrList = OldIntr.getAttributes();
165 NewCall->setAttributes(OldAttrList);
166
167 // Erase and replace uses
168 if (!InstToReplace.getType()->isVoidTy())
169 IC.replaceInstUsesWith(I&: InstToReplace, V: NewCall);
170
171 bool RemoveOldIntr = &OldIntr != &InstToReplace;
172
173 auto *RetValue = IC.eraseInstFromFunction(I&: InstToReplace);
174 if (RemoveOldIntr)
175 IC.eraseInstFromFunction(I&: OldIntr);
176
177 return RetValue;
178}
179
180static std::optional<Instruction *>
181simplifyAMDGCNImageIntrinsic(const GCNSubtarget *ST,
182 const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr,
183 IntrinsicInst &II, InstCombiner &IC) {
184 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
185 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: ImageDimIntr->BaseOpcode);
186
187 // Optimize _L to _LZ when _L is zero
188 if (const auto *LZMappingInfo =
189 AMDGPU::getMIMGLZMappingInfo(L: ImageDimIntr->BaseOpcode)) {
190 if (auto *ConstantLod =
191 dyn_cast<ConstantFP>(Val: II.getOperand(i_nocapture: ImageDimIntr->LodIndex))) {
192 if (ConstantLod->isZero() || ConstantLod->isNegative()) {
193 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
194 AMDGPU::getImageDimIntrinsicByBaseOpcode(BaseOpcode: LZMappingInfo->LZ,
195 Dim: ImageDimIntr->Dim);
196 return modifyIntrinsicCall(
197 OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC, Func: [&](auto &Args, auto &ArgTys) {
198 Args.erase(Args.begin() + ImageDimIntr->LodIndex);
199 });
200 }
201 }
202 }
203
204 // Optimize _mip away, when 'lod' is zero
205 if (const auto *MIPMappingInfo =
206 AMDGPU::getMIMGMIPMappingInfo(MIP: ImageDimIntr->BaseOpcode)) {
207 if (auto *ConstantMip =
208 dyn_cast<ConstantInt>(Val: II.getOperand(i_nocapture: ImageDimIntr->MipIndex))) {
209 if (ConstantMip->isZero()) {
210 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
211 AMDGPU::getImageDimIntrinsicByBaseOpcode(BaseOpcode: MIPMappingInfo->NONMIP,
212 Dim: ImageDimIntr->Dim);
213 return modifyIntrinsicCall(
214 OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC, Func: [&](auto &Args, auto &ArgTys) {
215 Args.erase(Args.begin() + ImageDimIntr->MipIndex);
216 });
217 }
218 }
219 }
220
221 // Optimize _bias away when 'bias' is zero
222 if (const auto *BiasMappingInfo =
223 AMDGPU::getMIMGBiasMappingInfo(Bias: ImageDimIntr->BaseOpcode)) {
224 if (auto *ConstantBias =
225 dyn_cast<ConstantFP>(Val: II.getOperand(i_nocapture: ImageDimIntr->BiasIndex))) {
226 if (ConstantBias->isZero()) {
227 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
228 AMDGPU::getImageDimIntrinsicByBaseOpcode(BaseOpcode: BiasMappingInfo->NoBias,
229 Dim: ImageDimIntr->Dim);
230 return modifyIntrinsicCall(
231 OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC, Func: [&](auto &Args, auto &ArgTys) {
232 Args.erase(Args.begin() + ImageDimIntr->BiasIndex);
233 ArgTys.erase(ArgTys.begin() + ImageDimIntr->BiasTyArg);
234 });
235 }
236 }
237 }
238
239 // Optimize _offset away when 'offset' is zero
240 if (const auto *OffsetMappingInfo =
241 AMDGPU::getMIMGOffsetMappingInfo(Offset: ImageDimIntr->BaseOpcode)) {
242 if (auto *ConstantOffset =
243 dyn_cast<ConstantInt>(Val: II.getOperand(i_nocapture: ImageDimIntr->OffsetIndex))) {
244 if (ConstantOffset->isZero()) {
245 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
246 AMDGPU::getImageDimIntrinsicByBaseOpcode(
247 BaseOpcode: OffsetMappingInfo->NoOffset, Dim: ImageDimIntr->Dim);
248 return modifyIntrinsicCall(
249 OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC, Func: [&](auto &Args, auto &ArgTys) {
250 Args.erase(Args.begin() + ImageDimIntr->OffsetIndex);
251 });
252 }
253 }
254 }
255
256 // Optimize the arrayed dim away when the array slice is zero, since slice 0
257 // is the base layer. Restricted to non-atomic, non-sampled image loads and
258 // stores for now.
259 const AMDGPU::MIMGDimInfo *DimInfo =
260 AMDGPU::getMIMGDimInfo(DimEnum: ImageDimIntr->Dim);
261 if (!BaseOpcode->Atomic && !BaseOpcode->Sampler && BaseOpcode->Coordinates &&
262 DimInfo->NonArrayDim != ImageDimIntr->Dim) {
263 // Address is [coords..., slice, (fragid)] plus an optional mip operand.
264 // The slice is the last coordinate, so index it from CoordStart.
265 unsigned SliceIndex = ImageDimIntr->CoordStart + DimInfo->NumCoords - 1 -
266 (DimInfo->MSAA ? 1 : 0);
267 auto *ConstantSlice = dyn_cast<ConstantInt>(Val: II.getOperand(i_nocapture: SliceIndex));
268 if (ConstantSlice && ConstantSlice->isZero()) {
269 if (const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
270 AMDGPU::getImageDimIntrinsicByBaseOpcode(BaseOpcode: ImageDimIntr->BaseOpcode,
271 Dim: DimInfo->NonArrayDim)) {
272 return modifyIntrinsicCall(OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC,
273 Func: [&](auto &Args, auto &ArgTys) {
274 Args.erase(Args.begin() + SliceIndex);
275 });
276 }
277 }
278 }
279
280 // Try to use D16
281 if (ST->hasD16Images()) {
282 if (BaseOpcode->HasD16) {
283
284 // If the only use of image intrinsic is a fptrunc (with conversion to
285 // half) then both fptrunc and image intrinsic will be replaced with image
286 // intrinsic with D16 flag.
287 if (II.hasOneUse()) {
288 Instruction *User = II.user_back();
289
290 if (User->getOpcode() == Instruction::FPTrunc &&
291 User->getType()->getScalarType()->isHalfTy()) {
292
293 return modifyIntrinsicCall(OldIntr&: II, InstToReplace&: *User, NewIntr: ImageDimIntr->Intr, IC,
294 Func: [&](auto &Args, auto &ArgTys) {
295 // Change return type of image intrinsic.
296 // Set it to return type of fptrunc.
297 ArgTys[0] = User->getType();
298 });
299 }
300 }
301
302 // Only perform D16 folding if every user of the image sample is
303 // an ExtractElementInst immediately followed by an FPTrunc to half.
304 SmallVector<std::pair<ExtractElementInst *, FPTruncInst *>, 4>
305 ExtractTruncPairs;
306 bool AllHalfExtracts = true;
307
308 for (User *U : II.users()) {
309 auto *Ext = dyn_cast<ExtractElementInst>(Val: U);
310 if (!Ext || !Ext->hasOneUse()) {
311 AllHalfExtracts = false;
312 break;
313 }
314
315 auto *Tr = dyn_cast<FPTruncInst>(Val: *Ext->user_begin());
316 if (!Tr || !Tr->getType()->isHalfTy()) {
317 AllHalfExtracts = false;
318 break;
319 }
320
321 ExtractTruncPairs.emplace_back(Args&: Ext, Args&: Tr);
322 }
323
324 if (!ExtractTruncPairs.empty() && AllHalfExtracts) {
325 auto *VecTy = cast<VectorType>(Val: II.getType());
326 Type *HalfVecTy =
327 VecTy->getWithNewType(EltTy: Type::getHalfTy(C&: II.getContext()));
328
329 // Obtain the original image sample intrinsic's signature
330 // and replace its return type with the half-vector for D16 folding
331 SmallVector<Type *, 8> OverloadTys;
332 if (!Intrinsic::isSignatureValid(F: II.getCalledFunction(), OverloadTys))
333 return std::nullopt;
334
335 OverloadTys[0] = HalfVecTy;
336 Module *M = II.getModule();
337 Function *HalfDecl = Intrinsic::getOrInsertDeclaration(
338 M, id: ImageDimIntr->Intr, OverloadTys);
339
340 II.mutateType(Ty: HalfVecTy);
341 II.setCalledFunction(HalfDecl);
342
343 IRBuilder<> Builder(*M);
344 for (auto &[Ext, Tr] : ExtractTruncPairs) {
345 Value *Idx = Ext->getIndexOperand();
346
347 Builder.SetInsertPoint(Tr);
348
349 Value *HalfExtract = Builder.CreateExtractElement(Vec: &II, Idx);
350 HalfExtract->takeName(V: Tr);
351
352 Tr->replaceAllUsesWith(V: HalfExtract);
353 }
354
355 for (auto &[Ext, Tr] : ExtractTruncPairs) {
356 IC.eraseInstFromFunction(I&: *Tr);
357 IC.eraseInstFromFunction(I&: *Ext);
358 }
359
360 return &II;
361 }
362 }
363 }
364
365 // Try to use A16 or G16
366 if (!ST->hasA16() && !ST->hasG16())
367 return std::nullopt;
368
369 // Address is interpreted as float if the instruction has a sampler or as
370 // unsigned int if there is no sampler.
371 bool HasSampler = BaseOpcode->Sampler;
372 bool FloatCoord = false;
373 // true means derivatives can be converted to 16 bit, coordinates not
374 bool OnlyDerivatives = false;
375
376 // Sampler-less addresses are unsigned, so a sext from i16 folds to a16 like a
377 // zext: they only disagree for a negative i16 (>= 0x8000), which is out of
378 // bounds while the max image dimension is <= 0x8000.
379 bool AllowI16SExt = !HasSampler;
380
381 for (unsigned OperandIndex = ImageDimIntr->GradientStart;
382 OperandIndex < ImageDimIntr->VAddrEnd; OperandIndex++) {
383 Value *Coord = II.getOperand(i_nocapture: OperandIndex);
384 // If the values are not derived from 16-bit values, we cannot optimize.
385 if (!canSafelyConvertTo16Bit(V&: *Coord, IsFloat: HasSampler, AllowI16SExt)) {
386 if (OperandIndex < ImageDimIntr->CoordStart ||
387 ImageDimIntr->GradientStart == ImageDimIntr->CoordStart) {
388 return std::nullopt;
389 }
390 // All gradients can be converted, so convert only them
391 OnlyDerivatives = true;
392 break;
393 }
394
395 assert(OperandIndex == ImageDimIntr->GradientStart ||
396 FloatCoord == Coord->getType()->isFloatingPointTy());
397 FloatCoord = Coord->getType()->isFloatingPointTy();
398 }
399
400 if (!OnlyDerivatives && !ST->hasA16())
401 OnlyDerivatives = true; // Only supports G16
402
403 // Check if there is a bias parameter and if it can be converted to f16
404 if (!OnlyDerivatives && ImageDimIntr->NumBiasArgs != 0) {
405 Value *Bias = II.getOperand(i_nocapture: ImageDimIntr->BiasIndex);
406 assert(HasSampler &&
407 "Only image instructions with a sampler can have a bias");
408 if (!canSafelyConvertTo16Bit(V&: *Bias, IsFloat: HasSampler))
409 OnlyDerivatives = true;
410 }
411
412 if (OnlyDerivatives && (!ST->hasG16() || ImageDimIntr->GradientStart ==
413 ImageDimIntr->CoordStart))
414 return std::nullopt;
415
416 Type *CoordType = FloatCoord ? Type::getHalfTy(C&: II.getContext())
417 : Type::getInt16Ty(C&: II.getContext());
418
419 return modifyIntrinsicCall(
420 OldIntr&: II, InstToReplace&: II, NewIntr: II.getIntrinsicID(), IC, Func: [&](auto &Args, auto &ArgTys) {
421 ArgTys[ImageDimIntr->GradientTyArg] = CoordType;
422 if (!OnlyDerivatives) {
423 ArgTys[ImageDimIntr->CoordTyArg] = CoordType;
424
425 // Change the bias type
426 if (ImageDimIntr->NumBiasArgs != 0)
427 ArgTys[ImageDimIntr->BiasTyArg] = Type::getHalfTy(C&: II.getContext());
428 }
429
430 unsigned EndIndex =
431 OnlyDerivatives ? ImageDimIntr->CoordStart : ImageDimIntr->VAddrEnd;
432 for (unsigned OperandIndex = ImageDimIntr->GradientStart;
433 OperandIndex < EndIndex; OperandIndex++) {
434 Args[OperandIndex] =
435 convertTo16Bit(V&: *II.getOperand(i_nocapture: OperandIndex), Builder&: IC.Builder);
436 }
437
438 // Convert the bias
439 if (!OnlyDerivatives && ImageDimIntr->NumBiasArgs != 0) {
440 Value *Bias = II.getOperand(i_nocapture: ImageDimIntr->BiasIndex);
441 Args[ImageDimIntr->BiasIndex] = convertTo16Bit(V&: *Bias, Builder&: IC.Builder);
442 }
443 });
444}
445
446bool GCNTTIImpl::canSimplifyLegacyMulToMul(const Instruction &I,
447 const Value *Op0, const Value *Op1,
448 InstCombiner &IC) const {
449 // The legacy behaviour is that multiplying +/-0.0 by anything, even NaN or
450 // infinity, gives +0.0. If we can prove we don't have one of the special
451 // cases then we can use a normal multiply instead.
452 SimplifyQuery SQ = IC.getSimplifyQuery().getWithInstruction(I: &I);
453 KnownFPClass Known0 =
454 computeKnownFPClass(V: Op0, InterestedClasses: fcZero | fcSubnormal | fcInf | fcNan, SQ);
455 DenormalMode Mode = I.getFunction()->getDenormalMode(FPType: APFloat::IEEEsingle());
456
457 // Bail early if Op0 may be zero and nsz is not set -- Op1 cannot help.
458 if (!Known0.isKnownNeverLogicalZero(Mode) && !I.hasNoSignedZeros())
459 return false;
460
461 KnownFPClass Known1 =
462 computeKnownFPClass(V: Op1, InterestedClasses: fcZero | fcSubnormal | fcInf | fcNan, SQ);
463
464 // Simplify if both operands are known non-zero.
465 if (Known0.isKnownNeverLogicalZero(Mode) &&
466 Known1.isKnownNeverLogicalZero(Mode))
467 return true;
468
469 // With nsz, two additional cases allow simplification:
470 // 1. One operand is not zero or infinity or NaN:
471 // Op0 NeverLogicalZero && NeverInfOrNaN, or symmetric for Op1.
472 // 2. Neither operand is infinity or NaN:
473 // Op0 NeverInfOrNaN && Op1 NeverInfOrNaN.
474 // The following condition captures both cases.
475 if (I.hasNoSignedZeros() &&
476 (Known0.isKnownNeverLogicalZero(Mode) || Known1.isKnownNeverInfOrNaN()) &&
477 (Known1.isKnownNeverLogicalZero(Mode) || Known0.isKnownNeverInfOrNaN()))
478 return true;
479
480 return false;
481}
482
483/// Match an fpext from half to float, or a constant we can convert.
484static Value *matchFPExtFromF16(Value *Arg) {
485 Value *Src = nullptr;
486 ConstantFP *CFP = nullptr;
487 if (match(V: Arg, P: m_OneUse(SubPattern: m_FPExt(Op: m_Value(V&: Src))))) {
488 if (Src->getType()->isHalfTy())
489 return Src;
490 } else if (match(V: Arg, P: m_ConstantFP(C&: CFP))) {
491 bool LosesInfo;
492 APFloat Val(CFP->getValueAPF());
493 Val.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmNearestTiesToEven, losesInfo: &LosesInfo);
494 if (!LosesInfo)
495 return ConstantFP::get(Ty: Type::getHalfTy(C&: Arg->getContext()), V: Val);
496 }
497 return nullptr;
498}
499
500// Trim all zero components from the end of the vector \p UseV and return
501// an appropriate bitset with known elements.
502static APInt trimTrailingZerosInVector(InstCombiner &IC, Value *UseV,
503 Instruction *I) {
504 auto *VTy = cast<FixedVectorType>(Val: UseV->getType());
505 unsigned VWidth = VTy->getNumElements();
506 APInt DemandedElts = APInt::getAllOnes(numBits: VWidth);
507
508 for (int i = VWidth - 1; i > 0; --i) {
509 auto *Elt = findScalarElement(V: UseV, EltNo: i);
510 if (!Elt)
511 break;
512
513 if (auto *ConstElt = dyn_cast<Constant>(Val: Elt)) {
514 if (!ConstElt->isNullValue() && !isa<UndefValue>(Val: Elt))
515 break;
516 } else {
517 break;
518 }
519
520 DemandedElts.clearBit(BitPosition: i);
521 }
522
523 return DemandedElts;
524}
525
526// Trim elements of the end of the vector \p V, if they are
527// equal to the first element of the vector.
528static APInt defaultComponentBroadcast(Value *V) {
529 auto *VTy = cast<FixedVectorType>(Val: V->getType());
530 unsigned VWidth = VTy->getNumElements();
531 APInt DemandedElts = APInt::getAllOnes(numBits: VWidth);
532 Value *FirstComponent = findScalarElement(V, EltNo: 0);
533
534 SmallVector<int> ShuffleMask;
535 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: V))
536 SVI->getShuffleMask(Result&: ShuffleMask);
537
538 for (int I = VWidth - 1; I > 0; --I) {
539 if (ShuffleMask.empty()) {
540 auto *Elt = findScalarElement(V, EltNo: I);
541 if (!Elt || (Elt != FirstComponent && !isa<UndefValue>(Val: Elt)))
542 break;
543 } else {
544 // Detect identical elements in the shufflevector result, even though
545 // findScalarElement cannot tell us what that element is.
546 if (ShuffleMask[I] != ShuffleMask[0] && ShuffleMask[I] != PoisonMaskElem)
547 break;
548 }
549 DemandedElts.clearBit(BitPosition: I);
550 }
551
552 return DemandedElts;
553}
554
555static Value *simplifyAMDGCNMemoryIntrinsicDemanded(InstCombiner &IC,
556 IntrinsicInst &II,
557 APInt DemandedElts,
558 int DMaskIdx = -1,
559 bool IsLoad = true);
560
561/// Return true if it's legal to contract llvm.amdgcn.rcp(llvm.sqrt)
562static bool canContractSqrtToRsq(const FPMathOperator *SqrtOp) {
563 return (SqrtOp->getType()->isFloatTy() &&
564 (SqrtOp->hasApproxFunc() || SqrtOp->getFPAccuracy() >= 1.0f)) ||
565 SqrtOp->getType()->isHalfTy();
566}
567
568/// Return true if we can easily prove that use U is uniform.
569static bool isTriviallyUniform(const Use &U) {
570 Value *V = U.get();
571 if (isa<Constant>(Val: V))
572 return true;
573 if (const auto *A = dyn_cast<Argument>(Val: V))
574 return AMDGPU::isArgPassedInSGPR(Arg: A);
575 if (const auto *II = dyn_cast<IntrinsicInst>(Val: V)) {
576 if (!AMDGPU::isIntrinsicAlwaysUniform(IntrID: II->getIntrinsicID()))
577 return false;
578 // If II and U are in different blocks then there is a possibility of
579 // temporal divergence.
580 return II->getParent() == cast<Instruction>(Val: U.getUser())->getParent();
581 }
582 return false;
583}
584
585/// Simplify a lane index operand (e.g. llvm.amdgcn.readlane src1).
586///
587/// The instruction only reads the low 5 bits for wave32, and 6 bits for wave64.
588bool GCNTTIImpl::simplifyDemandedLaneMaskArg(InstCombiner &IC,
589 IntrinsicInst &II,
590 unsigned LaneArgIdx) const {
591 unsigned MaskBits = ST->getWavefrontSizeLog2();
592 APInt DemandedMask(32, maskTrailingOnes<unsigned>(N: MaskBits));
593
594 KnownBits Known(32);
595 if (IC.SimplifyDemandedBits(I: &II, OpNo: LaneArgIdx, DemandedMask, Known))
596 return true;
597
598 if (!Known.isConstant())
599 return false;
600
601 // Out of bounds indexes may appear in wave64 code compiled for wave32.
602 // Unlike the DAG version, SimplifyDemandedBits does not change constants, so
603 // manually fix it up.
604
605 Value *LaneArg = II.getArgOperand(i: LaneArgIdx);
606 Constant *MaskedConst =
607 ConstantInt::get(Ty: LaneArg->getType(), V: Known.getConstant() & DemandedMask);
608 if (MaskedConst != LaneArg) {
609 II.getOperandUse(i: LaneArgIdx).set(MaskedConst);
610 return true;
611 }
612
613 return false;
614}
615
616static CallInst *rewriteCall(IRBuilderBase &B, CallInst &Old,
617 Function &NewCallee, ArrayRef<Value *> Ops) {
618 SmallVector<OperandBundleDef, 2> OpBundles;
619 Old.getOperandBundlesAsDefs(Defs&: OpBundles);
620
621 CallInst *NewCall = B.CreateCall(Callee: &NewCallee, Args: Ops, OpBundles);
622 NewCall->takeName(V: &Old);
623 return NewCall;
624}
625
626// Return true for sequences of instructions that effectively assign
627// each lane to its thread ID
628static bool isThreadID(const GCNSubtarget &ST, Value *V) {
629 // Case 1:
630 // wave32: mbcnt_lo(-1, 0)
631 // wave64: mbcnt_hi(-1, mbcnt_lo(-1, 0))
632 auto W32Pred = m_Intrinsic<Intrinsic::amdgcn_mbcnt_lo>(Ops: m_ConstantInt<-1>(),
633 Ops: m_ConstantInt<0>());
634 auto W64Pred = m_Intrinsic<Intrinsic::amdgcn_mbcnt_hi>(
635 Ops: m_ConstantInt<-1>(), Ops: m_Intrinsic<Intrinsic::amdgcn_mbcnt_lo>(
636 Ops: m_ConstantInt<-1>(), Ops: m_ConstantInt<0>()));
637 if (ST.isWave32() && match(V, P: W32Pred))
638 return true;
639 if (ST.isWave64() && match(V, P: W64Pred))
640 return true;
641
642 return false;
643}
644
645Instruction *
646GCNTTIImpl::hoistLaneIntrinsicThroughOperand(InstCombiner &IC,
647 IntrinsicInst &II) const {
648 const auto IID = II.getIntrinsicID();
649 assert(IID == Intrinsic::amdgcn_readlane ||
650 IID == Intrinsic::amdgcn_readfirstlane ||
651 IID == Intrinsic::amdgcn_permlane64);
652
653 Instruction *OpInst = dyn_cast<Instruction>(Val: II.getOperand(i_nocapture: 0));
654
655 // Only do this if both instructions are in the same block
656 // (so the exec mask won't change) and the readlane is the only user of its
657 // operand.
658 if (!OpInst || !OpInst->hasOneUser() || OpInst->getParent() != II.getParent())
659 return nullptr;
660
661 const bool IsReadLane = (IID == Intrinsic::amdgcn_readlane);
662
663 // If this is a readlane, check that the second operand is a constant, or is
664 // defined before OpInst so we know it's safe to move this intrinsic higher.
665 Value *LaneID = nullptr;
666 if (IsReadLane) {
667 LaneID = II.getOperand(i_nocapture: 1);
668
669 // readlane take an extra operand for the lane ID, so we must check if that
670 // LaneID value can be used at the point where we want to move the
671 // intrinsic.
672 if (auto *LaneIDInst = dyn_cast<Instruction>(Val: LaneID)) {
673 if (!IC.getDominatorTree().dominates(Def: LaneIDInst, User: OpInst))
674 return nullptr;
675 }
676 }
677
678 // Hoist the intrinsic (II) through OpInst.
679 //
680 // (II (OpInst x)) -> (OpInst (II x))
681 const auto DoIt = [&](unsigned OpIdx,
682 Function *NewIntrinsic) -> Instruction * {
683 SmallVector<Value *, 2> Ops{OpInst->getOperand(i: OpIdx)};
684 if (IsReadLane)
685 Ops.push_back(Elt: LaneID);
686
687 // Rewrite the intrinsic call.
688 CallInst *NewII = rewriteCall(B&: IC.Builder, Old&: II, NewCallee&: *NewIntrinsic, Ops);
689
690 // Rewrite OpInst so it takes the result of the intrinsic now.
691 Instruction &NewOp = *OpInst->clone();
692 NewOp.setOperand(i: OpIdx, Val: NewII);
693 return &NewOp;
694 };
695
696 // TODO(?): Should we do more with permlane64?
697 if (IID == Intrinsic::amdgcn_permlane64 && !isa<BitCastInst>(Val: OpInst))
698 return nullptr;
699
700 if (isa<UnaryOperator>(Val: OpInst))
701 return DoIt(0, II.getCalledFunction());
702
703 if (isa<CastInst>(Val: OpInst)) {
704 Value *Src = OpInst->getOperand(i: 0);
705 Type *SrcTy = Src->getType();
706 if (!isTypeLegal(Ty: SrcTy))
707 return nullptr;
708
709 Function *Remangled =
710 Intrinsic::getOrInsertDeclaration(M: II.getModule(), id: IID, OverloadTys: {SrcTy});
711 return DoIt(0, Remangled);
712 }
713
714 // We can also hoist through binary operators if the other operand is uniform.
715 if (isa<BinaryOperator>(Val: OpInst)) {
716 // FIXME: If we had access to UniformityInfo here we could just check
717 // if the operand is uniform.
718 if (isTriviallyUniform(U: OpInst->getOperandUse(i: 0)))
719 return DoIt(1, II.getCalledFunction());
720 if (isTriviallyUniform(U: OpInst->getOperandUse(i: 1)))
721 return DoIt(0, II.getCalledFunction());
722 }
723
724 return nullptr;
725}
726
727/// Evaluate V as a function of the lane ID and return its value on Lane, or
728/// std::nullopt if V is not a closed-form expression of the lane ID.
729static std::optional<unsigned> evalLaneExpr(Value *V, unsigned Lane,
730 const GCNSubtarget &ST,
731 const DataLayout &DL,
732 unsigned Depth = 0) {
733 if (Depth >= MaxAnalysisRecursionDepth)
734 return std::nullopt;
735
736 // Poison/undef in the index expression: bail and let InstCombine fold the
737 // intrinsic the usual way.
738 if (isa<UndefValue>(Val: V))
739 return std::nullopt;
740
741 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: V))
742 return CI->getZExtValue();
743
744 if (isThreadID(ST, V))
745 return Lane;
746
747 const BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: V);
748 if (!BO)
749 return std::nullopt;
750
751 std::optional<unsigned> LHS =
752 evalLaneExpr(V: BO->getOperand(i_nocapture: 0), Lane, ST, DL, Depth: Depth + 1);
753 if (!LHS)
754 return std::nullopt;
755 std::optional<unsigned> RHS =
756 evalLaneExpr(V: BO->getOperand(i_nocapture: 1), Lane, ST, DL, Depth: Depth + 1);
757 if (!RHS)
758 return std::nullopt;
759
760 Type *Ty = BO->getType();
761 Constant *Ops[] = {ConstantInt::get(Ty, V: *LHS), ConstantInt::get(Ty, V: *RHS)};
762 auto *CI =
763 dyn_cast_or_null<ConstantInt>(Val: ConstantFoldInstOperands(I: BO, Ops, DL));
764 return CI ? std::optional<unsigned>(CI->getZExtValue()) : std::nullopt;
765}
766
767/// Build the per-lane shuffle map by evaluating Index for every lane in the
768/// wave. Returns false if any lane index is non-constant or out of range.
769static bool tryBuildShuffleMap(Value *Index, const GCNSubtarget &ST,
770 SmallVectorImpl<uint8_t> &Ids,
771 const DataLayout &DL) {
772 unsigned WaveSize = ST.getWavefrontSize();
773 Ids.resize(N: WaveSize);
774 for (unsigned Lane : seq(Size: WaveSize)) {
775 std::optional<unsigned> Val = evalLaneExpr(V: Index, Lane, ST, DL);
776 if (!Val || *Val >= WaveSize)
777 return false;
778 Ids[Lane] = *Val;
779 }
780 return true;
781}
782
783/// Lanes are partitioned into groups of Period; each group is a translated
784/// copy of the first: Ids[I] = Ids[I % Period] + (I & ~(Period - 1)).
785template <unsigned Period>
786static bool hasPeriodicLayout(ArrayRef<uint8_t> Ids) {
787 static_assert(isPowerOf2_32(Value: Period), "Period must be a power of two");
788 for (unsigned I = Period, E = Ids.size(); I < E; ++I)
789 if (Ids[I] != Ids[I % Period] + (I & ~(Period - 1)))
790 return false;
791 return true;
792}
793
794/// Match an N-lane row pattern: each lane in [0, N) reads from a source lane
795/// in the same N-lane row, and the pattern repeats periodically across rows.
796template <unsigned N> static bool isRowPattern(ArrayRef<uint8_t> Ids) {
797 for (unsigned I = 0; I < N; ++I)
798 if (Ids[I] >= N)
799 return false;
800 return hasPeriodicLayout<N>(Ids);
801}
802
803static constexpr auto isQuadPattern = isRowPattern<4>;
804static constexpr auto isHalfRowPattern = isRowPattern<8>;
805static constexpr auto isFullRowPattern = isRowPattern<16>;
806
807/// Match a 4-lane (quad) permutation, encoded as the v_mov_b32_dpp
808/// QUAD_PERM control word: bits[1:0]=Ids[0], [3:2]=Ids[1], [5:4]=Ids[2],
809/// [7:6]=Ids[3].
810static std::optional<unsigned> matchQuadPermPattern(ArrayRef<uint8_t> Ids) {
811 if (!isQuadPattern(Ids))
812 return std::nullopt;
813 return Ids[3] << 6 | Ids[2] << 4 | Ids[1] << 2 | Ids[0];
814}
815
816/// Match an N-lane reversal (mirror) pattern.
817template <unsigned N> static bool matchMirrorPattern(ArrayRef<uint8_t> Ids) {
818 if (!isRowPattern<N>(Ids))
819 return false;
820 for (unsigned J = 0; J < N; ++J)
821 if (Ids[J] != (N - 1) - J)
822 return false;
823 return true;
824}
825
826static constexpr auto matchHalfRowMirrorPattern = matchMirrorPattern<8>;
827static constexpr auto matchFullRowMirrorPattern = matchMirrorPattern<16>;
828
829/// Match a 16-lane cyclic rotation; returns the rotation amount in [1, 15].
830static std::optional<unsigned> matchRowRotatePattern(ArrayRef<uint8_t> Ids) {
831 if (Ids[0] == 0 || !isFullRowPattern(Ids))
832 return std::nullopt;
833 for (unsigned J = 1; J < 16; ++J)
834 if (Ids[J] != (Ids[0] + J) % 16)
835 return std::nullopt;
836 return 16u - Ids[0];
837}
838
839/// Match a row-share pattern: all 16 lanes of each row read the same source
840/// lane. Returns the shared source lane index in [0, 16).
841static std::optional<unsigned> matchRowSharePattern(ArrayRef<uint8_t> Ids) {
842 if (!isFullRowPattern(Ids))
843 return std::nullopt;
844 if (!all_equal(Range: Ids.take_front(N: 16)))
845 return std::nullopt;
846 return Ids[0];
847}
848
849/// Match an XOR mask pattern within each 16-lane row: Ids[J] == Mask ^ J,
850/// with Mask in [1, 15].
851static std::optional<unsigned> matchRowXMaskPattern(ArrayRef<uint8_t> Ids) {
852 unsigned Mask = Ids[0];
853 if (Mask == 0 || !isFullRowPattern(Ids))
854 return std::nullopt;
855 for (unsigned J = 0; J < 16; ++J)
856 if (Ids[J] != (Mask ^ J))
857 return std::nullopt;
858 return Mask;
859}
860
861/// Match an 8-lane arbitrary permutation, encoded as the v_mov_b32_dpp8
862/// 24-bit selector (three bits per output lane).
863static std::optional<unsigned> matchHalfRowPermPattern(ArrayRef<uint8_t> Ids) {
864 if (!isHalfRowPattern(Ids))
865 return std::nullopt;
866 unsigned Selector = 0;
867 for (unsigned J = 0; J < 8; ++J)
868 Selector |= Ids[J] << (J * 3);
869 return Selector;
870}
871
872/// Pack a 16-lane permutation into a single 64-bit value: four bits per output
873/// lane, lane J in bits [J*4 + 3 : J*4]. The caller splits it into the low and
874/// high 32-bit selector operands of v_permlane16 / v_permlanex16.
875static uint64_t computePermlane16Masks(ArrayRef<uint8_t> Ids) {
876 uint64_t Sel = 0;
877 for (unsigned J = 0; J < 16; ++J)
878 Sel |= static_cast<uint64_t>(Ids[J] & 0xF) << (J * 4);
879 return Sel;
880}
881
882/// Match a half-wave swap: lane J reads from lane J ^ 32. Only meaningful on
883/// wave64 targets.
884static bool matchHalfWaveSwapPattern(ArrayRef<uint8_t> Ids) {
885 if (Ids.size() != 64)
886 return false;
887 for (unsigned J = 0; J < 64; ++J)
888 if (Ids[J] != (J ^ 32))
889 return false;
890 return true;
891}
892
893/// Match a cross-row permutation suitable for v_permlanex16: every lane in
894/// the low 16-lane half reads from the high half of its own row, and vice
895/// versa.
896static bool isCrossRowPattern(ArrayRef<uint8_t> Ids) {
897 if (!hasPeriodicLayout<32>(Ids))
898 return false;
899 for (unsigned J = 0; J < 16; ++J) {
900 if (Ids[J] < 16 || Ids[J] >= 32)
901 return false;
902 if (Ids[J + 16] != Ids[J] - 16)
903 return false;
904 }
905 return true;
906}
907
908/// Match a DS_SWIZZLE bitmask-mode permutation:
909/// dst_lane = ((src_lane & AND) | OR) ^ XOR
910/// with each mask being five bits. Returns the encoded swizzle immediate.
911/// The hardware applies the formula independently within each 32-lane group,
912/// so on wave64 the high group must replicate the low one (translated by 32).
913static std::optional<unsigned>
914matchDsSwizzleBitmaskPattern(ArrayRef<uint8_t> Ids) {
915 if (!hasPeriodicLayout<32>(Ids))
916 return std::nullopt;
917
918 // The formula is per-bit: output bit B depends only on input bit B. Probe
919 // each bit with src=0 and src=(1<<B); if the output bit flipped, AND[B]=1
920 // and XOR[B] carries the constant offset; otherwise it is a constant bit
921 // encoded in OR (with AND[B]=0, XOR[B]=0).
922 unsigned AndMask = 0, OrMask = 0, XorMask = 0;
923 for (unsigned B = 0; B < 5; ++B) {
924 unsigned Bit0 = (Ids[0] >> B) & 1;
925 unsigned Bit1 = (Ids[1u << B] >> B) & 1;
926 if (Bit0 != Bit1) {
927 AndMask |= 1u << B;
928 XorMask |= Bit0 << B;
929 } else {
930 OrMask |= Bit0 << B;
931 }
932 }
933
934 // The per-bit derivation assumes bit independence; verify the masks
935 // actually reproduce every lane in the 32-lane group.
936 for (unsigned I : seq(Size: 32u)) {
937 unsigned Expected = ((I & AndMask) | OrMask) ^ XorMask;
938 if (Ids[I] != Expected)
939 return std::nullopt;
940 }
941
942 return AMDGPU::Swizzle::BITMASK_PERM_ENC |
943 AndMask << AMDGPU::Swizzle::BITMASK_AND_SHIFT |
944 OrMask << AMDGPU::Swizzle::BITMASK_OR_SHIFT |
945 XorMask << AMDGPU::Swizzle::BITMASK_XOR_SHIFT;
946}
947
948/// Match a GFX9+ DS_SWIZZLE rotate-mode permutation: a cyclic left-rotation
949/// of all 32 lanes within each 32-lane group by a constant N in [0, 31],
950/// i.e. dst_lane = (src_lane + N) % 32. On wave64, hasPeriodicLayout<32>
951/// ensures both 32-lane groups rotate by the same amount.
952static std::optional<unsigned>
953matchDsSwizzleRotatePattern(ArrayRef<uint8_t> Ids) {
954 if (!hasPeriodicLayout<32>(Ids))
955 return std::nullopt;
956
957 // Determine the rotation amount from lane 0: every lane must read from
958 // lane (I + N) % 32 where N = Ids[0] and 0 <= N <= 31.
959 unsigned N = Ids[0];
960 if (N >= 32)
961 return std::nullopt;
962
963 for (unsigned I = 0; I < 32; ++I)
964 if (Ids[I] != (I + N) % 32)
965 return std::nullopt;
966
967 return AMDGPU::Swizzle::ROTATE_MODE_ENC |
968 (N << AMDGPU::Swizzle::ROTATE_SIZE_SHIFT);
969}
970
971/// Emit v_mov_b32_dpp with the given control word, row/bank masks 0xF, and
972/// bound_ctrl=1 so out-of-bounds lanes are well-defined and the DPP mov can
973/// be folded into a consuming VALU op by GCNDPPCombine.
974static Value *createUpdateDpp(IRBuilderBase &B, Value *Val, unsigned Ctrl) {
975 Type *Ty = Val->getType();
976 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_update_dpp, OverloadTypes: {Ty},
977 Args: {PoisonValue::get(T: Ty), Val, B.getInt32(C: Ctrl),
978 B.getInt32(C: 0xF), B.getInt32(C: 0xF), B.getTrue()});
979}
980
981/// Emit v_mov_b32_dpp8 with the given 24-bit lane selector.
982static Value *createMovDpp8(IRBuilderBase &B, Value *Val, unsigned Selector) {
983 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_mov_dpp8, OverloadTypes: {Val->getType()},
984 Args: {Val, B.getInt32(C: Selector)});
985}
986
987/// Emit v_permlane16 with the precomputed lane-select halves.
988static Value *createPermlane16(IRBuilderBase &B, Value *Val, uint32_t Lo,
989 uint32_t Hi) {
990 Type *Ty = Val->getType();
991 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_permlane16, OverloadTypes: {Ty},
992 Args: {PoisonValue::get(T: Ty), Val, B.getInt32(C: Lo),
993 B.getInt32(C: Hi), B.getFalse(), B.getFalse()});
994}
995
996/// Emit v_permlanex16 with the precomputed lane-select halves. Each output
997/// lane reads from the other 16-lane half of the same row.
998static Value *createPermlaneX16(IRBuilderBase &B, Value *Val, uint32_t Lo,
999 uint32_t Hi) {
1000 Type *Ty = Val->getType();
1001 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_permlanex16, OverloadTypes: {Ty},
1002 Args: {PoisonValue::get(T: Ty), Val, B.getInt32(C: Lo),
1003 B.getInt32(C: Hi), B.getFalse(), B.getFalse()});
1004}
1005
1006/// Emit ds_swizzle with the given immediate, bitcasting/converting between
1007/// pointer/float types and i32 as required by the intrinsic signature.
1008static Value *createDsSwizzle(IRBuilderBase &B, Value *Val, unsigned Offset,
1009 const DataLayout &DL) {
1010 Type *OrigTy = Val->getType();
1011 assert(DL.getTypeSizeInBits(OrigTy) == 32 &&
1012 "ds_swizzle only supports 32-bit operands");
1013 IntegerType *I32Ty = B.getInt32Ty();
1014 Value *Src = Val;
1015 if (OrigTy->isPointerTy())
1016 Src = B.CreatePtrToInt(V: Src, DestTy: I32Ty);
1017 else if (OrigTy != I32Ty)
1018 Src = B.CreateBitCast(V: Src, DestTy: I32Ty);
1019 Value *Result = B.CreateIntrinsic(ID: Intrinsic::amdgcn_ds_swizzle, OverloadTypes: {},
1020 Args: {Src, B.getInt32(C: Offset)});
1021 if (OrigTy->isPointerTy())
1022 return B.CreateIntToPtr(V: Result, DestTy: OrigTy);
1023 if (OrigTy != I32Ty)
1024 return B.CreateBitCast(V: Result, DestTy: OrigTy);
1025 return Result;
1026}
1027
1028/// Emit v_permlane64 (swap of the two 32-lane halves of a wave64).
1029static Value *createPermlane64(IRBuilderBase &B, Value *Val) {
1030 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_permlane64, OverloadTypes: {Val->getType()},
1031 Args: {Val});
1032}
1033
1034/// Given a shuffle map, try to emit the best hardware intrinsic.
1035static Value *matchShuffleToHWIntrinsic(IRBuilderBase &B, Value *Src,
1036 ArrayRef<uint8_t> Ids,
1037 const GCNSubtarget &ST,
1038 const DataLayout &DL) {
1039 // Identity shuffle (every lane reads itself) folds to the source value.
1040 if (all_of(Range: enumerate(First&: Ids),
1041 P: [](const auto &E) { return E.value() == E.index(); }))
1042 return Src;
1043
1044 // Uniform shuffle (all lanes read the same value) is handled by cheaper
1045 // broadcast/readlane intrinsics.
1046 if (all_equal(Range&: Ids))
1047 return nullptr;
1048
1049 if (std::optional<unsigned> QP = matchQuadPermPattern(Ids)) {
1050 if (ST.hasDPP())
1051 return createUpdateDpp(B, Val: Src, Ctrl: *QP);
1052 return createDsSwizzle(B, Val: Src, Offset: AMDGPU::Swizzle::QUAD_PERM_ENC | *QP, DL);
1053 }
1054
1055 if (ST.hasDPP()) {
1056 if (matchHalfRowMirrorPattern(Ids))
1057 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_HALF_MIRROR);
1058 if (matchFullRowMirrorPattern(Ids))
1059 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_MIRROR);
1060 if (std::optional<unsigned> Amt = matchRowRotatePattern(Ids))
1061 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_ROR_FIRST + *Amt - 1);
1062 }
1063
1064 // row_share is supported on GFX90A and GFX10+; row_xmask is GFX10+ only.
1065 if (ST.hasDPPRowShare()) {
1066 if (std::optional<unsigned> Lane = matchRowSharePattern(Ids))
1067 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_SHARE_FIRST + *Lane);
1068 }
1069
1070 if (ST.hasDPP() && ST.hasGFX10Insts()) {
1071 if (std::optional<unsigned> Mask = matchRowXMaskPattern(Ids))
1072 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_XMASK_FIRST + *Mask);
1073 }
1074
1075 if (ST.hasDPP8()) {
1076 if (std::optional<unsigned> Sel = matchHalfRowPermPattern(Ids))
1077 return createMovDpp8(B, Val: Src, Selector: *Sel);
1078 }
1079
1080 if (ST.hasPermlane16Insts()) {
1081 if (isFullRowPattern(Ids)) {
1082 uint64_t Sel = computePermlane16Masks(Ids);
1083 return createPermlane16(B, Val: Src, Lo: Lo_32(Value: Sel), Hi: Hi_32(Value: Sel));
1084 }
1085 // Cross-row shuffles (e.g. XOR 16..31) — covered by permlanex16.
1086 if (isCrossRowPattern(Ids)) {
1087 uint64_t Sel = computePermlane16Masks(Ids);
1088 return createPermlaneX16(B, Val: Src, Lo: Lo_32(Value: Sel), Hi: Hi_32(Value: Sel));
1089 }
1090 }
1091
1092 // Generic DS_SWIZZLE bitmask-mode fallback: handles any 32-lane shuffle that
1093 // can be expressed as dst = ((src & AND) | OR) ^ XOR with 5-bit masks. This
1094 // is available on every target that has ds_swizzle.
1095 if (std::optional<unsigned> Imm = matchDsSwizzleBitmaskPattern(Ids))
1096 return createDsSwizzle(B, Val: Src, Offset: *Imm, DL);
1097
1098 // DS_SWIZZLE rotate mode (GFX9+): handles cyclic 32-lane rotations that
1099 // bitmask mode cannot express (e.g. +1 mod 32 requires inter-bit carry).
1100 if (ST.hasDsSwizzleRotateMode()) {
1101 if (std::optional<unsigned> Imm = matchDsSwizzleRotatePattern(Ids))
1102 return createDsSwizzle(B, Val: Src, Offset: *Imm, DL);
1103 }
1104
1105 if (ST.hasPermLane64() && matchHalfWaveSwapPattern(Ids))
1106 return createPermlane64(B, Val: Src);
1107
1108 return nullptr;
1109}
1110
1111/// Try to fold a wave_shuffle/ds_bpermute whose lane index is a constant
1112/// function of the lane ID into a hardware-specific lane permutation intrinsic.
1113static std::optional<Instruction *>
1114tryOptimizeShufflePattern(InstCombiner &IC, IntrinsicInst &II,
1115 const GCNSubtarget &ST) {
1116 const DataLayout &DL = IC.getDataLayout();
1117 if (DL.getTypeSizeInBits(Ty: II.getType()) != 32)
1118 return std::nullopt;
1119
1120 if (!ST.isWaveSizeKnown())
1121 return std::nullopt;
1122
1123 unsigned WaveSize = ST.getWavefrontSize();
1124 bool IsBpermute = II.getIntrinsicID() == Intrinsic::amdgcn_ds_bpermute;
1125 Value *Src = II.getArgOperand(i: IsBpermute ? 1 : 0);
1126 Value *Index = II.getArgOperand(i: IsBpermute ? 0 : 1);
1127
1128 SmallVector<uint8_t, 64> Ids;
1129 if (IsBpermute) {
1130 Ids.resize(N: WaveSize);
1131 for (unsigned Lane : seq(Size: WaveSize)) {
1132 std::optional<unsigned> Val = evalLaneExpr(V: Index, Lane, ST, DL);
1133 if (!Val || (*Val & 3) || (*Val >> 2) >= WaveSize)
1134 return std::nullopt;
1135 Ids[Lane] = *Val >> 2;
1136 }
1137 } else {
1138 if (!tryBuildShuffleMap(Index, ST, Ids, DL))
1139 return std::nullopt;
1140 }
1141
1142 Value *Result = matchShuffleToHWIntrinsic(B&: IC.Builder, Src, Ids, ST, DL);
1143 if (!Result)
1144 return std::nullopt;
1145
1146 return IC.replaceInstUsesWith(I&: II, V: Result);
1147}
1148
1149/// Try to fold a constant addition into the accumulator when saturation is
1150/// disabled.
1151static Instruction *foldConstantIntoDotAccumulator(IntrinsicInst &II,
1152 unsigned AccIdx,
1153 unsigned ClampIdx,
1154 InstCombiner &IC) {
1155 // Reassociating across a saturating accumulation is not valid.
1156 if (!match(V: II.getArgOperand(i: ClampIdx), P: m_Zero()) || !II.hasOneUse())
1157 return nullptr;
1158
1159 const APInt *Acc = nullptr;
1160 if (!match(V: II.getArgOperand(i: AccIdx), P: m_APInt(Res&: Acc)))
1161 return nullptr;
1162
1163 auto *AccumUser = dyn_cast<BinaryOperator>(Val: II.user_back());
1164 if (!AccumUser)
1165 return nullptr;
1166
1167 const APInt *AccumDelta = nullptr;
1168 if (!match(V: AccumUser, P: m_c_Add(L: m_Specific(V: &II), R: m_APInt(Res&: AccumDelta))))
1169 return nullptr;
1170
1171 Constant *NewAcc = ConstantInt::get(Ty: II.getType(), V: *Acc + *AccumDelta);
1172
1173 IC.replaceInstUsesWith(I&: *AccumUser, V: &II);
1174 IC.eraseInstFromFunction(I&: *AccumUser);
1175 return IC.replaceOperand(I&: II, OpNum: AccIdx, V: NewAcc);
1176}
1177
1178std::optional<Instruction *>
1179GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
1180 Intrinsic::ID IID = II.getIntrinsicID();
1181 switch (IID) {
1182 case Intrinsic::amdgcn_implicitarg_ptr: {
1183 if (II.getFunction()->hasFnAttribute(Kind: "amdgpu-no-implicitarg-ptr"))
1184 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1185 uint64_t ImplicitArgBytes = ST->getImplicitArgNumBytes(F: *II.getFunction());
1186
1187 uint64_t CurrentOrNullBytes =
1188 II.getAttributes().getRetDereferenceableOrNullBytes();
1189 if (CurrentOrNullBytes != 0) {
1190 // Refine "dereferenceable (A) meets dereferenceable_or_null(B)"
1191 // into dereferenceable(max(A, B))
1192 uint64_t NewBytes = std::max(a: CurrentOrNullBytes, b: ImplicitArgBytes);
1193 II.addRetAttr(
1194 Attr: Attribute::getWithDereferenceableBytes(Context&: II.getContext(), Bytes: NewBytes));
1195 II.removeRetAttr(Kind: Attribute::DereferenceableOrNull);
1196 return &II;
1197 }
1198
1199 uint64_t CurrentBytes = II.getAttributes().getRetDereferenceableBytes();
1200 uint64_t NewBytes = std::max(a: CurrentBytes, b: ImplicitArgBytes);
1201 if (NewBytes != CurrentBytes) {
1202 II.addRetAttr(
1203 Attr: Attribute::getWithDereferenceableBytes(Context&: II.getContext(), Bytes: NewBytes));
1204 return &II;
1205 }
1206
1207 return std::nullopt;
1208 }
1209 case Intrinsic::amdgcn_rcp: {
1210 Value *Src = II.getArgOperand(i: 0);
1211 if (isa<PoisonValue>(Val: Src))
1212 return IC.replaceInstUsesWith(I&: II, V: Src);
1213
1214 // TODO: Move to ConstantFolding/InstSimplify?
1215 if (isa<UndefValue>(Val: Src)) {
1216 Type *Ty = II.getType();
1217 auto *QNaN = ConstantFP::get(Ty, V: APFloat::getQNaN(Sem: Ty->getFltSemantics()));
1218 return IC.replaceInstUsesWith(I&: II, V: QNaN);
1219 }
1220
1221 if (II.isStrictFP())
1222 break;
1223
1224 if (const ConstantFP *C = dyn_cast<ConstantFP>(Val: Src)) {
1225 std::optional<APFloat> Val = AMDGPU::evaluateRcp(Val: C->getValueAPF());
1226 if (!Val)
1227 break;
1228
1229 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::get(Context&: II.getContext(), V: *Val));
1230 }
1231
1232 FastMathFlags FMF = cast<FPMathOperator>(Val&: II).getFastMathFlags();
1233 if (!FMF.allowContract())
1234 break;
1235 auto *SrcCI = dyn_cast<IntrinsicInst>(Val: Src);
1236 if (!SrcCI)
1237 break;
1238
1239 auto IID = SrcCI->getIntrinsicID();
1240 // llvm.amdgcn.rcp(llvm.amdgcn.sqrt(x)) -> llvm.amdgcn.rsq(x) if contractable
1241 //
1242 // llvm.amdgcn.rcp(llvm.sqrt(x)) -> llvm.amdgcn.rsq(x) if contractable and
1243 // relaxed.
1244 if (IID == Intrinsic::amdgcn_sqrt || IID == Intrinsic::sqrt) {
1245 const FPMathOperator *SqrtOp = cast<FPMathOperator>(Val: SrcCI);
1246 FastMathFlags InnerFMF = SqrtOp->getFastMathFlags();
1247 if (!InnerFMF.allowContract() || !SrcCI->hasOneUse())
1248 break;
1249
1250 if (IID == Intrinsic::sqrt && !canContractSqrtToRsq(SqrtOp))
1251 break;
1252
1253 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
1254 M: SrcCI->getModule(), id: Intrinsic::amdgcn_rsq, OverloadTys: {SrcCI->getType()});
1255
1256 InnerFMF |= FMF;
1257 II.setFastMathFlags(InnerFMF);
1258
1259 II.setCalledFunction(NewDecl);
1260 return IC.replaceOperand(I&: II, OpNum: 0, V: SrcCI->getArgOperand(i: 0));
1261 }
1262
1263 break;
1264 }
1265 case Intrinsic::amdgcn_sqrt:
1266 case Intrinsic::amdgcn_rsq:
1267 case Intrinsic::amdgcn_tanh: {
1268 Value *Src = II.getArgOperand(i: 0);
1269 if (isa<PoisonValue>(Val: Src))
1270 return IC.replaceInstUsesWith(I&: II, V: Src);
1271
1272 // TODO: Move to ConstantFolding/InstSimplify?
1273 if (isa<UndefValue>(Val: Src)) {
1274 Type *Ty = II.getType();
1275 auto *QNaN = ConstantFP::get(Ty, V: APFloat::getQNaN(Sem: Ty->getFltSemantics()));
1276 return IC.replaceInstUsesWith(I&: II, V: QNaN);
1277 }
1278
1279 // f16 amdgcn.sqrt is identical to regular sqrt.
1280 if (IID == Intrinsic::amdgcn_sqrt && Src->getType()->isHalfTy()) {
1281 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
1282 M: II.getModule(), id: Intrinsic::sqrt, OverloadTys: {II.getType()});
1283 II.setCalledFunction(NewDecl);
1284 return &II;
1285 }
1286
1287 break;
1288 }
1289 case Intrinsic::amdgcn_log:
1290 case Intrinsic::amdgcn_exp2: {
1291 const bool IsLog = IID == Intrinsic::amdgcn_log;
1292 const bool IsExp = IID == Intrinsic::amdgcn_exp2;
1293 Value *Src = II.getArgOperand(i: 0);
1294 Type *Ty = II.getType();
1295
1296 if (isa<PoisonValue>(Val: Src))
1297 return IC.replaceInstUsesWith(I&: II, V: Src);
1298
1299 if (IC.getSimplifyQuery().isUndefValue(V: Src))
1300 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getNaN(Ty));
1301
1302 if (ConstantFP *C = dyn_cast<ConstantFP>(Val: Src)) {
1303 if (C->isInfinity()) {
1304 // exp2(+inf) -> +inf
1305 // log2(+inf) -> +inf
1306 if (!C->isNegative())
1307 return IC.replaceInstUsesWith(I&: II, V: C);
1308
1309 // exp2(-inf) -> 0
1310 if (IsExp && C->isNegative())
1311 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getZero(Ty));
1312 }
1313
1314 if (II.isStrictFP())
1315 break;
1316
1317 if (C->isNaN()) {
1318 Constant *Quieted = ConstantFP::get(Ty, V: C->getValue().makeQuiet());
1319 return IC.replaceInstUsesWith(I&: II, V: Quieted);
1320 }
1321
1322 // f32 instruction doesn't handle denormals, f16 does.
1323 if (C->isZero() || (C->getValue().isDenormal() && Ty->isFloatTy())) {
1324 Constant *FoldedValue = IsLog ? ConstantFP::getInfinity(Ty, Negative: true)
1325 : ConstantFP::get(Ty, V: 1.0);
1326 return IC.replaceInstUsesWith(I&: II, V: FoldedValue);
1327 }
1328
1329 if (IsLog && C->isNegative())
1330 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getNaN(Ty));
1331
1332 // TODO: Full constant folding matching hardware behavior.
1333 }
1334
1335 break;
1336 }
1337 case Intrinsic::amdgcn_frexp_mant:
1338 case Intrinsic::amdgcn_frexp_exp: {
1339 Value *Src = II.getArgOperand(i: 0);
1340 if (const ConstantFP *C = dyn_cast<ConstantFP>(Val: Src)) {
1341 int Exp;
1342 APFloat Significand =
1343 frexp(X: C->getValueAPF(), Exp, RM: APFloat::rmNearestTiesToEven);
1344
1345 if (IID == Intrinsic::amdgcn_frexp_mant) {
1346 return IC.replaceInstUsesWith(
1347 I&: II, V: ConstantFP::get(Context&: II.getContext(), V: Significand));
1348 }
1349
1350 // Match instruction special case behavior.
1351 if (Exp == APFloat::IEK_NaN || Exp == APFloat::IEK_Inf)
1352 Exp = 0;
1353
1354 return IC.replaceInstUsesWith(I&: II,
1355 V: ConstantInt::getSigned(Ty: II.getType(), V: Exp));
1356 }
1357
1358 if (isa<PoisonValue>(Val: Src))
1359 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1360
1361 if (isa<UndefValue>(Val: Src)) {
1362 return IC.replaceInstUsesWith(I&: II, V: UndefValue::get(T: II.getType()));
1363 }
1364
1365 break;
1366 }
1367 case Intrinsic::amdgcn_class: {
1368 Value *Src0 = II.getArgOperand(i: 0);
1369 Value *Src1 = II.getArgOperand(i: 1);
1370 const ConstantInt *CMask = dyn_cast<ConstantInt>(Val: Src1);
1371 if (CMask) {
1372 II.setCalledOperand(Intrinsic::getOrInsertDeclaration(
1373 M: II.getModule(), id: Intrinsic::is_fpclass, OverloadTys: Src0->getType()));
1374
1375 // Clamp any excess bits, as they're illegal for the generic intrinsic.
1376 II.setArgOperand(i: 1, v: ConstantInt::get(Ty: Src1->getType(),
1377 V: CMask->getZExtValue() & fcAllFlags));
1378 return &II;
1379 }
1380
1381 // Propagate poison.
1382 if (isa<PoisonValue>(Val: Src0) || isa<PoisonValue>(Val: Src1))
1383 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1384
1385 // llvm.amdgcn.class(_, undef) -> false
1386 if (IC.getSimplifyQuery().isUndefValue(V: Src1))
1387 return IC.replaceInstUsesWith(I&: II, V: ConstantInt::get(Ty: II.getType(), V: false));
1388
1389 // llvm.amdgcn.class(undef, mask) -> mask != 0
1390 if (IC.getSimplifyQuery().isUndefValue(V: Src0)) {
1391 Value *CmpMask = IC.Builder.CreateICmpNE(
1392 LHS: Src1, RHS: ConstantInt::getNullValue(Ty: Src1->getType()));
1393 return IC.replaceInstUsesWith(I&: II, V: CmpMask);
1394 }
1395 break;
1396 }
1397 case Intrinsic::amdgcn_cvt_pkrtz: {
1398 auto foldFPTruncToF16RTZ = [](Value *Arg) -> Value * {
1399 Type *HalfTy = Type::getHalfTy(C&: Arg->getContext());
1400
1401 if (isa<PoisonValue>(Val: Arg))
1402 return PoisonValue::get(T: HalfTy);
1403 if (isa<UndefValue>(Val: Arg))
1404 return UndefValue::get(T: HalfTy);
1405
1406 ConstantFP *CFP = nullptr;
1407 if (match(V: Arg, P: m_ConstantFP(C&: CFP))) {
1408 bool LosesInfo;
1409 APFloat Val(CFP->getValueAPF());
1410 Val.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmTowardZero, losesInfo: &LosesInfo);
1411 return ConstantFP::get(Ty: HalfTy, V: Val);
1412 }
1413
1414 Value *Src = nullptr;
1415 if (match(V: Arg, P: m_FPExt(Op: m_Value(V&: Src)))) {
1416 if (Src->getType()->isHalfTy())
1417 return Src;
1418 }
1419
1420 return nullptr;
1421 };
1422
1423 if (Value *Src0 = foldFPTruncToF16RTZ(II.getArgOperand(i: 0))) {
1424 if (Value *Src1 = foldFPTruncToF16RTZ(II.getArgOperand(i: 1))) {
1425 Value *V = PoisonValue::get(T: II.getType());
1426 V = IC.Builder.CreateInsertElement(Vec: V, NewElt: Src0, Idx: (uint64_t)0);
1427 V = IC.Builder.CreateInsertElement(Vec: V, NewElt: Src1, Idx: (uint64_t)1);
1428 return IC.replaceInstUsesWith(I&: II, V);
1429 }
1430 }
1431
1432 break;
1433 }
1434 case Intrinsic::amdgcn_cvt_pknorm_i16:
1435 case Intrinsic::amdgcn_cvt_pknorm_u16:
1436 case Intrinsic::amdgcn_cvt_pk_i16:
1437 case Intrinsic::amdgcn_cvt_pk_u16: {
1438 Value *Src0 = II.getArgOperand(i: 0);
1439 Value *Src1 = II.getArgOperand(i: 1);
1440
1441 // TODO: Replace call with scalar operation if only one element is poison.
1442 if (isa<PoisonValue>(Val: Src0) && isa<PoisonValue>(Val: Src1))
1443 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1444
1445 if (isa<UndefValue>(Val: Src0) && isa<UndefValue>(Val: Src1)) {
1446 return IC.replaceInstUsesWith(I&: II, V: UndefValue::get(T: II.getType()));
1447 }
1448
1449 break;
1450 }
1451 case Intrinsic::amdgcn_cvt_off_f32_i4: {
1452 Value* Arg = II.getArgOperand(i: 0);
1453 Type *Ty = II.getType();
1454
1455 if (isa<PoisonValue>(Val: Arg))
1456 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: Ty));
1457
1458 if(IC.getSimplifyQuery().isUndefValue(V: Arg))
1459 return IC.replaceInstUsesWith(I&: II, V: Constant::getNullValue(Ty));
1460
1461 ConstantInt *CArg = dyn_cast<ConstantInt>(Val: II.getArgOperand(i: 0));
1462 if (!CArg)
1463 break;
1464
1465 // Tabulated 0.0625 * (sext (CArg & 0xf)).
1466 constexpr size_t ResValsSize = 16;
1467 static constexpr float ResVals[ResValsSize] = {
1468 0.0, 0.0625, 0.125, 0.1875, 0.25, 0.3125, 0.375, 0.4375,
1469 -0.5, -0.4375, -0.375, -0.3125, -0.25, -0.1875, -0.125, -0.0625};
1470 Constant *Res =
1471 ConstantFP::get(Ty, V: ResVals[CArg->getZExtValue() & (ResValsSize - 1)]);
1472 return IC.replaceInstUsesWith(I&: II, V: Res);
1473 }
1474 case Intrinsic::amdgcn_ubfe:
1475 case Intrinsic::amdgcn_sbfe: {
1476 // Decompose simple cases into standard shifts.
1477 Value *Src = II.getArgOperand(i: 0);
1478 if (isa<UndefValue>(Val: Src)) {
1479 return IC.replaceInstUsesWith(I&: II, V: Src);
1480 }
1481
1482 unsigned Width;
1483 Type *Ty = II.getType();
1484 unsigned IntSize = Ty->getIntegerBitWidth();
1485
1486 ConstantInt *CWidth = dyn_cast<ConstantInt>(Val: II.getArgOperand(i: 2));
1487 if (CWidth) {
1488 Width = CWidth->getZExtValue();
1489 if ((Width & (IntSize - 1)) == 0) {
1490 return IC.replaceInstUsesWith(I&: II, V: ConstantInt::getNullValue(Ty));
1491 }
1492
1493 // Hardware ignores high bits, so remove those.
1494 if (Width >= IntSize) {
1495 return IC.replaceOperand(
1496 I&: II, OpNum: 2, V: ConstantInt::get(Ty: CWidth->getType(), V: Width & (IntSize - 1)));
1497 }
1498 }
1499
1500 unsigned Offset;
1501 ConstantInt *COffset = dyn_cast<ConstantInt>(Val: II.getArgOperand(i: 1));
1502 if (COffset) {
1503 Offset = COffset->getZExtValue();
1504 if (Offset >= IntSize) {
1505 return IC.replaceOperand(
1506 I&: II, OpNum: 1,
1507 V: ConstantInt::get(Ty: COffset->getType(), V: Offset & (IntSize - 1)));
1508 }
1509 }
1510
1511 bool Signed = IID == Intrinsic::amdgcn_sbfe;
1512
1513 if (!CWidth || !COffset)
1514 break;
1515
1516 // The case of Width == 0 is handled above, which makes this transformation
1517 // safe. If Width == 0, then the ashr and lshr instructions become poison
1518 // value since the shift amount would be equal to the bit size.
1519 assert(Width != 0);
1520
1521 // TODO: This allows folding to undef when the hardware has specific
1522 // behavior?
1523 if (Offset + Width < IntSize) {
1524 Value *Shl = IC.Builder.CreateShl(LHS: Src, RHS: IntSize - Offset - Width);
1525 Value *RightShift = Signed ? IC.Builder.CreateAShr(LHS: Shl, RHS: IntSize - Width)
1526 : IC.Builder.CreateLShr(LHS: Shl, RHS: IntSize - Width);
1527 RightShift->takeName(V: &II);
1528 return IC.replaceInstUsesWith(I&: II, V: RightShift);
1529 }
1530
1531 Value *RightShift = Signed ? IC.Builder.CreateAShr(LHS: Src, RHS: Offset)
1532 : IC.Builder.CreateLShr(LHS: Src, RHS: Offset);
1533
1534 RightShift->takeName(V: &II);
1535 return IC.replaceInstUsesWith(I&: II, V: RightShift);
1536 }
1537 case Intrinsic::amdgcn_exp:
1538 case Intrinsic::amdgcn_exp_row:
1539 case Intrinsic::amdgcn_exp_compr: {
1540 ConstantInt *En = cast<ConstantInt>(Val: II.getArgOperand(i: 1));
1541 unsigned EnBits = En->getZExtValue();
1542 if (EnBits == 0xf)
1543 break; // All inputs enabled.
1544
1545 bool IsCompr = IID == Intrinsic::amdgcn_exp_compr;
1546 bool Changed = false;
1547 for (int I = 0; I < (IsCompr ? 2 : 4); ++I) {
1548 if ((!IsCompr && (EnBits & (1 << I)) == 0) ||
1549 (IsCompr && ((EnBits & (0x3 << (2 * I))) == 0))) {
1550 Value *Src = II.getArgOperand(i: I + 2);
1551 if (!isa<PoisonValue>(Val: Src)) {
1552 IC.replaceOperand(I&: II, OpNum: I + 2, V: PoisonValue::get(T: Src->getType()));
1553 Changed = true;
1554 }
1555 }
1556 }
1557
1558 if (Changed) {
1559 return &II;
1560 }
1561
1562 break;
1563 }
1564 case Intrinsic::amdgcn_fmed3: {
1565 Value *Src0 = II.getArgOperand(i: 0);
1566 Value *Src1 = II.getArgOperand(i: 1);
1567 Value *Src2 = II.getArgOperand(i: 2);
1568
1569 for (Value *Src : {Src0, Src1, Src2}) {
1570 if (isa<PoisonValue>(Val: Src))
1571 return IC.replaceInstUsesWith(I&: II, V: Src);
1572 }
1573
1574 if (II.isStrictFP())
1575 break;
1576
1577 // med3 with a nan input acts like
1578 // v_min_f32(v_min_f32(s0, s1), s2)
1579 //
1580 // Signalingness is ignored with ieee=0, so we fold to
1581 // minimumnum/maximumnum. With ieee=1, the v_min_f32 acts like llvm.minnum
1582 // with signaling nan handling. With ieee=0, like llvm.minimumnum except a
1583 // returned signaling nan will not be quieted.
1584
1585 // ieee=1
1586 // s0 snan: s2
1587 // s1 snan: s2
1588 // s2 snan: qnan
1589
1590 // s0 qnan: min(s1, s2)
1591 // s1 qnan: min(s0, s2)
1592 // s2 qnan: min(s0, s1)
1593
1594 // ieee=0
1595 // s0 _nan: min(s1, s2)
1596 // s1 _nan: min(s0, s2)
1597 // s2 _nan: min(s0, s1)
1598
1599 // med3 behavior with infinity
1600 // s0 +inf: max(s1, s2)
1601 // s1 +inf: max(s0, s2)
1602 // s2 +inf: max(s0, s1)
1603 // s0 -inf: min(s1, s2)
1604 // s1 -inf: min(s0, s2)
1605 // s2 -inf: min(s0, s1)
1606
1607 // Checking for NaN before canonicalization provides better fidelity when
1608 // mapping other operations onto fmed3 since the order of operands is
1609 // unchanged.
1610 Value *V = nullptr;
1611 const APFloat *ConstSrc0 = nullptr;
1612 const APFloat *ConstSrc1 = nullptr;
1613 const APFloat *ConstSrc2 = nullptr;
1614
1615 if ((match(V: Src0, P: m_APFloat(Res&: ConstSrc0)) &&
1616 (ConstSrc0->isNaN() || ConstSrc0->isInfinity())) ||
1617 isa<UndefValue>(Val: Src0)) {
1618 const bool IsPosInfinity = ConstSrc0 && ConstSrc0->isPosInfinity();
1619 switch (fpenvIEEEMode(I: II)) {
1620 case KnownIEEEMode::On:
1621 // TODO: If Src2 is snan, does it need quieting?
1622 if (ConstSrc0 && ConstSrc0->isNaN() && ConstSrc0->isSignaling())
1623 return IC.replaceInstUsesWith(I&: II, V: Src2);
1624
1625 V = IsPosInfinity ? IC.Builder.CreateMaxNum(LHS: Src1, RHS: Src2)
1626 : IC.Builder.CreateMinNum(LHS: Src1, RHS: Src2);
1627 break;
1628 case KnownIEEEMode::Off:
1629 V = IsPosInfinity ? IC.Builder.CreateMaximumNum(LHS: Src1, RHS: Src2)
1630 : IC.Builder.CreateMinimumNum(LHS: Src1, RHS: Src2);
1631 break;
1632 case KnownIEEEMode::Unknown:
1633 break;
1634 }
1635 } else if ((match(V: Src1, P: m_APFloat(Res&: ConstSrc1)) &&
1636 (ConstSrc1->isNaN() || ConstSrc1->isInfinity())) ||
1637 isa<UndefValue>(Val: Src1)) {
1638 const bool IsPosInfinity = ConstSrc1 && ConstSrc1->isPosInfinity();
1639 switch (fpenvIEEEMode(I: II)) {
1640 case KnownIEEEMode::On:
1641 // TODO: If Src2 is snan, does it need quieting?
1642 if (ConstSrc1 && ConstSrc1->isNaN() && ConstSrc1->isSignaling())
1643 return IC.replaceInstUsesWith(I&: II, V: Src2);
1644
1645 V = IsPosInfinity ? IC.Builder.CreateMaxNum(LHS: Src0, RHS: Src2)
1646 : IC.Builder.CreateMinNum(LHS: Src0, RHS: Src2);
1647 break;
1648 case KnownIEEEMode::Off:
1649 V = IsPosInfinity ? IC.Builder.CreateMaximumNum(LHS: Src0, RHS: Src2)
1650 : IC.Builder.CreateMinimumNum(LHS: Src0, RHS: Src2);
1651 break;
1652 case KnownIEEEMode::Unknown:
1653 break;
1654 }
1655 } else if ((match(V: Src2, P: m_APFloat(Res&: ConstSrc2)) &&
1656 (ConstSrc2->isNaN() || ConstSrc2->isInfinity())) ||
1657 isa<UndefValue>(Val: Src2)) {
1658 switch (fpenvIEEEMode(I: II)) {
1659 case KnownIEEEMode::On:
1660 if (ConstSrc2 && ConstSrc2->isNaN() && ConstSrc2->isSignaling()) {
1661 auto *Quieted = ConstantFP::get(Ty: II.getType(), V: ConstSrc2->makeQuiet());
1662 return IC.replaceInstUsesWith(I&: II, V: Quieted);
1663 }
1664
1665 V = (ConstSrc2 && ConstSrc2->isPosInfinity())
1666 ? IC.Builder.CreateMaxNum(LHS: Src0, RHS: Src1)
1667 : IC.Builder.CreateMinNum(LHS: Src0, RHS: Src1);
1668 break;
1669 case KnownIEEEMode::Off:
1670 V = (ConstSrc2 && ConstSrc2->isPosInfinity())
1671 ? IC.Builder.CreateMaximumNum(LHS: Src0, RHS: Src1)
1672 : IC.Builder.CreateMinimumNum(LHS: Src0, RHS: Src1);
1673 break;
1674 case KnownIEEEMode::Unknown:
1675 break;
1676 }
1677 }
1678
1679 if (V) {
1680 if (auto *CI = dyn_cast<CallInst>(Val: V)) {
1681 CI->copyFastMathFlags(I: &II);
1682 CI->takeName(V: &II);
1683 }
1684 return IC.replaceInstUsesWith(I&: II, V);
1685 }
1686
1687 bool Swap = false;
1688 // Canonicalize constants to RHS operands.
1689 //
1690 // fmed3(c0, x, c1) -> fmed3(x, c0, c1)
1691 if (isa<Constant>(Val: Src0) && !isa<Constant>(Val: Src1)) {
1692 std::swap(a&: Src0, b&: Src1);
1693 Swap = true;
1694 }
1695
1696 if (isa<Constant>(Val: Src1) && !isa<Constant>(Val: Src2)) {
1697 std::swap(a&: Src1, b&: Src2);
1698 Swap = true;
1699 }
1700
1701 if (isa<Constant>(Val: Src0) && !isa<Constant>(Val: Src1)) {
1702 std::swap(a&: Src0, b&: Src1);
1703 Swap = true;
1704 }
1705
1706 if (Swap) {
1707 II.setArgOperand(i: 0, v: Src0);
1708 II.setArgOperand(i: 1, v: Src1);
1709 II.setArgOperand(i: 2, v: Src2);
1710 return &II;
1711 }
1712
1713 if (const ConstantFP *C0 = dyn_cast<ConstantFP>(Val: Src0)) {
1714 if (const ConstantFP *C1 = dyn_cast<ConstantFP>(Val: Src1)) {
1715 if (const ConstantFP *C2 = dyn_cast<ConstantFP>(Val: Src2)) {
1716 APFloat Result = fmed3AMDGCN(Src0: C0->getValueAPF(), Src1: C1->getValueAPF(),
1717 Src2: C2->getValueAPF());
1718 return IC.replaceInstUsesWith(I&: II,
1719 V: ConstantFP::get(Ty: II.getType(), V: Result));
1720 }
1721 }
1722 }
1723
1724 if (!ST->hasMed3_16())
1725 break;
1726
1727 // Repeat floating-point width reduction done for minnum/maxnum.
1728 // fmed3((fpext X), (fpext Y), (fpext Z)) -> fpext (fmed3(X, Y, Z))
1729 if (Value *X = matchFPExtFromF16(Arg: Src0)) {
1730 if (Value *Y = matchFPExtFromF16(Arg: Src1)) {
1731 if (Value *Z = matchFPExtFromF16(Arg: Src2)) {
1732 Value *NewCall = IC.Builder.CreateIntrinsic(
1733 ID: IID, OverloadTypes: {X->getType()}, Args: {X, Y, Z}, FMFSource: &II, Name: II.getName());
1734 return new FPExtInst(NewCall, II.getType());
1735 }
1736 }
1737 }
1738
1739 break;
1740 }
1741 case Intrinsic::amdgcn_mbcnt_hi:
1742 // exec_hi is all 0, so this is just a copy.
1743 if (ST->isWave32())
1744 return IC.replaceInstUsesWith(I&: II, V: II.getArgOperand(i: 1));
1745 [[fallthrough]];
1746 case Intrinsic::amdgcn_mbcnt_lo: {
1747 ConstantRange AccRange =
1748 computeConstantRange(V: II.getArgOperand(i: 1),
1749 /*ForSigned=*/false, SQ: IC.getSimplifyQuery());
1750 if (AccRange.isFullSet())
1751 return nullptr;
1752
1753 // TODO: Can raise lower bound by inspecting first argument.
1754 ConstantRange MbcntRange(APInt(32, 0), APInt(32, 32 + 1));
1755 ConstantRange ComputedRange = AccRange.add(Other: MbcntRange);
1756 if (ComputedRange.isFullSet())
1757 return nullptr;
1758
1759 if (std::optional<ConstantRange> ExistingRange = II.getRange()) {
1760 ComputedRange = ComputedRange.intersectWith(CR: *ExistingRange);
1761 if (ComputedRange == *ExistingRange)
1762 return nullptr;
1763 }
1764
1765 II.addRangeRetAttr(CR: ComputedRange);
1766 return nullptr;
1767 }
1768 case Intrinsic::amdgcn_ballot: {
1769 Value *Arg = II.getArgOperand(i: 0);
1770 if (isa<PoisonValue>(Val: Arg))
1771 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1772
1773 if (auto *Src = dyn_cast<ConstantInt>(Val: Arg)) {
1774 if (Src->isZero()) {
1775 // amdgcn.ballot(i1 0) is zero.
1776 return IC.replaceInstUsesWith(I&: II, V: Constant::getNullValue(Ty: II.getType()));
1777 }
1778 }
1779 if (ST->isWave32() && II.getType()->getIntegerBitWidth() == 64) {
1780 // %b64 = call i64 ballot.i64(...)
1781 // =>
1782 // %b32 = call i32 ballot.i32(...)
1783 // %b64 = zext i32 %b32 to i64
1784 Value *Call = IC.Builder.CreateZExt(
1785 V: IC.Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_ballot,
1786 OverloadTypes: {IC.Builder.getInt32Ty()},
1787 Args: {II.getArgOperand(i: 0)}),
1788 DestTy: II.getType());
1789 Call->takeName(V: &II);
1790 return IC.replaceInstUsesWith(I&: II, V: Call);
1791 }
1792 break;
1793 }
1794 case Intrinsic::amdgcn_wavefrontsize: {
1795 if (ST->isWaveSizeKnown())
1796 return IC.replaceInstUsesWith(
1797 I&: II, V: ConstantInt::get(Ty: II.getType(), V: ST->getWavefrontSize()));
1798 break;
1799 }
1800 case Intrinsic::amdgcn_wqm_vote: {
1801 // wqm_vote is identity when the argument is constant.
1802 if (!isa<Constant>(Val: II.getArgOperand(i: 0)))
1803 break;
1804
1805 return IC.replaceInstUsesWith(I&: II, V: II.getArgOperand(i: 0));
1806 }
1807 case Intrinsic::amdgcn_kill: {
1808 const ConstantInt *C = dyn_cast<ConstantInt>(Val: II.getArgOperand(i: 0));
1809 if (!C || !C->getZExtValue())
1810 break;
1811
1812 // amdgcn.kill(i1 1) is a no-op
1813 return IC.eraseInstFromFunction(I&: II);
1814 }
1815 case Intrinsic::amdgcn_s_sendmsg:
1816 case Intrinsic::amdgcn_s_sendmsghalt: {
1817 // The second operand is copied to m0, but is only actually used for
1818 // certain message types. For message types that are known to not use m0,
1819 // fold it to poison.
1820 using namespace AMDGPU::SendMsg;
1821
1822 Value *M0Val = II.getArgOperand(i: 1);
1823 if (isa<PoisonValue>(Val: M0Val))
1824 break;
1825
1826 auto *MsgImm = cast<ConstantInt>(Val: II.getArgOperand(i: 0));
1827 uint16_t MsgId, OpId, StreamId;
1828 decodeMsg(Val: MsgImm->getZExtValue(), MsgId, OpId, StreamId, STI: *ST);
1829
1830 if (!msgDoesNotUseM0(MsgId, STI: *ST))
1831 break;
1832
1833 // Drop UB-implying attributes since we're replacing with poison.
1834 II.dropUBImplyingAttrsAndMetadata();
1835 IC.replaceOperand(I&: II, OpNum: 1, V: PoisonValue::get(T: M0Val->getType()));
1836 return nullptr;
1837 }
1838 case Intrinsic::amdgcn_update_dpp: {
1839 Value *Old = II.getArgOperand(i: 0);
1840
1841 auto *BC = cast<ConstantInt>(Val: II.getArgOperand(i: 5));
1842 auto *RM = cast<ConstantInt>(Val: II.getArgOperand(i: 3));
1843 auto *BM = cast<ConstantInt>(Val: II.getArgOperand(i: 4));
1844 if (BC->isNullValue() || RM->getZExtValue() != 0xF ||
1845 BM->getZExtValue() != 0xF || isa<PoisonValue>(Val: Old))
1846 break;
1847
1848 // If bound_ctrl = 1, row mask = bank mask = 0xf we can omit old value.
1849 return IC.replaceOperand(I&: II, OpNum: 0, V: PoisonValue::get(T: Old->getType()));
1850 }
1851 case Intrinsic::amdgcn_permlane16:
1852 case Intrinsic::amdgcn_permlane16_var:
1853 case Intrinsic::amdgcn_permlanex16:
1854 case Intrinsic::amdgcn_permlanex16_var: {
1855 // Discard vdst_in if it's not going to be read.
1856 Value *VDstIn = II.getArgOperand(i: 0);
1857 if (isa<PoisonValue>(Val: VDstIn))
1858 break;
1859
1860 // FetchInvalid operand idx.
1861 unsigned int FiIdx = (IID == Intrinsic::amdgcn_permlane16 ||
1862 IID == Intrinsic::amdgcn_permlanex16)
1863 ? 4 /* for permlane16 and permlanex16 */
1864 : 3; /* for permlane16_var and permlanex16_var */
1865
1866 // BoundCtrl operand idx.
1867 // For permlane16 and permlanex16 it should be 5
1868 // For Permlane16_var and permlanex16_var it should be 4
1869 unsigned int BcIdx = FiIdx + 1;
1870
1871 ConstantInt *FetchInvalid = cast<ConstantInt>(Val: II.getArgOperand(i: FiIdx));
1872 ConstantInt *BoundCtrl = cast<ConstantInt>(Val: II.getArgOperand(i: BcIdx));
1873 if (!FetchInvalid->getZExtValue() && !BoundCtrl->getZExtValue())
1874 break;
1875
1876 return IC.replaceOperand(I&: II, OpNum: 0, V: PoisonValue::get(T: VDstIn->getType()));
1877 }
1878 case Intrinsic::amdgcn_wave_shuffle:
1879 return tryOptimizeShufflePattern(IC, II, ST: *ST);
1880 case Intrinsic::amdgcn_permlane64:
1881 case Intrinsic::amdgcn_readfirstlane:
1882 case Intrinsic::amdgcn_readlane:
1883 case Intrinsic::amdgcn_ds_bpermute: {
1884 // If the data argument is uniform these intrinsics return it unchanged.
1885 unsigned SrcIdx = IID == Intrinsic::amdgcn_ds_bpermute ? 1 : 0;
1886 const Use &Src = II.getArgOperandUse(i: SrcIdx);
1887 if (isTriviallyUniform(U: Src))
1888 return IC.replaceInstUsesWith(I&: II, V: Src.get());
1889
1890 if (IID == Intrinsic::amdgcn_readlane &&
1891 simplifyDemandedLaneMaskArg(IC, II, LaneArgIdx: 1))
1892 return &II;
1893
1894 // If the lane argument of bpermute is uniform, change it to readlane. This
1895 // generates better code and can enable further optimizations because
1896 // readlane is AlwaysUniform.
1897 if (IID == Intrinsic::amdgcn_ds_bpermute) {
1898 const Use &Lane = II.getArgOperandUse(i: 0);
1899 if (isTriviallyUniform(U: Lane)) {
1900 Value *NewLane = IC.Builder.CreateLShr(LHS: Lane, RHS: 2);
1901 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
1902 M: II.getModule(), id: Intrinsic::amdgcn_readlane, OverloadTys: II.getType());
1903 II.setCalledFunction(NewDecl);
1904 II.setOperand(i_nocapture: 0, Val_nocapture: Src);
1905 II.setOperand(i_nocapture: 1, Val_nocapture: NewLane);
1906 return &II;
1907 }
1908 }
1909
1910 if (IID == Intrinsic::amdgcn_ds_bpermute)
1911 return tryOptimizeShufflePattern(IC, II, ST: *ST);
1912
1913 if (Instruction *Res = hoistLaneIntrinsicThroughOperand(IC, II))
1914 return Res;
1915
1916 return std::nullopt;
1917 }
1918 case Intrinsic::amdgcn_wave_match_b32: {
1919 const Use &Src0 = II.getArgOperandUse(i: 0);
1920 const Use &Src1 = II.getArgOperandUse(i: 1);
1921 if (Src0.get() == Src1.get() && isTriviallyUniform(U: Src0)) {
1922 Function *NewF = Intrinsic::getOrInsertDeclaration(
1923 M: II.getModule(), id: Intrinsic::amdgcn_ballot, OverloadTys: II.getType());
1924 CallInst *NewCall =
1925 IC.Builder.CreateCall(Callee: NewF, Args: {IC.Builder.getInt1(V: true)});
1926 return IC.replaceInstUsesWith(I&: II, V: NewCall);
1927 }
1928 break;
1929 }
1930 case Intrinsic::amdgcn_writelane: {
1931 // TODO: Fold bitcast like readlane.
1932 if (simplifyDemandedLaneMaskArg(IC, II, LaneArgIdx: 1))
1933 return &II;
1934 return std::nullopt;
1935 }
1936 case Intrinsic::amdgcn_trig_preop: {
1937 // The intrinsic is declared with name mangling, but currently the
1938 // instruction only exists for f64
1939 if (!II.getType()->isDoubleTy())
1940 break;
1941
1942 Value *Src = II.getArgOperand(i: 0);
1943 Value *Segment = II.getArgOperand(i: 1);
1944 if (isa<PoisonValue>(Val: Src) || isa<PoisonValue>(Val: Segment))
1945 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1946
1947 if (isa<UndefValue>(Val: Segment))
1948 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getZero(Ty: II.getType()));
1949
1950 // Sign bit is not used.
1951 Value *StrippedSign = InstCombiner::stripSignOnlyFPOps(Val: Src);
1952 if (StrippedSign != Src)
1953 return IC.replaceOperand(I&: II, OpNum: 0, V: StrippedSign);
1954
1955 if (II.isStrictFP())
1956 break;
1957
1958 const ConstantFP *CSrc = dyn_cast<ConstantFP>(Val: Src);
1959 if (!CSrc && !isa<UndefValue>(Val: Src))
1960 break;
1961
1962 // The instruction ignores special cases, and literally just extracts the
1963 // exponents. Fold undef to nan, and index the table as normal.
1964 APInt FSrcInt = CSrc ? CSrc->getValueAPF().bitcastToAPInt()
1965 : APFloat::getQNaN(Sem: II.getType()->getFltSemantics())
1966 .bitcastToAPInt();
1967
1968 const ConstantInt *Cseg = dyn_cast<ConstantInt>(Val: Segment);
1969 if (!Cseg) {
1970 if (isa<UndefValue>(Val: Src))
1971 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getZero(Ty: II.getType()));
1972 break;
1973 }
1974
1975 unsigned Exponent = FSrcInt.extractBitsAsZExtValue(numBits: 11, bitPosition: 52);
1976 unsigned SegmentVal = Cseg->getValue().trunc(width: 5).getZExtValue();
1977 unsigned Shift = SegmentVal * 53;
1978 if (Exponent > 1077)
1979 Shift += Exponent - 1077;
1980
1981 // 2.0/PI table.
1982 static const uint32_t TwoByPi[] = {
1983 0xa2f9836e, 0x4e441529, 0xfc2757d1, 0xf534ddc0, 0xdb629599, 0x3c439041,
1984 0xfe5163ab, 0xdebbc561, 0xb7246e3a, 0x424dd2e0, 0x06492eea, 0x09d1921c,
1985 0xfe1deb1c, 0xb129a73e, 0xe88235f5, 0x2ebb4484, 0xe99c7026, 0xb45f7e41,
1986 0x3991d639, 0x835339f4, 0x9c845f8b, 0xbdf9283b, 0x1ff897ff, 0xde05980f,
1987 0xef2f118b, 0x5a0a6d1f, 0x6d367ecf, 0x27cb09b7, 0x4f463f66, 0x9e5fea2d,
1988 0x7527bac7, 0xebe5f17b, 0x3d0739f7, 0x8a5292ea, 0x6bfb5fb1, 0x1f8d5d08,
1989 0x56033046};
1990
1991 // Return 0 for outbound segment (hardware behavior).
1992 unsigned Idx = Shift >> 5;
1993 if (Idx + 2 >= std::size(TwoByPi)) {
1994 APFloat Zero = APFloat::getZero(Sem: II.getType()->getFltSemantics());
1995 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::get(Ty: II.getType(), V: Zero));
1996 }
1997
1998 unsigned BShift = Shift & 0x1f;
1999 uint64_t Thi = Make_64(High: TwoByPi[Idx], Low: TwoByPi[Idx + 1]);
2000 uint64_t Tlo = Make_64(High: TwoByPi[Idx + 2], Low: 0);
2001 if (BShift)
2002 Thi = (Thi << BShift) | (Tlo >> (64 - BShift));
2003 Thi = Thi >> 11;
2004 APFloat Result = APFloat((double)Thi);
2005
2006 int Scale = -53 - Shift;
2007 if (Exponent >= 1968)
2008 Scale += 128;
2009
2010 Result = scalbn(X: Result, Exp: Scale, RM: RoundingMode::NearestTiesToEven);
2011 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::get(Ty: Src->getType(), V: Result));
2012 }
2013 case Intrinsic::amdgcn_sdot2:
2014 case Intrinsic::amdgcn_udot2:
2015 case Intrinsic::amdgcn_sdot4:
2016 case Intrinsic::amdgcn_udot4:
2017 case Intrinsic::amdgcn_sdot8:
2018 case Intrinsic::amdgcn_udot8: {
2019 Value *Src0 = II.getArgOperand(i: 0);
2020 Value *Src1 = II.getArgOperand(i: 1);
2021
2022 // Canonicalize the constant multiplicand to Src1.
2023 if (isa<Constant>(Val: Src0) && !isa<Constant>(Val: Src1)) {
2024 II.setArgOperand(i: 0, v: Src1);
2025 II.setArgOperand(i: 1, v: Src0);
2026 return &II;
2027 }
2028
2029 if (match(V: Src1, P: m_Zero()))
2030 return IC.replaceInstUsesWith(I&: II, V: II.getArgOperand(i: 2));
2031
2032 if (Instruction *I = foldConstantIntoDotAccumulator(II, AccIdx: 2, ClampIdx: 3, IC))
2033 return I;
2034
2035 break;
2036 }
2037 case Intrinsic::amdgcn_sudot4:
2038 case Intrinsic::amdgcn_sudot8: {
2039 Value *Src0 = II.getArgOperand(i: 1);
2040 Value *Src1 = II.getArgOperand(i: 3);
2041
2042 // Canonicalize the constant multiplicand to Src1, moving its sign flag with
2043 // it.
2044 if (isa<Constant>(Val: Src0) && !isa<Constant>(Val: Src1)) {
2045 Value *Sign0 = II.getArgOperand(i: 0);
2046 Value *Sign1 = II.getArgOperand(i: 2);
2047 II.setArgOperand(i: 0, v: Sign1);
2048 II.setArgOperand(i: 1, v: Src1);
2049 II.setArgOperand(i: 2, v: Sign0);
2050 II.setArgOperand(i: 3, v: Src0);
2051 return &II;
2052 }
2053
2054 if (match(V: Src1, P: m_Zero()))
2055 return IC.replaceInstUsesWith(I&: II, V: II.getArgOperand(i: 4));
2056
2057 if (Instruction *I = foldConstantIntoDotAccumulator(II, AccIdx: 4, ClampIdx: 5, IC))
2058 return I;
2059
2060 break;
2061 }
2062 case Intrinsic::amdgcn_fmul_legacy: {
2063 Value *Op0 = II.getArgOperand(i: 0);
2064 Value *Op1 = II.getArgOperand(i: 1);
2065
2066 for (Value *Src : {Op0, Op1}) {
2067 if (isa<PoisonValue>(Val: Src))
2068 return IC.replaceInstUsesWith(I&: II, V: Src);
2069 }
2070
2071 // The legacy behaviour is that multiplying +/-0.0 by anything, even NaN or
2072 // infinity, gives +0.0.
2073 // TODO: Move to InstSimplify?
2074 if (match(V: Op0, P: PatternMatch::m_AnyZeroFP()) ||
2075 match(V: Op1, P: PatternMatch::m_AnyZeroFP()))
2076 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getZero(Ty: II.getType()));
2077
2078 // If we can prove we don't have one of the special cases then we can use a
2079 // normal fmul instruction instead.
2080 if (canSimplifyLegacyMulToMul(I: II, Op0, Op1, IC)) {
2081 auto *FMul = IC.Builder.CreateFMulFMF(L: Op0, R: Op1, FMFSource: &II);
2082 FMul->takeName(V: &II);
2083 return IC.replaceInstUsesWith(I&: II, V: FMul);
2084 }
2085 break;
2086 }
2087 case Intrinsic::amdgcn_fma_legacy: {
2088 Value *Op0 = II.getArgOperand(i: 0);
2089 Value *Op1 = II.getArgOperand(i: 1);
2090 Value *Op2 = II.getArgOperand(i: 2);
2091
2092 for (Value *Src : {Op0, Op1, Op2}) {
2093 if (isa<PoisonValue>(Val: Src))
2094 return IC.replaceInstUsesWith(I&: II, V: Src);
2095 }
2096
2097 // The legacy behaviour is that multiplying +/-0.0 by anything, even NaN or
2098 // infinity, gives +0.0.
2099 // TODO: Move to InstSimplify?
2100 if (match(V: Op0, P: PatternMatch::m_AnyZeroFP()) ||
2101 match(V: Op1, P: PatternMatch::m_AnyZeroFP())) {
2102 // It's tempting to just return Op2 here, but that would give the wrong
2103 // result if Op2 was -0.0.
2104 auto *Zero = ConstantFP::getZero(Ty: II.getType());
2105 auto *FAdd = IC.Builder.CreateFAddFMF(L: Zero, R: Op2, FMFSource: &II);
2106 FAdd->takeName(V: &II);
2107 return IC.replaceInstUsesWith(I&: II, V: FAdd);
2108 }
2109
2110 // If we can prove we don't have one of the special cases then we can use a
2111 // normal fma instead.
2112 if (canSimplifyLegacyMulToMul(I: II, Op0, Op1, IC)) {
2113 II.setCalledOperand(Intrinsic::getOrInsertDeclaration(
2114 M: II.getModule(), id: Intrinsic::fma, OverloadTys: II.getType()));
2115 return &II;
2116 }
2117 break;
2118 }
2119 case Intrinsic::amdgcn_is_shared:
2120 case Intrinsic::amdgcn_is_private: {
2121 Value *Src = II.getArgOperand(i: 0);
2122 if (isa<PoisonValue>(Val: Src))
2123 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
2124 if (isa<UndefValue>(Val: Src))
2125 return IC.replaceInstUsesWith(I&: II, V: UndefValue::get(T: II.getType()));
2126
2127 if (isa<ConstantPointerNull>(Val: II.getArgOperand(i: 0)))
2128 return IC.replaceInstUsesWith(I&: II, V: ConstantInt::getFalse(Ty: II.getType()));
2129 break;
2130 }
2131 case Intrinsic::amdgcn_make_buffer_rsrc: {
2132 Value *Src = II.getArgOperand(i: 0);
2133 if (isa<PoisonValue>(Val: Src))
2134 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
2135
2136 // Normalize num_records to the correct width.
2137 std::optional<unsigned> Width = ST->getBufferResourceNumRecordsWidth();
2138 if (!Width)
2139 return std::nullopt;
2140 Type *NumRecordsTy = IC.Builder.getIntNTy(N: *Width);
2141 if (II.getArgOperand(i: 2)->getType() == NumRecordsTy)
2142 return std::nullopt;
2143 SmallVector<Value *, 4> Args(II.args());
2144 Args[2] = IC.Builder.CreateZExtOrTrunc(V: Args[2], DestTy: NumRecordsTy);
2145 CallInst *NewCall = IC.Builder.CreateIntrinsicWithoutFolding(
2146 ID: Intrinsic::amdgcn_make_buffer_rsrc,
2147 OverloadTypes: {II.getType(), Src->getType(), NumRecordsTy}, Args);
2148 NewCall->copyMetadata(SrcInst: II);
2149 NewCall->setTailCallKind(II.getTailCallKind());
2150 // Copy over all attributes except those on num_records, which may no longer
2151 // be valid.
2152 NewCall->setAttributes(
2153 II.getAttributes().removeParamAttributes(C&: II.getContext(), ArgNo: 2));
2154 NewCall->takeName(V: &II);
2155 return IC.replaceInstUsesWith(I&: II, V: NewCall);
2156 }
2157 case Intrinsic::amdgcn_raw_buffer_store_format:
2158 case Intrinsic::amdgcn_struct_buffer_store_format:
2159 case Intrinsic::amdgcn_raw_tbuffer_store:
2160 case Intrinsic::amdgcn_struct_tbuffer_store:
2161 case Intrinsic::amdgcn_image_store_1d:
2162 case Intrinsic::amdgcn_image_store_1darray:
2163 case Intrinsic::amdgcn_image_store_2d:
2164 case Intrinsic::amdgcn_image_store_2darray:
2165 case Intrinsic::amdgcn_image_store_2darraymsaa:
2166 case Intrinsic::amdgcn_image_store_2dmsaa:
2167 case Intrinsic::amdgcn_image_store_3d:
2168 case Intrinsic::amdgcn_image_store_cube:
2169 case Intrinsic::amdgcn_image_store_mip_1d:
2170 case Intrinsic::amdgcn_image_store_mip_1darray:
2171 case Intrinsic::amdgcn_image_store_mip_2d:
2172 case Intrinsic::amdgcn_image_store_mip_2darray:
2173 case Intrinsic::amdgcn_image_store_mip_3d:
2174 case Intrinsic::amdgcn_image_store_mip_cube: {
2175 if (!isa<FixedVectorType>(Val: II.getArgOperand(i: 0)->getType()))
2176 break;
2177
2178 APInt DemandedElts;
2179 if (ST->hasDefaultComponentBroadcast())
2180 DemandedElts = defaultComponentBroadcast(V: II.getArgOperand(i: 0));
2181 else if (ST->hasDefaultComponentZero())
2182 DemandedElts = trimTrailingZerosInVector(IC, UseV: II.getArgOperand(i: 0), I: &II);
2183 else
2184 break;
2185
2186 int DMaskIdx = getAMDGPUImageDMaskIntrinsic(Intr: II.getIntrinsicID()) ? 1 : -1;
2187 if (simplifyAMDGCNMemoryIntrinsicDemanded(IC, II, DemandedElts, DMaskIdx,
2188 IsLoad: false)) {
2189 return IC.eraseInstFromFunction(I&: II);
2190 }
2191
2192 break;
2193 }
2194 case Intrinsic::amdgcn_prng_b32: {
2195 auto *Src = II.getArgOperand(i: 0);
2196 if (isa<UndefValue>(Val: Src)) {
2197 return IC.replaceInstUsesWith(I&: II, V: Src);
2198 }
2199 return std::nullopt;
2200 }
2201 case Intrinsic::amdgcn_mfma_scale_f32_16x16x128_f8f6f4:
2202 case Intrinsic::amdgcn_mfma_scale_f32_32x32x64_f8f6f4: {
2203 Value *Src0 = II.getArgOperand(i: 0);
2204 Value *Src1 = II.getArgOperand(i: 1);
2205 uint64_t CBSZ = cast<ConstantInt>(Val: II.getArgOperand(i: 3))->getZExtValue();
2206 uint64_t BLGP = cast<ConstantInt>(Val: II.getArgOperand(i: 4))->getZExtValue();
2207 auto *Src0Ty = cast<FixedVectorType>(Val: Src0->getType());
2208 auto *Src1Ty = cast<FixedVectorType>(Val: Src1->getType());
2209
2210 auto getFormatNumRegs = [](unsigned FormatVal) {
2211 switch (FormatVal) {
2212 case AMDGPU::MFMAScaleFormats::FP6_E2M3:
2213 case AMDGPU::MFMAScaleFormats::FP6_E3M2:
2214 return 6u;
2215 case AMDGPU::MFMAScaleFormats::FP4_E2M1:
2216 return 4u;
2217 case AMDGPU::MFMAScaleFormats::FP8_E4M3:
2218 case AMDGPU::MFMAScaleFormats::FP8_E5M2:
2219 return 8u;
2220 default:
2221 llvm_unreachable("invalid format value");
2222 }
2223 };
2224
2225 bool MadeChange = false;
2226 unsigned Src0NumElts = getFormatNumRegs(CBSZ);
2227 unsigned Src1NumElts = getFormatNumRegs(BLGP);
2228
2229 // Depending on the used format, fewer registers are required so shrink the
2230 // vector type.
2231 if (Src0Ty->getNumElements() > Src0NumElts) {
2232 Src0 = IC.Builder.CreateExtractVector(
2233 DstType: FixedVectorType::get(ElementType: Src0Ty->getElementType(), NumElts: Src0NumElts), SrcVec: Src0,
2234 Idx: uint64_t(0));
2235 MadeChange = true;
2236 }
2237
2238 if (Src1Ty->getNumElements() > Src1NumElts) {
2239 Src1 = IC.Builder.CreateExtractVector(
2240 DstType: FixedVectorType::get(ElementType: Src1Ty->getElementType(), NumElts: Src1NumElts), SrcVec: Src1,
2241 Idx: uint64_t(0));
2242 MadeChange = true;
2243 }
2244
2245 if (!MadeChange)
2246 return std::nullopt;
2247
2248 SmallVector<Value *, 10> Args(II.args());
2249 Args[0] = Src0;
2250 Args[1] = Src1;
2251
2252 Value *NewII = IC.Builder.CreateIntrinsic(
2253 ID: IID, OverloadTypes: {Src0->getType(), Src1->getType()}, Args, FMFSource: &II);
2254 NewII->takeName(V: &II);
2255 return IC.replaceInstUsesWith(I&: II, V: NewII);
2256 }
2257 case Intrinsic::amdgcn_wmma_f32_16x16x128_f8f6f4:
2258 case Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
2259 case Intrinsic::amdgcn_wmma_scale16_f32_16x16x128_f8f6f4: {
2260 Value *Src0 = II.getArgOperand(i: 1);
2261 Value *Src1 = II.getArgOperand(i: 3);
2262 unsigned FmtA = cast<ConstantInt>(Val: II.getArgOperand(i: 0))->getZExtValue();
2263 uint64_t FmtB = cast<ConstantInt>(Val: II.getArgOperand(i: 2))->getZExtValue();
2264 auto *Src0Ty = cast<FixedVectorType>(Val: Src0->getType());
2265 auto *Src1Ty = cast<FixedVectorType>(Val: Src1->getType());
2266
2267 bool MadeChange = false;
2268 unsigned Src0NumElts = AMDGPU::wmmaScaleF8F6F4FormatToNumRegs(Fmt: FmtA);
2269 unsigned Src1NumElts = AMDGPU::wmmaScaleF8F6F4FormatToNumRegs(Fmt: FmtB);
2270
2271 // Depending on the used format, fewer registers are required so shrink the
2272 // vector type.
2273 if (Src0Ty->getNumElements() > Src0NumElts) {
2274 Src0 = IC.Builder.CreateExtractVector(
2275 DstType: FixedVectorType::get(ElementType: Src0Ty->getElementType(), NumElts: Src0NumElts), SrcVec: Src0,
2276 Idx: IC.Builder.getInt64(C: 0));
2277 MadeChange = true;
2278 }
2279
2280 if (Src1Ty->getNumElements() > Src1NumElts) {
2281 Src1 = IC.Builder.CreateExtractVector(
2282 DstType: FixedVectorType::get(ElementType: Src1Ty->getElementType(), NumElts: Src1NumElts), SrcVec: Src1,
2283 Idx: IC.Builder.getInt64(C: 0));
2284 MadeChange = true;
2285 }
2286
2287 if (!MadeChange)
2288 return std::nullopt;
2289
2290 SmallVector<Value *, 13> Args(II.args());
2291 Args[1] = Src0;
2292 Args[3] = Src1;
2293
2294 Value *NewII = IC.Builder.CreateIntrinsic(
2295 ID: IID, OverloadTypes: {II.getArgOperand(i: 5)->getType(), Src0->getType(), Src1->getType()},
2296 Args, FMFSource: &II);
2297 NewII->takeName(V: &II);
2298 return IC.replaceInstUsesWith(I&: II, V: NewII);
2299 }
2300 }
2301 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
2302 AMDGPU::getImageDimIntrinsicInfo(Intr: II.getIntrinsicID())) {
2303 return simplifyAMDGCNImageIntrinsic(ST, ImageDimIntr, II, IC);
2304 }
2305 return std::nullopt;
2306}
2307
2308/// Implement SimplifyDemandedVectorElts for amdgcn buffer and image intrinsics.
2309///
2310/// The result of simplifying amdgcn image and buffer store intrinsics is updating
2311/// definitions of the intrinsics vector argument, not Uses of the result like
2312/// image and buffer loads.
2313/// Note: This only supports non-TFE/LWE image intrinsic calls; those have
2314/// struct returns.
2315static Value *simplifyAMDGCNMemoryIntrinsicDemanded(InstCombiner &IC,
2316 IntrinsicInst &II,
2317 APInt DemandedElts,
2318 int DMaskIdx, bool IsLoad) {
2319
2320 auto *IIVTy = cast<FixedVectorType>(Val: IsLoad ? II.getType()
2321 : II.getOperand(i_nocapture: 0)->getType());
2322 unsigned VWidth = IIVTy->getNumElements();
2323 if (VWidth == 1)
2324 return nullptr;
2325 Type *EltTy = IIVTy->getElementType();
2326
2327 IRBuilderBase::InsertPointGuard Guard(IC.Builder);
2328 IC.Builder.SetInsertPoint(&II);
2329
2330 // Assume the arguments are unchanged and later override them, if needed.
2331 SmallVector<Value *, 16> Args(II.args());
2332
2333 if (DMaskIdx < 0) {
2334 // Buffer case.
2335
2336 const unsigned ActiveBits = DemandedElts.getActiveBits();
2337 const unsigned UnusedComponentsAtFront = DemandedElts.countr_zero();
2338
2339 // Start assuming the prefix of elements is demanded, but possibly clear
2340 // some other bits if there are trailing zeros (unused components at front)
2341 // and update offset.
2342 DemandedElts = (1 << ActiveBits) - 1;
2343
2344 if (UnusedComponentsAtFront > 0) {
2345 static const unsigned InvalidOffsetIdx = 0xf;
2346
2347 unsigned OffsetIdx;
2348 switch (II.getIntrinsicID()) {
2349 case Intrinsic::amdgcn_raw_buffer_load:
2350 case Intrinsic::amdgcn_raw_ptr_buffer_load:
2351 OffsetIdx = 1;
2352 break;
2353 case Intrinsic::amdgcn_s_buffer_load:
2354 case Intrinsic::amdgcn_ptr_s_buffer_load:
2355 // If resulting type is vec3, there is no point in trimming the
2356 // load with updated offset, as the vec3 would most likely be widened to
2357 // vec4 anyway during lowering.
2358 if (ActiveBits == 4 && UnusedComponentsAtFront == 1)
2359 OffsetIdx = InvalidOffsetIdx;
2360 else
2361 OffsetIdx = 1;
2362 break;
2363 case Intrinsic::amdgcn_struct_buffer_load:
2364 case Intrinsic::amdgcn_struct_ptr_buffer_load:
2365 OffsetIdx = 2;
2366 break;
2367 default:
2368 // TODO: handle tbuffer* intrinsics.
2369 OffsetIdx = InvalidOffsetIdx;
2370 break;
2371 }
2372
2373 if (OffsetIdx != InvalidOffsetIdx) {
2374 // Clear demanded bits and update the offset.
2375 DemandedElts &= ~((1 << UnusedComponentsAtFront) - 1);
2376 auto *Offset = Args[OffsetIdx];
2377 unsigned SingleComponentSizeInBits =
2378 IC.getDataLayout().getTypeSizeInBits(Ty: EltTy);
2379 unsigned OffsetAdd =
2380 UnusedComponentsAtFront * SingleComponentSizeInBits / 8;
2381 auto *OffsetAddVal = ConstantInt::get(Ty: Offset->getType(), V: OffsetAdd);
2382 Args[OffsetIdx] = IC.Builder.CreateAdd(LHS: Offset, RHS: OffsetAddVal);
2383 }
2384 }
2385 } else {
2386 // Image case.
2387
2388 ConstantInt *DMask = cast<ConstantInt>(Val: Args[DMaskIdx]);
2389 unsigned DMaskVal = DMask->getZExtValue() & 0xf;
2390
2391 // dmask 0 has special semantics, do not simplify.
2392 if (DMaskVal == 0)
2393 return nullptr;
2394
2395 if (!IsLoad && !isMask_32(Value: DMaskVal))
2396 return nullptr;
2397
2398 // Mask off values that are undefined because the dmask doesn't cover them
2399 DemandedElts &= (1 << llvm::popcount(Value: DMaskVal)) - 1;
2400
2401 unsigned NewDMaskVal = 0;
2402 unsigned OrigLdStIdx = 0;
2403 for (unsigned SrcIdx = 0; SrcIdx < 4; ++SrcIdx) {
2404 const unsigned Bit = 1 << SrcIdx;
2405 if (!!(DMaskVal & Bit)) {
2406 if (!!DemandedElts[OrigLdStIdx])
2407 NewDMaskVal |= Bit;
2408 OrigLdStIdx++;
2409 }
2410 }
2411
2412 if (DMaskVal != NewDMaskVal)
2413 Args[DMaskIdx] = ConstantInt::get(Ty: DMask->getType(), V: NewDMaskVal);
2414 }
2415
2416 unsigned NewNumElts = DemandedElts.popcount();
2417 if (!NewNumElts)
2418 return PoisonValue::get(T: IIVTy);
2419
2420 if (NewNumElts >= VWidth && DemandedElts.isMask()) {
2421 if (DMaskIdx >= 0)
2422 II.setArgOperand(i: DMaskIdx, v: Args[DMaskIdx]);
2423 return nullptr;
2424 }
2425
2426 // Validate function argument and return types, extracting overloaded types
2427 // along the way.
2428 SmallVector<Type *, 6> OverloadTys;
2429 if (!Intrinsic::isSignatureValid(F: II.getCalledFunction(), OverloadTys))
2430 return nullptr;
2431
2432 Type *NewTy =
2433 (NewNumElts == 1) ? EltTy : FixedVectorType::get(ElementType: EltTy, NumElts: NewNumElts);
2434 OverloadTys[0] = NewTy;
2435
2436 if (!IsLoad) {
2437 SmallVector<int, 8> EltMask;
2438 for (unsigned OrigStoreIdx = 0; OrigStoreIdx < VWidth; ++OrigStoreIdx)
2439 if (DemandedElts[OrigStoreIdx])
2440 EltMask.push_back(Elt: OrigStoreIdx);
2441
2442 if (NewNumElts == 1)
2443 Args[0] = IC.Builder.CreateExtractElement(Vec: II.getOperand(i_nocapture: 0), Idx: EltMask[0]);
2444 else
2445 Args[0] = IC.Builder.CreateShuffleVector(V: II.getOperand(i_nocapture: 0), Mask: EltMask);
2446 }
2447
2448 CallInst *NewCall = IC.Builder.CreateIntrinsicWithoutFolding(
2449 ID: II.getIntrinsicID(), OverloadTypes: OverloadTys, Args);
2450 NewCall->takeName(V: &II);
2451 NewCall->copyMetadata(SrcInst: II);
2452 AttributeList OldAttrList = II.getAttributes();
2453 NewCall->setAttributes(OldAttrList);
2454
2455 if (IsLoad) {
2456 if (NewNumElts == 1) {
2457 return IC.Builder.CreateInsertElement(Vec: PoisonValue::get(T: IIVTy), NewElt: NewCall,
2458 Idx: DemandedElts.countr_zero());
2459 }
2460
2461 SmallVector<int, 8> EltMask;
2462 unsigned NewLoadIdx = 0;
2463 for (unsigned OrigLoadIdx = 0; OrigLoadIdx < VWidth; ++OrigLoadIdx) {
2464 if (!!DemandedElts[OrigLoadIdx])
2465 EltMask.push_back(Elt: NewLoadIdx++);
2466 else
2467 EltMask.push_back(Elt: NewNumElts);
2468 }
2469
2470 auto *Shuffle = IC.Builder.CreateShuffleVector(V: NewCall, Mask: EltMask);
2471
2472 return Shuffle;
2473 }
2474
2475 return NewCall;
2476}
2477
2478Value *GCNTTIImpl::simplifyAMDGCNLaneIntrinsicDemanded(
2479 InstCombiner &IC, IntrinsicInst &II, const APInt &DemandedElts,
2480 APInt &UndefElts) const {
2481 auto *VT = dyn_cast<FixedVectorType>(Val: II.getType());
2482 if (!VT)
2483 return nullptr;
2484
2485 const unsigned FirstElt = DemandedElts.countr_zero();
2486 const unsigned LastElt = DemandedElts.getActiveBits() - 1;
2487 const unsigned MaskLen = LastElt - FirstElt + 1;
2488
2489 unsigned OldNumElts = VT->getNumElements();
2490 if (MaskLen == OldNumElts && MaskLen != 1)
2491 return nullptr;
2492
2493 Type *EltTy = VT->getElementType();
2494 Type *NewVT = MaskLen == 1 ? EltTy : FixedVectorType::get(ElementType: EltTy, NumElts: MaskLen);
2495
2496 // Theoretically we should support these intrinsics for any legal type. Avoid
2497 // introducing cases that aren't direct register types like v3i16.
2498 if (!isTypeLegal(Ty: NewVT))
2499 return nullptr;
2500
2501 Value *Src = II.getArgOperand(i: 0);
2502
2503 // Make sure convergence tokens are preserved.
2504 // TODO: CreateIntrinsic should allow directly copying bundles
2505 SmallVector<OperandBundleDef, 2> OpBundles;
2506 II.getOperandBundlesAsDefs(Defs&: OpBundles);
2507
2508 Module *M = IC.Builder.getModule();
2509 Function *Remangled =
2510 Intrinsic::getOrInsertDeclaration(M, id: II.getIntrinsicID(), OverloadTys: {NewVT});
2511
2512 if (MaskLen == 1) {
2513 Value *Extract = IC.Builder.CreateExtractElement(Vec: Src, Idx: FirstElt);
2514
2515 // TODO: Preserve callsite attributes?
2516 CallInst *NewCall = IC.Builder.CreateCall(Callee: Remangled, Args: {Extract}, OpBundles);
2517
2518 return IC.Builder.CreateInsertElement(Vec: PoisonValue::get(T: II.getType()),
2519 NewElt: NewCall, Idx: FirstElt);
2520 }
2521
2522 SmallVector<int> ExtractMask(MaskLen, -1);
2523 for (unsigned I = 0; I != MaskLen; ++I) {
2524 if (DemandedElts[FirstElt + I])
2525 ExtractMask[I] = FirstElt + I;
2526 }
2527
2528 Value *Extract = IC.Builder.CreateShuffleVector(V: Src, Mask: ExtractMask);
2529
2530 // TODO: Preserve callsite attributes?
2531 CallInst *NewCall = IC.Builder.CreateCall(Callee: Remangled, Args: {Extract}, OpBundles);
2532
2533 SmallVector<int> InsertMask(OldNumElts, -1);
2534 for (unsigned I = 0; I != MaskLen; ++I) {
2535 if (DemandedElts[FirstElt + I])
2536 InsertMask[FirstElt + I] = I;
2537 }
2538
2539 // FIXME: If the call has a convergence bundle, we end up leaving the dead
2540 // call behind.
2541 return IC.Builder.CreateShuffleVector(V: NewCall, Mask: InsertMask);
2542}
2543
2544std::optional<Value *> GCNTTIImpl::simplifyDemandedVectorEltsIntrinsic(
2545 InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
2546 APInt &UndefElts2, APInt &UndefElts3,
2547 std::function<void(Instruction *, unsigned, APInt, APInt &)>
2548 SimplifyAndSetOp) const {
2549 switch (II.getIntrinsicID()) {
2550 case Intrinsic::amdgcn_readfirstlane:
2551 SimplifyAndSetOp(&II, 0, DemandedElts, UndefElts);
2552 return simplifyAMDGCNLaneIntrinsicDemanded(IC, II, DemandedElts, UndefElts);
2553 case Intrinsic::amdgcn_raw_buffer_load:
2554 case Intrinsic::amdgcn_raw_ptr_buffer_load:
2555 case Intrinsic::amdgcn_raw_buffer_load_format:
2556 case Intrinsic::amdgcn_raw_ptr_buffer_load_format:
2557 case Intrinsic::amdgcn_raw_tbuffer_load:
2558 case Intrinsic::amdgcn_raw_ptr_tbuffer_load:
2559 case Intrinsic::amdgcn_s_buffer_load:
2560 case Intrinsic::amdgcn_ptr_s_buffer_load:
2561 case Intrinsic::amdgcn_struct_buffer_load:
2562 case Intrinsic::amdgcn_struct_ptr_buffer_load:
2563 case Intrinsic::amdgcn_struct_buffer_load_format:
2564 case Intrinsic::amdgcn_struct_ptr_buffer_load_format:
2565 case Intrinsic::amdgcn_struct_tbuffer_load:
2566 case Intrinsic::amdgcn_struct_ptr_tbuffer_load:
2567 return simplifyAMDGCNMemoryIntrinsicDemanded(IC, II, DemandedElts);
2568 default: {
2569 if (getAMDGPUImageDMaskIntrinsic(Intr: II.getIntrinsicID())) {
2570 return simplifyAMDGCNMemoryIntrinsicDemanded(IC, II, DemandedElts, DMaskIdx: 0);
2571 }
2572 break;
2573 }
2574 }
2575 return std::nullopt;
2576}
2577