1//===- AMDGPInstCombineIntrinsic.cpp - AMDGPU specific InstCombine pass ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// \file
10// This file implements a TargetTransformInfo analysis pass specific to the
11// AMDGPU target machine. It uses the target's detailed information to provide
12// more precise answers to certain TTI queries, while letting the target
13// independent and default TTI implementations handle the rest.
14//
15//===----------------------------------------------------------------------===//
16
17#include "AMDGPUInstrInfo.h"
18#include "AMDGPUTargetTransformInfo.h"
19#include "GCNSubtarget.h"
20#include "SIDefines.h"
21#include "llvm/ADT/FloatingPointMode.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/Analysis/ConstantFolding.h"
24#include "llvm/Analysis/ValueTracking.h"
25#include "llvm/IR/Constants.h"
26#include "llvm/IR/Dominators.h"
27#include "llvm/IR/IntrinsicsAMDGPU.h"
28#include "llvm/Support/MathExtras.h"
29#include "llvm/Transforms/InstCombine/InstCombiner.h"
30#include <optional>
31
32using namespace llvm;
33using namespace llvm::PatternMatch;
34
35#define DEBUG_TYPE "AMDGPUtti"
36
37namespace {
38
39struct AMDGPUImageDMaskIntrinsic {
40 unsigned Intr;
41};
42
43#define GET_AMDGPUImageDMaskIntrinsicTable_IMPL
44#include "AMDGPUGenSearchableTables.inc"
45
46} // end anonymous namespace
47
48// Constant fold llvm.amdgcn.fmed3 intrinsics for standard inputs.
49//
50// A single NaN input is folded to minnum, so we rely on that folding for
51// handling NaNs.
52static APFloat fmed3AMDGCN(const APFloat &Src0, const APFloat &Src1,
53 const APFloat &Src2) {
54 assert(!Src0.isNaN() && !Src1.isNaN() && !Src2.isNaN() &&
55 "nans handled separately");
56 APFloat Max3 = maxnum(A: maxnum(A: Src0, B: Src1), B: Src2);
57
58 if (Max3.bitwiseIsEqual(RHS: Src0))
59 return maxnum(A: Src1, B: Src2);
60
61 if (Max3.bitwiseIsEqual(RHS: Src1))
62 return maxnum(A: Src0, B: Src2);
63
64 return maxnum(A: Src0, B: Src1);
65}
66
67// Check if a value can be converted to a 16-bit value without losing precision.
68// The value is expected to be either a float (IsFloat = true) or an unsigned
69// integer (IsFloat = false). When AllowI16SExt is set, a sext from i16 is also
70// accepted: for unsigned addresses sext and zext only differ for a negative
71// i16, which is out of bounds anyway (see caller).
72static bool canSafelyConvertTo16Bit(Value &V, bool IsFloat,
73 bool AllowI16SExt = false) {
74 Type *VTy = V.getType();
75 if (VTy->isHalfTy() || VTy->isIntegerTy(BitWidth: 16)) {
76 // The value is already 16-bit, so we don't want to convert to 16-bit again!
77 return false;
78 }
79 if (IsFloat) {
80 if (ConstantFP *ConstFloat = dyn_cast<ConstantFP>(Val: &V)) {
81 // We need to check that if we cast the index down to a half, we do not
82 // lose precision.
83 APFloat FloatValue(ConstFloat->getValueAPF());
84 bool LosesInfo = true;
85 FloatValue.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmTowardZero,
86 losesInfo: &LosesInfo);
87 return !LosesInfo;
88 }
89 } else {
90 if (ConstantInt *ConstInt = dyn_cast<ConstantInt>(Val: &V)) {
91 // We need to check that if we cast the index down to an i16, we do not
92 // lose precision.
93 APInt IntValue(ConstInt->getValue());
94 return IntValue.getActiveBits() <= 16;
95 }
96 }
97
98 // Coordinates may arrive as extractelement((s|z|fp)ext Vec), Idx. The
99 // widening cast has one use per lane, so it is never sunk into the extract;
100 // strip the extract here so the cast check below is common to scalar and
101 // vector coords.
102 Value *CastCandidate;
103 if (!match(V: &V, P: m_ExtractElt(Val: m_Value(V&: CastCandidate), Idx: m_Value())))
104 CastCandidate = &V;
105
106 Value *CastSrc;
107 bool IsExt = IsFloat ? match(V: CastCandidate, P: m_FPExt(Op: m_Value(V&: CastSrc)))
108 : match(V: CastCandidate, P: m_ZExt(Op: m_Value(V&: CastSrc)));
109 if (!IsExt && !IsFloat && AllowI16SExt)
110 IsExt = match(V: CastCandidate, P: m_SExt(Op: m_Value(V&: CastSrc)));
111 if (IsExt) {
112 Type *CastSrcTy = CastSrc->getType()->getScalarType();
113 if (CastSrcTy->isHalfTy() || CastSrcTy->isIntegerTy(BitWidth: 16))
114 return true;
115 }
116
117 return false;
118}
119
120// Convert a value to 16-bit.
121static Value *convertTo16Bit(Value &V, InstCombiner::BuilderTy &Builder) {
122 Type *VTy = V.getType();
123 if (isa<FPExtInst, SExtInst, ZExtInst>(Val: &V))
124 return cast<Instruction>(Val: &V)->getOperand(i: 0);
125 // Vector form: extractelement((s|z|fp)ext Vec), Idx -> extractelement(Vec,
126 // Idx), taking the narrow lane directly so the widening cast can be removed.
127 Instruction *VecCast;
128 Value *Idx;
129 if (match(V: &V, P: m_ExtractElt(Val: m_Instruction(I&: VecCast), Idx: m_Value(V&: Idx))) &&
130 isa<FPExtInst, SExtInst, ZExtInst>(Val: VecCast))
131 return Builder.CreateExtractElement(Vec: VecCast->getOperand(i: 0), Idx);
132 if (VTy->isIntegerTy())
133 return Builder.CreateIntCast(V: &V, DestTy: Type::getInt16Ty(C&: V.getContext()), isSigned: false);
134 if (VTy->isFloatingPointTy())
135 return Builder.CreateFPCast(V: &V, DestTy: Type::getHalfTy(C&: V.getContext()));
136
137 llvm_unreachable("Should never be called!");
138}
139
140/// Applies Func(OldIntr.Args, OldIntr.ArgTys), creates intrinsic call with
141/// modified arguments (based on OldIntr) and replaces InstToReplace with
142/// this newly created intrinsic call.
143static std::optional<Instruction *> modifyIntrinsicCall(
144 IntrinsicInst &OldIntr, Instruction &InstToReplace, unsigned NewIntr,
145 InstCombiner &IC,
146 std::function<void(SmallVectorImpl<Value *> &, SmallVectorImpl<Type *> &)>
147 Func) {
148 SmallVector<Type *, 4> OverloadTys;
149 if (!Intrinsic::isSignatureValid(F: OldIntr.getCalledFunction(), OverloadTys))
150 return std::nullopt;
151
152 SmallVector<Value *, 8> Args(OldIntr.args());
153
154 // Modify arguments and types
155 Func(Args, OverloadTys);
156
157 CallInst *NewCall =
158 IC.Builder.CreateIntrinsicWithoutFolding(ID: NewIntr, OverloadTypes: OverloadTys, Args);
159 NewCall->takeName(V: &OldIntr);
160 NewCall->copyMetadata(SrcInst: OldIntr);
161 if (isa<FPMathOperator>(Val: NewCall))
162 NewCall->copyFastMathFlags(I: &OldIntr);
163 // Copy attributes
164 AttributeList OldAttrList = OldIntr.getAttributes();
165 NewCall->setAttributes(OldAttrList);
166
167 // Erase and replace uses
168 if (!InstToReplace.getType()->isVoidTy())
169 IC.replaceInstUsesWith(I&: InstToReplace, V: NewCall);
170
171 bool RemoveOldIntr = &OldIntr != &InstToReplace;
172
173 auto *RetValue = IC.eraseInstFromFunction(I&: InstToReplace);
174 if (RemoveOldIntr)
175 IC.eraseInstFromFunction(I&: OldIntr);
176
177 return RetValue;
178}
179
180static std::optional<Instruction *>
181simplifyAMDGCNImageIntrinsic(const GCNSubtarget *ST,
182 const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr,
183 IntrinsicInst &II, InstCombiner &IC) {
184 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
185 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: ImageDimIntr->BaseOpcode);
186
187 // Optimize _L to _LZ when _L is zero
188 if (const auto *LZMappingInfo =
189 AMDGPU::getMIMGLZMappingInfo(L: ImageDimIntr->BaseOpcode)) {
190 if (auto *ConstantLod =
191 dyn_cast<ConstantFP>(Val: II.getOperand(i_nocapture: ImageDimIntr->LodIndex))) {
192 if (ConstantLod->isZero() || ConstantLod->isNegative()) {
193 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
194 AMDGPU::getImageDimIntrinsicByBaseOpcode(BaseOpcode: LZMappingInfo->LZ,
195 Dim: ImageDimIntr->Dim);
196 return modifyIntrinsicCall(
197 OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC, Func: [&](auto &Args, auto &ArgTys) {
198 Args.erase(Args.begin() + ImageDimIntr->LodIndex);
199 });
200 }
201 }
202 }
203
204 // Optimize _mip away, when 'lod' is zero
205 if (const auto *MIPMappingInfo =
206 AMDGPU::getMIMGMIPMappingInfo(MIP: ImageDimIntr->BaseOpcode)) {
207 if (auto *ConstantMip =
208 dyn_cast<ConstantInt>(Val: II.getOperand(i_nocapture: ImageDimIntr->MipIndex))) {
209 if (ConstantMip->isZero()) {
210 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
211 AMDGPU::getImageDimIntrinsicByBaseOpcode(BaseOpcode: MIPMappingInfo->NONMIP,
212 Dim: ImageDimIntr->Dim);
213 return modifyIntrinsicCall(
214 OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC, Func: [&](auto &Args, auto &ArgTys) {
215 Args.erase(Args.begin() + ImageDimIntr->MipIndex);
216 });
217 }
218 }
219 }
220
221 // Optimize _bias away when 'bias' is zero
222 if (const auto *BiasMappingInfo =
223 AMDGPU::getMIMGBiasMappingInfo(Bias: ImageDimIntr->BaseOpcode)) {
224 if (auto *ConstantBias =
225 dyn_cast<ConstantFP>(Val: II.getOperand(i_nocapture: ImageDimIntr->BiasIndex))) {
226 if (ConstantBias->isZero()) {
227 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
228 AMDGPU::getImageDimIntrinsicByBaseOpcode(BaseOpcode: BiasMappingInfo->NoBias,
229 Dim: ImageDimIntr->Dim);
230 return modifyIntrinsicCall(
231 OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC, Func: [&](auto &Args, auto &ArgTys) {
232 Args.erase(Args.begin() + ImageDimIntr->BiasIndex);
233 ArgTys.erase(ArgTys.begin() + ImageDimIntr->BiasTyArg);
234 });
235 }
236 }
237 }
238
239 // Optimize _offset away when 'offset' is zero
240 if (const auto *OffsetMappingInfo =
241 AMDGPU::getMIMGOffsetMappingInfo(Offset: ImageDimIntr->BaseOpcode)) {
242 if (auto *ConstantOffset =
243 dyn_cast<ConstantInt>(Val: II.getOperand(i_nocapture: ImageDimIntr->OffsetIndex))) {
244 if (ConstantOffset->isZero()) {
245 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
246 AMDGPU::getImageDimIntrinsicByBaseOpcode(
247 BaseOpcode: OffsetMappingInfo->NoOffset, Dim: ImageDimIntr->Dim);
248 return modifyIntrinsicCall(
249 OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC, Func: [&](auto &Args, auto &ArgTys) {
250 Args.erase(Args.begin() + ImageDimIntr->OffsetIndex);
251 });
252 }
253 }
254 }
255
256 // Optimize the arrayed dim away when the array slice is zero, since slice 0
257 // is the base layer. Restricted to non-atomic, non-sampled image loads and
258 // stores for now.
259 const AMDGPU::MIMGDimInfo *DimInfo =
260 AMDGPU::getMIMGDimInfo(DimEnum: ImageDimIntr->Dim);
261 if (!BaseOpcode->Atomic && !BaseOpcode->Sampler && BaseOpcode->Coordinates &&
262 DimInfo->NonArrayDim != ImageDimIntr->Dim) {
263 // Address is [coords..., slice, (fragid)] plus an optional mip operand.
264 // The slice is the last coordinate, so index it from CoordStart.
265 unsigned SliceIndex = ImageDimIntr->CoordStart + DimInfo->NumCoords - 1 -
266 (DimInfo->MSAA ? 1 : 0);
267 auto *ConstantSlice = dyn_cast<ConstantInt>(Val: II.getOperand(i_nocapture: SliceIndex));
268 if (ConstantSlice && ConstantSlice->isZero()) {
269 if (const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
270 AMDGPU::getImageDimIntrinsicByBaseOpcode(BaseOpcode: ImageDimIntr->BaseOpcode,
271 Dim: DimInfo->NonArrayDim)) {
272 return modifyIntrinsicCall(OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC,
273 Func: [&](auto &Args, auto &ArgTys) {
274 Args.erase(Args.begin() + SliceIndex);
275 });
276 }
277 }
278 }
279
280 // Try to use D16
281 if (ST->hasD16Images()) {
282 if (BaseOpcode->HasD16) {
283
284 // If the only use of image intrinsic is a fptrunc (with conversion to
285 // half) then both fptrunc and image intrinsic will be replaced with image
286 // intrinsic with D16 flag.
287 if (II.hasOneUse()) {
288 Instruction *User = II.user_back();
289
290 if (User->getOpcode() == Instruction::FPTrunc &&
291 User->getType()->getScalarType()->isHalfTy()) {
292
293 return modifyIntrinsicCall(OldIntr&: II, InstToReplace&: *User, NewIntr: ImageDimIntr->Intr, IC,
294 Func: [&](auto &Args, auto &ArgTys) {
295 // Change return type of image intrinsic.
296 // Set it to return type of fptrunc.
297 ArgTys[0] = User->getType();
298 });
299 }
300 }
301
302 // Only perform D16 folding if every user of the image sample is
303 // an ExtractElementInst immediately followed by an FPTrunc to half.
304 SmallVector<std::pair<ExtractElementInst *, FPTruncInst *>, 4>
305 ExtractTruncPairs;
306 bool AllHalfExtracts = true;
307
308 for (User *U : II.users()) {
309 auto *Ext = dyn_cast<ExtractElementInst>(Val: U);
310 if (!Ext || !Ext->hasOneUse()) {
311 AllHalfExtracts = false;
312 break;
313 }
314
315 auto *Tr = dyn_cast<FPTruncInst>(Val: *Ext->user_begin());
316 if (!Tr || !Tr->getType()->isHalfTy()) {
317 AllHalfExtracts = false;
318 break;
319 }
320
321 ExtractTruncPairs.emplace_back(Args&: Ext, Args&: Tr);
322 }
323
324 if (!ExtractTruncPairs.empty() && AllHalfExtracts) {
325 auto *VecTy = cast<VectorType>(Val: II.getType());
326 Type *HalfVecTy =
327 VecTy->getWithNewType(EltTy: Type::getHalfTy(C&: II.getContext()));
328
329 // Obtain the original image sample intrinsic's signature
330 // and replace its return type with the half-vector for D16 folding
331 SmallVector<Type *, 8> OverloadTys;
332 if (!Intrinsic::isSignatureValid(F: II.getCalledFunction(), OverloadTys))
333 return std::nullopt;
334
335 OverloadTys[0] = HalfVecTy;
336 Module *M = II.getModule();
337 Function *HalfDecl = Intrinsic::getOrInsertDeclaration(
338 M, id: ImageDimIntr->Intr, OverloadTys);
339
340 II.mutateType(Ty: HalfVecTy);
341 II.setCalledFunction(HalfDecl);
342
343 IRBuilder<> Builder(*M);
344 for (auto &[Ext, Tr] : ExtractTruncPairs) {
345 Value *Idx = Ext->getIndexOperand();
346
347 Builder.SetInsertPoint(Tr);
348
349 Value *HalfExtract = Builder.CreateExtractElement(Vec: &II, Idx);
350 HalfExtract->takeName(V: Tr);
351
352 Tr->replaceAllUsesWith(V: HalfExtract);
353 }
354
355 for (auto &[Ext, Tr] : ExtractTruncPairs) {
356 IC.eraseInstFromFunction(I&: *Tr);
357 IC.eraseInstFromFunction(I&: *Ext);
358 }
359
360 return &II;
361 }
362 }
363 }
364
365 // Try to use A16 or G16
366 if (!ST->hasA16() && !ST->hasG16())
367 return std::nullopt;
368
369 // Address is interpreted as float if the instruction has a sampler or as
370 // unsigned int if there is no sampler.
371 bool HasSampler = BaseOpcode->Sampler;
372 bool FloatCoord = false;
373 // true means derivatives can be converted to 16 bit, coordinates not
374 bool OnlyDerivatives = false;
375
376 // Sampler-less addresses are unsigned, so a sext from i16 folds to a16 like a
377 // zext: they only disagree for a negative i16 (>= 0x8000), which is out of
378 // bounds while the max image dimension is <= 0x8000.
379 bool AllowI16SExt = !HasSampler;
380
381 for (unsigned OperandIndex = ImageDimIntr->GradientStart;
382 OperandIndex < ImageDimIntr->VAddrEnd; OperandIndex++) {
383 Value *Coord = II.getOperand(i_nocapture: OperandIndex);
384 // If the values are not derived from 16-bit values, we cannot optimize.
385 if (!canSafelyConvertTo16Bit(V&: *Coord, IsFloat: HasSampler, AllowI16SExt)) {
386 if (OperandIndex < ImageDimIntr->CoordStart ||
387 ImageDimIntr->GradientStart == ImageDimIntr->CoordStart) {
388 return std::nullopt;
389 }
390 // All gradients can be converted, so convert only them
391 OnlyDerivatives = true;
392 break;
393 }
394
395 assert(OperandIndex == ImageDimIntr->GradientStart ||
396 FloatCoord == Coord->getType()->isFloatingPointTy());
397 FloatCoord = Coord->getType()->isFloatingPointTy();
398 }
399
400 if (!OnlyDerivatives && !ST->hasA16())
401 OnlyDerivatives = true; // Only supports G16
402
403 // Check if there is a bias parameter and if it can be converted to f16
404 if (!OnlyDerivatives && ImageDimIntr->NumBiasArgs != 0) {
405 Value *Bias = II.getOperand(i_nocapture: ImageDimIntr->BiasIndex);
406 assert(HasSampler &&
407 "Only image instructions with a sampler can have a bias");
408 if (!canSafelyConvertTo16Bit(V&: *Bias, IsFloat: HasSampler))
409 OnlyDerivatives = true;
410 }
411
412 if (OnlyDerivatives && (!ST->hasG16() || ImageDimIntr->GradientStart ==
413 ImageDimIntr->CoordStart))
414 return std::nullopt;
415
416 Type *CoordType = FloatCoord ? Type::getHalfTy(C&: II.getContext())
417 : Type::getInt16Ty(C&: II.getContext());
418
419 return modifyIntrinsicCall(
420 OldIntr&: II, InstToReplace&: II, NewIntr: II.getIntrinsicID(), IC, Func: [&](auto &Args, auto &ArgTys) {
421 ArgTys[ImageDimIntr->GradientTyArg] = CoordType;
422 if (!OnlyDerivatives) {
423 ArgTys[ImageDimIntr->CoordTyArg] = CoordType;
424
425 // Change the bias type
426 if (ImageDimIntr->NumBiasArgs != 0)
427 ArgTys[ImageDimIntr->BiasTyArg] = Type::getHalfTy(C&: II.getContext());
428 }
429
430 unsigned EndIndex =
431 OnlyDerivatives ? ImageDimIntr->CoordStart : ImageDimIntr->VAddrEnd;
432 for (unsigned OperandIndex = ImageDimIntr->GradientStart;
433 OperandIndex < EndIndex; OperandIndex++) {
434 Args[OperandIndex] =
435 convertTo16Bit(V&: *II.getOperand(i_nocapture: OperandIndex), Builder&: IC.Builder);
436 }
437
438 // Convert the bias
439 if (!OnlyDerivatives && ImageDimIntr->NumBiasArgs != 0) {
440 Value *Bias = II.getOperand(i_nocapture: ImageDimIntr->BiasIndex);
441 Args[ImageDimIntr->BiasIndex] = convertTo16Bit(V&: *Bias, Builder&: IC.Builder);
442 }
443 });
444}
445
446bool GCNTTIImpl::canSimplifyLegacyMulToMul(const Instruction &I,
447 const Value *Op0, const Value *Op1,
448 InstCombiner &IC) const {
449 // The legacy behaviour is that multiplying +/-0.0 by anything, even NaN or
450 // infinity, gives +0.0. If we can prove we don't have one of the special
451 // cases then we can use a normal multiply instead.
452 SimplifyQuery SQ = IC.getSimplifyQuery().getWithInstruction(I: &I);
453 KnownFPClass Known0 =
454 computeKnownFPClass(V: Op0, InterestedClasses: fcZero | fcSubnormal | fcInf | fcNan, SQ);
455 DenormalMode Mode = I.getFunction()->getDenormalMode(FPType: APFloat::IEEEsingle());
456
457 // Bail early if Op0 may be zero and nsz is not set -- Op1 cannot help.
458 if (!Known0.isKnownNeverLogicalZero(Mode) && !I.hasNoSignedZeros())
459 return false;
460
461 KnownFPClass Known1 =
462 computeKnownFPClass(V: Op1, InterestedClasses: fcZero | fcSubnormal | fcInf | fcNan, SQ);
463
464 // Simplify if both operands are known non-zero.
465 if (Known0.isKnownNeverLogicalZero(Mode) &&
466 Known1.isKnownNeverLogicalZero(Mode))
467 return true;
468
469 // With nsz, two additional cases allow simplification:
470 // 1. One operand is not zero or infinity or NaN:
471 // Op0 NeverLogicalZero && NeverInfOrNaN, or symmetric for Op1.
472 // 2. Neither operand is infinity or NaN:
473 // Op0 NeverInfOrNaN && Op1 NeverInfOrNaN.
474 // The following condition captures both cases.
475 if (I.hasNoSignedZeros() &&
476 (Known0.isKnownNeverLogicalZero(Mode) || Known1.isKnownNeverInfOrNaN()) &&
477 (Known1.isKnownNeverLogicalZero(Mode) || Known0.isKnownNeverInfOrNaN()))
478 return true;
479
480 return false;
481}
482
483/// Match an fpext from half to float, or a constant we can convert.
484static Value *matchFPExtFromF16(Value *Arg) {
485 Value *Src = nullptr;
486 ConstantFP *CFP = nullptr;
487 if (match(V: Arg, P: m_OneUse(SubPattern: m_FPExt(Op: m_Value(V&: Src))))) {
488 if (Src->getType()->isHalfTy())
489 return Src;
490 } else if (match(V: Arg, P: m_ConstantFP(C&: CFP))) {
491 bool LosesInfo;
492 APFloat Val(CFP->getValueAPF());
493 Val.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmNearestTiesToEven, losesInfo: &LosesInfo);
494 if (!LosesInfo)
495 return ConstantFP::get(Ty: Type::getHalfTy(C&: Arg->getContext()), V: Val);
496 }
497 return nullptr;
498}
499
500// Trim all zero components from the end of the vector \p UseV and return
501// an appropriate bitset with known elements.
502static APInt trimTrailingZerosInVector(InstCombiner &IC, Value *UseV,
503 Instruction *I) {
504 auto *VTy = cast<FixedVectorType>(Val: UseV->getType());
505 unsigned VWidth = VTy->getNumElements();
506 APInt DemandedElts = APInt::getAllOnes(numBits: VWidth);
507
508 for (int i = VWidth - 1; i > 0; --i) {
509 auto *Elt = findScalarElement(V: UseV, EltNo: i);
510 if (!Elt)
511 break;
512
513 if (auto *ConstElt = dyn_cast<Constant>(Val: Elt)) {
514 if (!ConstElt->isNullValue() && !isa<UndefValue>(Val: Elt))
515 break;
516 } else {
517 break;
518 }
519
520 DemandedElts.clearBit(BitPosition: i);
521 }
522
523 return DemandedElts;
524}
525
526// Trim elements of the end of the vector \p V, if they are
527// equal to the first element of the vector.
528static APInt defaultComponentBroadcast(Value *V) {
529 auto *VTy = cast<FixedVectorType>(Val: V->getType());
530 unsigned VWidth = VTy->getNumElements();
531 APInt DemandedElts = APInt::getAllOnes(numBits: VWidth);
532 Value *FirstComponent = findScalarElement(V, EltNo: 0);
533
534 SmallVector<int> ShuffleMask;
535 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: V))
536 SVI->getShuffleMask(Result&: ShuffleMask);
537
538 for (int I = VWidth - 1; I > 0; --I) {
539 if (ShuffleMask.empty()) {
540 auto *Elt = findScalarElement(V, EltNo: I);
541 if (!Elt || (Elt != FirstComponent && !isa<UndefValue>(Val: Elt)))
542 break;
543 } else {
544 // Detect identical elements in the shufflevector result, even though
545 // findScalarElement cannot tell us what that element is.
546 if (ShuffleMask[I] != ShuffleMask[0] && ShuffleMask[I] != PoisonMaskElem)
547 break;
548 }
549 DemandedElts.clearBit(BitPosition: I);
550 }
551
552 return DemandedElts;
553}
554
555/// Return true if it's legal to contract llvm.amdgcn.rcp(llvm.sqrt)
556static bool canContractSqrtToRsq(const FPMathOperator *SqrtOp) {
557 return (SqrtOp->getType()->isFloatTy() &&
558 (SqrtOp->hasApproxFunc() || SqrtOp->getFPAccuracy() >= 1.0f)) ||
559 SqrtOp->getType()->isHalfTy();
560}
561
562/// Simplify a lane index operand (e.g. llvm.amdgcn.readlane src1).
563///
564/// The instruction only reads the low 5 bits for wave32, and 6 bits for wave64.
565bool GCNTTIImpl::simplifyDemandedLaneMaskArg(InstCombiner &IC,
566 IntrinsicInst &II,
567 unsigned LaneArgIdx) const {
568 unsigned MaskBits = ST->getWavefrontSizeLog2();
569 APInt DemandedMask(32, maskTrailingOnes<unsigned>(N: MaskBits));
570
571 KnownBits Known(32);
572 if (IC.SimplifyDemandedBits(I: &II, OpNo: LaneArgIdx, DemandedMask, Known))
573 return true;
574
575 if (!Known.isConstant())
576 return false;
577
578 // Out of bounds indexes may appear in wave64 code compiled for wave32.
579 // Unlike the DAG version, SimplifyDemandedBits does not change constants, so
580 // manually fix it up.
581
582 Value *LaneArg = II.getArgOperand(i: LaneArgIdx);
583 Constant *MaskedConst =
584 ConstantInt::get(Ty: LaneArg->getType(), V: Known.getConstant() & DemandedMask);
585 if (MaskedConst != LaneArg) {
586 II.getOperandUse(i: LaneArgIdx).set(MaskedConst);
587 return true;
588 }
589
590 return false;
591}
592
593/// Simplify the f32 scale operand of a V_CVT_SCALEF32_* conversion, of which
594/// only the exponent field is read.
595static bool simplifyExponentOnlyScaleArg(InstCombiner &IC, IntrinsicInst &II,
596 unsigned ScaleArgIdx) {
597 Value *Scale = II.getArgOperand(i: ScaleArgIdx);
598 Value *StrippedSign = InstCombiner::stripSignOnlyFPOps(Val: Scale);
599 if (StrippedSign != Scale) {
600 IC.replaceOperand(I&: II, OpNum: ScaleArgIdx, V: StrippedSign);
601 return true;
602 }
603
604 auto *BC = dyn_cast<BitCastInst>(Val: Scale);
605 if (!BC || !BC->hasOneUse() || !BC->getSrcTy()->isIntegerTy(BitWidth: 32))
606 return false;
607
608 KnownBits Known(32);
609 return IC.SimplifyDemandedBits(
610 I: BC, OpNo: 0, DemandedMask: AMDGPU::getExponentOnlyScaleDemandedBits(), Known);
611}
612
613static CallInst *rewriteCall(IRBuilderBase &B, CallInst &Old,
614 Function &NewCallee, ArrayRef<Value *> Ops) {
615 SmallVector<OperandBundleDef, 2> OpBundles;
616 Old.getOperandBundlesAsDefs(Defs&: OpBundles);
617
618 CallInst *NewCall = B.CreateCall(Callee: &NewCallee, Args: Ops, OpBundles);
619 NewCall->takeName(V: &Old);
620 return NewCall;
621}
622
623// Return true for sequences of instructions that effectively assign
624// each lane to its thread ID
625static bool isThreadID(const GCNSubtarget &ST, Value *V) {
626 // Case 1:
627 // wave32: mbcnt_lo(-1, 0)
628 // wave64: mbcnt_hi(-1, mbcnt_lo(-1, 0))
629 auto W32Pred = m_Intrinsic<Intrinsic::amdgcn_mbcnt_lo>(Ops: m_ConstantInt<-1>(),
630 Ops: m_ConstantInt<0>());
631 auto W64Pred = m_Intrinsic<Intrinsic::amdgcn_mbcnt_hi>(
632 Ops: m_ConstantInt<-1>(), Ops: m_Intrinsic<Intrinsic::amdgcn_mbcnt_lo>(
633 Ops: m_ConstantInt<-1>(), Ops: m_ConstantInt<0>()));
634 if (ST.isWave32() && match(V, P: W32Pred))
635 return true;
636 if (ST.isWave64() && match(V, P: W64Pred))
637 return true;
638
639 // Case 2:
640 // workitem.id.x & (wavefront size - 1), or bare workitem.id.x when the
641 // workgroup is one wave long or less.
642 auto *I = dyn_cast<Instruction>(Val: V);
643 if (!I || !ST.isWaveSizeKnown())
644 return false;
645
646 const Function &F = *I->getFunction();
647 if (!ST.hasWavefrontsEvenlySplittingXDim(F, /*RequiresUniformYZ=*/true))
648 return false;
649
650 unsigned WaveSize = ST.getWavefrontSize();
651 auto TidX = m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>();
652 if (match(V, P: m_c_And(L: TidX, R: m_SpecificInt(V: WaveSize - 1))))
653 return true;
654
655 return match(V, P: TidX) && ST.getMaxWorkitemID(Kernel: F, Dimension: 0) < WaveSize;
656}
657
658Instruction *
659GCNTTIImpl::hoistLaneIntrinsicThroughOperand(InstCombiner &IC,
660 IntrinsicInst &II) const {
661 const auto IID = II.getIntrinsicID();
662 assert(IID == Intrinsic::amdgcn_readlane ||
663 IID == Intrinsic::amdgcn_readfirstlane ||
664 IID == Intrinsic::amdgcn_permlane64);
665
666 Instruction *OpInst = dyn_cast<Instruction>(Val: II.getOperand(i_nocapture: 0));
667
668 // Only do this if both instructions are in the same block
669 // (so the exec mask won't change) and the readlane is the only user of its
670 // operand.
671 if (!OpInst || !OpInst->hasOneUser() || OpInst->getParent() != II.getParent())
672 return nullptr;
673
674 const bool IsReadLane = (IID == Intrinsic::amdgcn_readlane);
675
676 // If this is a readlane, check that the second operand is a constant, or is
677 // defined before OpInst so we know it's safe to move this intrinsic higher.
678 Value *LaneID = nullptr;
679 if (IsReadLane) {
680 LaneID = II.getOperand(i_nocapture: 1);
681
682 // readlane take an extra operand for the lane ID, so we must check if that
683 // LaneID value can be used at the point where we want to move the
684 // intrinsic.
685 if (auto *LaneIDInst = dyn_cast<Instruction>(Val: LaneID)) {
686 if (!IC.getDominatorTree().dominates(Def: LaneIDInst, User: OpInst))
687 return nullptr;
688 }
689 }
690
691 // Hoist the intrinsic (II) through OpInst.
692 //
693 // (II (OpInst x)) -> (OpInst (II x))
694 const auto DoIt = [&](unsigned OpIdx,
695 Function *NewIntrinsic) -> Instruction * {
696 SmallVector<Value *, 2> Ops{OpInst->getOperand(i: OpIdx)};
697 if (IsReadLane)
698 Ops.push_back(Elt: LaneID);
699
700 // Rewrite the intrinsic call.
701 CallInst *NewII = rewriteCall(B&: IC.Builder, Old&: II, NewCallee&: *NewIntrinsic, Ops);
702
703 // Rewrite OpInst so it takes the result of the intrinsic now.
704 Instruction &NewOp = *OpInst->clone();
705 NewOp.setOperand(i: OpIdx, Val: NewII);
706 return &NewOp;
707 };
708
709 // TODO(?): Should we do more with permlane64?
710 if (IID == Intrinsic::amdgcn_permlane64 && !isa<BitCastInst>(Val: OpInst))
711 return nullptr;
712
713 if (isa<UnaryOperator>(Val: OpInst))
714 return DoIt(0, II.getCalledFunction());
715
716 if (isa<CastInst>(Val: OpInst)) {
717 Value *Src = OpInst->getOperand(i: 0);
718 Type *SrcTy = Src->getType();
719 if (!isTypeLegal(Ty: SrcTy))
720 return nullptr;
721
722 Function *Remangled =
723 Intrinsic::getOrInsertDeclaration(M: II.getModule(), id: IID, OverloadTys: {SrcTy});
724 return DoIt(0, Remangled);
725 }
726
727 // We can also hoist through binary operators if the other operand is uniform.
728 if (isa<BinaryOperator>(Val: OpInst)) {
729 // FIXME: If we had access to UniformityInfo here we could just check
730 // if the operand is uniform.
731 if (AMDGPU::isTriviallyUniform(U: OpInst->getOperandUse(i: 0)))
732 return DoIt(1, II.getCalledFunction());
733 if (AMDGPU::isTriviallyUniform(U: OpInst->getOperandUse(i: 1)))
734 return DoIt(0, II.getCalledFunction());
735 }
736
737 return nullptr;
738}
739
740/// Evaluate V as a function of the lane ID and return its value on Lane, or
741/// std::nullopt if V is not a closed-form expression of the lane ID.
742static std::optional<uint64_t> evalLaneExpr(Value *V, unsigned Lane,
743 const GCNSubtarget &ST,
744 const DataLayout &DL,
745 unsigned Depth = 0) {
746 if (Depth >= MaxAnalysisRecursionDepth)
747 return std::nullopt;
748
749 // Poison/undef in the index expression: bail and let InstCombine fold the
750 // intrinsic the usual way.
751 if (isa<UndefValue>(Val: V))
752 return std::nullopt;
753
754 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: V))
755 return CI->getZExtValue();
756
757 if (isThreadID(ST, V))
758 return Lane;
759
760 const BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: V);
761 if (!BO)
762 return std::nullopt;
763
764 std::optional<uint64_t> LHS =
765 evalLaneExpr(V: BO->getOperand(i_nocapture: 0), Lane, ST, DL, Depth: Depth + 1);
766 if (!LHS)
767 return std::nullopt;
768 std::optional<uint64_t> RHS =
769 evalLaneExpr(V: BO->getOperand(i_nocapture: 1), Lane, ST, DL, Depth: Depth + 1);
770 if (!RHS)
771 return std::nullopt;
772
773 Type *Ty = BO->getType();
774 Constant *Ops[] = {ConstantInt::get(Ty, V: *LHS), ConstantInt::get(Ty, V: *RHS)};
775 auto *CI =
776 dyn_cast_or_null<ConstantInt>(Val: ConstantFoldInstOperands(I: BO, Ops, DL));
777 return CI ? std::optional<uint64_t>(CI->getZExtValue()) : std::nullopt;
778}
779
780/// Build the per-lane shuffle map by evaluating Index for every lane in the
781/// wave. Returns false if any lane index is non-constant or out of range.
782static bool tryBuildShuffleMap(Value *Index, const GCNSubtarget &ST,
783 SmallVectorImpl<uint8_t> &Ids,
784 const DataLayout &DL) {
785 unsigned WaveSize = ST.getWavefrontSize();
786 Ids.resize(N: WaveSize);
787 for (unsigned Lane : seq(Size: WaveSize)) {
788 std::optional<uint64_t> Val = evalLaneExpr(V: Index, Lane, ST, DL);
789 if (!Val || *Val >= WaveSize)
790 return false;
791 Ids[Lane] = static_cast<uint8_t>(*Val);
792 }
793 return true;
794}
795
796/// Lanes are partitioned into groups of Period; each group is a translated
797/// copy of the first: Ids[I] = Ids[I % Period] + (I & ~(Period - 1)).
798template <unsigned Period>
799static bool hasPeriodicLayout(ArrayRef<uint8_t> Ids) {
800 static_assert(isPowerOf2_32(Value: Period), "Period must be a power of two");
801 for (size_t I = Period, E = Ids.size(); I < E; ++I)
802 if (Ids[I] != Ids[I % Period] + (I & ~(Period - 1)))
803 return false;
804 return true;
805}
806
807/// Match an N-lane row pattern: each lane in [0, N) reads from a source lane
808/// in the same N-lane row, and the pattern repeats periodically across rows.
809template <unsigned N> static bool isRowPattern(ArrayRef<uint8_t> Ids) {
810 for (unsigned I = 0; I < N; ++I)
811 if (Ids[I] >= N)
812 return false;
813 return hasPeriodicLayout<N>(Ids);
814}
815
816static constexpr auto isQuadPattern = isRowPattern<4>;
817static constexpr auto isHalfRowPattern = isRowPattern<8>;
818static constexpr auto isFullRowPattern = isRowPattern<16>;
819
820/// Match a 4-lane (quad) permutation, encoded as the v_mov_b32_dpp
821/// QUAD_PERM control word: bits[1:0]=Ids[0], [3:2]=Ids[1], [5:4]=Ids[2],
822/// [7:6]=Ids[3].
823static std::optional<unsigned> matchQuadPermPattern(ArrayRef<uint8_t> Ids) {
824 if (!isQuadPattern(Ids))
825 return std::nullopt;
826 return Ids[3] << 6 | Ids[2] << 4 | Ids[1] << 2 | Ids[0];
827}
828
829/// Match an N-lane reversal (mirror) pattern.
830template <unsigned N> static bool matchMirrorPattern(ArrayRef<uint8_t> Ids) {
831 if (!isRowPattern<N>(Ids))
832 return false;
833 for (unsigned J = 0; J < N; ++J)
834 if (Ids[J] != (N - 1) - J)
835 return false;
836 return true;
837}
838
839static constexpr auto matchHalfRowMirrorPattern = matchMirrorPattern<8>;
840static constexpr auto matchFullRowMirrorPattern = matchMirrorPattern<16>;
841
842/// Match a 16-lane cyclic rotation; returns the rotation amount in [1, 15].
843static std::optional<unsigned> matchRowRotatePattern(ArrayRef<uint8_t> Ids) {
844 if (Ids[0] == 0 || !isFullRowPattern(Ids))
845 return std::nullopt;
846 for (unsigned J = 1; J < 16; ++J)
847 if (Ids[J] != (Ids[0] + J) % 16)
848 return std::nullopt;
849 return 16u - Ids[0];
850}
851
852/// Match a row-share pattern: all 16 lanes of each row read the same source
853/// lane. Returns the shared source lane index in [0, 16).
854static std::optional<unsigned> matchRowSharePattern(ArrayRef<uint8_t> Ids) {
855 if (!isFullRowPattern(Ids))
856 return std::nullopt;
857 if (!all_equal(Range: Ids.take_front(N: 16)))
858 return std::nullopt;
859 return Ids[0];
860}
861
862/// Match an XOR mask pattern within each 16-lane row: Ids[J] == Mask ^ J,
863/// with Mask in [1, 15].
864static std::optional<unsigned> matchRowXMaskPattern(ArrayRef<uint8_t> Ids) {
865 unsigned Mask = Ids[0];
866 if (Mask == 0 || !isFullRowPattern(Ids))
867 return std::nullopt;
868 for (unsigned J = 0; J < 16; ++J)
869 if (Ids[J] != (Mask ^ J))
870 return std::nullopt;
871 return Mask;
872}
873
874/// Match an 8-lane arbitrary permutation, encoded as the v_mov_b32_dpp8
875/// 24-bit selector (three bits per output lane).
876static std::optional<unsigned> matchHalfRowPermPattern(ArrayRef<uint8_t> Ids) {
877 if (!isHalfRowPattern(Ids))
878 return std::nullopt;
879 unsigned Selector = 0;
880 for (unsigned J = 0; J < 8; ++J)
881 Selector |= Ids[J] << (J * 3);
882 return Selector;
883}
884
885/// Pack a 16-lane permutation into a single 64-bit value: four bits per output
886/// lane, lane J in bits [J*4 + 3 : J*4]. The caller splits it into the low and
887/// high 32-bit selector operands of v_permlane16 / v_permlanex16.
888static uint64_t computePermlane16Masks(ArrayRef<uint8_t> Ids) {
889 uint64_t Sel = 0;
890 for (unsigned J = 0; J < 16; ++J)
891 Sel |= static_cast<uint64_t>(Ids[J] & 0xF) << (J * 4);
892 return Sel;
893}
894
895/// Match a half-wave swap: lane J reads from lane J ^ 32. Only meaningful on
896/// wave64 targets.
897static bool matchHalfWaveSwapPattern(ArrayRef<uint8_t> Ids) {
898 if (Ids.size() != 64)
899 return false;
900 for (unsigned J = 0; J < 64; ++J)
901 if (Ids[J] != (J ^ 32))
902 return false;
903 return true;
904}
905
906/// Match a cross-row permutation suitable for v_permlanex16: every lane in
907/// the low 16-lane half reads from the high half of its own row, and vice
908/// versa.
909static bool isCrossRowPattern(ArrayRef<uint8_t> Ids) {
910 if (!hasPeriodicLayout<32>(Ids))
911 return false;
912 for (unsigned J = 0; J < 16; ++J) {
913 if (Ids[J] < 16 || Ids[J] >= 32)
914 return false;
915 if (Ids[J + 16] != Ids[J] - 16)
916 return false;
917 }
918 return true;
919}
920
921/// Match a DS_SWIZZLE bitmask-mode permutation:
922/// dst_lane = ((src_lane & AND) | OR) ^ XOR
923/// with each mask being five bits. Returns the encoded swizzle immediate.
924/// The hardware applies the formula independently within each 32-lane group,
925/// so on wave64 the high group must replicate the low one (translated by 32).
926static std::optional<unsigned>
927matchDsSwizzleBitmaskPattern(ArrayRef<uint8_t> Ids) {
928 if (!hasPeriodicLayout<32>(Ids))
929 return std::nullopt;
930
931 // The formula is per-bit: output bit B depends only on input bit B. Probe
932 // each bit with src=0 and src=(1<<B); if the output bit flipped, AND[B]=1
933 // and XOR[B] carries the constant offset; otherwise it is a constant bit
934 // encoded in OR (with AND[B]=0, XOR[B]=0).
935 unsigned AndMask = 0, OrMask = 0, XorMask = 0;
936 for (unsigned B = 0; B < 5; ++B) {
937 unsigned Bit0 = (Ids[0] >> B) & 1;
938 unsigned Bit1 = (Ids[1ull << B] >> B) & 1;
939 if (Bit0 != Bit1) {
940 AndMask |= 1u << B;
941 XorMask |= Bit0 << B;
942 } else {
943 OrMask |= Bit0 << B;
944 }
945 }
946
947 // The per-bit derivation assumes bit independence; verify the masks
948 // actually reproduce every lane in the 32-lane group.
949 for (unsigned I : seq(Size: 32u)) {
950 unsigned Expected = ((I & AndMask) | OrMask) ^ XorMask;
951 if (Ids[I] != Expected)
952 return std::nullopt;
953 }
954
955 return AMDGPU::Swizzle::BITMASK_PERM_ENC |
956 AndMask << AMDGPU::Swizzle::BITMASK_AND_SHIFT |
957 OrMask << AMDGPU::Swizzle::BITMASK_OR_SHIFT |
958 XorMask << AMDGPU::Swizzle::BITMASK_XOR_SHIFT;
959}
960
961/// Match a GFX9+ DS_SWIZZLE rotate-mode permutation: a cyclic left-rotation
962/// of all 32 lanes within each 32-lane group by a constant N in [0, 31],
963/// i.e. dst_lane = (src_lane + N) % 32. On wave64, hasPeriodicLayout<32>
964/// ensures both 32-lane groups rotate by the same amount.
965static std::optional<unsigned>
966matchDsSwizzleRotatePattern(ArrayRef<uint8_t> Ids) {
967 if (!hasPeriodicLayout<32>(Ids))
968 return std::nullopt;
969
970 // Determine the rotation amount from lane 0: every lane must read from
971 // lane (I + N) % 32 where N = Ids[0] and 0 <= N <= 31.
972 unsigned N = Ids[0];
973 if (N >= 32)
974 return std::nullopt;
975
976 for (unsigned I = 0; I < 32; ++I)
977 if (Ids[I] != (I + N) % 32)
978 return std::nullopt;
979
980 return AMDGPU::Swizzle::ROTATE_MODE_ENC |
981 (N << AMDGPU::Swizzle::ROTATE_SIZE_SHIFT);
982}
983
984/// Emit v_mov_b32_dpp with the given control word, row/bank masks 0xF, and
985/// bound_ctrl=1 so out-of-bounds lanes are well-defined and the DPP mov can
986/// be folded into a consuming VALU op by GCNDPPCombine.
987static Value *createUpdateDpp(IRBuilderBase &B, Value *Val, unsigned Ctrl) {
988 Type *Ty = Val->getType();
989 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_update_dpp, OverloadTypes: {Ty},
990 Args: {PoisonValue::get(T: Ty), Val, B.getInt32(C: Ctrl),
991 B.getInt32(C: 0xF), B.getInt32(C: 0xF), B.getTrue()});
992}
993
994/// Emit v_mov_b32_dpp8 with the given 24-bit lane selector.
995static Value *createMovDpp8(IRBuilderBase &B, Value *Val, unsigned Selector) {
996 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_mov_dpp8, OverloadTypes: {Val->getType()},
997 Args: {Val, B.getInt32(C: Selector)});
998}
999
1000/// Emit v_permlane16 with the precomputed lane-select halves.
1001static Value *createPermlane16(IRBuilderBase &B, Value *Val, uint32_t Lo,
1002 uint32_t Hi) {
1003 Type *Ty = Val->getType();
1004 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_permlane16, OverloadTypes: {Ty},
1005 Args: {PoisonValue::get(T: Ty), Val, B.getInt32(C: Lo),
1006 B.getInt32(C: Hi), B.getFalse(), B.getFalse()});
1007}
1008
1009/// Emit v_permlanex16 with the precomputed lane-select halves. Each output
1010/// lane reads from the other 16-lane half of the same row.
1011static Value *createPermlaneX16(IRBuilderBase &B, Value *Val, uint32_t Lo,
1012 uint32_t Hi) {
1013 Type *Ty = Val->getType();
1014 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_permlanex16, OverloadTypes: {Ty},
1015 Args: {PoisonValue::get(T: Ty), Val, B.getInt32(C: Lo),
1016 B.getInt32(C: Hi), B.getFalse(), B.getFalse()});
1017}
1018
1019/// Emit ds_swizzle with the given immediate, bitcasting/converting between
1020/// pointer/float types and i32 as required by the intrinsic signature.
1021static Value *createDsSwizzle(IRBuilderBase &B, Value *Val, unsigned Offset,
1022 const DataLayout &DL) {
1023 Type *OrigTy = Val->getType();
1024 assert(DL.getTypeSizeInBits(OrigTy) == 32 &&
1025 "ds_swizzle only supports 32-bit operands");
1026 IntegerType *I32Ty = B.getInt32Ty();
1027 Value *Src = Val;
1028 if (OrigTy->isPointerTy())
1029 Src = B.CreatePtrToInt(V: Src, DestTy: I32Ty);
1030 else if (OrigTy != I32Ty)
1031 Src = B.CreateBitCast(V: Src, DestTy: I32Ty);
1032 Value *Result = B.CreateIntrinsic(ID: Intrinsic::amdgcn_ds_swizzle, OverloadTypes: {},
1033 Args: {Src, B.getInt32(C: Offset)});
1034 if (OrigTy->isPointerTy())
1035 return B.CreateIntToPtr(V: Result, DestTy: OrigTy);
1036 if (OrigTy != I32Ty)
1037 return B.CreateBitCast(V: Result, DestTy: OrigTy);
1038 return Result;
1039}
1040
1041/// Emit v_permlane64 (swap of the two 32-lane halves of a wave64).
1042static Value *createPermlane64(IRBuilderBase &B, Value *Val) {
1043 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_permlane64, OverloadTypes: {Val->getType()},
1044 Args: {Val});
1045}
1046
1047/// Given a shuffle map, try to emit the best hardware intrinsic.
1048static Value *matchShuffleToHWIntrinsic(IRBuilderBase &B, Value *Src,
1049 ArrayRef<uint8_t> Ids,
1050 const GCNSubtarget &ST,
1051 const DataLayout &DL) {
1052 // Identity shuffle (every lane reads itself) folds to the source value.
1053 if (all_of(Range: enumerate(First&: Ids),
1054 P: [](const auto &E) { return E.value() == E.index(); }))
1055 return Src;
1056
1057 // Uniform shuffle (all lanes read the same value) is handled by cheaper
1058 // broadcast/readlane intrinsics.
1059 if (all_equal(Range&: Ids))
1060 return nullptr;
1061
1062 if (std::optional<unsigned> QP = matchQuadPermPattern(Ids)) {
1063 if (ST.hasDPP())
1064 return createUpdateDpp(B, Val: Src, Ctrl: *QP);
1065 return createDsSwizzle(B, Val: Src, Offset: AMDGPU::Swizzle::QUAD_PERM_ENC | *QP, DL);
1066 }
1067
1068 if (ST.hasDPP()) {
1069 if (matchHalfRowMirrorPattern(Ids))
1070 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_HALF_MIRROR);
1071 if (matchFullRowMirrorPattern(Ids))
1072 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_MIRROR);
1073 if (std::optional<unsigned> Amt = matchRowRotatePattern(Ids))
1074 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_ROR_FIRST + *Amt - 1);
1075 }
1076
1077 // row_share is supported on GFX90A and GFX10+; row_xmask is GFX10+ only.
1078 if (ST.hasDPPRowShare()) {
1079 if (std::optional<unsigned> Lane = matchRowSharePattern(Ids))
1080 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_SHARE_FIRST + *Lane);
1081 }
1082
1083 if (ST.hasDPP() && ST.hasGFX10Insts()) {
1084 if (std::optional<unsigned> Mask = matchRowXMaskPattern(Ids))
1085 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_XMASK_FIRST + *Mask);
1086 }
1087
1088 if (ST.hasDPP8()) {
1089 if (std::optional<unsigned> Sel = matchHalfRowPermPattern(Ids))
1090 return createMovDpp8(B, Val: Src, Selector: *Sel);
1091 }
1092
1093 if (ST.hasPermlane16Insts()) {
1094 if (isFullRowPattern(Ids)) {
1095 uint64_t Sel = computePermlane16Masks(Ids);
1096 return createPermlane16(B, Val: Src, Lo: Lo_32(Value: Sel), Hi: Hi_32(Value: Sel));
1097 }
1098 // Cross-row shuffles (e.g. XOR 16..31) — covered by permlanex16.
1099 if (isCrossRowPattern(Ids)) {
1100 uint64_t Sel = computePermlane16Masks(Ids);
1101 return createPermlaneX16(B, Val: Src, Lo: Lo_32(Value: Sel), Hi: Hi_32(Value: Sel));
1102 }
1103 }
1104
1105 // Generic DS_SWIZZLE bitmask-mode fallback: handles any 32-lane shuffle that
1106 // can be expressed as dst = ((src & AND) | OR) ^ XOR with 5-bit masks. This
1107 // is available on every target that has ds_swizzle.
1108 if (std::optional<unsigned> Imm = matchDsSwizzleBitmaskPattern(Ids))
1109 return createDsSwizzle(B, Val: Src, Offset: *Imm, DL);
1110
1111 // DS_SWIZZLE rotate mode (GFX9+): handles cyclic 32-lane rotations that
1112 // bitmask mode cannot express (e.g. +1 mod 32 requires inter-bit carry).
1113 if (ST.hasDsSwizzleRotateMode()) {
1114 if (std::optional<unsigned> Imm = matchDsSwizzleRotatePattern(Ids))
1115 return createDsSwizzle(B, Val: Src, Offset: *Imm, DL);
1116 }
1117
1118 if (ST.hasPermLane64() && matchHalfWaveSwapPattern(Ids))
1119 return createPermlane64(B, Val: Src);
1120
1121 return nullptr;
1122}
1123
1124/// Try to fold a wave_shuffle/ds_bpermute whose lane index is a constant
1125/// function of the lane ID into a hardware-specific lane permutation intrinsic.
1126static std::optional<Instruction *>
1127tryOptimizeShufflePattern(InstCombiner &IC, IntrinsicInst &II,
1128 const GCNSubtarget &ST) {
1129 const DataLayout &DL = IC.getDataLayout();
1130 if (DL.getTypeSizeInBits(Ty: II.getType()) != 32)
1131 return std::nullopt;
1132
1133 if (!ST.isWaveSizeKnown())
1134 return std::nullopt;
1135
1136 unsigned WaveSize = ST.getWavefrontSize();
1137 bool IsBpermute = II.getIntrinsicID() == Intrinsic::amdgcn_ds_bpermute;
1138 Value *Src = II.getArgOperand(i: IsBpermute ? 1 : 0);
1139 Value *Index = II.getArgOperand(i: IsBpermute ? 0 : 1);
1140
1141 SmallVector<uint8_t, 64> Ids;
1142 if (IsBpermute) {
1143 Ids.resize(N: WaveSize);
1144 for (unsigned Lane : seq(Size: WaveSize)) {
1145 std::optional<uint64_t> Val = evalLaneExpr(V: Index, Lane, ST, DL);
1146 if (!Val || (*Val & 3) || (*Val >> 2) >= WaveSize)
1147 return std::nullopt;
1148 Ids[Lane] = static_cast<uint8_t>(*Val >> 2);
1149 }
1150 } else {
1151 if (!tryBuildShuffleMap(Index, ST, Ids, DL))
1152 return std::nullopt;
1153 }
1154
1155 Value *Result = matchShuffleToHWIntrinsic(B&: IC.Builder, Src, Ids, ST, DL);
1156 if (!Result)
1157 return std::nullopt;
1158
1159 return IC.replaceInstUsesWith(I&: II, V: Result);
1160}
1161
1162/// Try to fold an addition into the accumulator when saturation is disabled.
1163static Instruction *foldAddIntoDotAccumulator(IntrinsicInst &II,
1164 unsigned AccIdx,
1165 unsigned ClampIdx,
1166 InstCombiner &IC) {
1167 // Reassociating across a saturating accumulation is not valid.
1168 if (!match(V: II.getArgOperand(i: ClampIdx), P: m_Zero()) || !II.hasOneUse())
1169 return nullptr;
1170
1171 const APInt *Acc = nullptr;
1172 if (!match(V: II.getArgOperand(i: AccIdx), P: m_APInt(Res&: Acc)))
1173 return nullptr;
1174
1175 auto *AccumUser = dyn_cast<BinaryOperator>(Val: II.user_back());
1176 if (!AccumUser)
1177 return nullptr;
1178
1179 Value *AccumDelta = nullptr;
1180 if (!match(V: AccumUser, P: m_c_Add(L: m_Specific(V: &II), R: m_Value(V&: AccumDelta))))
1181 return nullptr;
1182
1183 Value *NewAcc = nullptr;
1184 const APInt *AccumDeltaC = nullptr;
1185 if (Acc->isZero())
1186 NewAcc = AccumDelta;
1187 else if (match(V: AccumDelta, P: m_APInt(Res&: AccumDeltaC)))
1188 NewAcc = ConstantInt::get(Ty: II.getType(), V: *Acc + *AccumDeltaC);
1189 else
1190 return nullptr;
1191
1192 Instruction *NewII =
1193 IC.InsertNewInstWith(New: II.clone(), Old: AccumUser->getIterator());
1194 NewII->setOperand(i: AccIdx, Val: NewAcc);
1195 NewII->dropPoisonGeneratingAnnotations();
1196 NewII->dropUBImplyingAttrsAndMetadata();
1197 IC.replaceInstUsesWith(I&: *AccumUser, V: NewII);
1198 IC.eraseInstFromFunction(I&: *AccumUser);
1199 return &II;
1200}
1201
1202std::optional<Instruction *>
1203GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
1204 Intrinsic::ID IID = II.getIntrinsicID();
1205 switch (IID) {
1206 case Intrinsic::amdgcn_implicitarg_ptr: {
1207 if (II.getFunction()->hasFnAttribute(Kind: "amdgpu-no-implicitarg-ptr"))
1208 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1209 uint64_t ImplicitArgBytes = ST->getImplicitArgNumBytes(F: *II.getFunction());
1210
1211 uint64_t CurrentOrNullBytes =
1212 II.getAttributes().getRetDereferenceableOrNullBytes();
1213 if (CurrentOrNullBytes != 0) {
1214 // Refine "dereferenceable (A) meets dereferenceable_or_null(B)"
1215 // into dereferenceable(max(A, B))
1216 uint64_t NewBytes = std::max(a: CurrentOrNullBytes, b: ImplicitArgBytes);
1217 II.addRetAttr(
1218 Attr: Attribute::getWithDereferenceableBytes(Context&: II.getContext(), Bytes: NewBytes));
1219 II.removeRetAttr(Kind: Attribute::DereferenceableOrNull);
1220 return &II;
1221 }
1222
1223 uint64_t CurrentBytes = II.getAttributes().getRetDereferenceableBytes();
1224 uint64_t NewBytes = std::max(a: CurrentBytes, b: ImplicitArgBytes);
1225 if (NewBytes != CurrentBytes) {
1226 II.addRetAttr(
1227 Attr: Attribute::getWithDereferenceableBytes(Context&: II.getContext(), Bytes: NewBytes));
1228 return &II;
1229 }
1230
1231 return std::nullopt;
1232 }
1233 case Intrinsic::amdgcn_rcp: {
1234 Value *Src = II.getArgOperand(i: 0);
1235 if (isa<PoisonValue>(Val: Src))
1236 return IC.replaceInstUsesWith(I&: II, V: Src);
1237
1238 // TODO: Move to ConstantFolding/InstSimplify?
1239 if (isa<UndefValue>(Val: Src)) {
1240 Type *Ty = II.getType();
1241 auto *QNaN = ConstantFP::get(Ty, V: APFloat::getQNaN(Sem: Ty->getFltSemantics()));
1242 return IC.replaceInstUsesWith(I&: II, V: QNaN);
1243 }
1244
1245 if (II.isStrictFP())
1246 break;
1247
1248 if (const ConstantFP *C = dyn_cast<ConstantFP>(Val: Src)) {
1249 std::optional<APFloat> Val = AMDGPU::evaluateRcp(Val: C->getValueAPF());
1250 if (!Val)
1251 break;
1252
1253 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::get(Context&: II.getContext(), V: *Val));
1254 }
1255
1256 FastMathFlags FMF = cast<FPMathOperator>(Val&: II).getFastMathFlags();
1257 if (!FMF.allowContract())
1258 break;
1259 auto *SrcCI = dyn_cast<IntrinsicInst>(Val: Src);
1260 if (!SrcCI)
1261 break;
1262
1263 auto IID = SrcCI->getIntrinsicID();
1264 // llvm.amdgcn.rcp(llvm.amdgcn.sqrt(x)) -> llvm.amdgcn.rsq(x) if contractable
1265 //
1266 // llvm.amdgcn.rcp(llvm.sqrt(x)) -> llvm.amdgcn.rsq(x) if contractable and
1267 // relaxed.
1268 if (IID == Intrinsic::amdgcn_sqrt || IID == Intrinsic::sqrt) {
1269 const FPMathOperator *SqrtOp = cast<FPMathOperator>(Val: SrcCI);
1270 FastMathFlags InnerFMF = SqrtOp->getFastMathFlags();
1271 if (!InnerFMF.allowContract() || !SrcCI->hasOneUse())
1272 break;
1273
1274 if (IID == Intrinsic::sqrt && !canContractSqrtToRsq(SqrtOp))
1275 break;
1276
1277 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
1278 M: SrcCI->getModule(), id: Intrinsic::amdgcn_rsq, OverloadTys: {SrcCI->getType()});
1279
1280 InnerFMF |= FMF;
1281 II.setFastMathFlags(InnerFMF);
1282
1283 II.setCalledFunction(NewDecl);
1284 return IC.replaceOperand(I&: II, OpNum: 0, V: SrcCI->getArgOperand(i: 0));
1285 }
1286
1287 break;
1288 }
1289 case Intrinsic::amdgcn_sqrt:
1290 case Intrinsic::amdgcn_rsq:
1291 case Intrinsic::amdgcn_tanh: {
1292 Value *Src = II.getArgOperand(i: 0);
1293 if (isa<PoisonValue>(Val: Src))
1294 return IC.replaceInstUsesWith(I&: II, V: Src);
1295
1296 // TODO: Move to ConstantFolding/InstSimplify?
1297 if (isa<UndefValue>(Val: Src)) {
1298 Type *Ty = II.getType();
1299 auto *QNaN = ConstantFP::get(Ty, V: APFloat::getQNaN(Sem: Ty->getFltSemantics()));
1300 return IC.replaceInstUsesWith(I&: II, V: QNaN);
1301 }
1302
1303 // f16 amdgcn.sqrt is identical to regular sqrt.
1304 if (IID == Intrinsic::amdgcn_sqrt && Src->getType()->isHalfTy()) {
1305 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
1306 M: II.getModule(), id: Intrinsic::sqrt, OverloadTys: {II.getType()});
1307 II.setCalledFunction(NewDecl);
1308 return &II;
1309 }
1310
1311 break;
1312 }
1313 case Intrinsic::amdgcn_log:
1314 case Intrinsic::amdgcn_exp2: {
1315 const bool IsLog = IID == Intrinsic::amdgcn_log;
1316 const bool IsExp = IID == Intrinsic::amdgcn_exp2;
1317 Value *Src = II.getArgOperand(i: 0);
1318 Type *Ty = II.getType();
1319
1320 if (isa<PoisonValue>(Val: Src))
1321 return IC.replaceInstUsesWith(I&: II, V: Src);
1322
1323 if (IC.getSimplifyQuery().isUndefValue(V: Src))
1324 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getNaN(Ty));
1325
1326 if (ConstantFP *C = dyn_cast<ConstantFP>(Val: Src)) {
1327 if (C->isInfinity()) {
1328 // exp2(+inf) -> +inf
1329 // log2(+inf) -> +inf
1330 if (!C->isNegative())
1331 return IC.replaceInstUsesWith(I&: II, V: C);
1332
1333 // exp2(-inf) -> 0
1334 if (IsExp && C->isNegative())
1335 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getZero(Ty));
1336 }
1337
1338 if (II.isStrictFP())
1339 break;
1340
1341 if (C->isNaN()) {
1342 Constant *Quieted = ConstantFP::get(Ty, V: C->getValue().makeQuiet());
1343 return IC.replaceInstUsesWith(I&: II, V: Quieted);
1344 }
1345
1346 // f32 instruction doesn't handle denormals, f16 does.
1347 if (C->isZero() || (C->getValue().isDenormal() && Ty->isFloatTy())) {
1348 Constant *FoldedValue = IsLog ? ConstantFP::getInfinity(Ty, Negative: true)
1349 : ConstantFP::get(Ty, V: 1.0);
1350 return IC.replaceInstUsesWith(I&: II, V: FoldedValue);
1351 }
1352
1353 if (IsLog && C->isNegative())
1354 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getNaN(Ty));
1355
1356 // TODO: Full constant folding matching hardware behavior.
1357 }
1358
1359 break;
1360 }
1361 case Intrinsic::amdgcn_frexp_mant:
1362 case Intrinsic::amdgcn_frexp_exp: {
1363 Value *Src = II.getArgOperand(i: 0);
1364 if (const ConstantFP *C = dyn_cast<ConstantFP>(Val: Src)) {
1365 int Exp;
1366 APFloat Significand =
1367 frexp(X: C->getValueAPF(), Exp, RM: APFloat::rmNearestTiesToEven);
1368
1369 if (IID == Intrinsic::amdgcn_frexp_mant) {
1370 return IC.replaceInstUsesWith(
1371 I&: II, V: ConstantFP::get(Context&: II.getContext(), V: Significand));
1372 }
1373
1374 // Match instruction special case behavior.
1375 if (Exp == APFloat::IEK_NaN || Exp == APFloat::IEK_Inf)
1376 Exp = 0;
1377
1378 return IC.replaceInstUsesWith(I&: II,
1379 V: ConstantInt::getSigned(Ty: II.getType(), V: Exp));
1380 }
1381
1382 if (isa<PoisonValue>(Val: Src))
1383 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1384
1385 if (isa<UndefValue>(Val: Src)) {
1386 return IC.replaceInstUsesWith(I&: II, V: UndefValue::get(T: II.getType()));
1387 }
1388
1389 break;
1390 }
1391 case Intrinsic::amdgcn_class: {
1392 Value *Src0 = II.getArgOperand(i: 0);
1393 Value *Src1 = II.getArgOperand(i: 1);
1394 const ConstantInt *CMask = dyn_cast<ConstantInt>(Val: Src1);
1395 if (CMask) {
1396 II.setCalledOperand(Intrinsic::getOrInsertDeclaration(
1397 M: II.getModule(), id: Intrinsic::is_fpclass, OverloadTys: Src0->getType()));
1398
1399 // Clamp any excess bits, as they're illegal for the generic intrinsic.
1400 II.setArgOperand(i: 1, v: ConstantInt::get(Ty: Src1->getType(),
1401 V: CMask->getZExtValue() & fcAllFlags));
1402 return &II;
1403 }
1404
1405 // Propagate poison.
1406 if (isa<PoisonValue>(Val: Src0) || isa<PoisonValue>(Val: Src1))
1407 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1408
1409 // llvm.amdgcn.class(_, undef) -> false
1410 if (IC.getSimplifyQuery().isUndefValue(V: Src1))
1411 return IC.replaceInstUsesWith(I&: II, V: ConstantInt::get(Ty: II.getType(), V: false));
1412
1413 // llvm.amdgcn.class(undef, mask) -> mask != 0
1414 if (IC.getSimplifyQuery().isUndefValue(V: Src0)) {
1415 Value *CmpMask = IC.Builder.CreateICmpNE(
1416 LHS: Src1, RHS: ConstantInt::getNullValue(Ty: Src1->getType()));
1417 return IC.replaceInstUsesWith(I&: II, V: CmpMask);
1418 }
1419 break;
1420 }
1421 case Intrinsic::amdgcn_cvt_pkrtz: {
1422 auto foldFPTruncToF16RTZ = [](Value *Arg) -> Value * {
1423 Type *HalfTy = Type::getHalfTy(C&: Arg->getContext());
1424
1425 if (isa<PoisonValue>(Val: Arg))
1426 return PoisonValue::get(T: HalfTy);
1427 if (isa<UndefValue>(Val: Arg))
1428 return UndefValue::get(T: HalfTy);
1429
1430 ConstantFP *CFP = nullptr;
1431 if (match(V: Arg, P: m_ConstantFP(C&: CFP))) {
1432 bool LosesInfo;
1433 APFloat Val(CFP->getValueAPF());
1434 Val.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmTowardZero, losesInfo: &LosesInfo);
1435 return ConstantFP::get(Ty: HalfTy, V: Val);
1436 }
1437
1438 Value *Src = nullptr;
1439 if (match(V: Arg, P: m_FPExt(Op: m_Value(V&: Src)))) {
1440 if (Src->getType()->isHalfTy())
1441 return Src;
1442 }
1443
1444 return nullptr;
1445 };
1446
1447 if (Value *Src0 = foldFPTruncToF16RTZ(II.getArgOperand(i: 0))) {
1448 if (Value *Src1 = foldFPTruncToF16RTZ(II.getArgOperand(i: 1))) {
1449 Value *V = PoisonValue::get(T: II.getType());
1450 V = IC.Builder.CreateInsertElement(Vec: V, NewElt: Src0, Idx: (uint64_t)0);
1451 V = IC.Builder.CreateInsertElement(Vec: V, NewElt: Src1, Idx: (uint64_t)1);
1452 return IC.replaceInstUsesWith(I&: II, V);
1453 }
1454 }
1455
1456 break;
1457 }
1458 case Intrinsic::amdgcn_cvt_pknorm_i16:
1459 case Intrinsic::amdgcn_cvt_pknorm_u16:
1460 case Intrinsic::amdgcn_cvt_pk_i16:
1461 case Intrinsic::amdgcn_cvt_pk_u16: {
1462 Value *Src0 = II.getArgOperand(i: 0);
1463 Value *Src1 = II.getArgOperand(i: 1);
1464
1465 // TODO: Replace call with scalar operation if only one element is poison.
1466 if (isa<PoisonValue>(Val: Src0) && isa<PoisonValue>(Val: Src1))
1467 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1468
1469 if (isa<UndefValue>(Val: Src0) && isa<UndefValue>(Val: Src1)) {
1470 return IC.replaceInstUsesWith(I&: II, V: UndefValue::get(T: II.getType()));
1471 }
1472
1473 break;
1474 }
1475 case Intrinsic::amdgcn_cvt_off_f32_i4: {
1476 Value* Arg = II.getArgOperand(i: 0);
1477 Type *Ty = II.getType();
1478
1479 if (isa<PoisonValue>(Val: Arg))
1480 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: Ty));
1481
1482 if(IC.getSimplifyQuery().isUndefValue(V: Arg))
1483 return IC.replaceInstUsesWith(I&: II, V: Constant::getNullValue(Ty));
1484
1485 ConstantInt *CArg = dyn_cast<ConstantInt>(Val: II.getArgOperand(i: 0));
1486 if (!CArg)
1487 break;
1488
1489 // Tabulated 0.0625 * (sext (CArg & 0xf)).
1490 constexpr size_t ResValsSize = 16;
1491 static constexpr float ResVals[ResValsSize] = {
1492 0.0, 0.0625, 0.125, 0.1875, 0.25, 0.3125, 0.375, 0.4375,
1493 -0.5, -0.4375, -0.375, -0.3125, -0.25, -0.1875, -0.125, -0.0625};
1494 Constant *Res =
1495 ConstantFP::get(Ty, V: ResVals[CArg->getZExtValue() & (ResValsSize - 1)]);
1496 return IC.replaceInstUsesWith(I&: II, V: Res);
1497 }
1498 case Intrinsic::amdgcn_ubfe:
1499 case Intrinsic::amdgcn_sbfe: {
1500 // Decompose simple cases into standard shifts.
1501 Value *Src = II.getArgOperand(i: 0);
1502 if (isa<UndefValue>(Val: Src)) {
1503 return IC.replaceInstUsesWith(I&: II, V: Src);
1504 }
1505
1506 uint64_t Width;
1507 Type *Ty = II.getType();
1508 unsigned IntSize = Ty->getIntegerBitWidth();
1509
1510 ConstantInt *CWidth = dyn_cast<ConstantInt>(Val: II.getArgOperand(i: 2));
1511 if (CWidth) {
1512 Width = CWidth->getZExtValue();
1513 if ((Width & (IntSize - 1)) == 0) {
1514 return IC.replaceInstUsesWith(I&: II, V: ConstantInt::getNullValue(Ty));
1515 }
1516
1517 // Hardware ignores high bits, so remove those.
1518 if (Width >= IntSize) {
1519 return IC.replaceOperand(
1520 I&: II, OpNum: 2, V: ConstantInt::get(Ty: CWidth->getType(), V: Width & (IntSize - 1)));
1521 }
1522 }
1523
1524 uint64_t Offset;
1525 ConstantInt *COffset = dyn_cast<ConstantInt>(Val: II.getArgOperand(i: 1));
1526 if (COffset) {
1527 Offset = COffset->getZExtValue();
1528 if (Offset >= IntSize) {
1529 return IC.replaceOperand(
1530 I&: II, OpNum: 1,
1531 V: ConstantInt::get(Ty: COffset->getType(), V: Offset & (IntSize - 1)));
1532 }
1533 }
1534
1535 bool Signed = IID == Intrinsic::amdgcn_sbfe;
1536
1537 if (!CWidth || !COffset)
1538 break;
1539
1540 // The case of Width == 0 is handled above, which makes this transformation
1541 // safe. If Width == 0, then the ashr and lshr instructions become poison
1542 // value since the shift amount would be equal to the bit size.
1543 assert(Width != 0);
1544
1545 // TODO: This allows folding to undef when the hardware has specific
1546 // behavior?
1547 if (Offset + Width < IntSize) {
1548 Value *Shl = IC.Builder.CreateShl(LHS: Src, RHS: IntSize - Offset - Width);
1549 Value *RightShift = Signed ? IC.Builder.CreateAShr(LHS: Shl, RHS: IntSize - Width)
1550 : IC.Builder.CreateLShr(LHS: Shl, RHS: IntSize - Width);
1551 RightShift->takeName(V: &II);
1552 return IC.replaceInstUsesWith(I&: II, V: RightShift);
1553 }
1554
1555 Value *RightShift = Signed ? IC.Builder.CreateAShr(LHS: Src, RHS: Offset)
1556 : IC.Builder.CreateLShr(LHS: Src, RHS: Offset);
1557
1558 RightShift->takeName(V: &II);
1559 return IC.replaceInstUsesWith(I&: II, V: RightShift);
1560 }
1561 case Intrinsic::amdgcn_exp:
1562 case Intrinsic::amdgcn_exp_row:
1563 case Intrinsic::amdgcn_exp_compr: {
1564 ConstantInt *En = cast<ConstantInt>(Val: II.getArgOperand(i: 1));
1565 uint64_t EnBits = En->getZExtValue();
1566 if (EnBits == 0xf)
1567 break; // All inputs enabled.
1568
1569 bool IsCompr = IID == Intrinsic::amdgcn_exp_compr;
1570 bool Changed = false;
1571 for (int I = 0; I < (IsCompr ? 2 : 4); ++I) {
1572 if ((!IsCompr && (EnBits & (1ULL << I)) == 0) ||
1573 (IsCompr && ((EnBits & (0x3ULL << (2 * I))) == 0))) {
1574 Value *Src = II.getArgOperand(i: I + 2);
1575 if (!isa<PoisonValue>(Val: Src)) {
1576 IC.replaceOperand(I&: II, OpNum: I + 2, V: PoisonValue::get(T: Src->getType()));
1577 Changed = true;
1578 }
1579 }
1580 }
1581
1582 if (Changed) {
1583 return &II;
1584 }
1585
1586 break;
1587 }
1588 case Intrinsic::amdgcn_fmed3: {
1589 Value *Src0 = II.getArgOperand(i: 0);
1590 Value *Src1 = II.getArgOperand(i: 1);
1591 Value *Src2 = II.getArgOperand(i: 2);
1592
1593 for (Value *Src : {Src0, Src1, Src2}) {
1594 if (isa<PoisonValue>(Val: Src))
1595 return IC.replaceInstUsesWith(I&: II, V: Src);
1596 }
1597
1598 if (II.isStrictFP())
1599 break;
1600
1601 // med3 with a nan input acts like
1602 // v_min_f32(v_min_f32(s0, s1), s2)
1603 //
1604 // Signalingness is ignored with ieee=0, so we fold to
1605 // minimumnum/maximumnum. With ieee=1, the v_min_f32 acts like llvm.minnum
1606 // with signaling nan handling. With ieee=0, like llvm.minimumnum except a
1607 // returned signaling nan will not be quieted.
1608
1609 // ieee=1
1610 // s0 snan: s2
1611 // s1 snan: s2
1612 // s2 snan: qnan
1613
1614 // s0 qnan: min(s1, s2)
1615 // s1 qnan: min(s0, s2)
1616 // s2 qnan: min(s0, s1)
1617
1618 // ieee=0
1619 // s0 _nan: min(s1, s2)
1620 // s1 _nan: min(s0, s2)
1621 // s2 _nan: min(s0, s1)
1622
1623 // med3 behavior with infinity
1624 // s0 +inf: max(s1, s2)
1625 // s1 +inf: max(s0, s2)
1626 // s2 +inf: max(s0, s1)
1627 // s0 -inf: min(s1, s2)
1628 // s1 -inf: min(s0, s2)
1629 // s2 -inf: min(s0, s1)
1630
1631 // Checking for NaN before canonicalization provides better fidelity when
1632 // mapping other operations onto fmed3 since the order of operands is
1633 // unchanged.
1634 Value *V = nullptr;
1635 const APFloat *ConstSrc0 = nullptr;
1636 const APFloat *ConstSrc1 = nullptr;
1637 const APFloat *ConstSrc2 = nullptr;
1638
1639 if ((match(V: Src0, P: m_APFloat(Res&: ConstSrc0)) &&
1640 (ConstSrc0->isNaN() || ConstSrc0->isInfinity())) ||
1641 isa<UndefValue>(Val: Src0)) {
1642 const bool IsPosInfinity = ConstSrc0 && ConstSrc0->isPosInfinity();
1643 switch (fpenvIEEEMode(I: II)) {
1644 case KnownIEEEMode::On:
1645 // TODO: If Src2 is snan, does it need quieting?
1646 if (ConstSrc0 && ConstSrc0->isNaN() && ConstSrc0->isSignaling())
1647 return IC.replaceInstUsesWith(I&: II, V: Src2);
1648
1649 V = IsPosInfinity ? IC.Builder.CreateMaxNum(LHS: Src1, RHS: Src2)
1650 : IC.Builder.CreateMinNum(LHS: Src1, RHS: Src2);
1651 break;
1652 case KnownIEEEMode::Off:
1653 V = IsPosInfinity ? IC.Builder.CreateMaximumNum(LHS: Src1, RHS: Src2)
1654 : IC.Builder.CreateMinimumNum(LHS: Src1, RHS: Src2);
1655 break;
1656 case KnownIEEEMode::Unknown:
1657 break;
1658 }
1659 } else if ((match(V: Src1, P: m_APFloat(Res&: ConstSrc1)) &&
1660 (ConstSrc1->isNaN() || ConstSrc1->isInfinity())) ||
1661 isa<UndefValue>(Val: Src1)) {
1662 const bool IsPosInfinity = ConstSrc1 && ConstSrc1->isPosInfinity();
1663 switch (fpenvIEEEMode(I: II)) {
1664 case KnownIEEEMode::On:
1665 // TODO: If Src2 is snan, does it need quieting?
1666 if (ConstSrc1 && ConstSrc1->isNaN() && ConstSrc1->isSignaling())
1667 return IC.replaceInstUsesWith(I&: II, V: Src2);
1668
1669 V = IsPosInfinity ? IC.Builder.CreateMaxNum(LHS: Src0, RHS: Src2)
1670 : IC.Builder.CreateMinNum(LHS: Src0, RHS: Src2);
1671 break;
1672 case KnownIEEEMode::Off:
1673 V = IsPosInfinity ? IC.Builder.CreateMaximumNum(LHS: Src0, RHS: Src2)
1674 : IC.Builder.CreateMinimumNum(LHS: Src0, RHS: Src2);
1675 break;
1676 case KnownIEEEMode::Unknown:
1677 break;
1678 }
1679 } else if ((match(V: Src2, P: m_APFloat(Res&: ConstSrc2)) &&
1680 (ConstSrc2->isNaN() || ConstSrc2->isInfinity())) ||
1681 isa<UndefValue>(Val: Src2)) {
1682 switch (fpenvIEEEMode(I: II)) {
1683 case KnownIEEEMode::On:
1684 if (ConstSrc2 && ConstSrc2->isNaN() && ConstSrc2->isSignaling()) {
1685 auto *Quieted = ConstantFP::get(Ty: II.getType(), V: ConstSrc2->makeQuiet());
1686 return IC.replaceInstUsesWith(I&: II, V: Quieted);
1687 }
1688
1689 V = (ConstSrc2 && ConstSrc2->isPosInfinity())
1690 ? IC.Builder.CreateMaxNum(LHS: Src0, RHS: Src1)
1691 : IC.Builder.CreateMinNum(LHS: Src0, RHS: Src1);
1692 break;
1693 case KnownIEEEMode::Off:
1694 V = (ConstSrc2 && ConstSrc2->isPosInfinity())
1695 ? IC.Builder.CreateMaximumNum(LHS: Src0, RHS: Src1)
1696 : IC.Builder.CreateMinimumNum(LHS: Src0, RHS: Src1);
1697 break;
1698 case KnownIEEEMode::Unknown:
1699 break;
1700 }
1701 }
1702
1703 if (V) {
1704 if (auto *CI = dyn_cast<CallInst>(Val: V)) {
1705 CI->copyFastMathFlags(I: &II);
1706 CI->takeName(V: &II);
1707 }
1708 return IC.replaceInstUsesWith(I&: II, V);
1709 }
1710
1711 bool Swap = false;
1712 // Canonicalize constants to RHS operands.
1713 //
1714 // fmed3(c0, x, c1) -> fmed3(x, c0, c1)
1715 if (isa<Constant>(Val: Src0) && !isa<Constant>(Val: Src1)) {
1716 std::swap(a&: Src0, b&: Src1);
1717 Swap = true;
1718 }
1719
1720 if (isa<Constant>(Val: Src1) && !isa<Constant>(Val: Src2)) {
1721 std::swap(a&: Src1, b&: Src2);
1722 Swap = true;
1723 }
1724
1725 if (isa<Constant>(Val: Src0) && !isa<Constant>(Val: Src1)) {
1726 std::swap(a&: Src0, b&: Src1);
1727 Swap = true;
1728 }
1729
1730 if (Swap) {
1731 II.setArgOperand(i: 0, v: Src0);
1732 II.setArgOperand(i: 1, v: Src1);
1733 II.setArgOperand(i: 2, v: Src2);
1734 return &II;
1735 }
1736
1737 if (const ConstantFP *C0 = dyn_cast<ConstantFP>(Val: Src0)) {
1738 if (const ConstantFP *C1 = dyn_cast<ConstantFP>(Val: Src1)) {
1739 if (const ConstantFP *C2 = dyn_cast<ConstantFP>(Val: Src2)) {
1740 APFloat Result = fmed3AMDGCN(Src0: C0->getValueAPF(), Src1: C1->getValueAPF(),
1741 Src2: C2->getValueAPF());
1742 return IC.replaceInstUsesWith(I&: II,
1743 V: ConstantFP::get(Ty: II.getType(), V: Result));
1744 }
1745 }
1746 }
1747
1748 if (!ST->hasMed3_16())
1749 break;
1750
1751 // Repeat floating-point width reduction done for minnum/maxnum.
1752 // fmed3((fpext X), (fpext Y), (fpext Z)) -> fpext (fmed3(X, Y, Z))
1753 if (Value *X = matchFPExtFromF16(Arg: Src0)) {
1754 if (Value *Y = matchFPExtFromF16(Arg: Src1)) {
1755 if (Value *Z = matchFPExtFromF16(Arg: Src2)) {
1756 Value *NewCall = IC.Builder.CreateIntrinsic(
1757 ID: IID, OverloadTypes: {X->getType()}, Args: {X, Y, Z}, FMFSource: &II, Name: II.getName());
1758 return new FPExtInst(NewCall, II.getType());
1759 }
1760 }
1761 }
1762
1763 break;
1764 }
1765 case Intrinsic::amdgcn_mbcnt_hi:
1766 // exec_hi is all 0, so this is just a copy.
1767 if (ST->isWave32())
1768 return IC.replaceInstUsesWith(I&: II, V: II.getArgOperand(i: 1));
1769 [[fallthrough]];
1770 case Intrinsic::amdgcn_mbcnt_lo: {
1771 ConstantRange AccRange =
1772 computeConstantRange(V: II.getArgOperand(i: 1),
1773 /*ForSigned=*/false, SQ: IC.getSimplifyQuery());
1774 if (AccRange.isFullSet())
1775 return nullptr;
1776
1777 // TODO: Can raise lower bound by inspecting first argument.
1778 // Wave64 mbcnt_lo adds at most 32. Otherwise these add at most 31.
1779 unsigned MaxCount =
1780 IID == Intrinsic::amdgcn_mbcnt_lo && ST->isWave64() ? 32 : 31;
1781 ConstantRange MbcntRange(APInt(32, 0), APInt(32, MaxCount + 1));
1782 ConstantRange ComputedRange = AccRange.add(Other: MbcntRange);
1783 if (ComputedRange.isFullSet())
1784 return nullptr;
1785
1786 if (std::optional<ConstantRange> ExistingRange = II.getRange()) {
1787 ComputedRange = ComputedRange.intersectWith(CR: *ExistingRange);
1788 if (ComputedRange == *ExistingRange)
1789 return nullptr;
1790 }
1791
1792 II.addRangeRetAttr(CR: ComputedRange);
1793 return nullptr;
1794 }
1795 case Intrinsic::amdgcn_ballot: {
1796 Value *Arg = II.getArgOperand(i: 0);
1797 if (isa<PoisonValue>(Val: Arg))
1798 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1799
1800 if (auto *Src = dyn_cast<ConstantInt>(Val: Arg)) {
1801 if (Src->isZero()) {
1802 // amdgcn.ballot(i1 0) is zero.
1803 return IC.replaceInstUsesWith(I&: II, V: Constant::getNullValue(Ty: II.getType()));
1804 }
1805 }
1806 if (ST->isWave32() && II.getType()->getIntegerBitWidth() == 64) {
1807 // %b64 = call i64 ballot.i64(...)
1808 // =>
1809 // %b32 = call i32 ballot.i32(...)
1810 // %b64 = zext i32 %b32 to i64
1811 Value *Call = IC.Builder.CreateZExt(
1812 V: IC.Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_ballot,
1813 OverloadTypes: {IC.Builder.getInt32Ty()},
1814 Args: {II.getArgOperand(i: 0)}),
1815 DestTy: II.getType());
1816 Call->takeName(V: &II);
1817 return IC.replaceInstUsesWith(I&: II, V: Call);
1818 }
1819 break;
1820 }
1821 case Intrinsic::amdgcn_wavefrontsize: {
1822 if (ST->isWaveSizeKnown())
1823 return IC.replaceInstUsesWith(
1824 I&: II, V: ConstantInt::get(Ty: II.getType(), V: ST->getWavefrontSize()));
1825 break;
1826 }
1827 case Intrinsic::amdgcn_wqm_vote: {
1828 // wqm_vote is identity when the argument is constant.
1829 if (!isa<Constant>(Val: II.getArgOperand(i: 0)))
1830 break;
1831
1832 return IC.replaceInstUsesWith(I&: II, V: II.getArgOperand(i: 0));
1833 }
1834 case Intrinsic::amdgcn_kill: {
1835 const ConstantInt *C = dyn_cast<ConstantInt>(Val: II.getArgOperand(i: 0));
1836 if (!C || !C->getZExtValue())
1837 break;
1838
1839 // amdgcn.kill(i1 1) is a no-op
1840 return IC.eraseInstFromFunction(I&: II);
1841 }
1842 case Intrinsic::amdgcn_s_sendmsg:
1843 case Intrinsic::amdgcn_s_sendmsghalt: {
1844 // The second operand is copied to m0, but is only actually used for
1845 // certain message types. For message types that are known to not use m0,
1846 // fold it to poison.
1847 using namespace AMDGPU::SendMsg;
1848
1849 Value *M0Val = II.getArgOperand(i: 1);
1850 if (isa<PoisonValue>(Val: M0Val))
1851 break;
1852
1853 auto *MsgImm = cast<ConstantInt>(Val: II.getArgOperand(i: 0));
1854 uint16_t MsgId, OpId, StreamId;
1855 decodeMsg(Val: static_cast<unsigned>(MsgImm->getZExtValue()), MsgId, OpId,
1856 StreamId, STI: *ST);
1857
1858 if (!msgDoesNotUseM0(MsgId, STI: *ST))
1859 break;
1860
1861 // Drop UB-implying attributes since we're replacing with poison.
1862 II.dropUBImplyingAttrsAndMetadata();
1863 IC.replaceOperand(I&: II, OpNum: 1, V: PoisonValue::get(T: M0Val->getType()));
1864 return nullptr;
1865 }
1866 case Intrinsic::amdgcn_update_dpp: {
1867 Value *Old = II.getArgOperand(i: 0);
1868
1869 auto *BC = cast<ConstantInt>(Val: II.getArgOperand(i: 5));
1870 auto *RM = cast<ConstantInt>(Val: II.getArgOperand(i: 3));
1871 auto *BM = cast<ConstantInt>(Val: II.getArgOperand(i: 4));
1872 if (BC->isNullValue() || RM->getZExtValue() != 0xF ||
1873 BM->getZExtValue() != 0xF || isa<PoisonValue>(Val: Old))
1874 break;
1875
1876 // If bound_ctrl = 1, row mask = bank mask = 0xf we can omit old value.
1877 return IC.replaceOperand(I&: II, OpNum: 0, V: PoisonValue::get(T: Old->getType()));
1878 }
1879 case Intrinsic::amdgcn_permlane16:
1880 case Intrinsic::amdgcn_permlane16_var:
1881 case Intrinsic::amdgcn_permlanex16:
1882 case Intrinsic::amdgcn_permlanex16_var: {
1883 // Discard vdst_in if it's not going to be read.
1884 Value *VDstIn = II.getArgOperand(i: 0);
1885 if (isa<PoisonValue>(Val: VDstIn))
1886 break;
1887
1888 // FetchInvalid operand idx.
1889 unsigned int FiIdx = (IID == Intrinsic::amdgcn_permlane16 ||
1890 IID == Intrinsic::amdgcn_permlanex16)
1891 ? 4 /* for permlane16 and permlanex16 */
1892 : 3; /* for permlane16_var and permlanex16_var */
1893
1894 // BoundCtrl operand idx.
1895 // For permlane16 and permlanex16 it should be 5
1896 // For Permlane16_var and permlanex16_var it should be 4
1897 unsigned int BcIdx = FiIdx + 1;
1898
1899 ConstantInt *FetchInvalid = cast<ConstantInt>(Val: II.getArgOperand(i: FiIdx));
1900 ConstantInt *BoundCtrl = cast<ConstantInt>(Val: II.getArgOperand(i: BcIdx));
1901 if (!FetchInvalid->getZExtValue() && !BoundCtrl->getZExtValue())
1902 break;
1903
1904 return IC.replaceOperand(I&: II, OpNum: 0, V: PoisonValue::get(T: VDstIn->getType()));
1905 }
1906 case Intrinsic::amdgcn_wave_shuffle:
1907 return tryOptimizeShufflePattern(IC, II, ST: *ST);
1908 case Intrinsic::amdgcn_permlane64:
1909 case Intrinsic::amdgcn_readfirstlane:
1910 case Intrinsic::amdgcn_readlane:
1911 case Intrinsic::amdgcn_ds_bpermute: {
1912 // If the data argument is uniform these intrinsics return it unchanged.
1913 unsigned SrcIdx = IID == Intrinsic::amdgcn_ds_bpermute ? 1 : 0;
1914 const Use &Src = II.getArgOperandUse(i: SrcIdx);
1915 if (AMDGPU::isTriviallyUniform(U: Src))
1916 return IC.replaceInstUsesWith(I&: II, V: Src.get());
1917
1918 if (IID == Intrinsic::amdgcn_readlane &&
1919 simplifyDemandedLaneMaskArg(IC, II, LaneArgIdx: 1))
1920 return &II;
1921
1922 // If the lane argument of bpermute is uniform, change it to readlane. This
1923 // generates better code and can enable further optimizations because
1924 // readlane is AlwaysUniform.
1925 if (IID == Intrinsic::amdgcn_ds_bpermute) {
1926 const Use &Lane = II.getArgOperandUse(i: 0);
1927 if (AMDGPU::isTriviallyUniform(U: Lane)) {
1928 Value *NewLane = IC.Builder.CreateLShr(LHS: Lane, RHS: 2);
1929 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
1930 M: II.getModule(), id: Intrinsic::amdgcn_readlane, OverloadTys: II.getType());
1931 II.setCalledFunction(NewDecl);
1932 II.setOperand(i_nocapture: 0, Val_nocapture: Src);
1933 II.setOperand(i_nocapture: 1, Val_nocapture: NewLane);
1934 return &II;
1935 }
1936 }
1937
1938 if (IID == Intrinsic::amdgcn_ds_bpermute)
1939 return tryOptimizeShufflePattern(IC, II, ST: *ST);
1940
1941 if (Instruction *Res = hoistLaneIntrinsicThroughOperand(IC, II))
1942 return Res;
1943
1944 return std::nullopt;
1945 }
1946 case Intrinsic::amdgcn_wave_match_b32: {
1947 const Use &Src0 = II.getArgOperandUse(i: 0);
1948 const Use &Src1 = II.getArgOperandUse(i: 1);
1949 if (Src0.get() == Src1.get() && AMDGPU::isTriviallyUniform(U: Src0)) {
1950 Function *NewF = Intrinsic::getOrInsertDeclaration(
1951 M: II.getModule(), id: Intrinsic::amdgcn_ballot, OverloadTys: II.getType());
1952 CallInst *NewCall =
1953 IC.Builder.CreateCall(Callee: NewF, Args: {IC.Builder.getInt1(V: true)});
1954 return IC.replaceInstUsesWith(I&: II, V: NewCall);
1955 }
1956 break;
1957 }
1958 case Intrinsic::amdgcn_writelane: {
1959 // TODO: Fold bitcast like readlane.
1960 if (simplifyDemandedLaneMaskArg(IC, II, LaneArgIdx: 1))
1961 return &II;
1962 return std::nullopt;
1963 }
1964 case Intrinsic::amdgcn_trig_preop: {
1965 // The intrinsic is declared with name mangling, but currently the
1966 // instruction only exists for f64
1967 if (!II.getType()->isDoubleTy())
1968 break;
1969
1970 Value *Src = II.getArgOperand(i: 0);
1971 Value *Segment = II.getArgOperand(i: 1);
1972 if (isa<PoisonValue>(Val: Src) || isa<PoisonValue>(Val: Segment))
1973 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1974
1975 if (isa<UndefValue>(Val: Segment))
1976 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getZero(Ty: II.getType()));
1977
1978 // Sign bit is not used.
1979 Value *StrippedSign = InstCombiner::stripSignOnlyFPOps(Val: Src);
1980 if (StrippedSign != Src)
1981 return IC.replaceOperand(I&: II, OpNum: 0, V: StrippedSign);
1982
1983 if (II.isStrictFP())
1984 break;
1985
1986 const ConstantFP *CSrc = dyn_cast<ConstantFP>(Val: Src);
1987 if (!CSrc && !isa<UndefValue>(Val: Src))
1988 break;
1989
1990 // The instruction ignores special cases, and literally just extracts the
1991 // exponents. Fold undef to nan, and index the table as normal.
1992 APInt FSrcInt = CSrc ? CSrc->getValueAPF().bitcastToAPInt()
1993 : APFloat::getQNaN(Sem: II.getType()->getFltSemantics())
1994 .bitcastToAPInt();
1995
1996 const ConstantInt *Cseg = dyn_cast<ConstantInt>(Val: Segment);
1997 if (!Cseg) {
1998 if (isa<UndefValue>(Val: Src))
1999 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getZero(Ty: II.getType()));
2000 break;
2001 }
2002
2003 unsigned Exponent =
2004 static_cast<unsigned>(FSrcInt.extractBitsAsZExtValue(numBits: 11, bitPosition: 52));
2005 unsigned SegmentVal =
2006 static_cast<unsigned>(Cseg->getValue().trunc(width: 5).getZExtValue());
2007 unsigned Shift = SegmentVal * 53;
2008 if (Exponent > 1077)
2009 Shift += Exponent - 1077;
2010
2011 // 2.0/PI table.
2012 static const uint32_t TwoByPi[] = {
2013 0xa2f9836e, 0x4e441529, 0xfc2757d1, 0xf534ddc0, 0xdb629599, 0x3c439041,
2014 0xfe5163ab, 0xdebbc561, 0xb7246e3a, 0x424dd2e0, 0x06492eea, 0x09d1921c,
2015 0xfe1deb1c, 0xb129a73e, 0xe88235f5, 0x2ebb4484, 0xe99c7026, 0xb45f7e41,
2016 0x3991d639, 0x835339f4, 0x9c845f8b, 0xbdf9283b, 0x1ff897ff, 0xde05980f,
2017 0xef2f118b, 0x5a0a6d1f, 0x6d367ecf, 0x27cb09b7, 0x4f463f66, 0x9e5fea2d,
2018 0x7527bac7, 0xebe5f17b, 0x3d0739f7, 0x8a5292ea, 0x6bfb5fb1, 0x1f8d5d08,
2019 0x56033046};
2020
2021 // Return 0 for outbound segment (hardware behavior).
2022 unsigned Idx = Shift >> 5;
2023 if (Idx + 2 >= std::size(TwoByPi)) {
2024 APFloat Zero = APFloat::getZero(Sem: II.getType()->getFltSemantics());
2025 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::get(Ty: II.getType(), V: Zero));
2026 }
2027
2028 unsigned BShift = Shift & 0x1f;
2029 uint64_t Thi = Make_64(High: TwoByPi[Idx], Low: TwoByPi[Idx + 1]);
2030 uint64_t Tlo = Make_64(High: TwoByPi[Idx + 2], Low: 0);
2031 if (BShift)
2032 Thi = (Thi << BShift) | (Tlo >> (64 - BShift));
2033 Thi = Thi >> 11;
2034 APFloat Result = APFloat((double)Thi);
2035
2036 int Scale = -53 - Shift;
2037 if (Exponent >= 1968)
2038 Scale += 128;
2039
2040 Result = scalbn(X: Result, Exp: Scale, RM: RoundingMode::NearestTiesToEven);
2041 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::get(Ty: Src->getType(), V: Result));
2042 }
2043 case Intrinsic::amdgcn_sdot2:
2044 case Intrinsic::amdgcn_udot2:
2045 case Intrinsic::amdgcn_sdot4:
2046 case Intrinsic::amdgcn_udot4:
2047 case Intrinsic::amdgcn_sdot8:
2048 case Intrinsic::amdgcn_udot8: {
2049 Value *Src0 = II.getArgOperand(i: 0);
2050 Value *Src1 = II.getArgOperand(i: 1);
2051
2052 // Canonicalize the constant multiplicand to Src1.
2053 if (isa<Constant>(Val: Src0) && !isa<Constant>(Val: Src1)) {
2054 II.setArgOperand(i: 0, v: Src1);
2055 II.setArgOperand(i: 1, v: Src0);
2056 return &II;
2057 }
2058
2059 if (match(V: Src1, P: m_Zero()))
2060 return IC.replaceInstUsesWith(I&: II, V: II.getArgOperand(i: 2));
2061
2062 if (Instruction *I = foldAddIntoDotAccumulator(II, AccIdx: 2, ClampIdx: 3, IC))
2063 return I;
2064
2065 break;
2066 }
2067 case Intrinsic::amdgcn_sudot4:
2068 case Intrinsic::amdgcn_sudot8: {
2069 Value *Src0 = II.getArgOperand(i: 1);
2070 Value *Src1 = II.getArgOperand(i: 3);
2071
2072 // Canonicalize the constant multiplicand to Src1, moving its sign flag with
2073 // it.
2074 if (isa<Constant>(Val: Src0) && !isa<Constant>(Val: Src1)) {
2075 Value *Sign0 = II.getArgOperand(i: 0);
2076 Value *Sign1 = II.getArgOperand(i: 2);
2077 II.setArgOperand(i: 0, v: Sign1);
2078 II.setArgOperand(i: 1, v: Src1);
2079 II.setArgOperand(i: 2, v: Sign0);
2080 II.setArgOperand(i: 3, v: Src0);
2081 return &II;
2082 }
2083
2084 if (match(V: Src1, P: m_Zero()))
2085 return IC.replaceInstUsesWith(I&: II, V: II.getArgOperand(i: 4));
2086
2087 if (Instruction *I = foldAddIntoDotAccumulator(II, AccIdx: 4, ClampIdx: 5, IC))
2088 return I;
2089
2090 break;
2091 }
2092 case Intrinsic::amdgcn_fmul_legacy: {
2093 Value *Op0 = II.getArgOperand(i: 0);
2094 Value *Op1 = II.getArgOperand(i: 1);
2095
2096 for (Value *Src : {Op0, Op1}) {
2097 if (isa<PoisonValue>(Val: Src))
2098 return IC.replaceInstUsesWith(I&: II, V: Src);
2099 }
2100
2101 // The legacy behaviour is that multiplying +/-0.0 by anything, even NaN or
2102 // infinity, gives +0.0.
2103 // TODO: Move to InstSimplify?
2104 if (match(V: Op0, P: PatternMatch::m_AnyZeroFP()) ||
2105 match(V: Op1, P: PatternMatch::m_AnyZeroFP()))
2106 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getZero(Ty: II.getType()));
2107
2108 // If we can prove we don't have one of the special cases then we can use a
2109 // normal fmul instruction instead.
2110 if (canSimplifyLegacyMulToMul(I: II, Op0, Op1, IC)) {
2111 auto *FMul = IC.Builder.CreateFMulFMF(L: Op0, R: Op1, FMFSource: &II);
2112 FMul->takeName(V: &II);
2113 return IC.replaceInstUsesWith(I&: II, V: FMul);
2114 }
2115 break;
2116 }
2117 case Intrinsic::amdgcn_fma_legacy: {
2118 Value *Op0 = II.getArgOperand(i: 0);
2119 Value *Op1 = II.getArgOperand(i: 1);
2120 Value *Op2 = II.getArgOperand(i: 2);
2121
2122 for (Value *Src : {Op0, Op1, Op2}) {
2123 if (isa<PoisonValue>(Val: Src))
2124 return IC.replaceInstUsesWith(I&: II, V: Src);
2125 }
2126
2127 // The legacy behaviour is that multiplying +/-0.0 by anything, even NaN or
2128 // infinity, gives +0.0.
2129 // TODO: Move to InstSimplify?
2130 if (match(V: Op0, P: PatternMatch::m_AnyZeroFP()) ||
2131 match(V: Op1, P: PatternMatch::m_AnyZeroFP())) {
2132 // It's tempting to just return Op2 here, but that would give the wrong
2133 // result if Op2 was -0.0.
2134 auto *Zero = ConstantFP::getZero(Ty: II.getType());
2135 auto *FAdd = IC.Builder.CreateFAddFMF(L: Zero, R: Op2, FMFSource: &II);
2136 FAdd->takeName(V: &II);
2137 return IC.replaceInstUsesWith(I&: II, V: FAdd);
2138 }
2139
2140 // If we can prove we don't have one of the special cases then we can use a
2141 // normal fma instead.
2142 if (canSimplifyLegacyMulToMul(I: II, Op0, Op1, IC)) {
2143 II.setCalledOperand(Intrinsic::getOrInsertDeclaration(
2144 M: II.getModule(), id: Intrinsic::fma, OverloadTys: II.getType()));
2145 return &II;
2146 }
2147 break;
2148 }
2149 case Intrinsic::amdgcn_is_shared:
2150 case Intrinsic::amdgcn_is_private: {
2151 Value *Src = II.getArgOperand(i: 0);
2152 if (isa<PoisonValue>(Val: Src))
2153 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
2154 if (isa<UndefValue>(Val: Src))
2155 return IC.replaceInstUsesWith(I&: II, V: UndefValue::get(T: II.getType()));
2156
2157 if (isa<ConstantPointerNull>(Val: II.getArgOperand(i: 0)))
2158 return IC.replaceInstUsesWith(I&: II, V: ConstantInt::getFalse(Ty: II.getType()));
2159 break;
2160 }
2161 case Intrinsic::amdgcn_make_buffer_rsrc: {
2162 Value *Src = II.getArgOperand(i: 0);
2163 if (isa<PoisonValue>(Val: Src))
2164 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
2165
2166 // Normalize num_records to the correct width.
2167 std::optional<unsigned> Width = ST->getBufferResourceNumRecordsWidth();
2168 if (!Width)
2169 return std::nullopt;
2170 Type *NumRecordsTy = IC.Builder.getIntNTy(N: *Width);
2171 if (II.getArgOperand(i: 2)->getType() == NumRecordsTy)
2172 return std::nullopt;
2173 SmallVector<Value *, 4> Args(II.args());
2174 Args[2] = IC.Builder.CreateZExtOrTrunc(V: Args[2], DestTy: NumRecordsTy);
2175 CallInst *NewCall = IC.Builder.CreateIntrinsicWithoutFolding(
2176 ID: Intrinsic::amdgcn_make_buffer_rsrc,
2177 OverloadTypes: {II.getType(), Src->getType(), NumRecordsTy}, Args);
2178 NewCall->copyMetadata(SrcInst: II);
2179 NewCall->setTailCallKind(II.getTailCallKind());
2180 // Copy over all attributes except those on num_records, which may no longer
2181 // be valid.
2182 NewCall->setAttributes(
2183 II.getAttributes().removeParamAttributes(C&: II.getContext(), ArgNo: 2));
2184 NewCall->takeName(V: &II);
2185 return IC.replaceInstUsesWith(I&: II, V: NewCall);
2186 }
2187 case Intrinsic::amdgcn_raw_buffer_store_format:
2188 case Intrinsic::amdgcn_raw_ptr_buffer_store_format:
2189 case Intrinsic::amdgcn_struct_buffer_store_format:
2190 case Intrinsic::amdgcn_struct_ptr_buffer_store_format:
2191 case Intrinsic::amdgcn_raw_tbuffer_store:
2192 case Intrinsic::amdgcn_raw_ptr_tbuffer_store:
2193 case Intrinsic::amdgcn_struct_tbuffer_store:
2194 case Intrinsic::amdgcn_struct_ptr_tbuffer_store:
2195 case Intrinsic::amdgcn_image_store_1d:
2196 case Intrinsic::amdgcn_image_store_1darray:
2197 case Intrinsic::amdgcn_image_store_2d:
2198 case Intrinsic::amdgcn_image_store_2darray:
2199 case Intrinsic::amdgcn_image_store_2darraymsaa:
2200 case Intrinsic::amdgcn_image_store_2dmsaa:
2201 case Intrinsic::amdgcn_image_store_3d:
2202 case Intrinsic::amdgcn_image_store_cube:
2203 case Intrinsic::amdgcn_image_store_mip_1d:
2204 case Intrinsic::amdgcn_image_store_mip_1darray:
2205 case Intrinsic::amdgcn_image_store_mip_2d:
2206 case Intrinsic::amdgcn_image_store_mip_2darray:
2207 case Intrinsic::amdgcn_image_store_mip_3d:
2208 case Intrinsic::amdgcn_image_store_mip_cube: {
2209 if (!isa<FixedVectorType>(Val: II.getArgOperand(i: 0)->getType()))
2210 break;
2211
2212 APInt DemandedElts;
2213 if (ST->hasDefaultComponentBroadcast())
2214 DemandedElts = defaultComponentBroadcast(V: II.getArgOperand(i: 0));
2215 else if (ST->hasDefaultComponentZero())
2216 DemandedElts = trimTrailingZerosInVector(IC, UseV: II.getArgOperand(i: 0), I: &II);
2217 else
2218 break;
2219
2220 if (simplifyAMDGCNMemoryIntrinsicDemanded(IC, II, DemandedElts))
2221 return IC.eraseInstFromFunction(I&: II);
2222
2223 break;
2224 }
2225 case Intrinsic::amdgcn_prng_b32: {
2226 auto *Src = II.getArgOperand(i: 0);
2227 if (isa<UndefValue>(Val: Src)) {
2228 return IC.replaceInstUsesWith(I&: II, V: Src);
2229 }
2230 return std::nullopt;
2231 }
2232 case Intrinsic::amdgcn_mfma_scale_f32_16x16x128_f8f6f4:
2233 case Intrinsic::amdgcn_mfma_scale_f32_32x32x64_f8f6f4: {
2234 Value *Src0 = II.getArgOperand(i: 0);
2235 Value *Src1 = II.getArgOperand(i: 1);
2236 uint64_t CBSZ = cast<ConstantInt>(Val: II.getArgOperand(i: 3))->getZExtValue();
2237 uint64_t BLGP = cast<ConstantInt>(Val: II.getArgOperand(i: 4))->getZExtValue();
2238 auto *Src0Ty = cast<FixedVectorType>(Val: Src0->getType());
2239 auto *Src1Ty = cast<FixedVectorType>(Val: Src1->getType());
2240
2241 auto getFormatNumRegs = [](uint64_t FormatVal) {
2242 switch (FormatVal) {
2243 case AMDGPU::MFMAScaleFormats::FP6_E2M3:
2244 case AMDGPU::MFMAScaleFormats::FP6_E3M2:
2245 return 6u;
2246 case AMDGPU::MFMAScaleFormats::FP4_E2M1:
2247 return 4u;
2248 case AMDGPU::MFMAScaleFormats::FP8_E4M3:
2249 case AMDGPU::MFMAScaleFormats::FP8_E5M2:
2250 return 8u;
2251 default:
2252 llvm_unreachable("invalid format value");
2253 }
2254 };
2255
2256 bool MadeChange = false;
2257 unsigned Src0NumElts = getFormatNumRegs(CBSZ);
2258 unsigned Src1NumElts = getFormatNumRegs(BLGP);
2259
2260 // Depending on the used format, fewer registers are required so shrink the
2261 // vector type.
2262 if (Src0Ty->getNumElements() > Src0NumElts) {
2263 Src0 = IC.Builder.CreateExtractVector(
2264 DstType: FixedVectorType::get(ElementType: Src0Ty->getElementType(), NumElts: Src0NumElts), SrcVec: Src0,
2265 Idx: uint64_t(0));
2266 MadeChange = true;
2267 }
2268
2269 if (Src1Ty->getNumElements() > Src1NumElts) {
2270 Src1 = IC.Builder.CreateExtractVector(
2271 DstType: FixedVectorType::get(ElementType: Src1Ty->getElementType(), NumElts: Src1NumElts), SrcVec: Src1,
2272 Idx: uint64_t(0));
2273 MadeChange = true;
2274 }
2275
2276 if (!MadeChange)
2277 return std::nullopt;
2278
2279 SmallVector<Value *, 10> Args(II.args());
2280 Args[0] = Src0;
2281 Args[1] = Src1;
2282
2283 Value *NewII = IC.Builder.CreateIntrinsic(
2284 ID: IID, OverloadTypes: {Src0->getType(), Src1->getType()}, Args, FMFSource: &II);
2285 NewII->takeName(V: &II);
2286 return IC.replaceInstUsesWith(I&: II, V: NewII);
2287 }
2288 case Intrinsic::amdgcn_wmma_f32_16x16x128_f8f6f4:
2289 case Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
2290 case Intrinsic::amdgcn_wmma_scale16_f32_16x16x128_f8f6f4: {
2291 Value *Src0 = II.getArgOperand(i: 1);
2292 Value *Src1 = II.getArgOperand(i: 3);
2293 unsigned FmtA = cast<ConstantInt>(Val: II.getArgOperand(i: 0))->getZExtValue();
2294 uint64_t FmtB = cast<ConstantInt>(Val: II.getArgOperand(i: 2))->getZExtValue();
2295 auto *Src0Ty = cast<FixedVectorType>(Val: Src0->getType());
2296 auto *Src1Ty = cast<FixedVectorType>(Val: Src1->getType());
2297
2298 bool MadeChange = false;
2299 unsigned Src0NumElts = AMDGPU::wmmaScaleF8F6F4FormatToNumRegs(Fmt: FmtA);
2300 unsigned Src1NumElts = AMDGPU::wmmaScaleF8F6F4FormatToNumRegs(Fmt: FmtB);
2301
2302 // Depending on the used format, fewer registers are required so shrink the
2303 // vector type.
2304 if (Src0Ty->getNumElements() > Src0NumElts) {
2305 Src0 = IC.Builder.CreateExtractVector(
2306 DstType: FixedVectorType::get(ElementType: Src0Ty->getElementType(), NumElts: Src0NumElts), SrcVec: Src0,
2307 Idx: IC.Builder.getInt64(C: 0));
2308 MadeChange = true;
2309 }
2310
2311 if (Src1Ty->getNumElements() > Src1NumElts) {
2312 Src1 = IC.Builder.CreateExtractVector(
2313 DstType: FixedVectorType::get(ElementType: Src1Ty->getElementType(), NumElts: Src1NumElts), SrcVec: Src1,
2314 Idx: IC.Builder.getInt64(C: 0));
2315 MadeChange = true;
2316 }
2317
2318 if (!MadeChange)
2319 return std::nullopt;
2320
2321 SmallVector<Value *, 13> Args(II.args());
2322 Args[1] = Src0;
2323 Args[3] = Src1;
2324
2325 Value *NewII = IC.Builder.CreateIntrinsic(
2326 ID: IID, OverloadTypes: {II.getArgOperand(i: 5)->getType(), Src0->getType(), Src1->getType()},
2327 Args, FMFSource: &II);
2328 NewII->takeName(V: &II);
2329 return IC.replaceInstUsesWith(I&: II, V: NewII);
2330 }
2331 }
2332 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
2333 AMDGPU::getImageDimIntrinsicInfo(Intr: II.getIntrinsicID())) {
2334 return simplifyAMDGCNImageIntrinsic(ST, ImageDimIntr, II, IC);
2335 }
2336 if (std::optional<unsigned> ScaleArgIdx =
2337 AMDGPU::getExponentOnlyScaleArgIdx(IntrID: IID)) {
2338 if (simplifyExponentOnlyScaleArg(IC, II, ScaleArgIdx: *ScaleArgIdx))
2339 return &II;
2340 }
2341 return std::nullopt;
2342}
2343
2344/// Implement SimplifyDemandedVectorElts for amdgcn buffer and image intrinsics.
2345///
2346/// The result of simplifying amdgcn image and buffer store intrinsics is updating
2347/// definitions of the intrinsics vector argument, not Uses of the result like
2348/// image and buffer loads.
2349/// Note: This only supports non-TFE/LWE image intrinsic calls; those have
2350/// struct returns.
2351Value *GCNTTIImpl::simplifyAMDGCNMemoryIntrinsicDemanded(
2352 InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts) const {
2353 // Only stores return void here; image stores take the dmask after the
2354 // stored value.
2355 const bool IsLoad = !II.getType()->isVoidTy();
2356 int DMaskIdx = -1;
2357 if (getAMDGPUImageDMaskIntrinsic(Intr: II.getIntrinsicID()))
2358 DMaskIdx = IsLoad ? 0 : 1;
2359
2360 auto *IIVTy = cast<FixedVectorType>(Val: IsLoad ? II.getType()
2361 : II.getOperand(i_nocapture: 0)->getType());
2362 unsigned VWidth = IIVTy->getNumElements();
2363 if (VWidth == 1)
2364 return nullptr;
2365 Type *EltTy = IIVTy->getElementType();
2366
2367 IRBuilderBase::InsertPointGuard Guard(IC.Builder);
2368 IC.Builder.SetInsertPoint(&II);
2369
2370 // Assume the arguments are unchanged and later override them, if needed.
2371 SmallVector<Value *, 16> Args(II.args());
2372 uint64_t OffsetAdd = 0;
2373
2374 if (DMaskIdx < 0) {
2375 // Buffer case.
2376
2377 const unsigned ActiveBits = DemandedElts.getActiveBits();
2378 const unsigned UnusedComponentsAtFront = DemandedElts.countr_zero();
2379
2380 // Start assuming the prefix of elements is demanded, but possibly clear
2381 // some other bits if there are trailing zeros (unused components at front)
2382 // and update offset.
2383 DemandedElts = (1 << ActiveBits) - 1;
2384
2385 if (UnusedComponentsAtFront > 0) {
2386 const AMDGPU::BufferIntrinsicInfo *BufInfo =
2387 AMDGPU::lookupBufferIntrinsic(Intr: II.getIntrinsicID());
2388 // buffer_load_format takes its format from the resource, which is
2389 // unknown here.
2390 bool CanAdjustOffset = BufInfo &&
2391 (BufInfo->Kind == AMDGPU::BufferKindLoad ||
2392 BufInfo->Kind == AMDGPU::BufferKindSLoad) &&
2393 !BufInfo->isFormatFromRsrc();
2394 // If resulting type is vec3, there is no point in trimming the scalar
2395 // load with updated offset, as the vec3 would most likely be widened to
2396 // vec4 anyway during lowering.
2397 if (CanAdjustOffset && BufInfo->Kind == AMDGPU::BufferKindSLoad &&
2398 ActiveBits == 4 && UnusedComponentsAtFront == 1)
2399 CanAdjustOffset = false;
2400
2401 // Set for tbuffer loads, whose component size comes from the format.
2402 unsigned FormatBitsPerComp = 0;
2403 if (CanAdjustOffset && BufInfo->isTyped()) {
2404 const unsigned FormatIdx = *BufInfo->getFormatArg();
2405 // The format describes the whole element, so also drop the leading
2406 // components from it to keep the element ending at the same byte.
2407 // Lanes past the end of the format are filled in by the hardware
2408 // rather than loaded; leave such loads alone.
2409 const AMDGPU::GcnBufferFormatInfo *OldFormat =
2410 AMDGPU::getGcnBufferFormatInfo(
2411 Format: cast<ConstantInt>(Val: Args[FormatIdx])->getZExtValue(), STI: *ST);
2412 const AMDGPU::GcnBufferFormatInfo *NewFormat = nullptr;
2413 if (OldFormat && ActiveBits <= OldFormat->NumComponents)
2414 NewFormat = AMDGPU::getGcnBufferFormatInfo(
2415 BitsPerComp: OldFormat->BitsPerComp,
2416 NumComponents: OldFormat->NumComponents - UnusedComponentsAtFront,
2417 NumFormat: OldFormat->NumFormat, STI: *ST);
2418 if (NewFormat) {
2419 Args[FormatIdx] =
2420 ConstantInt::get(Ty: Args[FormatIdx]->getType(), V: NewFormat->Format);
2421 FormatBitsPerComp = OldFormat->BitsPerComp;
2422 } else {
2423 CanAdjustOffset = false;
2424 }
2425 }
2426
2427 if (CanAdjustOffset) {
2428 unsigned OffsetIdx = BufInfo->getOffsetArg();
2429 // Clear demanded bits and update the offset.
2430 DemandedElts &= ~((1 << UnusedComponentsAtFront) - 1);
2431 auto *Offset = Args[OffsetIdx];
2432 uint64_t SingleComponentSizeInBits =
2433 FormatBitsPerComp
2434 ? FormatBitsPerComp
2435 : IC.getDataLayout().getTypeSizeInBits(Ty: EltTy).getFixedValue();
2436 OffsetAdd = UnusedComponentsAtFront * SingleComponentSizeInBits / 8;
2437 auto *OffsetAddVal = ConstantInt::get(Ty: Offset->getType(), V: OffsetAdd);
2438 Args[OffsetIdx] = IC.Builder.CreateAdd(LHS: Offset, RHS: OffsetAddVal);
2439 }
2440 }
2441 } else {
2442 // Image case.
2443
2444 ConstantInt *DMask = cast<ConstantInt>(Val: Args[DMaskIdx]);
2445 unsigned DMaskVal = DMask->getZExtValue() & 0xf;
2446
2447 // dmask 0 has special semantics, do not simplify.
2448 if (DMaskVal == 0)
2449 return nullptr;
2450
2451 if (!IsLoad && !isMask_32(Value: DMaskVal))
2452 return nullptr;
2453
2454 // Mask off values that are undefined because the dmask doesn't cover them
2455 DemandedElts &= (1 << llvm::popcount(Value: DMaskVal)) - 1;
2456
2457 unsigned NewDMaskVal = 0;
2458 unsigned OrigLdStIdx = 0;
2459 for (unsigned SrcIdx = 0; SrcIdx < 4; ++SrcIdx) {
2460 const unsigned Bit = 1 << SrcIdx;
2461 if (!!(DMaskVal & Bit)) {
2462 if (!!DemandedElts[OrigLdStIdx])
2463 NewDMaskVal |= Bit;
2464 OrigLdStIdx++;
2465 }
2466 }
2467
2468 if (DMaskVal != NewDMaskVal)
2469 Args[DMaskIdx] = ConstantInt::get(Ty: DMask->getType(), V: NewDMaskVal);
2470 }
2471
2472 unsigned NewNumElts = DemandedElts.popcount();
2473 if (!NewNumElts)
2474 return PoisonValue::get(T: IIVTy);
2475
2476 if (NewNumElts >= VWidth && DemandedElts.isMask()) {
2477 if (DMaskIdx >= 0)
2478 II.setArgOperand(i: DMaskIdx, v: Args[DMaskIdx]);
2479 return nullptr;
2480 }
2481
2482 // Validate function argument and return types, extracting overloaded types
2483 // along the way.
2484 SmallVector<Type *, 6> OverloadTys;
2485 if (!Intrinsic::isSignatureValid(F: II.getCalledFunction(), OverloadTys))
2486 return nullptr;
2487
2488 Type *NewTy =
2489 (NewNumElts == 1) ? EltTy : FixedVectorType::get(ElementType: EltTy, NumElts: NewNumElts);
2490 OverloadTys[0] = NewTy;
2491
2492 if (!IsLoad) {
2493 SmallVector<int, 8> EltMask;
2494 for (unsigned OrigStoreIdx = 0; OrigStoreIdx < VWidth; ++OrigStoreIdx)
2495 if (DemandedElts[OrigStoreIdx])
2496 EltMask.push_back(Elt: OrigStoreIdx);
2497
2498 if (NewNumElts == 1)
2499 Args[0] = IC.Builder.CreateExtractElement(Vec: II.getOperand(i_nocapture: 0), Idx: EltMask[0]);
2500 else
2501 Args[0] = IC.Builder.CreateShuffleVector(V: II.getOperand(i_nocapture: 0), Mask: EltMask);
2502 }
2503
2504 CallInst *NewCall = IC.Builder.CreateIntrinsicWithoutFolding(
2505 ID: II.getIntrinsicID(), OverloadTypes: OverloadTys, Args);
2506 NewCall->takeName(V: &II);
2507 NewCall->copyMetadata(SrcInst: II);
2508 AttributeList OldAttrList = II.getAttributes();
2509 NewCall->setAttributes(OldAttrList);
2510
2511 if (OffsetAdd != 0) {
2512 unsigned RsrcArg =
2513 AMDGPU::lookupRsrcIntrinsic(Intr: II.getIntrinsicID())->RsrcArg;
2514 if (MaybeAlign RsrcAlign = NewCall->getParamAlign(ArgNo: RsrcArg)) {
2515 NewCall->removeParamAttr(ArgNo: RsrcArg, Kind: Attribute::Alignment);
2516 NewCall->addParamAttr(
2517 ArgNo: RsrcArg,
2518 Attr: Attribute::getWithAlignment(Context&: NewCall->getContext(),
2519 Alignment: commonAlignment(A: *RsrcAlign, Offset: OffsetAdd)));
2520 }
2521 }
2522
2523 if (IsLoad) {
2524 if (NewNumElts == 1) {
2525 return IC.Builder.CreateInsertElement(Vec: PoisonValue::get(T: IIVTy), NewElt: NewCall,
2526 Idx: DemandedElts.countr_zero());
2527 }
2528
2529 SmallVector<int, 8> EltMask;
2530 unsigned NewLoadIdx = 0;
2531 for (unsigned OrigLoadIdx = 0; OrigLoadIdx < VWidth; ++OrigLoadIdx) {
2532 if (!!DemandedElts[OrigLoadIdx])
2533 EltMask.push_back(Elt: NewLoadIdx++);
2534 else
2535 EltMask.push_back(Elt: NewNumElts);
2536 }
2537
2538 auto *Shuffle = IC.Builder.CreateShuffleVector(V: NewCall, Mask: EltMask);
2539
2540 return Shuffle;
2541 }
2542
2543 return NewCall;
2544}
2545
2546Value *GCNTTIImpl::simplifyAMDGCNLaneIntrinsicDemanded(
2547 InstCombiner &IC, IntrinsicInst &II, const APInt &DemandedElts,
2548 APInt &UndefElts) const {
2549 auto *VT = dyn_cast<FixedVectorType>(Val: II.getType());
2550 if (!VT)
2551 return nullptr;
2552
2553 const unsigned FirstElt = DemandedElts.countr_zero();
2554 const unsigned LastElt = DemandedElts.getActiveBits() - 1;
2555 const unsigned MaskLen = LastElt - FirstElt + 1;
2556
2557 unsigned OldNumElts = VT->getNumElements();
2558 if (MaskLen == OldNumElts && MaskLen != 1)
2559 return nullptr;
2560
2561 Type *EltTy = VT->getElementType();
2562 Type *NewVT = MaskLen == 1 ? EltTy : FixedVectorType::get(ElementType: EltTy, NumElts: MaskLen);
2563
2564 // Theoretically we should support these intrinsics for any legal type. Avoid
2565 // introducing cases that aren't direct register types like v3i16.
2566 if (!isTypeLegal(Ty: NewVT))
2567 return nullptr;
2568
2569 Value *Src = II.getArgOperand(i: 0);
2570
2571 // Make sure convergence tokens are preserved.
2572 // TODO: CreateIntrinsic should allow directly copying bundles
2573 SmallVector<OperandBundleDef, 2> OpBundles;
2574 II.getOperandBundlesAsDefs(Defs&: OpBundles);
2575
2576 Module *M = IC.Builder.getModule();
2577 Function *Remangled =
2578 Intrinsic::getOrInsertDeclaration(M, id: II.getIntrinsicID(), OverloadTys: {NewVT});
2579
2580 if (MaskLen == 1) {
2581 Value *Extract = IC.Builder.CreateExtractElement(Vec: Src, Idx: FirstElt);
2582
2583 // TODO: Preserve callsite attributes?
2584 CallInst *NewCall = IC.Builder.CreateCall(Callee: Remangled, Args: {Extract}, OpBundles);
2585
2586 return IC.Builder.CreateInsertElement(Vec: PoisonValue::get(T: II.getType()),
2587 NewElt: NewCall, Idx: FirstElt);
2588 }
2589
2590 SmallVector<int> ExtractMask(MaskLen, -1);
2591 for (unsigned I = 0; I != MaskLen; ++I) {
2592 if (DemandedElts[FirstElt + I])
2593 ExtractMask[I] = FirstElt + I;
2594 }
2595
2596 Value *Extract = IC.Builder.CreateShuffleVector(V: Src, Mask: ExtractMask);
2597
2598 // TODO: Preserve callsite attributes?
2599 CallInst *NewCall = IC.Builder.CreateCall(Callee: Remangled, Args: {Extract}, OpBundles);
2600
2601 SmallVector<int> InsertMask(OldNumElts, -1);
2602 for (unsigned I = 0; I != MaskLen; ++I) {
2603 if (DemandedElts[FirstElt + I])
2604 InsertMask[FirstElt + I] = I;
2605 }
2606
2607 // FIXME: If the call has a convergence bundle, we end up leaving the dead
2608 // call behind.
2609 return IC.Builder.CreateShuffleVector(V: NewCall, Mask: InsertMask);
2610}
2611
2612std::optional<Value *> GCNTTIImpl::simplifyDemandedVectorEltsIntrinsic(
2613 InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
2614 APInt &UndefElts2, APInt &UndefElts3,
2615 std::function<void(Instruction *, unsigned, APInt, APInt &)>
2616 SimplifyAndSetOp) const {
2617 switch (II.getIntrinsicID()) {
2618 case Intrinsic::amdgcn_readfirstlane:
2619 SimplifyAndSetOp(&II, 0, DemandedElts, UndefElts);
2620 return simplifyAMDGCNLaneIntrinsicDemanded(IC, II, DemandedElts, UndefElts);
2621 default:
2622 if (const AMDGPU::BufferIntrinsicInfo *BufInfo =
2623 AMDGPU::lookupBufferIntrinsic(Intr: II.getIntrinsicID());
2624 BufInfo && (BufInfo->Kind == AMDGPU::BufferKindLoad ||
2625 BufInfo->Kind == AMDGPU::BufferKindSLoad))
2626 return simplifyAMDGCNMemoryIntrinsicDemanded(IC, II, DemandedElts);
2627 if (getAMDGPUImageDMaskIntrinsic(Intr: II.getIntrinsicID()))
2628 return simplifyAMDGCNMemoryIntrinsicDemanded(IC, II, DemandedElts);
2629 break;
2630 }
2631 return std::nullopt;
2632}
2633