1//===- AMDGPInstCombineIntrinsic.cpp - AMDGPU specific InstCombine pass ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// \file
10// This file implements a TargetTransformInfo analysis pass specific to the
11// AMDGPU target machine. It uses the target's detailed information to provide
12// more precise answers to certain TTI queries, while letting the target
13// independent and default TTI implementations handle the rest.
14//
15//===----------------------------------------------------------------------===//
16
17#include "AMDGPUInstrInfo.h"
18#include "AMDGPUTargetTransformInfo.h"
19#include "GCNSubtarget.h"
20#include "SIDefines.h"
21#include "llvm/ADT/FloatingPointMode.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/Sequence.h"
24#include "llvm/Analysis/ConstantFolding.h"
25#include "llvm/Analysis/ValueTracking.h"
26#include "llvm/IR/Constants.h"
27#include "llvm/IR/Dominators.h"
28#include "llvm/IR/IntrinsicsAMDGPU.h"
29#include "llvm/Support/MathExtras.h"
30#include "llvm/Transforms/InstCombine/InstCombiner.h"
31#include <optional>
32
33using namespace llvm;
34using namespace llvm::PatternMatch;
35
36#define DEBUG_TYPE "AMDGPUtti"
37
38namespace {
39
40struct AMDGPUImageDMaskIntrinsic {
41 unsigned Intr;
42};
43
44#define GET_AMDGPUImageDMaskIntrinsicTable_IMPL
45#include "AMDGPUGenSearchableTables.inc"
46
47} // end anonymous namespace
48
49// Constant fold llvm.amdgcn.fmed3 intrinsics for standard inputs.
50//
51// A single NaN input is folded to minnum, so we rely on that folding for
52// handling NaNs.
53static APFloat fmed3AMDGCN(const APFloat &Src0, const APFloat &Src1,
54 const APFloat &Src2) {
55 assert(!Src0.isNaN() && !Src1.isNaN() && !Src2.isNaN() &&
56 "nans handled separately");
57 APFloat Max3 = maxnum(A: maxnum(A: Src0, B: Src1), B: Src2);
58
59 if (Max3.bitwiseIsEqual(RHS: Src0))
60 return maxnum(A: Src1, B: Src2);
61
62 if (Max3.bitwiseIsEqual(RHS: Src1))
63 return maxnum(A: Src0, B: Src2);
64
65 return maxnum(A: Src0, B: Src1);
66}
67
68// Check if a value can be converted to a 16-bit value without losing precision.
69// The value is expected to be either a float (IsFloat = true) or an unsigned
70// integer (IsFloat = false). When AllowI16SExt is set, a sext from i16 is also
71// accepted: for unsigned addresses sext and zext only differ for a negative
72// i16, which is out of bounds anyway (see caller).
73static bool canSafelyConvertTo16Bit(Value &V, bool IsFloat,
74 bool AllowI16SExt = false) {
75 Type *VTy = V.getType();
76 if (VTy->isHalfTy() || VTy->isIntegerTy(BitWidth: 16)) {
77 // The value is already 16-bit, so we don't want to convert to 16-bit again!
78 return false;
79 }
80 if (IsFloat) {
81 if (ConstantFP *ConstFloat = dyn_cast<ConstantFP>(Val: &V)) {
82 // We need to check that if we cast the index down to a half, we do not
83 // lose precision.
84 APFloat FloatValue(ConstFloat->getValueAPF());
85 bool LosesInfo = true;
86 FloatValue.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmTowardZero,
87 losesInfo: &LosesInfo);
88 return !LosesInfo;
89 }
90 } else {
91 if (ConstantInt *ConstInt = dyn_cast<ConstantInt>(Val: &V)) {
92 // We need to check that if we cast the index down to an i16, we do not
93 // lose precision.
94 APInt IntValue(ConstInt->getValue());
95 return IntValue.getActiveBits() <= 16;
96 }
97 }
98
99 // Coordinates may arrive as extractelement((s|z|fp)ext Vec), Idx. The
100 // widening cast has one use per lane, so it is never sunk into the extract;
101 // strip the extract here so the cast check below is common to scalar and
102 // vector coords.
103 Value *CastCandidate;
104 if (!match(V: &V, P: m_ExtractElt(Val: m_Value(V&: CastCandidate), Idx: m_Value())))
105 CastCandidate = &V;
106
107 Value *CastSrc;
108 bool IsExt = IsFloat ? match(V: CastCandidate, P: m_FPExt(Op: m_Value(V&: CastSrc)))
109 : match(V: CastCandidate, P: m_ZExt(Op: m_Value(V&: CastSrc)));
110 if (!IsExt && !IsFloat && AllowI16SExt)
111 IsExt = match(V: CastCandidate, P: m_SExt(Op: m_Value(V&: CastSrc)));
112 if (IsExt) {
113 Type *CastSrcTy = CastSrc->getType()->getScalarType();
114 if (CastSrcTy->isHalfTy() || CastSrcTy->isIntegerTy(BitWidth: 16))
115 return true;
116 }
117
118 return false;
119}
120
121// Convert a value to 16-bit.
122static Value *convertTo16Bit(Value &V, InstCombiner::BuilderTy &Builder) {
123 Type *VTy = V.getType();
124 if (isa<FPExtInst, SExtInst, ZExtInst>(Val: &V))
125 return cast<Instruction>(Val: &V)->getOperand(i: 0);
126 // Vector form: extractelement((s|z|fp)ext Vec), Idx -> extractelement(Vec,
127 // Idx), taking the narrow lane directly so the widening cast can be removed.
128 Instruction *VecCast;
129 Value *Idx;
130 if (match(V: &V, P: m_ExtractElt(Val: m_Instruction(I&: VecCast), Idx: m_Value(V&: Idx))) &&
131 isa<FPExtInst, SExtInst, ZExtInst>(Val: VecCast))
132 return Builder.CreateExtractElement(Vec: VecCast->getOperand(i: 0), Idx);
133 if (VTy->isIntegerTy())
134 return Builder.CreateIntCast(V: &V, DestTy: Type::getInt16Ty(C&: V.getContext()), isSigned: false);
135 if (VTy->isFloatingPointTy())
136 return Builder.CreateFPCast(V: &V, DestTy: Type::getHalfTy(C&: V.getContext()));
137
138 llvm_unreachable("Should never be called!");
139}
140
141/// Applies Func(OldIntr.Args, OldIntr.ArgTys), creates intrinsic call with
142/// modified arguments (based on OldIntr) and replaces InstToReplace with
143/// this newly created intrinsic call.
144static std::optional<Instruction *> modifyIntrinsicCall(
145 IntrinsicInst &OldIntr, Instruction &InstToReplace, unsigned NewIntr,
146 InstCombiner &IC,
147 std::function<void(SmallVectorImpl<Value *> &, SmallVectorImpl<Type *> &)>
148 Func) {
149 SmallVector<Type *, 4> OverloadTys;
150 if (!Intrinsic::isSignatureValid(F: OldIntr.getCalledFunction(), OverloadTys))
151 return std::nullopt;
152
153 SmallVector<Value *, 8> Args(OldIntr.args());
154
155 // Modify arguments and types
156 Func(Args, OverloadTys);
157
158 CallInst *NewCall =
159 IC.Builder.CreateIntrinsicWithoutFolding(ID: NewIntr, OverloadTypes: OverloadTys, Args);
160 NewCall->takeName(V: &OldIntr);
161 NewCall->copyMetadata(SrcInst: OldIntr);
162 if (isa<FPMathOperator>(Val: NewCall))
163 NewCall->copyFastMathFlags(I: &OldIntr);
164 // Copy attributes
165 AttributeList OldAttrList = OldIntr.getAttributes();
166 NewCall->setAttributes(OldAttrList);
167
168 // Erase and replace uses
169 if (!InstToReplace.getType()->isVoidTy())
170 IC.replaceInstUsesWith(I&: InstToReplace, V: NewCall);
171
172 bool RemoveOldIntr = &OldIntr != &InstToReplace;
173
174 auto *RetValue = IC.eraseInstFromFunction(I&: InstToReplace);
175 if (RemoveOldIntr)
176 IC.eraseInstFromFunction(I&: OldIntr);
177
178 return RetValue;
179}
180
181static std::optional<Instruction *>
182simplifyAMDGCNImageIntrinsic(const GCNSubtarget *ST,
183 const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr,
184 IntrinsicInst &II, InstCombiner &IC) {
185 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
186 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: ImageDimIntr->BaseOpcode);
187
188 // Optimize _L to _LZ when _L is zero
189 if (const auto *LZMappingInfo =
190 AMDGPU::getMIMGLZMappingInfo(L: ImageDimIntr->BaseOpcode)) {
191 if (auto *ConstantLod =
192 dyn_cast<ConstantFP>(Val: II.getOperand(i_nocapture: ImageDimIntr->LodIndex))) {
193 if (ConstantLod->isZero() || ConstantLod->isNegative()) {
194 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
195 AMDGPU::getImageDimIntrinsicByBaseOpcode(BaseOpcode: LZMappingInfo->LZ,
196 Dim: ImageDimIntr->Dim);
197 return modifyIntrinsicCall(
198 OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC, Func: [&](auto &Args, auto &ArgTys) {
199 Args.erase(Args.begin() + ImageDimIntr->LodIndex);
200 });
201 }
202 }
203 }
204
205 // Optimize _mip away, when 'lod' is zero
206 if (const auto *MIPMappingInfo =
207 AMDGPU::getMIMGMIPMappingInfo(MIP: ImageDimIntr->BaseOpcode)) {
208 if (auto *ConstantMip =
209 dyn_cast<ConstantInt>(Val: II.getOperand(i_nocapture: ImageDimIntr->MipIndex))) {
210 if (ConstantMip->isZero()) {
211 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
212 AMDGPU::getImageDimIntrinsicByBaseOpcode(BaseOpcode: MIPMappingInfo->NONMIP,
213 Dim: ImageDimIntr->Dim);
214 return modifyIntrinsicCall(
215 OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC, Func: [&](auto &Args, auto &ArgTys) {
216 Args.erase(Args.begin() + ImageDimIntr->MipIndex);
217 });
218 }
219 }
220 }
221
222 // Optimize _bias away when 'bias' is zero
223 if (const auto *BiasMappingInfo =
224 AMDGPU::getMIMGBiasMappingInfo(Bias: ImageDimIntr->BaseOpcode)) {
225 if (auto *ConstantBias =
226 dyn_cast<ConstantFP>(Val: II.getOperand(i_nocapture: ImageDimIntr->BiasIndex))) {
227 if (ConstantBias->isZero()) {
228 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
229 AMDGPU::getImageDimIntrinsicByBaseOpcode(BaseOpcode: BiasMappingInfo->NoBias,
230 Dim: ImageDimIntr->Dim);
231 return modifyIntrinsicCall(
232 OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC, Func: [&](auto &Args, auto &ArgTys) {
233 Args.erase(Args.begin() + ImageDimIntr->BiasIndex);
234 ArgTys.erase(ArgTys.begin() + ImageDimIntr->BiasTyArg);
235 });
236 }
237 }
238 }
239
240 // Optimize _offset away when 'offset' is zero
241 if (const auto *OffsetMappingInfo =
242 AMDGPU::getMIMGOffsetMappingInfo(Offset: ImageDimIntr->BaseOpcode)) {
243 if (auto *ConstantOffset =
244 dyn_cast<ConstantInt>(Val: II.getOperand(i_nocapture: ImageDimIntr->OffsetIndex))) {
245 if (ConstantOffset->isZero()) {
246 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
247 AMDGPU::getImageDimIntrinsicByBaseOpcode(
248 BaseOpcode: OffsetMappingInfo->NoOffset, Dim: ImageDimIntr->Dim);
249 return modifyIntrinsicCall(
250 OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC, Func: [&](auto &Args, auto &ArgTys) {
251 Args.erase(Args.begin() + ImageDimIntr->OffsetIndex);
252 });
253 }
254 }
255 }
256
257 // Optimize the arrayed dim away when the array slice is zero, since slice 0
258 // is the base layer. Restricted to non-atomic, non-sampled image loads and
259 // stores for now.
260 const AMDGPU::MIMGDimInfo *DimInfo =
261 AMDGPU::getMIMGDimInfo(DimEnum: ImageDimIntr->Dim);
262 if (!BaseOpcode->Atomic && !BaseOpcode->Sampler && BaseOpcode->Coordinates &&
263 DimInfo->NonArrayDim != ImageDimIntr->Dim) {
264 // Address is [coords..., slice, (fragid)] plus an optional mip operand.
265 // The slice is the last coordinate, so index it from CoordStart.
266 unsigned SliceIndex = ImageDimIntr->CoordStart + DimInfo->NumCoords - 1 -
267 (DimInfo->MSAA ? 1 : 0);
268 auto *ConstantSlice = dyn_cast<ConstantInt>(Val: II.getOperand(i_nocapture: SliceIndex));
269 if (ConstantSlice && ConstantSlice->isZero()) {
270 if (const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
271 AMDGPU::getImageDimIntrinsicByBaseOpcode(BaseOpcode: ImageDimIntr->BaseOpcode,
272 Dim: DimInfo->NonArrayDim)) {
273 return modifyIntrinsicCall(OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC,
274 Func: [&](auto &Args, auto &ArgTys) {
275 Args.erase(Args.begin() + SliceIndex);
276 });
277 }
278 }
279 }
280
281 // Try to use D16
282 if (ST->hasD16Images()) {
283 if (BaseOpcode->HasD16) {
284
285 // If the only use of image intrinsic is a fptrunc (with conversion to
286 // half) then both fptrunc and image intrinsic will be replaced with image
287 // intrinsic with D16 flag.
288 if (II.hasOneUse()) {
289 Instruction *User = II.user_back();
290
291 if (User->getOpcode() == Instruction::FPTrunc &&
292 User->getType()->getScalarType()->isHalfTy()) {
293
294 return modifyIntrinsicCall(OldIntr&: II, InstToReplace&: *User, NewIntr: ImageDimIntr->Intr, IC,
295 Func: [&](auto &Args, auto &ArgTys) {
296 // Change return type of image intrinsic.
297 // Set it to return type of fptrunc.
298 ArgTys[0] = User->getType();
299 });
300 }
301 }
302
303 // Only perform D16 folding if every user of the image sample is
304 // an ExtractElementInst immediately followed by an FPTrunc to half.
305 SmallVector<std::pair<ExtractElementInst *, FPTruncInst *>, 4>
306 ExtractTruncPairs;
307 bool AllHalfExtracts = true;
308
309 for (User *U : II.users()) {
310 auto *Ext = dyn_cast<ExtractElementInst>(Val: U);
311 if (!Ext || !Ext->hasOneUse()) {
312 AllHalfExtracts = false;
313 break;
314 }
315
316 auto *Tr = dyn_cast<FPTruncInst>(Val: *Ext->user_begin());
317 if (!Tr || !Tr->getType()->isHalfTy()) {
318 AllHalfExtracts = false;
319 break;
320 }
321
322 ExtractTruncPairs.emplace_back(Args&: Ext, Args&: Tr);
323 }
324
325 if (!ExtractTruncPairs.empty() && AllHalfExtracts) {
326 auto *VecTy = cast<VectorType>(Val: II.getType());
327 Type *HalfVecTy =
328 VecTy->getWithNewType(EltTy: Type::getHalfTy(C&: II.getContext()));
329
330 // Obtain the original image sample intrinsic's signature
331 // and replace its return type with the half-vector for D16 folding
332 SmallVector<Type *, 8> OverloadTys;
333 if (!Intrinsic::isSignatureValid(F: II.getCalledFunction(), OverloadTys))
334 return std::nullopt;
335
336 OverloadTys[0] = HalfVecTy;
337 Module *M = II.getModule();
338 Function *HalfDecl = Intrinsic::getOrInsertDeclaration(
339 M, id: ImageDimIntr->Intr, OverloadTys);
340
341 II.mutateType(Ty: HalfVecTy);
342 II.setCalledFunction(HalfDecl);
343
344 IRBuilder<> Builder(II.getContext());
345 for (auto &[Ext, Tr] : ExtractTruncPairs) {
346 Value *Idx = Ext->getIndexOperand();
347
348 Builder.SetInsertPoint(Tr);
349
350 Value *HalfExtract = Builder.CreateExtractElement(Vec: &II, Idx);
351 HalfExtract->takeName(V: Tr);
352
353 Tr->replaceAllUsesWith(V: HalfExtract);
354 }
355
356 for (auto &[Ext, Tr] : ExtractTruncPairs) {
357 IC.eraseInstFromFunction(I&: *Tr);
358 IC.eraseInstFromFunction(I&: *Ext);
359 }
360
361 return &II;
362 }
363 }
364 }
365
366 // Try to use A16 or G16
367 if (!ST->hasA16() && !ST->hasG16())
368 return std::nullopt;
369
370 // Address is interpreted as float if the instruction has a sampler or as
371 // unsigned int if there is no sampler.
372 bool HasSampler = BaseOpcode->Sampler;
373 bool FloatCoord = false;
374 // true means derivatives can be converted to 16 bit, coordinates not
375 bool OnlyDerivatives = false;
376
377 // Sampler-less addresses are unsigned, so a sext from i16 folds to a16 like a
378 // zext: they only disagree for a negative i16 (>= 0x8000), which is out of
379 // bounds while the max image dimension is <= 0x8000.
380 bool AllowI16SExt = !HasSampler;
381
382 for (unsigned OperandIndex = ImageDimIntr->GradientStart;
383 OperandIndex < ImageDimIntr->VAddrEnd; OperandIndex++) {
384 Value *Coord = II.getOperand(i_nocapture: OperandIndex);
385 // If the values are not derived from 16-bit values, we cannot optimize.
386 if (!canSafelyConvertTo16Bit(V&: *Coord, IsFloat: HasSampler, AllowI16SExt)) {
387 if (OperandIndex < ImageDimIntr->CoordStart ||
388 ImageDimIntr->GradientStart == ImageDimIntr->CoordStart) {
389 return std::nullopt;
390 }
391 // All gradients can be converted, so convert only them
392 OnlyDerivatives = true;
393 break;
394 }
395
396 assert(OperandIndex == ImageDimIntr->GradientStart ||
397 FloatCoord == Coord->getType()->isFloatingPointTy());
398 FloatCoord = Coord->getType()->isFloatingPointTy();
399 }
400
401 if (!OnlyDerivatives && !ST->hasA16())
402 OnlyDerivatives = true; // Only supports G16
403
404 // Check if there is a bias parameter and if it can be converted to f16
405 if (!OnlyDerivatives && ImageDimIntr->NumBiasArgs != 0) {
406 Value *Bias = II.getOperand(i_nocapture: ImageDimIntr->BiasIndex);
407 assert(HasSampler &&
408 "Only image instructions with a sampler can have a bias");
409 if (!canSafelyConvertTo16Bit(V&: *Bias, IsFloat: HasSampler))
410 OnlyDerivatives = true;
411 }
412
413 if (OnlyDerivatives && (!ST->hasG16() || ImageDimIntr->GradientStart ==
414 ImageDimIntr->CoordStart))
415 return std::nullopt;
416
417 Type *CoordType = FloatCoord ? Type::getHalfTy(C&: II.getContext())
418 : Type::getInt16Ty(C&: II.getContext());
419
420 return modifyIntrinsicCall(
421 OldIntr&: II, InstToReplace&: II, NewIntr: II.getIntrinsicID(), IC, Func: [&](auto &Args, auto &ArgTys) {
422 ArgTys[ImageDimIntr->GradientTyArg] = CoordType;
423 if (!OnlyDerivatives) {
424 ArgTys[ImageDimIntr->CoordTyArg] = CoordType;
425
426 // Change the bias type
427 if (ImageDimIntr->NumBiasArgs != 0)
428 ArgTys[ImageDimIntr->BiasTyArg] = Type::getHalfTy(C&: II.getContext());
429 }
430
431 unsigned EndIndex =
432 OnlyDerivatives ? ImageDimIntr->CoordStart : ImageDimIntr->VAddrEnd;
433 for (unsigned OperandIndex = ImageDimIntr->GradientStart;
434 OperandIndex < EndIndex; OperandIndex++) {
435 Args[OperandIndex] =
436 convertTo16Bit(V&: *II.getOperand(i_nocapture: OperandIndex), Builder&: IC.Builder);
437 }
438
439 // Convert the bias
440 if (!OnlyDerivatives && ImageDimIntr->NumBiasArgs != 0) {
441 Value *Bias = II.getOperand(i_nocapture: ImageDimIntr->BiasIndex);
442 Args[ImageDimIntr->BiasIndex] = convertTo16Bit(V&: *Bias, Builder&: IC.Builder);
443 }
444 });
445}
446
447bool GCNTTIImpl::canSimplifyLegacyMulToMul(const Instruction &I,
448 const Value *Op0, const Value *Op1,
449 InstCombiner &IC) const {
450 // The legacy behaviour is that multiplying +/-0.0 by anything, even NaN or
451 // infinity, gives +0.0. If we can prove we don't have one of the special
452 // cases then we can use a normal multiply instead.
453 SimplifyQuery SQ = IC.getSimplifyQuery().getWithInstruction(I: &I);
454 KnownFPClass Known0 =
455 computeKnownFPClass(V: Op0, InterestedClasses: fcZero | fcSubnormal | fcInf | fcNan, SQ);
456 DenormalMode Mode = I.getFunction()->getDenormalMode(FPType: APFloat::IEEEsingle());
457
458 // Bail early if Op0 may be zero and nsz is not set -- Op1 cannot help.
459 if (!Known0.isKnownNeverLogicalZero(Mode) && !I.hasNoSignedZeros())
460 return false;
461
462 KnownFPClass Known1 =
463 computeKnownFPClass(V: Op1, InterestedClasses: fcZero | fcSubnormal | fcInf | fcNan, SQ);
464
465 // Simplify if both operands are known non-zero.
466 if (Known0.isKnownNeverLogicalZero(Mode) &&
467 Known1.isKnownNeverLogicalZero(Mode))
468 return true;
469
470 // With nsz, two additional cases allow simplification:
471 // 1. One operand is not zero or infinity or NaN:
472 // Op0 NeverLogicalZero && NeverInfOrNaN, or symmetric for Op1.
473 // 2. Neither operand is infinity or NaN:
474 // Op0 NeverInfOrNaN && Op1 NeverInfOrNaN.
475 // The following condition captures both cases.
476 if (I.hasNoSignedZeros() &&
477 (Known0.isKnownNeverLogicalZero(Mode) || Known1.isKnownNeverInfOrNaN()) &&
478 (Known1.isKnownNeverLogicalZero(Mode) || Known0.isKnownNeverInfOrNaN()))
479 return true;
480
481 return false;
482}
483
484/// Match an fpext from half to float, or a constant we can convert.
485static Value *matchFPExtFromF16(Value *Arg) {
486 Value *Src = nullptr;
487 ConstantFP *CFP = nullptr;
488 if (match(V: Arg, P: m_OneUse(SubPattern: m_FPExt(Op: m_Value(V&: Src))))) {
489 if (Src->getType()->isHalfTy())
490 return Src;
491 } else if (match(V: Arg, P: m_ConstantFP(C&: CFP))) {
492 bool LosesInfo;
493 APFloat Val(CFP->getValueAPF());
494 Val.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmNearestTiesToEven, losesInfo: &LosesInfo);
495 if (!LosesInfo)
496 return ConstantFP::get(Ty: Type::getHalfTy(C&: Arg->getContext()), V: Val);
497 }
498 return nullptr;
499}
500
501// Trim all zero components from the end of the vector \p UseV and return
502// an appropriate bitset with known elements.
503static APInt trimTrailingZerosInVector(InstCombiner &IC, Value *UseV,
504 Instruction *I) {
505 auto *VTy = cast<FixedVectorType>(Val: UseV->getType());
506 unsigned VWidth = VTy->getNumElements();
507 APInt DemandedElts = APInt::getAllOnes(numBits: VWidth);
508
509 for (int i = VWidth - 1; i > 0; --i) {
510 auto *Elt = findScalarElement(V: UseV, EltNo: i);
511 if (!Elt)
512 break;
513
514 if (auto *ConstElt = dyn_cast<Constant>(Val: Elt)) {
515 if (!ConstElt->isNullValue() && !isa<UndefValue>(Val: Elt))
516 break;
517 } else {
518 break;
519 }
520
521 DemandedElts.clearBit(BitPosition: i);
522 }
523
524 return DemandedElts;
525}
526
527// Trim elements of the end of the vector \p V, if they are
528// equal to the first element of the vector.
529static APInt defaultComponentBroadcast(Value *V) {
530 auto *VTy = cast<FixedVectorType>(Val: V->getType());
531 unsigned VWidth = VTy->getNumElements();
532 APInt DemandedElts = APInt::getAllOnes(numBits: VWidth);
533 Value *FirstComponent = findScalarElement(V, EltNo: 0);
534
535 SmallVector<int> ShuffleMask;
536 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: V))
537 SVI->getShuffleMask(Result&: ShuffleMask);
538
539 for (int I = VWidth - 1; I > 0; --I) {
540 if (ShuffleMask.empty()) {
541 auto *Elt = findScalarElement(V, EltNo: I);
542 if (!Elt || (Elt != FirstComponent && !isa<UndefValue>(Val: Elt)))
543 break;
544 } else {
545 // Detect identical elements in the shufflevector result, even though
546 // findScalarElement cannot tell us what that element is.
547 if (ShuffleMask[I] != ShuffleMask[0] && ShuffleMask[I] != PoisonMaskElem)
548 break;
549 }
550 DemandedElts.clearBit(BitPosition: I);
551 }
552
553 return DemandedElts;
554}
555
556static Value *simplifyAMDGCNMemoryIntrinsicDemanded(InstCombiner &IC,
557 IntrinsicInst &II,
558 APInt DemandedElts,
559 int DMaskIdx = -1,
560 bool IsLoad = true);
561
562/// Return true if it's legal to contract llvm.amdgcn.rcp(llvm.sqrt)
563static bool canContractSqrtToRsq(const FPMathOperator *SqrtOp) {
564 return (SqrtOp->getType()->isFloatTy() &&
565 (SqrtOp->hasApproxFunc() || SqrtOp->getFPAccuracy() >= 1.0f)) ||
566 SqrtOp->getType()->isHalfTy();
567}
568
569/// Return true if we can easily prove that use U is uniform.
570static bool isTriviallyUniform(const Use &U) {
571 Value *V = U.get();
572 if (isa<Constant>(Val: V))
573 return true;
574 if (const auto *A = dyn_cast<Argument>(Val: V))
575 return AMDGPU::isArgPassedInSGPR(Arg: A);
576 if (const auto *II = dyn_cast<IntrinsicInst>(Val: V)) {
577 if (!AMDGPU::isIntrinsicAlwaysUniform(IntrID: II->getIntrinsicID()))
578 return false;
579 // If II and U are in different blocks then there is a possibility of
580 // temporal divergence.
581 return II->getParent() == cast<Instruction>(Val: U.getUser())->getParent();
582 }
583 return false;
584}
585
586/// Simplify a lane index operand (e.g. llvm.amdgcn.readlane src1).
587///
588/// The instruction only reads the low 5 bits for wave32, and 6 bits for wave64.
589bool GCNTTIImpl::simplifyDemandedLaneMaskArg(InstCombiner &IC,
590 IntrinsicInst &II,
591 unsigned LaneArgIdx) const {
592 unsigned MaskBits = ST->getWavefrontSizeLog2();
593 APInt DemandedMask(32, maskTrailingOnes<unsigned>(N: MaskBits));
594
595 KnownBits Known(32);
596 if (IC.SimplifyDemandedBits(I: &II, OpNo: LaneArgIdx, DemandedMask, Known))
597 return true;
598
599 if (!Known.isConstant())
600 return false;
601
602 // Out of bounds indexes may appear in wave64 code compiled for wave32.
603 // Unlike the DAG version, SimplifyDemandedBits does not change constants, so
604 // manually fix it up.
605
606 Value *LaneArg = II.getArgOperand(i: LaneArgIdx);
607 Constant *MaskedConst =
608 ConstantInt::get(Ty: LaneArg->getType(), V: Known.getConstant() & DemandedMask);
609 if (MaskedConst != LaneArg) {
610 II.getOperandUse(i: LaneArgIdx).set(MaskedConst);
611 return true;
612 }
613
614 return false;
615}
616
617static CallInst *rewriteCall(IRBuilderBase &B, CallInst &Old,
618 Function &NewCallee, ArrayRef<Value *> Ops) {
619 SmallVector<OperandBundleDef, 2> OpBundles;
620 Old.getOperandBundlesAsDefs(Defs&: OpBundles);
621
622 CallInst *NewCall = B.CreateCall(Callee: &NewCallee, Args: Ops, OpBundles);
623 NewCall->takeName(V: &Old);
624 return NewCall;
625}
626
627// Return true for sequences of instructions that effectively assign
628// each lane to its thread ID
629static bool isThreadID(const GCNSubtarget &ST, Value *V) {
630 // Case 1:
631 // wave32: mbcnt_lo(-1, 0)
632 // wave64: mbcnt_hi(-1, mbcnt_lo(-1, 0))
633 auto W32Pred = m_Intrinsic<Intrinsic::amdgcn_mbcnt_lo>(Ops: m_ConstantInt<-1>(),
634 Ops: m_ConstantInt<0>());
635 auto W64Pred = m_Intrinsic<Intrinsic::amdgcn_mbcnt_hi>(
636 Ops: m_ConstantInt<-1>(), Ops: m_Intrinsic<Intrinsic::amdgcn_mbcnt_lo>(
637 Ops: m_ConstantInt<-1>(), Ops: m_ConstantInt<0>()));
638 if (ST.isWave32() && match(V, P: W32Pred))
639 return true;
640 if (ST.isWave64() && match(V, P: W64Pred))
641 return true;
642
643 return false;
644}
645
646Instruction *
647GCNTTIImpl::hoistLaneIntrinsicThroughOperand(InstCombiner &IC,
648 IntrinsicInst &II) const {
649 const auto IID = II.getIntrinsicID();
650 assert(IID == Intrinsic::amdgcn_readlane ||
651 IID == Intrinsic::amdgcn_readfirstlane ||
652 IID == Intrinsic::amdgcn_permlane64);
653
654 Instruction *OpInst = dyn_cast<Instruction>(Val: II.getOperand(i_nocapture: 0));
655
656 // Only do this if both instructions are in the same block
657 // (so the exec mask won't change) and the readlane is the only user of its
658 // operand.
659 if (!OpInst || !OpInst->hasOneUser() || OpInst->getParent() != II.getParent())
660 return nullptr;
661
662 const bool IsReadLane = (IID == Intrinsic::amdgcn_readlane);
663
664 // If this is a readlane, check that the second operand is a constant, or is
665 // defined before OpInst so we know it's safe to move this intrinsic higher.
666 Value *LaneID = nullptr;
667 if (IsReadLane) {
668 LaneID = II.getOperand(i_nocapture: 1);
669
670 // readlane take an extra operand for the lane ID, so we must check if that
671 // LaneID value can be used at the point where we want to move the
672 // intrinsic.
673 if (auto *LaneIDInst = dyn_cast<Instruction>(Val: LaneID)) {
674 if (!IC.getDominatorTree().dominates(Def: LaneIDInst, User: OpInst))
675 return nullptr;
676 }
677 }
678
679 // Hoist the intrinsic (II) through OpInst.
680 //
681 // (II (OpInst x)) -> (OpInst (II x))
682 const auto DoIt = [&](unsigned OpIdx,
683 Function *NewIntrinsic) -> Instruction * {
684 SmallVector<Value *, 2> Ops{OpInst->getOperand(i: OpIdx)};
685 if (IsReadLane)
686 Ops.push_back(Elt: LaneID);
687
688 // Rewrite the intrinsic call.
689 CallInst *NewII = rewriteCall(B&: IC.Builder, Old&: II, NewCallee&: *NewIntrinsic, Ops);
690
691 // Rewrite OpInst so it takes the result of the intrinsic now.
692 Instruction &NewOp = *OpInst->clone();
693 NewOp.setOperand(i: OpIdx, Val: NewII);
694 return &NewOp;
695 };
696
697 // TODO(?): Should we do more with permlane64?
698 if (IID == Intrinsic::amdgcn_permlane64 && !isa<BitCastInst>(Val: OpInst))
699 return nullptr;
700
701 if (isa<UnaryOperator>(Val: OpInst))
702 return DoIt(0, II.getCalledFunction());
703
704 if (isa<CastInst>(Val: OpInst)) {
705 Value *Src = OpInst->getOperand(i: 0);
706 Type *SrcTy = Src->getType();
707 if (!isTypeLegal(Ty: SrcTy))
708 return nullptr;
709
710 Function *Remangled =
711 Intrinsic::getOrInsertDeclaration(M: II.getModule(), id: IID, OverloadTys: {SrcTy});
712 return DoIt(0, Remangled);
713 }
714
715 // We can also hoist through binary operators if the other operand is uniform.
716 if (isa<BinaryOperator>(Val: OpInst)) {
717 // FIXME: If we had access to UniformityInfo here we could just check
718 // if the operand is uniform.
719 if (isTriviallyUniform(U: OpInst->getOperandUse(i: 0)))
720 return DoIt(1, II.getCalledFunction());
721 if (isTriviallyUniform(U: OpInst->getOperandUse(i: 1)))
722 return DoIt(0, II.getCalledFunction());
723 }
724
725 return nullptr;
726}
727
728/// Evaluate V as a function of the lane ID and return its value on Lane, or
729/// std::nullopt if V is not a closed-form expression of the lane ID.
730static std::optional<unsigned> evalLaneExpr(Value *V, unsigned Lane,
731 const GCNSubtarget &ST,
732 const DataLayout &DL,
733 unsigned Depth = 0) {
734 if (Depth >= MaxAnalysisRecursionDepth)
735 return std::nullopt;
736
737 // Poison/undef in the index expression: bail and let InstCombine fold the
738 // intrinsic the usual way.
739 if (isa<UndefValue>(Val: V))
740 return std::nullopt;
741
742 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: V))
743 return CI->getZExtValue();
744
745 if (isThreadID(ST, V))
746 return Lane;
747
748 const BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: V);
749 if (!BO)
750 return std::nullopt;
751
752 std::optional<unsigned> LHS =
753 evalLaneExpr(V: BO->getOperand(i_nocapture: 0), Lane, ST, DL, Depth: Depth + 1);
754 if (!LHS)
755 return std::nullopt;
756 std::optional<unsigned> RHS =
757 evalLaneExpr(V: BO->getOperand(i_nocapture: 1), Lane, ST, DL, Depth: Depth + 1);
758 if (!RHS)
759 return std::nullopt;
760
761 Type *Ty = BO->getType();
762 Constant *Ops[] = {ConstantInt::get(Ty, V: *LHS), ConstantInt::get(Ty, V: *RHS)};
763 auto *CI =
764 dyn_cast_or_null<ConstantInt>(Val: ConstantFoldInstOperands(I: BO, Ops, DL));
765 return CI ? std::optional<unsigned>(CI->getZExtValue()) : std::nullopt;
766}
767
768/// Build the per-lane shuffle map by evaluating Index for every lane in the
769/// wave. Returns false if any lane index is non-constant or out of range.
770static bool tryBuildShuffleMap(Value *Index, const GCNSubtarget &ST,
771 SmallVectorImpl<uint8_t> &Ids,
772 const DataLayout &DL) {
773 unsigned WaveSize = ST.getWavefrontSize();
774 Ids.resize(N: WaveSize);
775 for (unsigned Lane : seq(Size: WaveSize)) {
776 std::optional<unsigned> Val = evalLaneExpr(V: Index, Lane, ST, DL);
777 if (!Val || *Val >= WaveSize)
778 return false;
779 Ids[Lane] = *Val;
780 }
781 return true;
782}
783
784/// Lanes are partitioned into groups of Period; each group is a translated
785/// copy of the first: Ids[I] = Ids[I % Period] + (I & ~(Period - 1)).
786template <unsigned Period>
787static bool hasPeriodicLayout(ArrayRef<uint8_t> Ids) {
788 static_assert(isPowerOf2_32(Value: Period), "Period must be a power of two");
789 for (unsigned I = Period, E = Ids.size(); I < E; ++I)
790 if (Ids[I] != Ids[I % Period] + (I & ~(Period - 1)))
791 return false;
792 return true;
793}
794
795/// Match an N-lane row pattern: each lane in [0, N) reads from a source lane
796/// in the same N-lane row, and the pattern repeats periodically across rows.
797template <unsigned N> static bool isRowPattern(ArrayRef<uint8_t> Ids) {
798 for (unsigned I = 0; I < N; ++I)
799 if (Ids[I] >= N)
800 return false;
801 return hasPeriodicLayout<N>(Ids);
802}
803
804static constexpr auto isQuadPattern = isRowPattern<4>;
805static constexpr auto isHalfRowPattern = isRowPattern<8>;
806static constexpr auto isFullRowPattern = isRowPattern<16>;
807
808/// Match a 4-lane (quad) permutation, encoded as the v_mov_b32_dpp
809/// QUAD_PERM control word: bits[1:0]=Ids[0], [3:2]=Ids[1], [5:4]=Ids[2],
810/// [7:6]=Ids[3].
811static std::optional<unsigned> matchQuadPermPattern(ArrayRef<uint8_t> Ids) {
812 if (!isQuadPattern(Ids))
813 return std::nullopt;
814 return Ids[3] << 6 | Ids[2] << 4 | Ids[1] << 2 | Ids[0];
815}
816
817/// Match an N-lane reversal (mirror) pattern.
818template <unsigned N> static bool matchMirrorPattern(ArrayRef<uint8_t> Ids) {
819 if (!isRowPattern<N>(Ids))
820 return false;
821 for (unsigned J = 0; J < N; ++J)
822 if (Ids[J] != (N - 1) - J)
823 return false;
824 return true;
825}
826
827static constexpr auto matchHalfRowMirrorPattern = matchMirrorPattern<8>;
828static constexpr auto matchFullRowMirrorPattern = matchMirrorPattern<16>;
829
830/// Match a 16-lane cyclic rotation; returns the rotation amount in [1, 15].
831static std::optional<unsigned> matchRowRotatePattern(ArrayRef<uint8_t> Ids) {
832 if (Ids[0] == 0 || !isFullRowPattern(Ids))
833 return std::nullopt;
834 for (unsigned J = 1; J < 16; ++J)
835 if (Ids[J] != (Ids[0] + J) % 16)
836 return std::nullopt;
837 return 16u - Ids[0];
838}
839
840/// Match a row-share pattern: all 16 lanes of each row read the same source
841/// lane. Returns the shared source lane index in [0, 16).
842static std::optional<unsigned> matchRowSharePattern(ArrayRef<uint8_t> Ids) {
843 if (!isFullRowPattern(Ids))
844 return std::nullopt;
845 if (!all_equal(Range: Ids.take_front(N: 16)))
846 return std::nullopt;
847 return Ids[0];
848}
849
850/// Match an XOR mask pattern within each 16-lane row: Ids[J] == Mask ^ J,
851/// with Mask in [1, 15].
852static std::optional<unsigned> matchRowXMaskPattern(ArrayRef<uint8_t> Ids) {
853 unsigned Mask = Ids[0];
854 if (Mask == 0 || !isFullRowPattern(Ids))
855 return std::nullopt;
856 for (unsigned J = 0; J < 16; ++J)
857 if (Ids[J] != (Mask ^ J))
858 return std::nullopt;
859 return Mask;
860}
861
862/// Match an 8-lane arbitrary permutation, encoded as the v_mov_b32_dpp8
863/// 24-bit selector (three bits per output lane).
864static std::optional<unsigned> matchHalfRowPermPattern(ArrayRef<uint8_t> Ids) {
865 if (!isHalfRowPattern(Ids))
866 return std::nullopt;
867 unsigned Selector = 0;
868 for (unsigned J = 0; J < 8; ++J)
869 Selector |= Ids[J] << (J * 3);
870 return Selector;
871}
872
873/// Pack a 16-lane permutation into a single 64-bit value: four bits per output
874/// lane, lane J in bits [J*4 + 3 : J*4]. The caller splits it into the low and
875/// high 32-bit selector operands of v_permlane16 / v_permlanex16.
876static uint64_t computePermlane16Masks(ArrayRef<uint8_t> Ids) {
877 uint64_t Sel = 0;
878 for (unsigned J = 0; J < 16; ++J)
879 Sel |= static_cast<uint64_t>(Ids[J] & 0xF) << (J * 4);
880 return Sel;
881}
882
883/// Match a half-wave swap: lane J reads from lane J ^ 32. Only meaningful on
884/// wave64 targets.
885static bool matchHalfWaveSwapPattern(ArrayRef<uint8_t> Ids) {
886 if (Ids.size() != 64)
887 return false;
888 for (unsigned J = 0; J < 64; ++J)
889 if (Ids[J] != (J ^ 32))
890 return false;
891 return true;
892}
893
894/// Match a cross-row permutation suitable for v_permlanex16: every lane in
895/// the low 16-lane half reads from the high half of its own row, and vice
896/// versa.
897static bool isCrossRowPattern(ArrayRef<uint8_t> Ids) {
898 if (!hasPeriodicLayout<32>(Ids))
899 return false;
900 for (unsigned J = 0; J < 16; ++J) {
901 if (Ids[J] < 16 || Ids[J] >= 32)
902 return false;
903 if (Ids[J + 16] != Ids[J] - 16)
904 return false;
905 }
906 return true;
907}
908
909/// Match a DS_SWIZZLE bitmask-mode permutation:
910/// dst_lane = ((src_lane & AND) | OR) ^ XOR
911/// with each mask being five bits. Returns the encoded swizzle immediate.
912/// The hardware applies the formula independently within each 32-lane group,
913/// so on wave64 the high group must replicate the low one (translated by 32).
914static std::optional<unsigned>
915matchDsSwizzleBitmaskPattern(ArrayRef<uint8_t> Ids) {
916 if (!hasPeriodicLayout<32>(Ids))
917 return std::nullopt;
918
919 // The formula is per-bit: output bit B depends only on input bit B. Probe
920 // each bit with src=0 and src=(1<<B); if the output bit flipped, AND[B]=1
921 // and XOR[B] carries the constant offset; otherwise it is a constant bit
922 // encoded in OR (with AND[B]=0, XOR[B]=0).
923 unsigned AndMask = 0, OrMask = 0, XorMask = 0;
924 for (unsigned B = 0; B < 5; ++B) {
925 unsigned Bit0 = (Ids[0] >> B) & 1;
926 unsigned Bit1 = (Ids[1u << B] >> B) & 1;
927 if (Bit0 != Bit1) {
928 AndMask |= 1u << B;
929 XorMask |= Bit0 << B;
930 } else {
931 OrMask |= Bit0 << B;
932 }
933 }
934
935 // The per-bit derivation assumes bit independence; verify the masks
936 // actually reproduce every lane in the 32-lane group.
937 for (unsigned I : seq(Size: 32u)) {
938 unsigned Expected = ((I & AndMask) | OrMask) ^ XorMask;
939 if (Ids[I] != Expected)
940 return std::nullopt;
941 }
942
943 return AMDGPU::Swizzle::BITMASK_PERM_ENC |
944 AndMask << AMDGPU::Swizzle::BITMASK_AND_SHIFT |
945 OrMask << AMDGPU::Swizzle::BITMASK_OR_SHIFT |
946 XorMask << AMDGPU::Swizzle::BITMASK_XOR_SHIFT;
947}
948
949/// Match a GFX9+ DS_SWIZZLE rotate-mode permutation: a cyclic left-rotation
950/// of all 32 lanes within each 32-lane group by a constant N in [0, 31],
951/// i.e. dst_lane = (src_lane + N) % 32. On wave64, hasPeriodicLayout<32>
952/// ensures both 32-lane groups rotate by the same amount.
953static std::optional<unsigned>
954matchDsSwizzleRotatePattern(ArrayRef<uint8_t> Ids) {
955 if (!hasPeriodicLayout<32>(Ids))
956 return std::nullopt;
957
958 // Determine the rotation amount from lane 0: every lane must read from
959 // lane (I + N) % 32 where N = Ids[0] and 0 <= N <= 31.
960 unsigned N = Ids[0];
961 if (N >= 32)
962 return std::nullopt;
963
964 for (unsigned I = 0; I < 32; ++I)
965 if (Ids[I] != (I + N) % 32)
966 return std::nullopt;
967
968 return AMDGPU::Swizzle::ROTATE_MODE_ENC |
969 (N << AMDGPU::Swizzle::ROTATE_SIZE_SHIFT);
970}
971
972/// Emit v_mov_b32_dpp with the given control word, row/bank masks 0xF, and
973/// bound_ctrl=1 so out-of-bounds lanes are well-defined and the DPP mov can
974/// be folded into a consuming VALU op by GCNDPPCombine.
975static Value *createUpdateDpp(IRBuilderBase &B, Value *Val, unsigned Ctrl) {
976 Type *Ty = Val->getType();
977 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_update_dpp, OverloadTypes: {Ty},
978 Args: {PoisonValue::get(T: Ty), Val, B.getInt32(C: Ctrl),
979 B.getInt32(C: 0xF), B.getInt32(C: 0xF), B.getTrue()});
980}
981
982/// Emit v_mov_b32_dpp8 with the given 24-bit lane selector.
983static Value *createMovDpp8(IRBuilderBase &B, Value *Val, unsigned Selector) {
984 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_mov_dpp8, OverloadTypes: {Val->getType()},
985 Args: {Val, B.getInt32(C: Selector)});
986}
987
988/// Emit v_permlane16 with the precomputed lane-select halves.
989static Value *createPermlane16(IRBuilderBase &B, Value *Val, uint32_t Lo,
990 uint32_t Hi) {
991 Type *Ty = Val->getType();
992 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_permlane16, OverloadTypes: {Ty},
993 Args: {PoisonValue::get(T: Ty), Val, B.getInt32(C: Lo),
994 B.getInt32(C: Hi), B.getFalse(), B.getFalse()});
995}
996
997/// Emit v_permlanex16 with the precomputed lane-select halves. Each output
998/// lane reads from the other 16-lane half of the same row.
999static Value *createPermlaneX16(IRBuilderBase &B, Value *Val, uint32_t Lo,
1000 uint32_t Hi) {
1001 Type *Ty = Val->getType();
1002 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_permlanex16, OverloadTypes: {Ty},
1003 Args: {PoisonValue::get(T: Ty), Val, B.getInt32(C: Lo),
1004 B.getInt32(C: Hi), B.getFalse(), B.getFalse()});
1005}
1006
1007/// Emit ds_swizzle with the given immediate, bitcasting/converting between
1008/// pointer/float types and i32 as required by the intrinsic signature.
1009static Value *createDsSwizzle(IRBuilderBase &B, Value *Val, unsigned Offset,
1010 const DataLayout &DL) {
1011 Type *OrigTy = Val->getType();
1012 assert(DL.getTypeSizeInBits(OrigTy) == 32 &&
1013 "ds_swizzle only supports 32-bit operands");
1014 IntegerType *I32Ty = B.getInt32Ty();
1015 Value *Src = Val;
1016 if (OrigTy->isPointerTy())
1017 Src = B.CreatePtrToInt(V: Src, DestTy: I32Ty);
1018 else if (OrigTy != I32Ty)
1019 Src = B.CreateBitCast(V: Src, DestTy: I32Ty);
1020 Value *Result = B.CreateIntrinsic(ID: Intrinsic::amdgcn_ds_swizzle, OverloadTypes: {},
1021 Args: {Src, B.getInt32(C: Offset)});
1022 if (OrigTy->isPointerTy())
1023 return B.CreateIntToPtr(V: Result, DestTy: OrigTy);
1024 if (OrigTy != I32Ty)
1025 return B.CreateBitCast(V: Result, DestTy: OrigTy);
1026 return Result;
1027}
1028
1029/// Emit v_permlane64 (swap of the two 32-lane halves of a wave64).
1030static Value *createPermlane64(IRBuilderBase &B, Value *Val) {
1031 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_permlane64, OverloadTypes: {Val->getType()},
1032 Args: {Val});
1033}
1034
1035/// Given a shuffle map, try to emit the best hardware intrinsic.
1036static Value *matchShuffleToHWIntrinsic(IRBuilderBase &B, Value *Src,
1037 ArrayRef<uint8_t> Ids,
1038 const GCNSubtarget &ST,
1039 const DataLayout &DL) {
1040 // Identity shuffle (every lane reads itself) folds to the source value.
1041 if (all_of(Range: enumerate(First&: Ids),
1042 P: [](const auto &E) { return E.value() == E.index(); }))
1043 return Src;
1044
1045 // Uniform shuffle (all lanes read the same value) is handled by cheaper
1046 // broadcast/readlane intrinsics.
1047 if (all_equal(Range&: Ids))
1048 return nullptr;
1049
1050 if (std::optional<unsigned> QP = matchQuadPermPattern(Ids)) {
1051 if (ST.hasDPP())
1052 return createUpdateDpp(B, Val: Src, Ctrl: *QP);
1053 return createDsSwizzle(B, Val: Src, Offset: AMDGPU::Swizzle::QUAD_PERM_ENC | *QP, DL);
1054 }
1055
1056 if (ST.hasDPP()) {
1057 if (matchHalfRowMirrorPattern(Ids))
1058 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_HALF_MIRROR);
1059 if (matchFullRowMirrorPattern(Ids))
1060 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_MIRROR);
1061 if (std::optional<unsigned> Amt = matchRowRotatePattern(Ids))
1062 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_ROR_FIRST + *Amt - 1);
1063 }
1064
1065 // row_share is supported on GFX90A and GFX10+; row_xmask is GFX10+ only.
1066 if (ST.hasDPPRowShare()) {
1067 if (std::optional<unsigned> Lane = matchRowSharePattern(Ids))
1068 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_SHARE_FIRST + *Lane);
1069 }
1070
1071 if (ST.hasDPP() && ST.hasGFX10Insts()) {
1072 if (std::optional<unsigned> Mask = matchRowXMaskPattern(Ids))
1073 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_XMASK_FIRST + *Mask);
1074 }
1075
1076 if (ST.hasDPP8()) {
1077 if (std::optional<unsigned> Sel = matchHalfRowPermPattern(Ids))
1078 return createMovDpp8(B, Val: Src, Selector: *Sel);
1079 }
1080
1081 if (ST.hasPermlane16Insts()) {
1082 if (isFullRowPattern(Ids)) {
1083 uint64_t Sel = computePermlane16Masks(Ids);
1084 return createPermlane16(B, Val: Src, Lo: Lo_32(Value: Sel), Hi: Hi_32(Value: Sel));
1085 }
1086 // Cross-row shuffles (e.g. XOR 16..31) — covered by permlanex16.
1087 if (isCrossRowPattern(Ids)) {
1088 uint64_t Sel = computePermlane16Masks(Ids);
1089 return createPermlaneX16(B, Val: Src, Lo: Lo_32(Value: Sel), Hi: Hi_32(Value: Sel));
1090 }
1091 }
1092
1093 // Generic DS_SWIZZLE bitmask-mode fallback: handles any 32-lane shuffle that
1094 // can be expressed as dst = ((src & AND) | OR) ^ XOR with 5-bit masks. This
1095 // is available on every target that has ds_swizzle.
1096 if (std::optional<unsigned> Imm = matchDsSwizzleBitmaskPattern(Ids))
1097 return createDsSwizzle(B, Val: Src, Offset: *Imm, DL);
1098
1099 // DS_SWIZZLE rotate mode (GFX9+): handles cyclic 32-lane rotations that
1100 // bitmask mode cannot express (e.g. +1 mod 32 requires inter-bit carry).
1101 if (ST.hasDsSwizzleRotateMode()) {
1102 if (std::optional<unsigned> Imm = matchDsSwizzleRotatePattern(Ids))
1103 return createDsSwizzle(B, Val: Src, Offset: *Imm, DL);
1104 }
1105
1106 if (ST.hasPermLane64() && matchHalfWaveSwapPattern(Ids))
1107 return createPermlane64(B, Val: Src);
1108
1109 return nullptr;
1110}
1111
1112/// Try to fold a wave_shuffle/ds_bpermute whose lane index is a constant
1113/// function of the lane ID into a hardware-specific lane permutation intrinsic.
1114static std::optional<Instruction *>
1115tryOptimizeShufflePattern(InstCombiner &IC, IntrinsicInst &II,
1116 const GCNSubtarget &ST) {
1117 const DataLayout &DL = IC.getDataLayout();
1118 if (DL.getTypeSizeInBits(Ty: II.getType()) != 32)
1119 return std::nullopt;
1120
1121 if (!ST.isWaveSizeKnown())
1122 return std::nullopt;
1123
1124 unsigned WaveSize = ST.getWavefrontSize();
1125 bool IsBpermute = II.getIntrinsicID() == Intrinsic::amdgcn_ds_bpermute;
1126 Value *Src = II.getArgOperand(i: IsBpermute ? 1 : 0);
1127 Value *Index = II.getArgOperand(i: IsBpermute ? 0 : 1);
1128
1129 SmallVector<uint8_t, 64> Ids;
1130 if (IsBpermute) {
1131 Ids.resize(N: WaveSize);
1132 for (unsigned Lane : seq(Size: WaveSize)) {
1133 std::optional<unsigned> Val = evalLaneExpr(V: Index, Lane, ST, DL);
1134 if (!Val || (*Val & 3) || (*Val >> 2) >= WaveSize)
1135 return std::nullopt;
1136 Ids[Lane] = *Val >> 2;
1137 }
1138 } else {
1139 if (!tryBuildShuffleMap(Index, ST, Ids, DL))
1140 return std::nullopt;
1141 }
1142
1143 Value *Result = matchShuffleToHWIntrinsic(B&: IC.Builder, Src, Ids, ST, DL);
1144 if (!Result)
1145 return std::nullopt;
1146
1147 return IC.replaceInstUsesWith(I&: II, V: Result);
1148}
1149std::optional<Instruction *>
1150GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
1151 Intrinsic::ID IID = II.getIntrinsicID();
1152 switch (IID) {
1153 case Intrinsic::amdgcn_implicitarg_ptr: {
1154 if (II.getFunction()->hasFnAttribute(Kind: "amdgpu-no-implicitarg-ptr"))
1155 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1156 uint64_t ImplicitArgBytes = ST->getImplicitArgNumBytes(F: *II.getFunction());
1157
1158 uint64_t CurrentOrNullBytes =
1159 II.getAttributes().getRetDereferenceableOrNullBytes();
1160 if (CurrentOrNullBytes != 0) {
1161 // Refine "dereferenceable (A) meets dereferenceable_or_null(B)"
1162 // into dereferenceable(max(A, B))
1163 uint64_t NewBytes = std::max(a: CurrentOrNullBytes, b: ImplicitArgBytes);
1164 II.addRetAttr(
1165 Attr: Attribute::getWithDereferenceableBytes(Context&: II.getContext(), Bytes: NewBytes));
1166 II.removeRetAttr(Kind: Attribute::DereferenceableOrNull);
1167 return &II;
1168 }
1169
1170 uint64_t CurrentBytes = II.getAttributes().getRetDereferenceableBytes();
1171 uint64_t NewBytes = std::max(a: CurrentBytes, b: ImplicitArgBytes);
1172 if (NewBytes != CurrentBytes) {
1173 II.addRetAttr(
1174 Attr: Attribute::getWithDereferenceableBytes(Context&: II.getContext(), Bytes: NewBytes));
1175 return &II;
1176 }
1177
1178 return std::nullopt;
1179 }
1180 case Intrinsic::amdgcn_rcp: {
1181 Value *Src = II.getArgOperand(i: 0);
1182 if (isa<PoisonValue>(Val: Src))
1183 return IC.replaceInstUsesWith(I&: II, V: Src);
1184
1185 // TODO: Move to ConstantFolding/InstSimplify?
1186 if (isa<UndefValue>(Val: Src)) {
1187 Type *Ty = II.getType();
1188 auto *QNaN = ConstantFP::get(Ty, V: APFloat::getQNaN(Sem: Ty->getFltSemantics()));
1189 return IC.replaceInstUsesWith(I&: II, V: QNaN);
1190 }
1191
1192 if (II.isStrictFP())
1193 break;
1194
1195 if (const ConstantFP *C = dyn_cast<ConstantFP>(Val: Src)) {
1196 std::optional<APFloat> Val = AMDGPU::evaluateRcp(Val: C->getValueAPF());
1197 if (!Val)
1198 break;
1199
1200 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::get(Context&: II.getContext(), V: *Val));
1201 }
1202
1203 FastMathFlags FMF = cast<FPMathOperator>(Val&: II).getFastMathFlags();
1204 if (!FMF.allowContract())
1205 break;
1206 auto *SrcCI = dyn_cast<IntrinsicInst>(Val: Src);
1207 if (!SrcCI)
1208 break;
1209
1210 auto IID = SrcCI->getIntrinsicID();
1211 // llvm.amdgcn.rcp(llvm.amdgcn.sqrt(x)) -> llvm.amdgcn.rsq(x) if contractable
1212 //
1213 // llvm.amdgcn.rcp(llvm.sqrt(x)) -> llvm.amdgcn.rsq(x) if contractable and
1214 // relaxed.
1215 if (IID == Intrinsic::amdgcn_sqrt || IID == Intrinsic::sqrt) {
1216 const FPMathOperator *SqrtOp = cast<FPMathOperator>(Val: SrcCI);
1217 FastMathFlags InnerFMF = SqrtOp->getFastMathFlags();
1218 if (!InnerFMF.allowContract() || !SrcCI->hasOneUse())
1219 break;
1220
1221 if (IID == Intrinsic::sqrt && !canContractSqrtToRsq(SqrtOp))
1222 break;
1223
1224 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
1225 M: SrcCI->getModule(), id: Intrinsic::amdgcn_rsq, OverloadTys: {SrcCI->getType()});
1226
1227 InnerFMF |= FMF;
1228 II.setFastMathFlags(InnerFMF);
1229
1230 II.setCalledFunction(NewDecl);
1231 return IC.replaceOperand(I&: II, OpNum: 0, V: SrcCI->getArgOperand(i: 0));
1232 }
1233
1234 break;
1235 }
1236 case Intrinsic::amdgcn_sqrt:
1237 case Intrinsic::amdgcn_rsq:
1238 case Intrinsic::amdgcn_tanh: {
1239 Value *Src = II.getArgOperand(i: 0);
1240 if (isa<PoisonValue>(Val: Src))
1241 return IC.replaceInstUsesWith(I&: II, V: Src);
1242
1243 // TODO: Move to ConstantFolding/InstSimplify?
1244 if (isa<UndefValue>(Val: Src)) {
1245 Type *Ty = II.getType();
1246 auto *QNaN = ConstantFP::get(Ty, V: APFloat::getQNaN(Sem: Ty->getFltSemantics()));
1247 return IC.replaceInstUsesWith(I&: II, V: QNaN);
1248 }
1249
1250 // f16 amdgcn.sqrt is identical to regular sqrt.
1251 if (IID == Intrinsic::amdgcn_sqrt && Src->getType()->isHalfTy()) {
1252 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
1253 M: II.getModule(), id: Intrinsic::sqrt, OverloadTys: {II.getType()});
1254 II.setCalledFunction(NewDecl);
1255 return &II;
1256 }
1257
1258 break;
1259 }
1260 case Intrinsic::amdgcn_log:
1261 case Intrinsic::amdgcn_exp2: {
1262 const bool IsLog = IID == Intrinsic::amdgcn_log;
1263 const bool IsExp = IID == Intrinsic::amdgcn_exp2;
1264 Value *Src = II.getArgOperand(i: 0);
1265 Type *Ty = II.getType();
1266
1267 if (isa<PoisonValue>(Val: Src))
1268 return IC.replaceInstUsesWith(I&: II, V: Src);
1269
1270 if (IC.getSimplifyQuery().isUndefValue(V: Src))
1271 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getNaN(Ty));
1272
1273 if (ConstantFP *C = dyn_cast<ConstantFP>(Val: Src)) {
1274 if (C->isInfinity()) {
1275 // exp2(+inf) -> +inf
1276 // log2(+inf) -> +inf
1277 if (!C->isNegative())
1278 return IC.replaceInstUsesWith(I&: II, V: C);
1279
1280 // exp2(-inf) -> 0
1281 if (IsExp && C->isNegative())
1282 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getZero(Ty));
1283 }
1284
1285 if (II.isStrictFP())
1286 break;
1287
1288 if (C->isNaN()) {
1289 Constant *Quieted = ConstantFP::get(Ty, V: C->getValue().makeQuiet());
1290 return IC.replaceInstUsesWith(I&: II, V: Quieted);
1291 }
1292
1293 // f32 instruction doesn't handle denormals, f16 does.
1294 if (C->isZero() || (C->getValue().isDenormal() && Ty->isFloatTy())) {
1295 Constant *FoldedValue = IsLog ? ConstantFP::getInfinity(Ty, Negative: true)
1296 : ConstantFP::get(Ty, V: 1.0);
1297 return IC.replaceInstUsesWith(I&: II, V: FoldedValue);
1298 }
1299
1300 if (IsLog && C->isNegative())
1301 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getNaN(Ty));
1302
1303 // TODO: Full constant folding matching hardware behavior.
1304 }
1305
1306 break;
1307 }
1308 case Intrinsic::amdgcn_frexp_mant:
1309 case Intrinsic::amdgcn_frexp_exp: {
1310 Value *Src = II.getArgOperand(i: 0);
1311 if (const ConstantFP *C = dyn_cast<ConstantFP>(Val: Src)) {
1312 int Exp;
1313 APFloat Significand =
1314 frexp(X: C->getValueAPF(), Exp, RM: APFloat::rmNearestTiesToEven);
1315
1316 if (IID == Intrinsic::amdgcn_frexp_mant) {
1317 return IC.replaceInstUsesWith(
1318 I&: II, V: ConstantFP::get(Context&: II.getContext(), V: Significand));
1319 }
1320
1321 // Match instruction special case behavior.
1322 if (Exp == APFloat::IEK_NaN || Exp == APFloat::IEK_Inf)
1323 Exp = 0;
1324
1325 return IC.replaceInstUsesWith(I&: II,
1326 V: ConstantInt::getSigned(Ty: II.getType(), V: Exp));
1327 }
1328
1329 if (isa<PoisonValue>(Val: Src))
1330 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1331
1332 if (isa<UndefValue>(Val: Src)) {
1333 return IC.replaceInstUsesWith(I&: II, V: UndefValue::get(T: II.getType()));
1334 }
1335
1336 break;
1337 }
1338 case Intrinsic::amdgcn_class: {
1339 Value *Src0 = II.getArgOperand(i: 0);
1340 Value *Src1 = II.getArgOperand(i: 1);
1341 const ConstantInt *CMask = dyn_cast<ConstantInt>(Val: Src1);
1342 if (CMask) {
1343 II.setCalledOperand(Intrinsic::getOrInsertDeclaration(
1344 M: II.getModule(), id: Intrinsic::is_fpclass, OverloadTys: Src0->getType()));
1345
1346 // Clamp any excess bits, as they're illegal for the generic intrinsic.
1347 II.setArgOperand(i: 1, v: ConstantInt::get(Ty: Src1->getType(),
1348 V: CMask->getZExtValue() & fcAllFlags));
1349 return &II;
1350 }
1351
1352 // Propagate poison.
1353 if (isa<PoisonValue>(Val: Src0) || isa<PoisonValue>(Val: Src1))
1354 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1355
1356 // llvm.amdgcn.class(_, undef) -> false
1357 if (IC.getSimplifyQuery().isUndefValue(V: Src1))
1358 return IC.replaceInstUsesWith(I&: II, V: ConstantInt::get(Ty: II.getType(), V: false));
1359
1360 // llvm.amdgcn.class(undef, mask) -> mask != 0
1361 if (IC.getSimplifyQuery().isUndefValue(V: Src0)) {
1362 Value *CmpMask = IC.Builder.CreateICmpNE(
1363 LHS: Src1, RHS: ConstantInt::getNullValue(Ty: Src1->getType()));
1364 return IC.replaceInstUsesWith(I&: II, V: CmpMask);
1365 }
1366 break;
1367 }
1368 case Intrinsic::amdgcn_cvt_pkrtz: {
1369 auto foldFPTruncToF16RTZ = [](Value *Arg) -> Value * {
1370 Type *HalfTy = Type::getHalfTy(C&: Arg->getContext());
1371
1372 if (isa<PoisonValue>(Val: Arg))
1373 return PoisonValue::get(T: HalfTy);
1374 if (isa<UndefValue>(Val: Arg))
1375 return UndefValue::get(T: HalfTy);
1376
1377 ConstantFP *CFP = nullptr;
1378 if (match(V: Arg, P: m_ConstantFP(C&: CFP))) {
1379 bool LosesInfo;
1380 APFloat Val(CFP->getValueAPF());
1381 Val.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmTowardZero, losesInfo: &LosesInfo);
1382 return ConstantFP::get(Ty: HalfTy, V: Val);
1383 }
1384
1385 Value *Src = nullptr;
1386 if (match(V: Arg, P: m_FPExt(Op: m_Value(V&: Src)))) {
1387 if (Src->getType()->isHalfTy())
1388 return Src;
1389 }
1390
1391 return nullptr;
1392 };
1393
1394 if (Value *Src0 = foldFPTruncToF16RTZ(II.getArgOperand(i: 0))) {
1395 if (Value *Src1 = foldFPTruncToF16RTZ(II.getArgOperand(i: 1))) {
1396 Value *V = PoisonValue::get(T: II.getType());
1397 V = IC.Builder.CreateInsertElement(Vec: V, NewElt: Src0, Idx: (uint64_t)0);
1398 V = IC.Builder.CreateInsertElement(Vec: V, NewElt: Src1, Idx: (uint64_t)1);
1399 return IC.replaceInstUsesWith(I&: II, V);
1400 }
1401 }
1402
1403 break;
1404 }
1405 case Intrinsic::amdgcn_cvt_pknorm_i16:
1406 case Intrinsic::amdgcn_cvt_pknorm_u16:
1407 case Intrinsic::amdgcn_cvt_pk_i16:
1408 case Intrinsic::amdgcn_cvt_pk_u16: {
1409 Value *Src0 = II.getArgOperand(i: 0);
1410 Value *Src1 = II.getArgOperand(i: 1);
1411
1412 // TODO: Replace call with scalar operation if only one element is poison.
1413 if (isa<PoisonValue>(Val: Src0) && isa<PoisonValue>(Val: Src1))
1414 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1415
1416 if (isa<UndefValue>(Val: Src0) && isa<UndefValue>(Val: Src1)) {
1417 return IC.replaceInstUsesWith(I&: II, V: UndefValue::get(T: II.getType()));
1418 }
1419
1420 break;
1421 }
1422 case Intrinsic::amdgcn_cvt_off_f32_i4: {
1423 Value* Arg = II.getArgOperand(i: 0);
1424 Type *Ty = II.getType();
1425
1426 if (isa<PoisonValue>(Val: Arg))
1427 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: Ty));
1428
1429 if(IC.getSimplifyQuery().isUndefValue(V: Arg))
1430 return IC.replaceInstUsesWith(I&: II, V: Constant::getNullValue(Ty));
1431
1432 ConstantInt *CArg = dyn_cast<ConstantInt>(Val: II.getArgOperand(i: 0));
1433 if (!CArg)
1434 break;
1435
1436 // Tabulated 0.0625 * (sext (CArg & 0xf)).
1437 constexpr size_t ResValsSize = 16;
1438 static constexpr float ResVals[ResValsSize] = {
1439 0.0, 0.0625, 0.125, 0.1875, 0.25, 0.3125, 0.375, 0.4375,
1440 -0.5, -0.4375, -0.375, -0.3125, -0.25, -0.1875, -0.125, -0.0625};
1441 Constant *Res =
1442 ConstantFP::get(Ty, V: ResVals[CArg->getZExtValue() & (ResValsSize - 1)]);
1443 return IC.replaceInstUsesWith(I&: II, V: Res);
1444 }
1445 case Intrinsic::amdgcn_ubfe:
1446 case Intrinsic::amdgcn_sbfe: {
1447 // Decompose simple cases into standard shifts.
1448 Value *Src = II.getArgOperand(i: 0);
1449 if (isa<UndefValue>(Val: Src)) {
1450 return IC.replaceInstUsesWith(I&: II, V: Src);
1451 }
1452
1453 unsigned Width;
1454 Type *Ty = II.getType();
1455 unsigned IntSize = Ty->getIntegerBitWidth();
1456
1457 ConstantInt *CWidth = dyn_cast<ConstantInt>(Val: II.getArgOperand(i: 2));
1458 if (CWidth) {
1459 Width = CWidth->getZExtValue();
1460 if ((Width & (IntSize - 1)) == 0) {
1461 return IC.replaceInstUsesWith(I&: II, V: ConstantInt::getNullValue(Ty));
1462 }
1463
1464 // Hardware ignores high bits, so remove those.
1465 if (Width >= IntSize) {
1466 return IC.replaceOperand(
1467 I&: II, OpNum: 2, V: ConstantInt::get(Ty: CWidth->getType(), V: Width & (IntSize - 1)));
1468 }
1469 }
1470
1471 unsigned Offset;
1472 ConstantInt *COffset = dyn_cast<ConstantInt>(Val: II.getArgOperand(i: 1));
1473 if (COffset) {
1474 Offset = COffset->getZExtValue();
1475 if (Offset >= IntSize) {
1476 return IC.replaceOperand(
1477 I&: II, OpNum: 1,
1478 V: ConstantInt::get(Ty: COffset->getType(), V: Offset & (IntSize - 1)));
1479 }
1480 }
1481
1482 bool Signed = IID == Intrinsic::amdgcn_sbfe;
1483
1484 if (!CWidth || !COffset)
1485 break;
1486
1487 // The case of Width == 0 is handled above, which makes this transformation
1488 // safe. If Width == 0, then the ashr and lshr instructions become poison
1489 // value since the shift amount would be equal to the bit size.
1490 assert(Width != 0);
1491
1492 // TODO: This allows folding to undef when the hardware has specific
1493 // behavior?
1494 if (Offset + Width < IntSize) {
1495 Value *Shl = IC.Builder.CreateShl(LHS: Src, RHS: IntSize - Offset - Width);
1496 Value *RightShift = Signed ? IC.Builder.CreateAShr(LHS: Shl, RHS: IntSize - Width)
1497 : IC.Builder.CreateLShr(LHS: Shl, RHS: IntSize - Width);
1498 RightShift->takeName(V: &II);
1499 return IC.replaceInstUsesWith(I&: II, V: RightShift);
1500 }
1501
1502 Value *RightShift = Signed ? IC.Builder.CreateAShr(LHS: Src, RHS: Offset)
1503 : IC.Builder.CreateLShr(LHS: Src, RHS: Offset);
1504
1505 RightShift->takeName(V: &II);
1506 return IC.replaceInstUsesWith(I&: II, V: RightShift);
1507 }
1508 case Intrinsic::amdgcn_exp:
1509 case Intrinsic::amdgcn_exp_row:
1510 case Intrinsic::amdgcn_exp_compr: {
1511 ConstantInt *En = cast<ConstantInt>(Val: II.getArgOperand(i: 1));
1512 unsigned EnBits = En->getZExtValue();
1513 if (EnBits == 0xf)
1514 break; // All inputs enabled.
1515
1516 bool IsCompr = IID == Intrinsic::amdgcn_exp_compr;
1517 bool Changed = false;
1518 for (int I = 0; I < (IsCompr ? 2 : 4); ++I) {
1519 if ((!IsCompr && (EnBits & (1 << I)) == 0) ||
1520 (IsCompr && ((EnBits & (0x3 << (2 * I))) == 0))) {
1521 Value *Src = II.getArgOperand(i: I + 2);
1522 if (!isa<PoisonValue>(Val: Src)) {
1523 IC.replaceOperand(I&: II, OpNum: I + 2, V: PoisonValue::get(T: Src->getType()));
1524 Changed = true;
1525 }
1526 }
1527 }
1528
1529 if (Changed) {
1530 return &II;
1531 }
1532
1533 break;
1534 }
1535 case Intrinsic::amdgcn_fmed3: {
1536 Value *Src0 = II.getArgOperand(i: 0);
1537 Value *Src1 = II.getArgOperand(i: 1);
1538 Value *Src2 = II.getArgOperand(i: 2);
1539
1540 for (Value *Src : {Src0, Src1, Src2}) {
1541 if (isa<PoisonValue>(Val: Src))
1542 return IC.replaceInstUsesWith(I&: II, V: Src);
1543 }
1544
1545 if (II.isStrictFP())
1546 break;
1547
1548 // med3 with a nan input acts like
1549 // v_min_f32(v_min_f32(s0, s1), s2)
1550 //
1551 // Signalingness is ignored with ieee=0, so we fold to
1552 // minimumnum/maximumnum. With ieee=1, the v_min_f32 acts like llvm.minnum
1553 // with signaling nan handling. With ieee=0, like llvm.minimumnum except a
1554 // returned signaling nan will not be quieted.
1555
1556 // ieee=1
1557 // s0 snan: s2
1558 // s1 snan: s2
1559 // s2 snan: qnan
1560
1561 // s0 qnan: min(s1, s2)
1562 // s1 qnan: min(s0, s2)
1563 // s2 qnan: min(s0, s1)
1564
1565 // ieee=0
1566 // s0 _nan: min(s1, s2)
1567 // s1 _nan: min(s0, s2)
1568 // s2 _nan: min(s0, s1)
1569
1570 // med3 behavior with infinity
1571 // s0 +inf: max(s1, s2)
1572 // s1 +inf: max(s0, s2)
1573 // s2 +inf: max(s0, s1)
1574 // s0 -inf: min(s1, s2)
1575 // s1 -inf: min(s0, s2)
1576 // s2 -inf: min(s0, s1)
1577
1578 // Checking for NaN before canonicalization provides better fidelity when
1579 // mapping other operations onto fmed3 since the order of operands is
1580 // unchanged.
1581 Value *V = nullptr;
1582 const APFloat *ConstSrc0 = nullptr;
1583 const APFloat *ConstSrc1 = nullptr;
1584 const APFloat *ConstSrc2 = nullptr;
1585
1586 if ((match(V: Src0, P: m_APFloat(Res&: ConstSrc0)) &&
1587 (ConstSrc0->isNaN() || ConstSrc0->isInfinity())) ||
1588 isa<UndefValue>(Val: Src0)) {
1589 const bool IsPosInfinity = ConstSrc0 && ConstSrc0->isPosInfinity();
1590 switch (fpenvIEEEMode(I: II)) {
1591 case KnownIEEEMode::On:
1592 // TODO: If Src2 is snan, does it need quieting?
1593 if (ConstSrc0 && ConstSrc0->isNaN() && ConstSrc0->isSignaling())
1594 return IC.replaceInstUsesWith(I&: II, V: Src2);
1595
1596 V = IsPosInfinity ? IC.Builder.CreateMaxNum(LHS: Src1, RHS: Src2)
1597 : IC.Builder.CreateMinNum(LHS: Src1, RHS: Src2);
1598 break;
1599 case KnownIEEEMode::Off:
1600 V = IsPosInfinity ? IC.Builder.CreateMaximumNum(LHS: Src1, RHS: Src2)
1601 : IC.Builder.CreateMinimumNum(LHS: Src1, RHS: Src2);
1602 break;
1603 case KnownIEEEMode::Unknown:
1604 break;
1605 }
1606 } else if ((match(V: Src1, P: m_APFloat(Res&: ConstSrc1)) &&
1607 (ConstSrc1->isNaN() || ConstSrc1->isInfinity())) ||
1608 isa<UndefValue>(Val: Src1)) {
1609 const bool IsPosInfinity = ConstSrc1 && ConstSrc1->isPosInfinity();
1610 switch (fpenvIEEEMode(I: II)) {
1611 case KnownIEEEMode::On:
1612 // TODO: If Src2 is snan, does it need quieting?
1613 if (ConstSrc1 && ConstSrc1->isNaN() && ConstSrc1->isSignaling())
1614 return IC.replaceInstUsesWith(I&: II, V: Src2);
1615
1616 V = IsPosInfinity ? IC.Builder.CreateMaxNum(LHS: Src0, RHS: Src2)
1617 : IC.Builder.CreateMinNum(LHS: Src0, RHS: Src2);
1618 break;
1619 case KnownIEEEMode::Off:
1620 V = IsPosInfinity ? IC.Builder.CreateMaximumNum(LHS: Src0, RHS: Src2)
1621 : IC.Builder.CreateMinimumNum(LHS: Src0, RHS: Src2);
1622 break;
1623 case KnownIEEEMode::Unknown:
1624 break;
1625 }
1626 } else if ((match(V: Src2, P: m_APFloat(Res&: ConstSrc2)) &&
1627 (ConstSrc2->isNaN() || ConstSrc2->isInfinity())) ||
1628 isa<UndefValue>(Val: Src2)) {
1629 switch (fpenvIEEEMode(I: II)) {
1630 case KnownIEEEMode::On:
1631 if (ConstSrc2 && ConstSrc2->isNaN() && ConstSrc2->isSignaling()) {
1632 auto *Quieted = ConstantFP::get(Ty: II.getType(), V: ConstSrc2->makeQuiet());
1633 return IC.replaceInstUsesWith(I&: II, V: Quieted);
1634 }
1635
1636 V = (ConstSrc2 && ConstSrc2->isPosInfinity())
1637 ? IC.Builder.CreateMaxNum(LHS: Src0, RHS: Src1)
1638 : IC.Builder.CreateMinNum(LHS: Src0, RHS: Src1);
1639 break;
1640 case KnownIEEEMode::Off:
1641 V = (ConstSrc2 && ConstSrc2->isPosInfinity())
1642 ? IC.Builder.CreateMaximumNum(LHS: Src0, RHS: Src1)
1643 : IC.Builder.CreateMinimumNum(LHS: Src0, RHS: Src1);
1644 break;
1645 case KnownIEEEMode::Unknown:
1646 break;
1647 }
1648 }
1649
1650 if (V) {
1651 if (auto *CI = dyn_cast<CallInst>(Val: V)) {
1652 CI->copyFastMathFlags(I: &II);
1653 CI->takeName(V: &II);
1654 }
1655 return IC.replaceInstUsesWith(I&: II, V);
1656 }
1657
1658 bool Swap = false;
1659 // Canonicalize constants to RHS operands.
1660 //
1661 // fmed3(c0, x, c1) -> fmed3(x, c0, c1)
1662 if (isa<Constant>(Val: Src0) && !isa<Constant>(Val: Src1)) {
1663 std::swap(a&: Src0, b&: Src1);
1664 Swap = true;
1665 }
1666
1667 if (isa<Constant>(Val: Src1) && !isa<Constant>(Val: Src2)) {
1668 std::swap(a&: Src1, b&: Src2);
1669 Swap = true;
1670 }
1671
1672 if (isa<Constant>(Val: Src0) && !isa<Constant>(Val: Src1)) {
1673 std::swap(a&: Src0, b&: Src1);
1674 Swap = true;
1675 }
1676
1677 if (Swap) {
1678 II.setArgOperand(i: 0, v: Src0);
1679 II.setArgOperand(i: 1, v: Src1);
1680 II.setArgOperand(i: 2, v: Src2);
1681 return &II;
1682 }
1683
1684 if (const ConstantFP *C0 = dyn_cast<ConstantFP>(Val: Src0)) {
1685 if (const ConstantFP *C1 = dyn_cast<ConstantFP>(Val: Src1)) {
1686 if (const ConstantFP *C2 = dyn_cast<ConstantFP>(Val: Src2)) {
1687 APFloat Result = fmed3AMDGCN(Src0: C0->getValueAPF(), Src1: C1->getValueAPF(),
1688 Src2: C2->getValueAPF());
1689 return IC.replaceInstUsesWith(I&: II,
1690 V: ConstantFP::get(Ty: II.getType(), V: Result));
1691 }
1692 }
1693 }
1694
1695 if (!ST->hasMed3_16())
1696 break;
1697
1698 // Repeat floating-point width reduction done for minnum/maxnum.
1699 // fmed3((fpext X), (fpext Y), (fpext Z)) -> fpext (fmed3(X, Y, Z))
1700 if (Value *X = matchFPExtFromF16(Arg: Src0)) {
1701 if (Value *Y = matchFPExtFromF16(Arg: Src1)) {
1702 if (Value *Z = matchFPExtFromF16(Arg: Src2)) {
1703 Value *NewCall = IC.Builder.CreateIntrinsic(
1704 ID: IID, OverloadTypes: {X->getType()}, Args: {X, Y, Z}, FMFSource: &II, Name: II.getName());
1705 return new FPExtInst(NewCall, II.getType());
1706 }
1707 }
1708 }
1709
1710 break;
1711 }
1712 case Intrinsic::amdgcn_mbcnt_hi:
1713 // exec_hi is all 0, so this is just a copy.
1714 if (ST->isWave32())
1715 return IC.replaceInstUsesWith(I&: II, V: II.getArgOperand(i: 1));
1716 [[fallthrough]];
1717 case Intrinsic::amdgcn_mbcnt_lo: {
1718 ConstantRange AccRange =
1719 computeConstantRange(V: II.getArgOperand(i: 1),
1720 /*ForSigned=*/false, SQ: IC.getSimplifyQuery());
1721 if (AccRange.isFullSet())
1722 return nullptr;
1723
1724 // TODO: Can raise lower bound by inspecting first argument.
1725 ConstantRange MbcntRange(APInt(32, 0), APInt(32, 32 + 1));
1726 ConstantRange ComputedRange = AccRange.add(Other: MbcntRange);
1727 if (ComputedRange.isFullSet())
1728 return nullptr;
1729
1730 if (std::optional<ConstantRange> ExistingRange = II.getRange()) {
1731 ComputedRange = ComputedRange.intersectWith(CR: *ExistingRange);
1732 if (ComputedRange == *ExistingRange)
1733 return nullptr;
1734 }
1735
1736 II.addRangeRetAttr(CR: ComputedRange);
1737 return nullptr;
1738 }
1739 case Intrinsic::amdgcn_ballot: {
1740 Value *Arg = II.getArgOperand(i: 0);
1741 if (isa<PoisonValue>(Val: Arg))
1742 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1743
1744 if (auto *Src = dyn_cast<ConstantInt>(Val: Arg)) {
1745 if (Src->isZero()) {
1746 // amdgcn.ballot(i1 0) is zero.
1747 return IC.replaceInstUsesWith(I&: II, V: Constant::getNullValue(Ty: II.getType()));
1748 }
1749 }
1750 if (ST->isWave32() && II.getType()->getIntegerBitWidth() == 64) {
1751 // %b64 = call i64 ballot.i64(...)
1752 // =>
1753 // %b32 = call i32 ballot.i32(...)
1754 // %b64 = zext i32 %b32 to i64
1755 Value *Call = IC.Builder.CreateZExt(
1756 V: IC.Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_ballot,
1757 OverloadTypes: {IC.Builder.getInt32Ty()},
1758 Args: {II.getArgOperand(i: 0)}),
1759 DestTy: II.getType());
1760 Call->takeName(V: &II);
1761 return IC.replaceInstUsesWith(I&: II, V: Call);
1762 }
1763 break;
1764 }
1765 case Intrinsic::amdgcn_wavefrontsize: {
1766 if (ST->isWaveSizeKnown())
1767 return IC.replaceInstUsesWith(
1768 I&: II, V: ConstantInt::get(Ty: II.getType(), V: ST->getWavefrontSize()));
1769 break;
1770 }
1771 case Intrinsic::amdgcn_wqm_vote: {
1772 // wqm_vote is identity when the argument is constant.
1773 if (!isa<Constant>(Val: II.getArgOperand(i: 0)))
1774 break;
1775
1776 return IC.replaceInstUsesWith(I&: II, V: II.getArgOperand(i: 0));
1777 }
1778 case Intrinsic::amdgcn_kill: {
1779 const ConstantInt *C = dyn_cast<ConstantInt>(Val: II.getArgOperand(i: 0));
1780 if (!C || !C->getZExtValue())
1781 break;
1782
1783 // amdgcn.kill(i1 1) is a no-op
1784 return IC.eraseInstFromFunction(I&: II);
1785 }
1786 case Intrinsic::amdgcn_s_sendmsg:
1787 case Intrinsic::amdgcn_s_sendmsghalt: {
1788 // The second operand is copied to m0, but is only actually used for
1789 // certain message types. For message types that are known to not use m0,
1790 // fold it to poison.
1791 using namespace AMDGPU::SendMsg;
1792
1793 Value *M0Val = II.getArgOperand(i: 1);
1794 if (isa<PoisonValue>(Val: M0Val))
1795 break;
1796
1797 auto *MsgImm = cast<ConstantInt>(Val: II.getArgOperand(i: 0));
1798 uint16_t MsgId, OpId, StreamId;
1799 decodeMsg(Val: MsgImm->getZExtValue(), MsgId, OpId, StreamId, STI: *ST);
1800
1801 if (!msgDoesNotUseM0(MsgId, STI: *ST))
1802 break;
1803
1804 // Drop UB-implying attributes since we're replacing with poison.
1805 II.dropUBImplyingAttrsAndMetadata();
1806 IC.replaceOperand(I&: II, OpNum: 1, V: PoisonValue::get(T: M0Val->getType()));
1807 return nullptr;
1808 }
1809 case Intrinsic::amdgcn_update_dpp: {
1810 Value *Old = II.getArgOperand(i: 0);
1811
1812 auto *BC = cast<ConstantInt>(Val: II.getArgOperand(i: 5));
1813 auto *RM = cast<ConstantInt>(Val: II.getArgOperand(i: 3));
1814 auto *BM = cast<ConstantInt>(Val: II.getArgOperand(i: 4));
1815 if (BC->isNullValue() || RM->getZExtValue() != 0xF ||
1816 BM->getZExtValue() != 0xF || isa<PoisonValue>(Val: Old))
1817 break;
1818
1819 // If bound_ctrl = 1, row mask = bank mask = 0xf we can omit old value.
1820 return IC.replaceOperand(I&: II, OpNum: 0, V: PoisonValue::get(T: Old->getType()));
1821 }
1822 case Intrinsic::amdgcn_permlane16:
1823 case Intrinsic::amdgcn_permlane16_var:
1824 case Intrinsic::amdgcn_permlanex16:
1825 case Intrinsic::amdgcn_permlanex16_var: {
1826 // Discard vdst_in if it's not going to be read.
1827 Value *VDstIn = II.getArgOperand(i: 0);
1828 if (isa<PoisonValue>(Val: VDstIn))
1829 break;
1830
1831 // FetchInvalid operand idx.
1832 unsigned int FiIdx = (IID == Intrinsic::amdgcn_permlane16 ||
1833 IID == Intrinsic::amdgcn_permlanex16)
1834 ? 4 /* for permlane16 and permlanex16 */
1835 : 3; /* for permlane16_var and permlanex16_var */
1836
1837 // BoundCtrl operand idx.
1838 // For permlane16 and permlanex16 it should be 5
1839 // For Permlane16_var and permlanex16_var it should be 4
1840 unsigned int BcIdx = FiIdx + 1;
1841
1842 ConstantInt *FetchInvalid = cast<ConstantInt>(Val: II.getArgOperand(i: FiIdx));
1843 ConstantInt *BoundCtrl = cast<ConstantInt>(Val: II.getArgOperand(i: BcIdx));
1844 if (!FetchInvalid->getZExtValue() && !BoundCtrl->getZExtValue())
1845 break;
1846
1847 return IC.replaceOperand(I&: II, OpNum: 0, V: PoisonValue::get(T: VDstIn->getType()));
1848 }
1849 case Intrinsic::amdgcn_wave_shuffle:
1850 return tryOptimizeShufflePattern(IC, II, ST: *ST);
1851 case Intrinsic::amdgcn_permlane64:
1852 case Intrinsic::amdgcn_readfirstlane:
1853 case Intrinsic::amdgcn_readlane:
1854 case Intrinsic::amdgcn_ds_bpermute: {
1855 // If the data argument is uniform these intrinsics return it unchanged.
1856 unsigned SrcIdx = IID == Intrinsic::amdgcn_ds_bpermute ? 1 : 0;
1857 const Use &Src = II.getArgOperandUse(i: SrcIdx);
1858 if (isTriviallyUniform(U: Src))
1859 return IC.replaceInstUsesWith(I&: II, V: Src.get());
1860
1861 if (IID == Intrinsic::amdgcn_readlane &&
1862 simplifyDemandedLaneMaskArg(IC, II, LaneArgIdx: 1))
1863 return &II;
1864
1865 // If the lane argument of bpermute is uniform, change it to readlane. This
1866 // generates better code and can enable further optimizations because
1867 // readlane is AlwaysUniform.
1868 if (IID == Intrinsic::amdgcn_ds_bpermute) {
1869 const Use &Lane = II.getArgOperandUse(i: 0);
1870 if (isTriviallyUniform(U: Lane)) {
1871 Value *NewLane = IC.Builder.CreateLShr(LHS: Lane, RHS: 2);
1872 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
1873 M: II.getModule(), id: Intrinsic::amdgcn_readlane, OverloadTys: II.getType());
1874 II.setCalledFunction(NewDecl);
1875 II.setOperand(i_nocapture: 0, Val_nocapture: Src);
1876 II.setOperand(i_nocapture: 1, Val_nocapture: NewLane);
1877 return &II;
1878 }
1879 }
1880
1881 if (IID == Intrinsic::amdgcn_ds_bpermute)
1882 return tryOptimizeShufflePattern(IC, II, ST: *ST);
1883
1884 if (Instruction *Res = hoistLaneIntrinsicThroughOperand(IC, II))
1885 return Res;
1886
1887 return std::nullopt;
1888 }
1889 case Intrinsic::amdgcn_writelane: {
1890 // TODO: Fold bitcast like readlane.
1891 if (simplifyDemandedLaneMaskArg(IC, II, LaneArgIdx: 1))
1892 return &II;
1893 return std::nullopt;
1894 }
1895 case Intrinsic::amdgcn_trig_preop: {
1896 // The intrinsic is declared with name mangling, but currently the
1897 // instruction only exists for f64
1898 if (!II.getType()->isDoubleTy())
1899 break;
1900
1901 Value *Src = II.getArgOperand(i: 0);
1902 Value *Segment = II.getArgOperand(i: 1);
1903 if (isa<PoisonValue>(Val: Src) || isa<PoisonValue>(Val: Segment))
1904 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1905
1906 if (isa<UndefValue>(Val: Segment))
1907 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getZero(Ty: II.getType()));
1908
1909 // Sign bit is not used.
1910 Value *StrippedSign = InstCombiner::stripSignOnlyFPOps(Val: Src);
1911 if (StrippedSign != Src)
1912 return IC.replaceOperand(I&: II, OpNum: 0, V: StrippedSign);
1913
1914 if (II.isStrictFP())
1915 break;
1916
1917 const ConstantFP *CSrc = dyn_cast<ConstantFP>(Val: Src);
1918 if (!CSrc && !isa<UndefValue>(Val: Src))
1919 break;
1920
1921 // The instruction ignores special cases, and literally just extracts the
1922 // exponents. Fold undef to nan, and index the table as normal.
1923 APInt FSrcInt = CSrc ? CSrc->getValueAPF().bitcastToAPInt()
1924 : APFloat::getQNaN(Sem: II.getType()->getFltSemantics())
1925 .bitcastToAPInt();
1926
1927 const ConstantInt *Cseg = dyn_cast<ConstantInt>(Val: Segment);
1928 if (!Cseg) {
1929 if (isa<UndefValue>(Val: Src))
1930 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getZero(Ty: II.getType()));
1931 break;
1932 }
1933
1934 unsigned Exponent = FSrcInt.extractBitsAsZExtValue(numBits: 11, bitPosition: 52);
1935 unsigned SegmentVal = Cseg->getValue().trunc(width: 5).getZExtValue();
1936 unsigned Shift = SegmentVal * 53;
1937 if (Exponent > 1077)
1938 Shift += Exponent - 1077;
1939
1940 // 2.0/PI table.
1941 static const uint32_t TwoByPi[] = {
1942 0xa2f9836e, 0x4e441529, 0xfc2757d1, 0xf534ddc0, 0xdb629599, 0x3c439041,
1943 0xfe5163ab, 0xdebbc561, 0xb7246e3a, 0x424dd2e0, 0x06492eea, 0x09d1921c,
1944 0xfe1deb1c, 0xb129a73e, 0xe88235f5, 0x2ebb4484, 0xe99c7026, 0xb45f7e41,
1945 0x3991d639, 0x835339f4, 0x9c845f8b, 0xbdf9283b, 0x1ff897ff, 0xde05980f,
1946 0xef2f118b, 0x5a0a6d1f, 0x6d367ecf, 0x27cb09b7, 0x4f463f66, 0x9e5fea2d,
1947 0x7527bac7, 0xebe5f17b, 0x3d0739f7, 0x8a5292ea, 0x6bfb5fb1, 0x1f8d5d08,
1948 0x56033046};
1949
1950 // Return 0 for outbound segment (hardware behavior).
1951 unsigned Idx = Shift >> 5;
1952 if (Idx + 2 >= std::size(TwoByPi)) {
1953 APFloat Zero = APFloat::getZero(Sem: II.getType()->getFltSemantics());
1954 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::get(Ty: II.getType(), V: Zero));
1955 }
1956
1957 unsigned BShift = Shift & 0x1f;
1958 uint64_t Thi = Make_64(High: TwoByPi[Idx], Low: TwoByPi[Idx + 1]);
1959 uint64_t Tlo = Make_64(High: TwoByPi[Idx + 2], Low: 0);
1960 if (BShift)
1961 Thi = (Thi << BShift) | (Tlo >> (64 - BShift));
1962 Thi = Thi >> 11;
1963 APFloat Result = APFloat((double)Thi);
1964
1965 int Scale = -53 - Shift;
1966 if (Exponent >= 1968)
1967 Scale += 128;
1968
1969 Result = scalbn(X: Result, Exp: Scale, RM: RoundingMode::NearestTiesToEven);
1970 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::get(Ty: Src->getType(), V: Result));
1971 }
1972 case Intrinsic::amdgcn_fmul_legacy: {
1973 Value *Op0 = II.getArgOperand(i: 0);
1974 Value *Op1 = II.getArgOperand(i: 1);
1975
1976 for (Value *Src : {Op0, Op1}) {
1977 if (isa<PoisonValue>(Val: Src))
1978 return IC.replaceInstUsesWith(I&: II, V: Src);
1979 }
1980
1981 // The legacy behaviour is that multiplying +/-0.0 by anything, even NaN or
1982 // infinity, gives +0.0.
1983 // TODO: Move to InstSimplify?
1984 if (match(V: Op0, P: PatternMatch::m_AnyZeroFP()) ||
1985 match(V: Op1, P: PatternMatch::m_AnyZeroFP()))
1986 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getZero(Ty: II.getType()));
1987
1988 // If we can prove we don't have one of the special cases then we can use a
1989 // normal fmul instruction instead.
1990 if (canSimplifyLegacyMulToMul(I: II, Op0, Op1, IC)) {
1991 auto *FMul = IC.Builder.CreateFMulFMF(L: Op0, R: Op1, FMFSource: &II);
1992 FMul->takeName(V: &II);
1993 return IC.replaceInstUsesWith(I&: II, V: FMul);
1994 }
1995 break;
1996 }
1997 case Intrinsic::amdgcn_fma_legacy: {
1998 Value *Op0 = II.getArgOperand(i: 0);
1999 Value *Op1 = II.getArgOperand(i: 1);
2000 Value *Op2 = II.getArgOperand(i: 2);
2001
2002 for (Value *Src : {Op0, Op1, Op2}) {
2003 if (isa<PoisonValue>(Val: Src))
2004 return IC.replaceInstUsesWith(I&: II, V: Src);
2005 }
2006
2007 // The legacy behaviour is that multiplying +/-0.0 by anything, even NaN or
2008 // infinity, gives +0.0.
2009 // TODO: Move to InstSimplify?
2010 if (match(V: Op0, P: PatternMatch::m_AnyZeroFP()) ||
2011 match(V: Op1, P: PatternMatch::m_AnyZeroFP())) {
2012 // It's tempting to just return Op2 here, but that would give the wrong
2013 // result if Op2 was -0.0.
2014 auto *Zero = ConstantFP::getZero(Ty: II.getType());
2015 auto *FAdd = IC.Builder.CreateFAddFMF(L: Zero, R: Op2, FMFSource: &II);
2016 FAdd->takeName(V: &II);
2017 return IC.replaceInstUsesWith(I&: II, V: FAdd);
2018 }
2019
2020 // If we can prove we don't have one of the special cases then we can use a
2021 // normal fma instead.
2022 if (canSimplifyLegacyMulToMul(I: II, Op0, Op1, IC)) {
2023 II.setCalledOperand(Intrinsic::getOrInsertDeclaration(
2024 M: II.getModule(), id: Intrinsic::fma, OverloadTys: II.getType()));
2025 return &II;
2026 }
2027 break;
2028 }
2029 case Intrinsic::amdgcn_is_shared:
2030 case Intrinsic::amdgcn_is_private: {
2031 Value *Src = II.getArgOperand(i: 0);
2032 if (isa<PoisonValue>(Val: Src))
2033 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
2034 if (isa<UndefValue>(Val: Src))
2035 return IC.replaceInstUsesWith(I&: II, V: UndefValue::get(T: II.getType()));
2036
2037 if (isa<ConstantPointerNull>(Val: II.getArgOperand(i: 0)))
2038 return IC.replaceInstUsesWith(I&: II, V: ConstantInt::getFalse(Ty: II.getType()));
2039 break;
2040 }
2041 case Intrinsic::amdgcn_make_buffer_rsrc: {
2042 Value *Src = II.getArgOperand(i: 0);
2043 if (isa<PoisonValue>(Val: Src))
2044 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
2045 return std::nullopt;
2046 }
2047 case Intrinsic::amdgcn_raw_buffer_store_format:
2048 case Intrinsic::amdgcn_struct_buffer_store_format:
2049 case Intrinsic::amdgcn_raw_tbuffer_store:
2050 case Intrinsic::amdgcn_struct_tbuffer_store:
2051 case Intrinsic::amdgcn_image_store_1d:
2052 case Intrinsic::amdgcn_image_store_1darray:
2053 case Intrinsic::amdgcn_image_store_2d:
2054 case Intrinsic::amdgcn_image_store_2darray:
2055 case Intrinsic::amdgcn_image_store_2darraymsaa:
2056 case Intrinsic::amdgcn_image_store_2dmsaa:
2057 case Intrinsic::amdgcn_image_store_3d:
2058 case Intrinsic::amdgcn_image_store_cube:
2059 case Intrinsic::amdgcn_image_store_mip_1d:
2060 case Intrinsic::amdgcn_image_store_mip_1darray:
2061 case Intrinsic::amdgcn_image_store_mip_2d:
2062 case Intrinsic::amdgcn_image_store_mip_2darray:
2063 case Intrinsic::amdgcn_image_store_mip_3d:
2064 case Intrinsic::amdgcn_image_store_mip_cube: {
2065 if (!isa<FixedVectorType>(Val: II.getArgOperand(i: 0)->getType()))
2066 break;
2067
2068 APInt DemandedElts;
2069 if (ST->hasDefaultComponentBroadcast())
2070 DemandedElts = defaultComponentBroadcast(V: II.getArgOperand(i: 0));
2071 else if (ST->hasDefaultComponentZero())
2072 DemandedElts = trimTrailingZerosInVector(IC, UseV: II.getArgOperand(i: 0), I: &II);
2073 else
2074 break;
2075
2076 int DMaskIdx = getAMDGPUImageDMaskIntrinsic(Intr: II.getIntrinsicID()) ? 1 : -1;
2077 if (simplifyAMDGCNMemoryIntrinsicDemanded(IC, II, DemandedElts, DMaskIdx,
2078 IsLoad: false)) {
2079 return IC.eraseInstFromFunction(I&: II);
2080 }
2081
2082 break;
2083 }
2084 case Intrinsic::amdgcn_prng_b32: {
2085 auto *Src = II.getArgOperand(i: 0);
2086 if (isa<UndefValue>(Val: Src)) {
2087 return IC.replaceInstUsesWith(I&: II, V: Src);
2088 }
2089 return std::nullopt;
2090 }
2091 case Intrinsic::amdgcn_mfma_scale_f32_16x16x128_f8f6f4:
2092 case Intrinsic::amdgcn_mfma_scale_f32_32x32x64_f8f6f4: {
2093 Value *Src0 = II.getArgOperand(i: 0);
2094 Value *Src1 = II.getArgOperand(i: 1);
2095 uint64_t CBSZ = cast<ConstantInt>(Val: II.getArgOperand(i: 3))->getZExtValue();
2096 uint64_t BLGP = cast<ConstantInt>(Val: II.getArgOperand(i: 4))->getZExtValue();
2097 auto *Src0Ty = cast<FixedVectorType>(Val: Src0->getType());
2098 auto *Src1Ty = cast<FixedVectorType>(Val: Src1->getType());
2099
2100 auto getFormatNumRegs = [](unsigned FormatVal) {
2101 switch (FormatVal) {
2102 case AMDGPU::MFMAScaleFormats::FP6_E2M3:
2103 case AMDGPU::MFMAScaleFormats::FP6_E3M2:
2104 return 6u;
2105 case AMDGPU::MFMAScaleFormats::FP4_E2M1:
2106 return 4u;
2107 case AMDGPU::MFMAScaleFormats::FP8_E4M3:
2108 case AMDGPU::MFMAScaleFormats::FP8_E5M2:
2109 return 8u;
2110 default:
2111 llvm_unreachable("invalid format value");
2112 }
2113 };
2114
2115 bool MadeChange = false;
2116 unsigned Src0NumElts = getFormatNumRegs(CBSZ);
2117 unsigned Src1NumElts = getFormatNumRegs(BLGP);
2118
2119 // Depending on the used format, fewer registers are required so shrink the
2120 // vector type.
2121 if (Src0Ty->getNumElements() > Src0NumElts) {
2122 Src0 = IC.Builder.CreateExtractVector(
2123 DstType: FixedVectorType::get(ElementType: Src0Ty->getElementType(), NumElts: Src0NumElts), SrcVec: Src0,
2124 Idx: uint64_t(0));
2125 MadeChange = true;
2126 }
2127
2128 if (Src1Ty->getNumElements() > Src1NumElts) {
2129 Src1 = IC.Builder.CreateExtractVector(
2130 DstType: FixedVectorType::get(ElementType: Src1Ty->getElementType(), NumElts: Src1NumElts), SrcVec: Src1,
2131 Idx: uint64_t(0));
2132 MadeChange = true;
2133 }
2134
2135 if (!MadeChange)
2136 return std::nullopt;
2137
2138 SmallVector<Value *, 10> Args(II.args());
2139 Args[0] = Src0;
2140 Args[1] = Src1;
2141
2142 Value *NewII = IC.Builder.CreateIntrinsic(
2143 ID: IID, OverloadTypes: {Src0->getType(), Src1->getType()}, Args, FMFSource: &II);
2144 NewII->takeName(V: &II);
2145 return IC.replaceInstUsesWith(I&: II, V: NewII);
2146 }
2147 case Intrinsic::amdgcn_wmma_f32_16x16x128_f8f6f4:
2148 case Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
2149 case Intrinsic::amdgcn_wmma_scale16_f32_16x16x128_f8f6f4: {
2150 Value *Src0 = II.getArgOperand(i: 1);
2151 Value *Src1 = II.getArgOperand(i: 3);
2152 unsigned FmtA = cast<ConstantInt>(Val: II.getArgOperand(i: 0))->getZExtValue();
2153 uint64_t FmtB = cast<ConstantInt>(Val: II.getArgOperand(i: 2))->getZExtValue();
2154 auto *Src0Ty = cast<FixedVectorType>(Val: Src0->getType());
2155 auto *Src1Ty = cast<FixedVectorType>(Val: Src1->getType());
2156
2157 bool MadeChange = false;
2158 unsigned Src0NumElts = AMDGPU::wmmaScaleF8F6F4FormatToNumRegs(Fmt: FmtA);
2159 unsigned Src1NumElts = AMDGPU::wmmaScaleF8F6F4FormatToNumRegs(Fmt: FmtB);
2160
2161 // Depending on the used format, fewer registers are required so shrink the
2162 // vector type.
2163 if (Src0Ty->getNumElements() > Src0NumElts) {
2164 Src0 = IC.Builder.CreateExtractVector(
2165 DstType: FixedVectorType::get(ElementType: Src0Ty->getElementType(), NumElts: Src0NumElts), SrcVec: Src0,
2166 Idx: IC.Builder.getInt64(C: 0));
2167 MadeChange = true;
2168 }
2169
2170 if (Src1Ty->getNumElements() > Src1NumElts) {
2171 Src1 = IC.Builder.CreateExtractVector(
2172 DstType: FixedVectorType::get(ElementType: Src1Ty->getElementType(), NumElts: Src1NumElts), SrcVec: Src1,
2173 Idx: IC.Builder.getInt64(C: 0));
2174 MadeChange = true;
2175 }
2176
2177 if (!MadeChange)
2178 return std::nullopt;
2179
2180 SmallVector<Value *, 13> Args(II.args());
2181 Args[1] = Src0;
2182 Args[3] = Src1;
2183
2184 Value *NewII = IC.Builder.CreateIntrinsic(
2185 ID: IID, OverloadTypes: {II.getArgOperand(i: 5)->getType(), Src0->getType(), Src1->getType()},
2186 Args, FMFSource: &II);
2187 NewII->takeName(V: &II);
2188 return IC.replaceInstUsesWith(I&: II, V: NewII);
2189 }
2190 }
2191 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
2192 AMDGPU::getImageDimIntrinsicInfo(Intr: II.getIntrinsicID())) {
2193 return simplifyAMDGCNImageIntrinsic(ST, ImageDimIntr, II, IC);
2194 }
2195 return std::nullopt;
2196}
2197
2198/// Implement SimplifyDemandedVectorElts for amdgcn buffer and image intrinsics.
2199///
2200/// The result of simplifying amdgcn image and buffer store intrinsics is updating
2201/// definitions of the intrinsics vector argument, not Uses of the result like
2202/// image and buffer loads.
2203/// Note: This only supports non-TFE/LWE image intrinsic calls; those have
2204/// struct returns.
2205static Value *simplifyAMDGCNMemoryIntrinsicDemanded(InstCombiner &IC,
2206 IntrinsicInst &II,
2207 APInt DemandedElts,
2208 int DMaskIdx, bool IsLoad) {
2209
2210 auto *IIVTy = cast<FixedVectorType>(Val: IsLoad ? II.getType()
2211 : II.getOperand(i_nocapture: 0)->getType());
2212 unsigned VWidth = IIVTy->getNumElements();
2213 if (VWidth == 1)
2214 return nullptr;
2215 Type *EltTy = IIVTy->getElementType();
2216
2217 IRBuilderBase::InsertPointGuard Guard(IC.Builder);
2218 IC.Builder.SetInsertPoint(&II);
2219
2220 // Assume the arguments are unchanged and later override them, if needed.
2221 SmallVector<Value *, 16> Args(II.args());
2222
2223 if (DMaskIdx < 0) {
2224 // Buffer case.
2225
2226 const unsigned ActiveBits = DemandedElts.getActiveBits();
2227 const unsigned UnusedComponentsAtFront = DemandedElts.countr_zero();
2228
2229 // Start assuming the prefix of elements is demanded, but possibly clear
2230 // some other bits if there are trailing zeros (unused components at front)
2231 // and update offset.
2232 DemandedElts = (1 << ActiveBits) - 1;
2233
2234 if (UnusedComponentsAtFront > 0) {
2235 static const unsigned InvalidOffsetIdx = 0xf;
2236
2237 unsigned OffsetIdx;
2238 switch (II.getIntrinsicID()) {
2239 case Intrinsic::amdgcn_raw_buffer_load:
2240 case Intrinsic::amdgcn_raw_ptr_buffer_load:
2241 OffsetIdx = 1;
2242 break;
2243 case Intrinsic::amdgcn_s_buffer_load:
2244 case Intrinsic::amdgcn_ptr_s_buffer_load:
2245 // If resulting type is vec3, there is no point in trimming the
2246 // load with updated offset, as the vec3 would most likely be widened to
2247 // vec4 anyway during lowering.
2248 if (ActiveBits == 4 && UnusedComponentsAtFront == 1)
2249 OffsetIdx = InvalidOffsetIdx;
2250 else
2251 OffsetIdx = 1;
2252 break;
2253 case Intrinsic::amdgcn_struct_buffer_load:
2254 case Intrinsic::amdgcn_struct_ptr_buffer_load:
2255 OffsetIdx = 2;
2256 break;
2257 default:
2258 // TODO: handle tbuffer* intrinsics.
2259 OffsetIdx = InvalidOffsetIdx;
2260 break;
2261 }
2262
2263 if (OffsetIdx != InvalidOffsetIdx) {
2264 // Clear demanded bits and update the offset.
2265 DemandedElts &= ~((1 << UnusedComponentsAtFront) - 1);
2266 auto *Offset = Args[OffsetIdx];
2267 unsigned SingleComponentSizeInBits =
2268 IC.getDataLayout().getTypeSizeInBits(Ty: EltTy);
2269 unsigned OffsetAdd =
2270 UnusedComponentsAtFront * SingleComponentSizeInBits / 8;
2271 auto *OffsetAddVal = ConstantInt::get(Ty: Offset->getType(), V: OffsetAdd);
2272 Args[OffsetIdx] = IC.Builder.CreateAdd(LHS: Offset, RHS: OffsetAddVal);
2273 }
2274 }
2275 } else {
2276 // Image case.
2277
2278 ConstantInt *DMask = cast<ConstantInt>(Val: Args[DMaskIdx]);
2279 unsigned DMaskVal = DMask->getZExtValue() & 0xf;
2280
2281 // dmask 0 has special semantics, do not simplify.
2282 if (DMaskVal == 0)
2283 return nullptr;
2284
2285 if (!IsLoad && !isMask_32(Value: DMaskVal))
2286 return nullptr;
2287
2288 // Mask off values that are undefined because the dmask doesn't cover them
2289 DemandedElts &= (1 << llvm::popcount(Value: DMaskVal)) - 1;
2290
2291 unsigned NewDMaskVal = 0;
2292 unsigned OrigLdStIdx = 0;
2293 for (unsigned SrcIdx = 0; SrcIdx < 4; ++SrcIdx) {
2294 const unsigned Bit = 1 << SrcIdx;
2295 if (!!(DMaskVal & Bit)) {
2296 if (!!DemandedElts[OrigLdStIdx])
2297 NewDMaskVal |= Bit;
2298 OrigLdStIdx++;
2299 }
2300 }
2301
2302 if (DMaskVal != NewDMaskVal)
2303 Args[DMaskIdx] = ConstantInt::get(Ty: DMask->getType(), V: NewDMaskVal);
2304 }
2305
2306 unsigned NewNumElts = DemandedElts.popcount();
2307 if (!NewNumElts)
2308 return PoisonValue::get(T: IIVTy);
2309
2310 if (NewNumElts >= VWidth && DemandedElts.isMask()) {
2311 if (DMaskIdx >= 0)
2312 II.setArgOperand(i: DMaskIdx, v: Args[DMaskIdx]);
2313 return nullptr;
2314 }
2315
2316 // Validate function argument and return types, extracting overloaded types
2317 // along the way.
2318 SmallVector<Type *, 6> OverloadTys;
2319 if (!Intrinsic::isSignatureValid(F: II.getCalledFunction(), OverloadTys))
2320 return nullptr;
2321
2322 Type *NewTy =
2323 (NewNumElts == 1) ? EltTy : FixedVectorType::get(ElementType: EltTy, NumElts: NewNumElts);
2324 OverloadTys[0] = NewTy;
2325
2326 if (!IsLoad) {
2327 SmallVector<int, 8> EltMask;
2328 for (unsigned OrigStoreIdx = 0; OrigStoreIdx < VWidth; ++OrigStoreIdx)
2329 if (DemandedElts[OrigStoreIdx])
2330 EltMask.push_back(Elt: OrigStoreIdx);
2331
2332 if (NewNumElts == 1)
2333 Args[0] = IC.Builder.CreateExtractElement(Vec: II.getOperand(i_nocapture: 0), Idx: EltMask[0]);
2334 else
2335 Args[0] = IC.Builder.CreateShuffleVector(V: II.getOperand(i_nocapture: 0), Mask: EltMask);
2336 }
2337
2338 CallInst *NewCall = IC.Builder.CreateIntrinsicWithoutFolding(
2339 ID: II.getIntrinsicID(), OverloadTypes: OverloadTys, Args);
2340 NewCall->takeName(V: &II);
2341 NewCall->copyMetadata(SrcInst: II);
2342 AttributeList OldAttrList = II.getAttributes();
2343 NewCall->setAttributes(OldAttrList);
2344
2345 if (IsLoad) {
2346 if (NewNumElts == 1) {
2347 return IC.Builder.CreateInsertElement(Vec: PoisonValue::get(T: IIVTy), NewElt: NewCall,
2348 Idx: DemandedElts.countr_zero());
2349 }
2350
2351 SmallVector<int, 8> EltMask;
2352 unsigned NewLoadIdx = 0;
2353 for (unsigned OrigLoadIdx = 0; OrigLoadIdx < VWidth; ++OrigLoadIdx) {
2354 if (!!DemandedElts[OrigLoadIdx])
2355 EltMask.push_back(Elt: NewLoadIdx++);
2356 else
2357 EltMask.push_back(Elt: NewNumElts);
2358 }
2359
2360 auto *Shuffle = IC.Builder.CreateShuffleVector(V: NewCall, Mask: EltMask);
2361
2362 return Shuffle;
2363 }
2364
2365 return NewCall;
2366}
2367
2368Value *GCNTTIImpl::simplifyAMDGCNLaneIntrinsicDemanded(
2369 InstCombiner &IC, IntrinsicInst &II, const APInt &DemandedElts,
2370 APInt &UndefElts) const {
2371 auto *VT = dyn_cast<FixedVectorType>(Val: II.getType());
2372 if (!VT)
2373 return nullptr;
2374
2375 const unsigned FirstElt = DemandedElts.countr_zero();
2376 const unsigned LastElt = DemandedElts.getActiveBits() - 1;
2377 const unsigned MaskLen = LastElt - FirstElt + 1;
2378
2379 unsigned OldNumElts = VT->getNumElements();
2380 if (MaskLen == OldNumElts && MaskLen != 1)
2381 return nullptr;
2382
2383 Type *EltTy = VT->getElementType();
2384 Type *NewVT = MaskLen == 1 ? EltTy : FixedVectorType::get(ElementType: EltTy, NumElts: MaskLen);
2385
2386 // Theoretically we should support these intrinsics for any legal type. Avoid
2387 // introducing cases that aren't direct register types like v3i16.
2388 if (!isTypeLegal(Ty: NewVT))
2389 return nullptr;
2390
2391 Value *Src = II.getArgOperand(i: 0);
2392
2393 // Make sure convergence tokens are preserved.
2394 // TODO: CreateIntrinsic should allow directly copying bundles
2395 SmallVector<OperandBundleDef, 2> OpBundles;
2396 II.getOperandBundlesAsDefs(Defs&: OpBundles);
2397
2398 Module *M = IC.Builder.GetInsertBlock()->getModule();
2399 Function *Remangled =
2400 Intrinsic::getOrInsertDeclaration(M, id: II.getIntrinsicID(), OverloadTys: {NewVT});
2401
2402 if (MaskLen == 1) {
2403 Value *Extract = IC.Builder.CreateExtractElement(Vec: Src, Idx: FirstElt);
2404
2405 // TODO: Preserve callsite attributes?
2406 CallInst *NewCall = IC.Builder.CreateCall(Callee: Remangled, Args: {Extract}, OpBundles);
2407
2408 return IC.Builder.CreateInsertElement(Vec: PoisonValue::get(T: II.getType()),
2409 NewElt: NewCall, Idx: FirstElt);
2410 }
2411
2412 SmallVector<int> ExtractMask(MaskLen, -1);
2413 for (unsigned I = 0; I != MaskLen; ++I) {
2414 if (DemandedElts[FirstElt + I])
2415 ExtractMask[I] = FirstElt + I;
2416 }
2417
2418 Value *Extract = IC.Builder.CreateShuffleVector(V: Src, Mask: ExtractMask);
2419
2420 // TODO: Preserve callsite attributes?
2421 CallInst *NewCall = IC.Builder.CreateCall(Callee: Remangled, Args: {Extract}, OpBundles);
2422
2423 SmallVector<int> InsertMask(OldNumElts, -1);
2424 for (unsigned I = 0; I != MaskLen; ++I) {
2425 if (DemandedElts[FirstElt + I])
2426 InsertMask[FirstElt + I] = I;
2427 }
2428
2429 // FIXME: If the call has a convergence bundle, we end up leaving the dead
2430 // call behind.
2431 return IC.Builder.CreateShuffleVector(V: NewCall, Mask: InsertMask);
2432}
2433
2434std::optional<Value *> GCNTTIImpl::simplifyDemandedVectorEltsIntrinsic(
2435 InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
2436 APInt &UndefElts2, APInt &UndefElts3,
2437 std::function<void(Instruction *, unsigned, APInt, APInt &)>
2438 SimplifyAndSetOp) const {
2439 switch (II.getIntrinsicID()) {
2440 case Intrinsic::amdgcn_readfirstlane:
2441 SimplifyAndSetOp(&II, 0, DemandedElts, UndefElts);
2442 return simplifyAMDGCNLaneIntrinsicDemanded(IC, II, DemandedElts, UndefElts);
2443 case Intrinsic::amdgcn_raw_buffer_load:
2444 case Intrinsic::amdgcn_raw_ptr_buffer_load:
2445 case Intrinsic::amdgcn_raw_buffer_load_format:
2446 case Intrinsic::amdgcn_raw_ptr_buffer_load_format:
2447 case Intrinsic::amdgcn_raw_tbuffer_load:
2448 case Intrinsic::amdgcn_raw_ptr_tbuffer_load:
2449 case Intrinsic::amdgcn_s_buffer_load:
2450 case Intrinsic::amdgcn_ptr_s_buffer_load:
2451 case Intrinsic::amdgcn_struct_buffer_load:
2452 case Intrinsic::amdgcn_struct_ptr_buffer_load:
2453 case Intrinsic::amdgcn_struct_buffer_load_format:
2454 case Intrinsic::amdgcn_struct_ptr_buffer_load_format:
2455 case Intrinsic::amdgcn_struct_tbuffer_load:
2456 case Intrinsic::amdgcn_struct_ptr_tbuffer_load:
2457 return simplifyAMDGCNMemoryIntrinsicDemanded(IC, II, DemandedElts);
2458 default: {
2459 if (getAMDGPUImageDMaskIntrinsic(Intr: II.getIntrinsicID())) {
2460 return simplifyAMDGCNMemoryIntrinsicDemanded(IC, II, DemandedElts, DMaskIdx: 0);
2461 }
2462 break;
2463 }
2464 }
2465 return std::nullopt;
2466}
2467