1//===- AMDGPInstCombineIntrinsic.cpp - AMDGPU specific InstCombine pass ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// \file
10// This file implements a TargetTransformInfo analysis pass specific to the
11// AMDGPU target machine. It uses the target's detailed information to provide
12// more precise answers to certain TTI queries, while letting the target
13// independent and default TTI implementations handle the rest.
14//
15//===----------------------------------------------------------------------===//
16
17#include "AMDGPUInstrInfo.h"
18#include "AMDGPUTargetTransformInfo.h"
19#include "GCNSubtarget.h"
20#include "SIDefines.h"
21#include "llvm/ADT/FloatingPointMode.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/Analysis/ConstantFolding.h"
24#include "llvm/Analysis/ValueTracking.h"
25#include "llvm/IR/Constants.h"
26#include "llvm/IR/Dominators.h"
27#include "llvm/IR/IntrinsicsAMDGPU.h"
28#include "llvm/Support/MathExtras.h"
29#include "llvm/Transforms/InstCombine/InstCombiner.h"
30#include <optional>
31
32using namespace llvm;
33using namespace llvm::PatternMatch;
34
35#define DEBUG_TYPE "AMDGPUtti"
36
37namespace {
38
39struct AMDGPUImageDMaskIntrinsic {
40 unsigned Intr;
41};
42
43#define GET_AMDGPUImageDMaskIntrinsicTable_IMPL
44#include "AMDGPUGenSearchableTables.inc"
45
46} // end anonymous namespace
47
48// Constant fold llvm.amdgcn.fmed3 intrinsics for standard inputs.
49//
50// A single NaN input is folded to minnum, so we rely on that folding for
51// handling NaNs.
52static APFloat fmed3AMDGCN(const APFloat &Src0, const APFloat &Src1,
53 const APFloat &Src2) {
54 assert(!Src0.isNaN() && !Src1.isNaN() && !Src2.isNaN() &&
55 "nans handled separately");
56 APFloat Max3 = maxnum(A: maxnum(A: Src0, B: Src1), B: Src2);
57
58 if (Max3.bitwiseIsEqual(RHS: Src0))
59 return maxnum(A: Src1, B: Src2);
60
61 if (Max3.bitwiseIsEqual(RHS: Src1))
62 return maxnum(A: Src0, B: Src2);
63
64 return maxnum(A: Src0, B: Src1);
65}
66
67// Check if a value can be converted to a 16-bit value without losing precision.
68// The value is expected to be either a float (IsFloat = true) or an unsigned
69// integer (IsFloat = false). When AllowI16SExt is set, a sext from i16 is also
70// accepted: for unsigned addresses sext and zext only differ for a negative
71// i16, which is out of bounds anyway (see caller).
72static bool canSafelyConvertTo16Bit(Value &V, bool IsFloat,
73 bool AllowI16SExt = false) {
74 Type *VTy = V.getType();
75 if (VTy->isHalfTy() || VTy->isIntegerTy(BitWidth: 16)) {
76 // The value is already 16-bit, so we don't want to convert to 16-bit again!
77 return false;
78 }
79 if (IsFloat) {
80 if (ConstantFP *ConstFloat = dyn_cast<ConstantFP>(Val: &V)) {
81 // We need to check that if we cast the index down to a half, we do not
82 // lose precision.
83 APFloat FloatValue(ConstFloat->getValueAPF());
84 bool LosesInfo = true;
85 FloatValue.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmTowardZero,
86 losesInfo: &LosesInfo);
87 return !LosesInfo;
88 }
89 } else {
90 if (ConstantInt *ConstInt = dyn_cast<ConstantInt>(Val: &V)) {
91 // We need to check that if we cast the index down to an i16, we do not
92 // lose precision.
93 APInt IntValue(ConstInt->getValue());
94 return IntValue.getActiveBits() <= 16;
95 }
96 }
97
98 // Coordinates may arrive as extractelement((s|z|fp)ext Vec), Idx. The
99 // widening cast has one use per lane, so it is never sunk into the extract;
100 // strip the extract here so the cast check below is common to scalar and
101 // vector coords.
102 Value *CastCandidate;
103 if (!match(V: &V, P: m_ExtractElt(Val: m_Value(V&: CastCandidate), Idx: m_Value())))
104 CastCandidate = &V;
105
106 Value *CastSrc;
107 bool IsExt = IsFloat ? match(V: CastCandidate, P: m_FPExt(Op: m_Value(V&: CastSrc)))
108 : match(V: CastCandidate, P: m_ZExt(Op: m_Value(V&: CastSrc)));
109 if (!IsExt && !IsFloat && AllowI16SExt)
110 IsExt = match(V: CastCandidate, P: m_SExt(Op: m_Value(V&: CastSrc)));
111 if (IsExt) {
112 Type *CastSrcTy = CastSrc->getType()->getScalarType();
113 if (CastSrcTy->isHalfTy() || CastSrcTy->isIntegerTy(BitWidth: 16))
114 return true;
115 }
116
117 return false;
118}
119
120// Convert a value to 16-bit.
121static Value *convertTo16Bit(Value &V, InstCombiner::BuilderTy &Builder) {
122 Type *VTy = V.getType();
123 if (isa<FPExtInst, SExtInst, ZExtInst>(Val: &V))
124 return cast<Instruction>(Val: &V)->getOperand(i: 0);
125 // Vector form: extractelement((s|z|fp)ext Vec), Idx -> extractelement(Vec,
126 // Idx), taking the narrow lane directly so the widening cast can be removed.
127 Instruction *VecCast;
128 Value *Idx;
129 if (match(V: &V, P: m_ExtractElt(Val: m_Instruction(I&: VecCast), Idx: m_Value(V&: Idx))) &&
130 isa<FPExtInst, SExtInst, ZExtInst>(Val: VecCast))
131 return Builder.CreateExtractElement(Vec: VecCast->getOperand(i: 0), Idx);
132 if (VTy->isIntegerTy())
133 return Builder.CreateIntCast(V: &V, DestTy: Type::getInt16Ty(C&: V.getContext()), isSigned: false);
134 if (VTy->isFloatingPointTy())
135 return Builder.CreateFPCast(V: &V, DestTy: Type::getHalfTy(C&: V.getContext()));
136
137 llvm_unreachable("Should never be called!");
138}
139
140/// Applies Func(OldIntr.Args, OldIntr.ArgTys), creates intrinsic call with
141/// modified arguments (based on OldIntr) and replaces InstToReplace with
142/// this newly created intrinsic call.
143static std::optional<Instruction *> modifyIntrinsicCall(
144 IntrinsicInst &OldIntr, Instruction &InstToReplace, unsigned NewIntr,
145 InstCombiner &IC,
146 std::function<void(SmallVectorImpl<Value *> &, SmallVectorImpl<Type *> &)>
147 Func) {
148 SmallVector<Type *, 4> OverloadTys;
149 if (!Intrinsic::isSignatureValid(F: OldIntr.getCalledFunction(), OverloadTys))
150 return std::nullopt;
151
152 SmallVector<Value *, 8> Args(OldIntr.args());
153
154 // Modify arguments and types
155 Func(Args, OverloadTys);
156
157 CallInst *NewCall =
158 IC.Builder.CreateIntrinsicWithoutFolding(ID: NewIntr, OverloadTypes: OverloadTys, Args);
159 NewCall->takeName(V: &OldIntr);
160 NewCall->copyMetadata(SrcInst: OldIntr);
161 if (isa<FPMathOperator>(Val: NewCall))
162 NewCall->copyFastMathFlags(I: &OldIntr);
163 // Copy attributes
164 AttributeList OldAttrList = OldIntr.getAttributes();
165 NewCall->setAttributes(OldAttrList);
166
167 // Erase and replace uses
168 if (!InstToReplace.getType()->isVoidTy())
169 IC.replaceInstUsesWith(I&: InstToReplace, V: NewCall);
170
171 bool RemoveOldIntr = &OldIntr != &InstToReplace;
172
173 auto *RetValue = IC.eraseInstFromFunction(I&: InstToReplace);
174 if (RemoveOldIntr)
175 IC.eraseInstFromFunction(I&: OldIntr);
176
177 return RetValue;
178}
179
180static std::optional<Instruction *>
181simplifyAMDGCNImageIntrinsic(const GCNSubtarget *ST,
182 const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr,
183 IntrinsicInst &II, InstCombiner &IC) {
184 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
185 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: ImageDimIntr->BaseOpcode);
186
187 // Optimize _L to _LZ when _L is zero
188 if (const auto *LZMappingInfo =
189 AMDGPU::getMIMGLZMappingInfo(L: ImageDimIntr->BaseOpcode)) {
190 if (auto *ConstantLod =
191 dyn_cast<ConstantFP>(Val: II.getOperand(i_nocapture: ImageDimIntr->LodIndex))) {
192 if (ConstantLod->isZero() || ConstantLod->isNegative()) {
193 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
194 AMDGPU::getImageDimIntrinsicByBaseOpcode(BaseOpcode: LZMappingInfo->LZ,
195 Dim: ImageDimIntr->Dim);
196 return modifyIntrinsicCall(
197 OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC, Func: [&](auto &Args, auto &ArgTys) {
198 Args.erase(Args.begin() + ImageDimIntr->LodIndex);
199 });
200 }
201 }
202 }
203
204 // Optimize _mip away, when 'lod' is zero
205 if (const auto *MIPMappingInfo =
206 AMDGPU::getMIMGMIPMappingInfo(MIP: ImageDimIntr->BaseOpcode)) {
207 if (auto *ConstantMip =
208 dyn_cast<ConstantInt>(Val: II.getOperand(i_nocapture: ImageDimIntr->MipIndex))) {
209 if (ConstantMip->isZero()) {
210 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
211 AMDGPU::getImageDimIntrinsicByBaseOpcode(BaseOpcode: MIPMappingInfo->NONMIP,
212 Dim: ImageDimIntr->Dim);
213 return modifyIntrinsicCall(
214 OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC, Func: [&](auto &Args, auto &ArgTys) {
215 Args.erase(Args.begin() + ImageDimIntr->MipIndex);
216 });
217 }
218 }
219 }
220
221 // Optimize _bias away when 'bias' is zero
222 if (const auto *BiasMappingInfo =
223 AMDGPU::getMIMGBiasMappingInfo(Bias: ImageDimIntr->BaseOpcode)) {
224 if (auto *ConstantBias =
225 dyn_cast<ConstantFP>(Val: II.getOperand(i_nocapture: ImageDimIntr->BiasIndex))) {
226 if (ConstantBias->isZero()) {
227 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
228 AMDGPU::getImageDimIntrinsicByBaseOpcode(BaseOpcode: BiasMappingInfo->NoBias,
229 Dim: ImageDimIntr->Dim);
230 return modifyIntrinsicCall(
231 OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC, Func: [&](auto &Args, auto &ArgTys) {
232 Args.erase(Args.begin() + ImageDimIntr->BiasIndex);
233 ArgTys.erase(ArgTys.begin() + ImageDimIntr->BiasTyArg);
234 });
235 }
236 }
237 }
238
239 // Optimize _offset away when 'offset' is zero
240 if (const auto *OffsetMappingInfo =
241 AMDGPU::getMIMGOffsetMappingInfo(Offset: ImageDimIntr->BaseOpcode)) {
242 if (auto *ConstantOffset =
243 dyn_cast<ConstantInt>(Val: II.getOperand(i_nocapture: ImageDimIntr->OffsetIndex))) {
244 if (ConstantOffset->isZero()) {
245 const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
246 AMDGPU::getImageDimIntrinsicByBaseOpcode(
247 BaseOpcode: OffsetMappingInfo->NoOffset, Dim: ImageDimIntr->Dim);
248 return modifyIntrinsicCall(
249 OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC, Func: [&](auto &Args, auto &ArgTys) {
250 Args.erase(Args.begin() + ImageDimIntr->OffsetIndex);
251 });
252 }
253 }
254 }
255
256 // Optimize the arrayed dim away when the array slice is zero, since slice 0
257 // is the base layer. Restricted to non-atomic, non-sampled image loads and
258 // stores for now.
259 const AMDGPU::MIMGDimInfo *DimInfo =
260 AMDGPU::getMIMGDimInfo(DimEnum: ImageDimIntr->Dim);
261 if (!BaseOpcode->Atomic && !BaseOpcode->Sampler && BaseOpcode->Coordinates &&
262 DimInfo->NonArrayDim != ImageDimIntr->Dim) {
263 // Address is [coords..., slice, (fragid)] plus an optional mip operand.
264 // The slice is the last coordinate, so index it from CoordStart.
265 unsigned SliceIndex = ImageDimIntr->CoordStart + DimInfo->NumCoords - 1 -
266 (DimInfo->MSAA ? 1 : 0);
267 auto *ConstantSlice = dyn_cast<ConstantInt>(Val: II.getOperand(i_nocapture: SliceIndex));
268 if (ConstantSlice && ConstantSlice->isZero()) {
269 if (const AMDGPU::ImageDimIntrinsicInfo *NewImageDimIntr =
270 AMDGPU::getImageDimIntrinsicByBaseOpcode(BaseOpcode: ImageDimIntr->BaseOpcode,
271 Dim: DimInfo->NonArrayDim)) {
272 return modifyIntrinsicCall(OldIntr&: II, InstToReplace&: II, NewIntr: NewImageDimIntr->Intr, IC,
273 Func: [&](auto &Args, auto &ArgTys) {
274 Args.erase(Args.begin() + SliceIndex);
275 });
276 }
277 }
278 }
279
280 // Try to use D16
281 if (ST->hasD16Images()) {
282 if (BaseOpcode->HasD16) {
283
284 // If the only use of image intrinsic is a fptrunc (with conversion to
285 // half) then both fptrunc and image intrinsic will be replaced with image
286 // intrinsic with D16 flag.
287 if (II.hasOneUse()) {
288 Instruction *User = II.user_back();
289
290 if (User->getOpcode() == Instruction::FPTrunc &&
291 User->getType()->getScalarType()->isHalfTy()) {
292
293 return modifyIntrinsicCall(OldIntr&: II, InstToReplace&: *User, NewIntr: ImageDimIntr->Intr, IC,
294 Func: [&](auto &Args, auto &ArgTys) {
295 // Change return type of image intrinsic.
296 // Set it to return type of fptrunc.
297 ArgTys[0] = User->getType();
298 });
299 }
300 }
301
302 // Only perform D16 folding if every user of the image sample is
303 // an ExtractElementInst immediately followed by an FPTrunc to half.
304 SmallVector<std::pair<ExtractElementInst *, FPTruncInst *>, 4>
305 ExtractTruncPairs;
306 bool AllHalfExtracts = true;
307
308 for (User *U : II.users()) {
309 auto *Ext = dyn_cast<ExtractElementInst>(Val: U);
310 if (!Ext || !Ext->hasOneUse()) {
311 AllHalfExtracts = false;
312 break;
313 }
314
315 auto *Tr = dyn_cast<FPTruncInst>(Val: *Ext->user_begin());
316 if (!Tr || !Tr->getType()->isHalfTy()) {
317 AllHalfExtracts = false;
318 break;
319 }
320
321 ExtractTruncPairs.emplace_back(Args&: Ext, Args&: Tr);
322 }
323
324 if (!ExtractTruncPairs.empty() && AllHalfExtracts) {
325 auto *VecTy = cast<VectorType>(Val: II.getType());
326 Type *HalfVecTy =
327 VecTy->getWithNewType(EltTy: Type::getHalfTy(C&: II.getContext()));
328
329 // Obtain the original image sample intrinsic's signature
330 // and replace its return type with the half-vector for D16 folding
331 SmallVector<Type *, 8> OverloadTys;
332 if (!Intrinsic::isSignatureValid(F: II.getCalledFunction(), OverloadTys))
333 return std::nullopt;
334
335 OverloadTys[0] = HalfVecTy;
336 Module *M = II.getModule();
337 Function *HalfDecl = Intrinsic::getOrInsertDeclaration(
338 M, id: ImageDimIntr->Intr, OverloadTys);
339
340 II.mutateType(Ty: HalfVecTy);
341 II.setCalledFunction(HalfDecl);
342
343 IRBuilder<> Builder(II.getContext());
344 for (auto &[Ext, Tr] : ExtractTruncPairs) {
345 Value *Idx = Ext->getIndexOperand();
346
347 Builder.SetInsertPoint(Tr);
348
349 Value *HalfExtract = Builder.CreateExtractElement(Vec: &II, Idx);
350 HalfExtract->takeName(V: Tr);
351
352 Tr->replaceAllUsesWith(V: HalfExtract);
353 }
354
355 for (auto &[Ext, Tr] : ExtractTruncPairs) {
356 IC.eraseInstFromFunction(I&: *Tr);
357 IC.eraseInstFromFunction(I&: *Ext);
358 }
359
360 return &II;
361 }
362 }
363 }
364
365 // Try to use A16 or G16
366 if (!ST->hasA16() && !ST->hasG16())
367 return std::nullopt;
368
369 // Address is interpreted as float if the instruction has a sampler or as
370 // unsigned int if there is no sampler.
371 bool HasSampler = BaseOpcode->Sampler;
372 bool FloatCoord = false;
373 // true means derivatives can be converted to 16 bit, coordinates not
374 bool OnlyDerivatives = false;
375
376 // Sampler-less addresses are unsigned, so a sext from i16 folds to a16 like a
377 // zext: they only disagree for a negative i16 (>= 0x8000), which is out of
378 // bounds while the max image dimension is <= 0x8000.
379 bool AllowI16SExt = !HasSampler;
380
381 for (unsigned OperandIndex = ImageDimIntr->GradientStart;
382 OperandIndex < ImageDimIntr->VAddrEnd; OperandIndex++) {
383 Value *Coord = II.getOperand(i_nocapture: OperandIndex);
384 // If the values are not derived from 16-bit values, we cannot optimize.
385 if (!canSafelyConvertTo16Bit(V&: *Coord, IsFloat: HasSampler, AllowI16SExt)) {
386 if (OperandIndex < ImageDimIntr->CoordStart ||
387 ImageDimIntr->GradientStart == ImageDimIntr->CoordStart) {
388 return std::nullopt;
389 }
390 // All gradients can be converted, so convert only them
391 OnlyDerivatives = true;
392 break;
393 }
394
395 assert(OperandIndex == ImageDimIntr->GradientStart ||
396 FloatCoord == Coord->getType()->isFloatingPointTy());
397 FloatCoord = Coord->getType()->isFloatingPointTy();
398 }
399
400 if (!OnlyDerivatives && !ST->hasA16())
401 OnlyDerivatives = true; // Only supports G16
402
403 // Check if there is a bias parameter and if it can be converted to f16
404 if (!OnlyDerivatives && ImageDimIntr->NumBiasArgs != 0) {
405 Value *Bias = II.getOperand(i_nocapture: ImageDimIntr->BiasIndex);
406 assert(HasSampler &&
407 "Only image instructions with a sampler can have a bias");
408 if (!canSafelyConvertTo16Bit(V&: *Bias, IsFloat: HasSampler))
409 OnlyDerivatives = true;
410 }
411
412 if (OnlyDerivatives && (!ST->hasG16() || ImageDimIntr->GradientStart ==
413 ImageDimIntr->CoordStart))
414 return std::nullopt;
415
416 Type *CoordType = FloatCoord ? Type::getHalfTy(C&: II.getContext())
417 : Type::getInt16Ty(C&: II.getContext());
418
419 return modifyIntrinsicCall(
420 OldIntr&: II, InstToReplace&: II, NewIntr: II.getIntrinsicID(), IC, Func: [&](auto &Args, auto &ArgTys) {
421 ArgTys[ImageDimIntr->GradientTyArg] = CoordType;
422 if (!OnlyDerivatives) {
423 ArgTys[ImageDimIntr->CoordTyArg] = CoordType;
424
425 // Change the bias type
426 if (ImageDimIntr->NumBiasArgs != 0)
427 ArgTys[ImageDimIntr->BiasTyArg] = Type::getHalfTy(C&: II.getContext());
428 }
429
430 unsigned EndIndex =
431 OnlyDerivatives ? ImageDimIntr->CoordStart : ImageDimIntr->VAddrEnd;
432 for (unsigned OperandIndex = ImageDimIntr->GradientStart;
433 OperandIndex < EndIndex; OperandIndex++) {
434 Args[OperandIndex] =
435 convertTo16Bit(V&: *II.getOperand(i_nocapture: OperandIndex), Builder&: IC.Builder);
436 }
437
438 // Convert the bias
439 if (!OnlyDerivatives && ImageDimIntr->NumBiasArgs != 0) {
440 Value *Bias = II.getOperand(i_nocapture: ImageDimIntr->BiasIndex);
441 Args[ImageDimIntr->BiasIndex] = convertTo16Bit(V&: *Bias, Builder&: IC.Builder);
442 }
443 });
444}
445
446bool GCNTTIImpl::canSimplifyLegacyMulToMul(const Instruction &I,
447 const Value *Op0, const Value *Op1,
448 InstCombiner &IC) const {
449 // The legacy behaviour is that multiplying +/-0.0 by anything, even NaN or
450 // infinity, gives +0.0. If we can prove we don't have one of the special
451 // cases then we can use a normal multiply instead.
452 SimplifyQuery SQ = IC.getSimplifyQuery().getWithInstruction(I: &I);
453 KnownFPClass Known0 =
454 computeKnownFPClass(V: Op0, InterestedClasses: fcZero | fcSubnormal | fcInf | fcNan, SQ);
455 DenormalMode Mode = I.getFunction()->getDenormalMode(FPType: APFloat::IEEEsingle());
456
457 // Bail early if Op0 may be zero and nsz is not set -- Op1 cannot help.
458 if (!Known0.isKnownNeverLogicalZero(Mode) && !I.hasNoSignedZeros())
459 return false;
460
461 KnownFPClass Known1 =
462 computeKnownFPClass(V: Op1, InterestedClasses: fcZero | fcSubnormal | fcInf | fcNan, SQ);
463
464 // Simplify if both operands are known non-zero.
465 if (Known0.isKnownNeverLogicalZero(Mode) &&
466 Known1.isKnownNeverLogicalZero(Mode))
467 return true;
468
469 // With nsz, two additional cases allow simplification:
470 // 1. One operand is not zero or infinity or NaN:
471 // Op0 NeverLogicalZero && NeverInfOrNaN, or symmetric for Op1.
472 // 2. Neither operand is infinity or NaN:
473 // Op0 NeverInfOrNaN && Op1 NeverInfOrNaN.
474 // The following condition captures both cases.
475 if (I.hasNoSignedZeros() &&
476 (Known0.isKnownNeverLogicalZero(Mode) || Known1.isKnownNeverInfOrNaN()) &&
477 (Known1.isKnownNeverLogicalZero(Mode) || Known0.isKnownNeverInfOrNaN()))
478 return true;
479
480 return false;
481}
482
483/// Match an fpext from half to float, or a constant we can convert.
484static Value *matchFPExtFromF16(Value *Arg) {
485 Value *Src = nullptr;
486 ConstantFP *CFP = nullptr;
487 if (match(V: Arg, P: m_OneUse(SubPattern: m_FPExt(Op: m_Value(V&: Src))))) {
488 if (Src->getType()->isHalfTy())
489 return Src;
490 } else if (match(V: Arg, P: m_ConstantFP(C&: CFP))) {
491 bool LosesInfo;
492 APFloat Val(CFP->getValueAPF());
493 Val.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmNearestTiesToEven, losesInfo: &LosesInfo);
494 if (!LosesInfo)
495 return ConstantFP::get(Ty: Type::getHalfTy(C&: Arg->getContext()), V: Val);
496 }
497 return nullptr;
498}
499
500// Trim all zero components from the end of the vector \p UseV and return
501// an appropriate bitset with known elements.
502static APInt trimTrailingZerosInVector(InstCombiner &IC, Value *UseV,
503 Instruction *I) {
504 auto *VTy = cast<FixedVectorType>(Val: UseV->getType());
505 unsigned VWidth = VTy->getNumElements();
506 APInt DemandedElts = APInt::getAllOnes(numBits: VWidth);
507
508 for (int i = VWidth - 1; i > 0; --i) {
509 auto *Elt = findScalarElement(V: UseV, EltNo: i);
510 if (!Elt)
511 break;
512
513 if (auto *ConstElt = dyn_cast<Constant>(Val: Elt)) {
514 if (!ConstElt->isNullValue() && !isa<UndefValue>(Val: Elt))
515 break;
516 } else {
517 break;
518 }
519
520 DemandedElts.clearBit(BitPosition: i);
521 }
522
523 return DemandedElts;
524}
525
526// Trim elements of the end of the vector \p V, if they are
527// equal to the first element of the vector.
528static APInt defaultComponentBroadcast(Value *V) {
529 auto *VTy = cast<FixedVectorType>(Val: V->getType());
530 unsigned VWidth = VTy->getNumElements();
531 APInt DemandedElts = APInt::getAllOnes(numBits: VWidth);
532 Value *FirstComponent = findScalarElement(V, EltNo: 0);
533
534 SmallVector<int> ShuffleMask;
535 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: V))
536 SVI->getShuffleMask(Result&: ShuffleMask);
537
538 for (int I = VWidth - 1; I > 0; --I) {
539 if (ShuffleMask.empty()) {
540 auto *Elt = findScalarElement(V, EltNo: I);
541 if (!Elt || (Elt != FirstComponent && !isa<UndefValue>(Val: Elt)))
542 break;
543 } else {
544 // Detect identical elements in the shufflevector result, even though
545 // findScalarElement cannot tell us what that element is.
546 if (ShuffleMask[I] != ShuffleMask[0] && ShuffleMask[I] != PoisonMaskElem)
547 break;
548 }
549 DemandedElts.clearBit(BitPosition: I);
550 }
551
552 return DemandedElts;
553}
554
555static Value *simplifyAMDGCNMemoryIntrinsicDemanded(InstCombiner &IC,
556 IntrinsicInst &II,
557 APInt DemandedElts,
558 int DMaskIdx = -1,
559 bool IsLoad = true);
560
561/// Return true if it's legal to contract llvm.amdgcn.rcp(llvm.sqrt)
562static bool canContractSqrtToRsq(const FPMathOperator *SqrtOp) {
563 return (SqrtOp->getType()->isFloatTy() &&
564 (SqrtOp->hasApproxFunc() || SqrtOp->getFPAccuracy() >= 1.0f)) ||
565 SqrtOp->getType()->isHalfTy();
566}
567
568/// Return true if we can easily prove that use U is uniform.
569static bool isTriviallyUniform(const Use &U) {
570 Value *V = U.get();
571 if (isa<Constant>(Val: V))
572 return true;
573 if (const auto *A = dyn_cast<Argument>(Val: V))
574 return AMDGPU::isArgPassedInSGPR(Arg: A);
575 if (const auto *II = dyn_cast<IntrinsicInst>(Val: V)) {
576 if (!AMDGPU::isIntrinsicAlwaysUniform(IntrID: II->getIntrinsicID()))
577 return false;
578 // If II and U are in different blocks then there is a possibility of
579 // temporal divergence.
580 return II->getParent() == cast<Instruction>(Val: U.getUser())->getParent();
581 }
582 return false;
583}
584
585/// Simplify a lane index operand (e.g. llvm.amdgcn.readlane src1).
586///
587/// The instruction only reads the low 5 bits for wave32, and 6 bits for wave64.
588bool GCNTTIImpl::simplifyDemandedLaneMaskArg(InstCombiner &IC,
589 IntrinsicInst &II,
590 unsigned LaneArgIdx) const {
591 unsigned MaskBits = ST->getWavefrontSizeLog2();
592 APInt DemandedMask(32, maskTrailingOnes<unsigned>(N: MaskBits));
593
594 KnownBits Known(32);
595 if (IC.SimplifyDemandedBits(I: &II, OpNo: LaneArgIdx, DemandedMask, Known))
596 return true;
597
598 if (!Known.isConstant())
599 return false;
600
601 // Out of bounds indexes may appear in wave64 code compiled for wave32.
602 // Unlike the DAG version, SimplifyDemandedBits does not change constants, so
603 // manually fix it up.
604
605 Value *LaneArg = II.getArgOperand(i: LaneArgIdx);
606 Constant *MaskedConst =
607 ConstantInt::get(Ty: LaneArg->getType(), V: Known.getConstant() & DemandedMask);
608 if (MaskedConst != LaneArg) {
609 II.getOperandUse(i: LaneArgIdx).set(MaskedConst);
610 return true;
611 }
612
613 return false;
614}
615
616static CallInst *rewriteCall(IRBuilderBase &B, CallInst &Old,
617 Function &NewCallee, ArrayRef<Value *> Ops) {
618 SmallVector<OperandBundleDef, 2> OpBundles;
619 Old.getOperandBundlesAsDefs(Defs&: OpBundles);
620
621 CallInst *NewCall = B.CreateCall(Callee: &NewCallee, Args: Ops, OpBundles);
622 NewCall->takeName(V: &Old);
623 return NewCall;
624}
625
626// Return true for sequences of instructions that effectively assign
627// each lane to its thread ID
628static bool isThreadID(const GCNSubtarget &ST, Value *V) {
629 // Case 1:
630 // wave32: mbcnt_lo(-1, 0)
631 // wave64: mbcnt_hi(-1, mbcnt_lo(-1, 0))
632 auto W32Pred = m_Intrinsic<Intrinsic::amdgcn_mbcnt_lo>(Ops: m_ConstantInt<-1>(),
633 Ops: m_ConstantInt<0>());
634 auto W64Pred = m_Intrinsic<Intrinsic::amdgcn_mbcnt_hi>(
635 Ops: m_ConstantInt<-1>(), Ops: m_Intrinsic<Intrinsic::amdgcn_mbcnt_lo>(
636 Ops: m_ConstantInt<-1>(), Ops: m_ConstantInt<0>()));
637 if (ST.isWave32() && match(V, P: W32Pred))
638 return true;
639 if (ST.isWave64() && match(V, P: W64Pred))
640 return true;
641
642 return false;
643}
644
645Instruction *
646GCNTTIImpl::hoistLaneIntrinsicThroughOperand(InstCombiner &IC,
647 IntrinsicInst &II) const {
648 const auto IID = II.getIntrinsicID();
649 assert(IID == Intrinsic::amdgcn_readlane ||
650 IID == Intrinsic::amdgcn_readfirstlane ||
651 IID == Intrinsic::amdgcn_permlane64);
652
653 Instruction *OpInst = dyn_cast<Instruction>(Val: II.getOperand(i_nocapture: 0));
654
655 // Only do this if both instructions are in the same block
656 // (so the exec mask won't change) and the readlane is the only user of its
657 // operand.
658 if (!OpInst || !OpInst->hasOneUser() || OpInst->getParent() != II.getParent())
659 return nullptr;
660
661 const bool IsReadLane = (IID == Intrinsic::amdgcn_readlane);
662
663 // If this is a readlane, check that the second operand is a constant, or is
664 // defined before OpInst so we know it's safe to move this intrinsic higher.
665 Value *LaneID = nullptr;
666 if (IsReadLane) {
667 LaneID = II.getOperand(i_nocapture: 1);
668
669 // readlane take an extra operand for the lane ID, so we must check if that
670 // LaneID value can be used at the point where we want to move the
671 // intrinsic.
672 if (auto *LaneIDInst = dyn_cast<Instruction>(Val: LaneID)) {
673 if (!IC.getDominatorTree().dominates(Def: LaneIDInst, User: OpInst))
674 return nullptr;
675 }
676 }
677
678 // Hoist the intrinsic (II) through OpInst.
679 //
680 // (II (OpInst x)) -> (OpInst (II x))
681 const auto DoIt = [&](unsigned OpIdx,
682 Function *NewIntrinsic) -> Instruction * {
683 SmallVector<Value *, 2> Ops{OpInst->getOperand(i: OpIdx)};
684 if (IsReadLane)
685 Ops.push_back(Elt: LaneID);
686
687 // Rewrite the intrinsic call.
688 CallInst *NewII = rewriteCall(B&: IC.Builder, Old&: II, NewCallee&: *NewIntrinsic, Ops);
689
690 // Rewrite OpInst so it takes the result of the intrinsic now.
691 Instruction &NewOp = *OpInst->clone();
692 NewOp.setOperand(i: OpIdx, Val: NewII);
693 return &NewOp;
694 };
695
696 // TODO(?): Should we do more with permlane64?
697 if (IID == Intrinsic::amdgcn_permlane64 && !isa<BitCastInst>(Val: OpInst))
698 return nullptr;
699
700 if (isa<UnaryOperator>(Val: OpInst))
701 return DoIt(0, II.getCalledFunction());
702
703 if (isa<CastInst>(Val: OpInst)) {
704 Value *Src = OpInst->getOperand(i: 0);
705 Type *SrcTy = Src->getType();
706 if (!isTypeLegal(Ty: SrcTy))
707 return nullptr;
708
709 Function *Remangled =
710 Intrinsic::getOrInsertDeclaration(M: II.getModule(), id: IID, OverloadTys: {SrcTy});
711 return DoIt(0, Remangled);
712 }
713
714 // We can also hoist through binary operators if the other operand is uniform.
715 if (isa<BinaryOperator>(Val: OpInst)) {
716 // FIXME: If we had access to UniformityInfo here we could just check
717 // if the operand is uniform.
718 if (isTriviallyUniform(U: OpInst->getOperandUse(i: 0)))
719 return DoIt(1, II.getCalledFunction());
720 if (isTriviallyUniform(U: OpInst->getOperandUse(i: 1)))
721 return DoIt(0, II.getCalledFunction());
722 }
723
724 return nullptr;
725}
726
727/// Evaluate V as a function of the lane ID and return its value on Lane, or
728/// std::nullopt if V is not a closed-form expression of the lane ID.
729static std::optional<unsigned> evalLaneExpr(Value *V, unsigned Lane,
730 const GCNSubtarget &ST,
731 const DataLayout &DL,
732 unsigned Depth = 0) {
733 if (Depth >= MaxAnalysisRecursionDepth)
734 return std::nullopt;
735
736 // Poison/undef in the index expression: bail and let InstCombine fold the
737 // intrinsic the usual way.
738 if (isa<UndefValue>(Val: V))
739 return std::nullopt;
740
741 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: V))
742 return CI->getZExtValue();
743
744 if (isThreadID(ST, V))
745 return Lane;
746
747 const BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: V);
748 if (!BO)
749 return std::nullopt;
750
751 std::optional<unsigned> LHS =
752 evalLaneExpr(V: BO->getOperand(i_nocapture: 0), Lane, ST, DL, Depth: Depth + 1);
753 if (!LHS)
754 return std::nullopt;
755 std::optional<unsigned> RHS =
756 evalLaneExpr(V: BO->getOperand(i_nocapture: 1), Lane, ST, DL, Depth: Depth + 1);
757 if (!RHS)
758 return std::nullopt;
759
760 Type *Ty = BO->getType();
761 Constant *Ops[] = {ConstantInt::get(Ty, V: *LHS), ConstantInt::get(Ty, V: *RHS)};
762 auto *CI =
763 dyn_cast_or_null<ConstantInt>(Val: ConstantFoldInstOperands(I: BO, Ops, DL));
764 return CI ? std::optional<unsigned>(CI->getZExtValue()) : std::nullopt;
765}
766
767/// Build the per-lane shuffle map by evaluating Index for every lane in the
768/// wave. Returns false if any lane index is non-constant or out of range.
769static bool tryBuildShuffleMap(Value *Index, const GCNSubtarget &ST,
770 SmallVectorImpl<uint8_t> &Ids,
771 const DataLayout &DL) {
772 unsigned WaveSize = ST.getWavefrontSize();
773 Ids.resize(N: WaveSize);
774 for (unsigned Lane : seq(Size: WaveSize)) {
775 std::optional<unsigned> Val = evalLaneExpr(V: Index, Lane, ST, DL);
776 if (!Val || *Val >= WaveSize)
777 return false;
778 Ids[Lane] = *Val;
779 }
780 return true;
781}
782
783/// Lanes are partitioned into groups of Period; each group is a translated
784/// copy of the first: Ids[I] = Ids[I % Period] + (I & ~(Period - 1)).
785template <unsigned Period>
786static bool hasPeriodicLayout(ArrayRef<uint8_t> Ids) {
787 static_assert(isPowerOf2_32(Value: Period), "Period must be a power of two");
788 for (unsigned I = Period, E = Ids.size(); I < E; ++I)
789 if (Ids[I] != Ids[I % Period] + (I & ~(Period - 1)))
790 return false;
791 return true;
792}
793
794/// Match an N-lane row pattern: each lane in [0, N) reads from a source lane
795/// in the same N-lane row, and the pattern repeats periodically across rows.
796template <unsigned N> static bool isRowPattern(ArrayRef<uint8_t> Ids) {
797 for (unsigned I = 0; I < N; ++I)
798 if (Ids[I] >= N)
799 return false;
800 return hasPeriodicLayout<N>(Ids);
801}
802
803static constexpr auto isQuadPattern = isRowPattern<4>;
804static constexpr auto isHalfRowPattern = isRowPattern<8>;
805static constexpr auto isFullRowPattern = isRowPattern<16>;
806
807/// Match a 4-lane (quad) permutation, encoded as the v_mov_b32_dpp
808/// QUAD_PERM control word: bits[1:0]=Ids[0], [3:2]=Ids[1], [5:4]=Ids[2],
809/// [7:6]=Ids[3].
810static std::optional<unsigned> matchQuadPermPattern(ArrayRef<uint8_t> Ids) {
811 if (!isQuadPattern(Ids))
812 return std::nullopt;
813 return Ids[3] << 6 | Ids[2] << 4 | Ids[1] << 2 | Ids[0];
814}
815
816/// Match an N-lane reversal (mirror) pattern.
817template <unsigned N> static bool matchMirrorPattern(ArrayRef<uint8_t> Ids) {
818 if (!isRowPattern<N>(Ids))
819 return false;
820 for (unsigned J = 0; J < N; ++J)
821 if (Ids[J] != (N - 1) - J)
822 return false;
823 return true;
824}
825
826static constexpr auto matchHalfRowMirrorPattern = matchMirrorPattern<8>;
827static constexpr auto matchFullRowMirrorPattern = matchMirrorPattern<16>;
828
829/// Match a 16-lane cyclic rotation; returns the rotation amount in [1, 15].
830static std::optional<unsigned> matchRowRotatePattern(ArrayRef<uint8_t> Ids) {
831 if (Ids[0] == 0 || !isFullRowPattern(Ids))
832 return std::nullopt;
833 for (unsigned J = 1; J < 16; ++J)
834 if (Ids[J] != (Ids[0] + J) % 16)
835 return std::nullopt;
836 return 16u - Ids[0];
837}
838
839/// Match a row-share pattern: all 16 lanes of each row read the same source
840/// lane. Returns the shared source lane index in [0, 16).
841static std::optional<unsigned> matchRowSharePattern(ArrayRef<uint8_t> Ids) {
842 if (!isFullRowPattern(Ids))
843 return std::nullopt;
844 if (!all_equal(Range: Ids.take_front(N: 16)))
845 return std::nullopt;
846 return Ids[0];
847}
848
849/// Match an XOR mask pattern within each 16-lane row: Ids[J] == Mask ^ J,
850/// with Mask in [1, 15].
851static std::optional<unsigned> matchRowXMaskPattern(ArrayRef<uint8_t> Ids) {
852 unsigned Mask = Ids[0];
853 if (Mask == 0 || !isFullRowPattern(Ids))
854 return std::nullopt;
855 for (unsigned J = 0; J < 16; ++J)
856 if (Ids[J] != (Mask ^ J))
857 return std::nullopt;
858 return Mask;
859}
860
861/// Match an 8-lane arbitrary permutation, encoded as the v_mov_b32_dpp8
862/// 24-bit selector (three bits per output lane).
863static std::optional<unsigned> matchHalfRowPermPattern(ArrayRef<uint8_t> Ids) {
864 if (!isHalfRowPattern(Ids))
865 return std::nullopt;
866 unsigned Selector = 0;
867 for (unsigned J = 0; J < 8; ++J)
868 Selector |= Ids[J] << (J * 3);
869 return Selector;
870}
871
872/// Pack a 16-lane permutation into a single 64-bit value: four bits per output
873/// lane, lane J in bits [J*4 + 3 : J*4]. The caller splits it into the low and
874/// high 32-bit selector operands of v_permlane16 / v_permlanex16.
875static uint64_t computePermlane16Masks(ArrayRef<uint8_t> Ids) {
876 uint64_t Sel = 0;
877 for (unsigned J = 0; J < 16; ++J)
878 Sel |= static_cast<uint64_t>(Ids[J] & 0xF) << (J * 4);
879 return Sel;
880}
881
882/// Match a half-wave swap: lane J reads from lane J ^ 32. Only meaningful on
883/// wave64 targets.
884static bool matchHalfWaveSwapPattern(ArrayRef<uint8_t> Ids) {
885 if (Ids.size() != 64)
886 return false;
887 for (unsigned J = 0; J < 64; ++J)
888 if (Ids[J] != (J ^ 32))
889 return false;
890 return true;
891}
892
893/// Match a cross-row permutation suitable for v_permlanex16: every lane in
894/// the low 16-lane half reads from the high half of its own row, and vice
895/// versa.
896static bool isCrossRowPattern(ArrayRef<uint8_t> Ids) {
897 if (!hasPeriodicLayout<32>(Ids))
898 return false;
899 for (unsigned J = 0; J < 16; ++J) {
900 if (Ids[J] < 16 || Ids[J] >= 32)
901 return false;
902 if (Ids[J + 16] != Ids[J] - 16)
903 return false;
904 }
905 return true;
906}
907
908/// Match a DS_SWIZZLE bitmask-mode permutation:
909/// dst_lane = ((src_lane & AND) | OR) ^ XOR
910/// with each mask being five bits. Returns the encoded swizzle immediate.
911/// The hardware applies the formula independently within each 32-lane group,
912/// so on wave64 the high group must replicate the low one (translated by 32).
913static std::optional<unsigned>
914matchDsSwizzleBitmaskPattern(ArrayRef<uint8_t> Ids) {
915 if (!hasPeriodicLayout<32>(Ids))
916 return std::nullopt;
917
918 // The formula is per-bit: output bit B depends only on input bit B. Probe
919 // each bit with src=0 and src=(1<<B); if the output bit flipped, AND[B]=1
920 // and XOR[B] carries the constant offset; otherwise it is a constant bit
921 // encoded in OR (with AND[B]=0, XOR[B]=0).
922 unsigned AndMask = 0, OrMask = 0, XorMask = 0;
923 for (unsigned B = 0; B < 5; ++B) {
924 unsigned Bit0 = (Ids[0] >> B) & 1;
925 unsigned Bit1 = (Ids[1u << B] >> B) & 1;
926 if (Bit0 != Bit1) {
927 AndMask |= 1u << B;
928 XorMask |= Bit0 << B;
929 } else {
930 OrMask |= Bit0 << B;
931 }
932 }
933
934 // The per-bit derivation assumes bit independence; verify the masks
935 // actually reproduce every lane in the 32-lane group.
936 for (unsigned I : seq(Size: 32u)) {
937 unsigned Expected = ((I & AndMask) | OrMask) ^ XorMask;
938 if (Ids[I] != Expected)
939 return std::nullopt;
940 }
941
942 return AMDGPU::Swizzle::BITMASK_PERM_ENC |
943 AndMask << AMDGPU::Swizzle::BITMASK_AND_SHIFT |
944 OrMask << AMDGPU::Swizzle::BITMASK_OR_SHIFT |
945 XorMask << AMDGPU::Swizzle::BITMASK_XOR_SHIFT;
946}
947
948/// Match a GFX9+ DS_SWIZZLE rotate-mode permutation: a cyclic left-rotation
949/// of all 32 lanes within each 32-lane group by a constant N in [0, 31],
950/// i.e. dst_lane = (src_lane + N) % 32. On wave64, hasPeriodicLayout<32>
951/// ensures both 32-lane groups rotate by the same amount.
952static std::optional<unsigned>
953matchDsSwizzleRotatePattern(ArrayRef<uint8_t> Ids) {
954 if (!hasPeriodicLayout<32>(Ids))
955 return std::nullopt;
956
957 // Determine the rotation amount from lane 0: every lane must read from
958 // lane (I + N) % 32 where N = Ids[0] and 0 <= N <= 31.
959 unsigned N = Ids[0];
960 if (N >= 32)
961 return std::nullopt;
962
963 for (unsigned I = 0; I < 32; ++I)
964 if (Ids[I] != (I + N) % 32)
965 return std::nullopt;
966
967 return AMDGPU::Swizzle::ROTATE_MODE_ENC |
968 (N << AMDGPU::Swizzle::ROTATE_SIZE_SHIFT);
969}
970
971/// Emit v_mov_b32_dpp with the given control word, row/bank masks 0xF, and
972/// bound_ctrl=1 so out-of-bounds lanes are well-defined and the DPP mov can
973/// be folded into a consuming VALU op by GCNDPPCombine.
974static Value *createUpdateDpp(IRBuilderBase &B, Value *Val, unsigned Ctrl) {
975 Type *Ty = Val->getType();
976 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_update_dpp, OverloadTypes: {Ty},
977 Args: {PoisonValue::get(T: Ty), Val, B.getInt32(C: Ctrl),
978 B.getInt32(C: 0xF), B.getInt32(C: 0xF), B.getTrue()});
979}
980
981/// Emit v_mov_b32_dpp8 with the given 24-bit lane selector.
982static Value *createMovDpp8(IRBuilderBase &B, Value *Val, unsigned Selector) {
983 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_mov_dpp8, OverloadTypes: {Val->getType()},
984 Args: {Val, B.getInt32(C: Selector)});
985}
986
987/// Emit v_permlane16 with the precomputed lane-select halves.
988static Value *createPermlane16(IRBuilderBase &B, Value *Val, uint32_t Lo,
989 uint32_t Hi) {
990 Type *Ty = Val->getType();
991 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_permlane16, OverloadTypes: {Ty},
992 Args: {PoisonValue::get(T: Ty), Val, B.getInt32(C: Lo),
993 B.getInt32(C: Hi), B.getFalse(), B.getFalse()});
994}
995
996/// Emit v_permlanex16 with the precomputed lane-select halves. Each output
997/// lane reads from the other 16-lane half of the same row.
998static Value *createPermlaneX16(IRBuilderBase &B, Value *Val, uint32_t Lo,
999 uint32_t Hi) {
1000 Type *Ty = Val->getType();
1001 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_permlanex16, OverloadTypes: {Ty},
1002 Args: {PoisonValue::get(T: Ty), Val, B.getInt32(C: Lo),
1003 B.getInt32(C: Hi), B.getFalse(), B.getFalse()});
1004}
1005
1006/// Emit ds_swizzle with the given immediate, bitcasting/converting between
1007/// pointer/float types and i32 as required by the intrinsic signature.
1008static Value *createDsSwizzle(IRBuilderBase &B, Value *Val, unsigned Offset,
1009 const DataLayout &DL) {
1010 Type *OrigTy = Val->getType();
1011 assert(DL.getTypeSizeInBits(OrigTy) == 32 &&
1012 "ds_swizzle only supports 32-bit operands");
1013 IntegerType *I32Ty = B.getInt32Ty();
1014 Value *Src = Val;
1015 if (OrigTy->isPointerTy())
1016 Src = B.CreatePtrToInt(V: Src, DestTy: I32Ty);
1017 else if (OrigTy != I32Ty)
1018 Src = B.CreateBitCast(V: Src, DestTy: I32Ty);
1019 Value *Result = B.CreateIntrinsic(ID: Intrinsic::amdgcn_ds_swizzle, OverloadTypes: {},
1020 Args: {Src, B.getInt32(C: Offset)});
1021 if (OrigTy->isPointerTy())
1022 return B.CreateIntToPtr(V: Result, DestTy: OrigTy);
1023 if (OrigTy != I32Ty)
1024 return B.CreateBitCast(V: Result, DestTy: OrigTy);
1025 return Result;
1026}
1027
1028/// Emit v_permlane64 (swap of the two 32-lane halves of a wave64).
1029static Value *createPermlane64(IRBuilderBase &B, Value *Val) {
1030 return B.CreateIntrinsic(ID: Intrinsic::amdgcn_permlane64, OverloadTypes: {Val->getType()},
1031 Args: {Val});
1032}
1033
1034/// Given a shuffle map, try to emit the best hardware intrinsic.
1035static Value *matchShuffleToHWIntrinsic(IRBuilderBase &B, Value *Src,
1036 ArrayRef<uint8_t> Ids,
1037 const GCNSubtarget &ST,
1038 const DataLayout &DL) {
1039 // Identity shuffle (every lane reads itself) folds to the source value.
1040 if (all_of(Range: enumerate(First&: Ids),
1041 P: [](const auto &E) { return E.value() == E.index(); }))
1042 return Src;
1043
1044 // Uniform shuffle (all lanes read the same value) is handled by cheaper
1045 // broadcast/readlane intrinsics.
1046 if (all_equal(Range&: Ids))
1047 return nullptr;
1048
1049 if (std::optional<unsigned> QP = matchQuadPermPattern(Ids)) {
1050 if (ST.hasDPP())
1051 return createUpdateDpp(B, Val: Src, Ctrl: *QP);
1052 return createDsSwizzle(B, Val: Src, Offset: AMDGPU::Swizzle::QUAD_PERM_ENC | *QP, DL);
1053 }
1054
1055 if (ST.hasDPP()) {
1056 if (matchHalfRowMirrorPattern(Ids))
1057 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_HALF_MIRROR);
1058 if (matchFullRowMirrorPattern(Ids))
1059 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_MIRROR);
1060 if (std::optional<unsigned> Amt = matchRowRotatePattern(Ids))
1061 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_ROR_FIRST + *Amt - 1);
1062 }
1063
1064 // row_share is supported on GFX90A and GFX10+; row_xmask is GFX10+ only.
1065 if (ST.hasDPPRowShare()) {
1066 if (std::optional<unsigned> Lane = matchRowSharePattern(Ids))
1067 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_SHARE_FIRST + *Lane);
1068 }
1069
1070 if (ST.hasDPP() && ST.hasGFX10Insts()) {
1071 if (std::optional<unsigned> Mask = matchRowXMaskPattern(Ids))
1072 return createUpdateDpp(B, Val: Src, Ctrl: AMDGPU::DPP::ROW_XMASK_FIRST + *Mask);
1073 }
1074
1075 if (ST.hasDPP8()) {
1076 if (std::optional<unsigned> Sel = matchHalfRowPermPattern(Ids))
1077 return createMovDpp8(B, Val: Src, Selector: *Sel);
1078 }
1079
1080 if (ST.hasPermlane16Insts()) {
1081 if (isFullRowPattern(Ids)) {
1082 uint64_t Sel = computePermlane16Masks(Ids);
1083 return createPermlane16(B, Val: Src, Lo: Lo_32(Value: Sel), Hi: Hi_32(Value: Sel));
1084 }
1085 // Cross-row shuffles (e.g. XOR 16..31) — covered by permlanex16.
1086 if (isCrossRowPattern(Ids)) {
1087 uint64_t Sel = computePermlane16Masks(Ids);
1088 return createPermlaneX16(B, Val: Src, Lo: Lo_32(Value: Sel), Hi: Hi_32(Value: Sel));
1089 }
1090 }
1091
1092 // Generic DS_SWIZZLE bitmask-mode fallback: handles any 32-lane shuffle that
1093 // can be expressed as dst = ((src & AND) | OR) ^ XOR with 5-bit masks. This
1094 // is available on every target that has ds_swizzle.
1095 if (std::optional<unsigned> Imm = matchDsSwizzleBitmaskPattern(Ids))
1096 return createDsSwizzle(B, Val: Src, Offset: *Imm, DL);
1097
1098 // DS_SWIZZLE rotate mode (GFX9+): handles cyclic 32-lane rotations that
1099 // bitmask mode cannot express (e.g. +1 mod 32 requires inter-bit carry).
1100 if (ST.hasDsSwizzleRotateMode()) {
1101 if (std::optional<unsigned> Imm = matchDsSwizzleRotatePattern(Ids))
1102 return createDsSwizzle(B, Val: Src, Offset: *Imm, DL);
1103 }
1104
1105 if (ST.hasPermLane64() && matchHalfWaveSwapPattern(Ids))
1106 return createPermlane64(B, Val: Src);
1107
1108 return nullptr;
1109}
1110
1111/// Try to fold a wave_shuffle/ds_bpermute whose lane index is a constant
1112/// function of the lane ID into a hardware-specific lane permutation intrinsic.
1113static std::optional<Instruction *>
1114tryOptimizeShufflePattern(InstCombiner &IC, IntrinsicInst &II,
1115 const GCNSubtarget &ST) {
1116 const DataLayout &DL = IC.getDataLayout();
1117 if (DL.getTypeSizeInBits(Ty: II.getType()) != 32)
1118 return std::nullopt;
1119
1120 if (!ST.isWaveSizeKnown())
1121 return std::nullopt;
1122
1123 unsigned WaveSize = ST.getWavefrontSize();
1124 bool IsBpermute = II.getIntrinsicID() == Intrinsic::amdgcn_ds_bpermute;
1125 Value *Src = II.getArgOperand(i: IsBpermute ? 1 : 0);
1126 Value *Index = II.getArgOperand(i: IsBpermute ? 0 : 1);
1127
1128 SmallVector<uint8_t, 64> Ids;
1129 if (IsBpermute) {
1130 Ids.resize(N: WaveSize);
1131 for (unsigned Lane : seq(Size: WaveSize)) {
1132 std::optional<unsigned> Val = evalLaneExpr(V: Index, Lane, ST, DL);
1133 if (!Val || (*Val & 3) || (*Val >> 2) >= WaveSize)
1134 return std::nullopt;
1135 Ids[Lane] = *Val >> 2;
1136 }
1137 } else {
1138 if (!tryBuildShuffleMap(Index, ST, Ids, DL))
1139 return std::nullopt;
1140 }
1141
1142 Value *Result = matchShuffleToHWIntrinsic(B&: IC.Builder, Src, Ids, ST, DL);
1143 if (!Result)
1144 return std::nullopt;
1145
1146 return IC.replaceInstUsesWith(I&: II, V: Result);
1147}
1148std::optional<Instruction *>
1149GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
1150 Intrinsic::ID IID = II.getIntrinsicID();
1151 switch (IID) {
1152 case Intrinsic::amdgcn_implicitarg_ptr: {
1153 if (II.getFunction()->hasFnAttribute(Kind: "amdgpu-no-implicitarg-ptr"))
1154 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1155 uint64_t ImplicitArgBytes = ST->getImplicitArgNumBytes(F: *II.getFunction());
1156
1157 uint64_t CurrentOrNullBytes =
1158 II.getAttributes().getRetDereferenceableOrNullBytes();
1159 if (CurrentOrNullBytes != 0) {
1160 // Refine "dereferenceable (A) meets dereferenceable_or_null(B)"
1161 // into dereferenceable(max(A, B))
1162 uint64_t NewBytes = std::max(a: CurrentOrNullBytes, b: ImplicitArgBytes);
1163 II.addRetAttr(
1164 Attr: Attribute::getWithDereferenceableBytes(Context&: II.getContext(), Bytes: NewBytes));
1165 II.removeRetAttr(Kind: Attribute::DereferenceableOrNull);
1166 return &II;
1167 }
1168
1169 uint64_t CurrentBytes = II.getAttributes().getRetDereferenceableBytes();
1170 uint64_t NewBytes = std::max(a: CurrentBytes, b: ImplicitArgBytes);
1171 if (NewBytes != CurrentBytes) {
1172 II.addRetAttr(
1173 Attr: Attribute::getWithDereferenceableBytes(Context&: II.getContext(), Bytes: NewBytes));
1174 return &II;
1175 }
1176
1177 return std::nullopt;
1178 }
1179 case Intrinsic::amdgcn_rcp: {
1180 Value *Src = II.getArgOperand(i: 0);
1181 if (isa<PoisonValue>(Val: Src))
1182 return IC.replaceInstUsesWith(I&: II, V: Src);
1183
1184 // TODO: Move to ConstantFolding/InstSimplify?
1185 if (isa<UndefValue>(Val: Src)) {
1186 Type *Ty = II.getType();
1187 auto *QNaN = ConstantFP::get(Ty, V: APFloat::getQNaN(Sem: Ty->getFltSemantics()));
1188 return IC.replaceInstUsesWith(I&: II, V: QNaN);
1189 }
1190
1191 if (II.isStrictFP())
1192 break;
1193
1194 if (const ConstantFP *C = dyn_cast<ConstantFP>(Val: Src)) {
1195 std::optional<APFloat> Val = AMDGPU::evaluateRcp(Val: C->getValueAPF());
1196 if (!Val)
1197 break;
1198
1199 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::get(Context&: II.getContext(), V: *Val));
1200 }
1201
1202 FastMathFlags FMF = cast<FPMathOperator>(Val&: II).getFastMathFlags();
1203 if (!FMF.allowContract())
1204 break;
1205 auto *SrcCI = dyn_cast<IntrinsicInst>(Val: Src);
1206 if (!SrcCI)
1207 break;
1208
1209 auto IID = SrcCI->getIntrinsicID();
1210 // llvm.amdgcn.rcp(llvm.amdgcn.sqrt(x)) -> llvm.amdgcn.rsq(x) if contractable
1211 //
1212 // llvm.amdgcn.rcp(llvm.sqrt(x)) -> llvm.amdgcn.rsq(x) if contractable and
1213 // relaxed.
1214 if (IID == Intrinsic::amdgcn_sqrt || IID == Intrinsic::sqrt) {
1215 const FPMathOperator *SqrtOp = cast<FPMathOperator>(Val: SrcCI);
1216 FastMathFlags InnerFMF = SqrtOp->getFastMathFlags();
1217 if (!InnerFMF.allowContract() || !SrcCI->hasOneUse())
1218 break;
1219
1220 if (IID == Intrinsic::sqrt && !canContractSqrtToRsq(SqrtOp))
1221 break;
1222
1223 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
1224 M: SrcCI->getModule(), id: Intrinsic::amdgcn_rsq, OverloadTys: {SrcCI->getType()});
1225
1226 InnerFMF |= FMF;
1227 II.setFastMathFlags(InnerFMF);
1228
1229 II.setCalledFunction(NewDecl);
1230 return IC.replaceOperand(I&: II, OpNum: 0, V: SrcCI->getArgOperand(i: 0));
1231 }
1232
1233 break;
1234 }
1235 case Intrinsic::amdgcn_sqrt:
1236 case Intrinsic::amdgcn_rsq:
1237 case Intrinsic::amdgcn_tanh: {
1238 Value *Src = II.getArgOperand(i: 0);
1239 if (isa<PoisonValue>(Val: Src))
1240 return IC.replaceInstUsesWith(I&: II, V: Src);
1241
1242 // TODO: Move to ConstantFolding/InstSimplify?
1243 if (isa<UndefValue>(Val: Src)) {
1244 Type *Ty = II.getType();
1245 auto *QNaN = ConstantFP::get(Ty, V: APFloat::getQNaN(Sem: Ty->getFltSemantics()));
1246 return IC.replaceInstUsesWith(I&: II, V: QNaN);
1247 }
1248
1249 // f16 amdgcn.sqrt is identical to regular sqrt.
1250 if (IID == Intrinsic::amdgcn_sqrt && Src->getType()->isHalfTy()) {
1251 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
1252 M: II.getModule(), id: Intrinsic::sqrt, OverloadTys: {II.getType()});
1253 II.setCalledFunction(NewDecl);
1254 return &II;
1255 }
1256
1257 break;
1258 }
1259 case Intrinsic::amdgcn_log:
1260 case Intrinsic::amdgcn_exp2: {
1261 const bool IsLog = IID == Intrinsic::amdgcn_log;
1262 const bool IsExp = IID == Intrinsic::amdgcn_exp2;
1263 Value *Src = II.getArgOperand(i: 0);
1264 Type *Ty = II.getType();
1265
1266 if (isa<PoisonValue>(Val: Src))
1267 return IC.replaceInstUsesWith(I&: II, V: Src);
1268
1269 if (IC.getSimplifyQuery().isUndefValue(V: Src))
1270 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getNaN(Ty));
1271
1272 if (ConstantFP *C = dyn_cast<ConstantFP>(Val: Src)) {
1273 if (C->isInfinity()) {
1274 // exp2(+inf) -> +inf
1275 // log2(+inf) -> +inf
1276 if (!C->isNegative())
1277 return IC.replaceInstUsesWith(I&: II, V: C);
1278
1279 // exp2(-inf) -> 0
1280 if (IsExp && C->isNegative())
1281 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getZero(Ty));
1282 }
1283
1284 if (II.isStrictFP())
1285 break;
1286
1287 if (C->isNaN()) {
1288 Constant *Quieted = ConstantFP::get(Ty, V: C->getValue().makeQuiet());
1289 return IC.replaceInstUsesWith(I&: II, V: Quieted);
1290 }
1291
1292 // f32 instruction doesn't handle denormals, f16 does.
1293 if (C->isZero() || (C->getValue().isDenormal() && Ty->isFloatTy())) {
1294 Constant *FoldedValue = IsLog ? ConstantFP::getInfinity(Ty, Negative: true)
1295 : ConstantFP::get(Ty, V: 1.0);
1296 return IC.replaceInstUsesWith(I&: II, V: FoldedValue);
1297 }
1298
1299 if (IsLog && C->isNegative())
1300 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getNaN(Ty));
1301
1302 // TODO: Full constant folding matching hardware behavior.
1303 }
1304
1305 break;
1306 }
1307 case Intrinsic::amdgcn_frexp_mant:
1308 case Intrinsic::amdgcn_frexp_exp: {
1309 Value *Src = II.getArgOperand(i: 0);
1310 if (const ConstantFP *C = dyn_cast<ConstantFP>(Val: Src)) {
1311 int Exp;
1312 APFloat Significand =
1313 frexp(X: C->getValueAPF(), Exp, RM: APFloat::rmNearestTiesToEven);
1314
1315 if (IID == Intrinsic::amdgcn_frexp_mant) {
1316 return IC.replaceInstUsesWith(
1317 I&: II, V: ConstantFP::get(Context&: II.getContext(), V: Significand));
1318 }
1319
1320 // Match instruction special case behavior.
1321 if (Exp == APFloat::IEK_NaN || Exp == APFloat::IEK_Inf)
1322 Exp = 0;
1323
1324 return IC.replaceInstUsesWith(I&: II,
1325 V: ConstantInt::getSigned(Ty: II.getType(), V: Exp));
1326 }
1327
1328 if (isa<PoisonValue>(Val: Src))
1329 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1330
1331 if (isa<UndefValue>(Val: Src)) {
1332 return IC.replaceInstUsesWith(I&: II, V: UndefValue::get(T: II.getType()));
1333 }
1334
1335 break;
1336 }
1337 case Intrinsic::amdgcn_class: {
1338 Value *Src0 = II.getArgOperand(i: 0);
1339 Value *Src1 = II.getArgOperand(i: 1);
1340 const ConstantInt *CMask = dyn_cast<ConstantInt>(Val: Src1);
1341 if (CMask) {
1342 II.setCalledOperand(Intrinsic::getOrInsertDeclaration(
1343 M: II.getModule(), id: Intrinsic::is_fpclass, OverloadTys: Src0->getType()));
1344
1345 // Clamp any excess bits, as they're illegal for the generic intrinsic.
1346 II.setArgOperand(i: 1, v: ConstantInt::get(Ty: Src1->getType(),
1347 V: CMask->getZExtValue() & fcAllFlags));
1348 return &II;
1349 }
1350
1351 // Propagate poison.
1352 if (isa<PoisonValue>(Val: Src0) || isa<PoisonValue>(Val: Src1))
1353 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1354
1355 // llvm.amdgcn.class(_, undef) -> false
1356 if (IC.getSimplifyQuery().isUndefValue(V: Src1))
1357 return IC.replaceInstUsesWith(I&: II, V: ConstantInt::get(Ty: II.getType(), V: false));
1358
1359 // llvm.amdgcn.class(undef, mask) -> mask != 0
1360 if (IC.getSimplifyQuery().isUndefValue(V: Src0)) {
1361 Value *CmpMask = IC.Builder.CreateICmpNE(
1362 LHS: Src1, RHS: ConstantInt::getNullValue(Ty: Src1->getType()));
1363 return IC.replaceInstUsesWith(I&: II, V: CmpMask);
1364 }
1365 break;
1366 }
1367 case Intrinsic::amdgcn_cvt_pkrtz: {
1368 auto foldFPTruncToF16RTZ = [](Value *Arg) -> Value * {
1369 Type *HalfTy = Type::getHalfTy(C&: Arg->getContext());
1370
1371 if (isa<PoisonValue>(Val: Arg))
1372 return PoisonValue::get(T: HalfTy);
1373 if (isa<UndefValue>(Val: Arg))
1374 return UndefValue::get(T: HalfTy);
1375
1376 ConstantFP *CFP = nullptr;
1377 if (match(V: Arg, P: m_ConstantFP(C&: CFP))) {
1378 bool LosesInfo;
1379 APFloat Val(CFP->getValueAPF());
1380 Val.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmTowardZero, losesInfo: &LosesInfo);
1381 return ConstantFP::get(Ty: HalfTy, V: Val);
1382 }
1383
1384 Value *Src = nullptr;
1385 if (match(V: Arg, P: m_FPExt(Op: m_Value(V&: Src)))) {
1386 if (Src->getType()->isHalfTy())
1387 return Src;
1388 }
1389
1390 return nullptr;
1391 };
1392
1393 if (Value *Src0 = foldFPTruncToF16RTZ(II.getArgOperand(i: 0))) {
1394 if (Value *Src1 = foldFPTruncToF16RTZ(II.getArgOperand(i: 1))) {
1395 Value *V = PoisonValue::get(T: II.getType());
1396 V = IC.Builder.CreateInsertElement(Vec: V, NewElt: Src0, Idx: (uint64_t)0);
1397 V = IC.Builder.CreateInsertElement(Vec: V, NewElt: Src1, Idx: (uint64_t)1);
1398 return IC.replaceInstUsesWith(I&: II, V);
1399 }
1400 }
1401
1402 break;
1403 }
1404 case Intrinsic::amdgcn_cvt_pknorm_i16:
1405 case Intrinsic::amdgcn_cvt_pknorm_u16:
1406 case Intrinsic::amdgcn_cvt_pk_i16:
1407 case Intrinsic::amdgcn_cvt_pk_u16: {
1408 Value *Src0 = II.getArgOperand(i: 0);
1409 Value *Src1 = II.getArgOperand(i: 1);
1410
1411 // TODO: Replace call with scalar operation if only one element is poison.
1412 if (isa<PoisonValue>(Val: Src0) && isa<PoisonValue>(Val: Src1))
1413 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1414
1415 if (isa<UndefValue>(Val: Src0) && isa<UndefValue>(Val: Src1)) {
1416 return IC.replaceInstUsesWith(I&: II, V: UndefValue::get(T: II.getType()));
1417 }
1418
1419 break;
1420 }
1421 case Intrinsic::amdgcn_cvt_off_f32_i4: {
1422 Value* Arg = II.getArgOperand(i: 0);
1423 Type *Ty = II.getType();
1424
1425 if (isa<PoisonValue>(Val: Arg))
1426 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: Ty));
1427
1428 if(IC.getSimplifyQuery().isUndefValue(V: Arg))
1429 return IC.replaceInstUsesWith(I&: II, V: Constant::getNullValue(Ty));
1430
1431 ConstantInt *CArg = dyn_cast<ConstantInt>(Val: II.getArgOperand(i: 0));
1432 if (!CArg)
1433 break;
1434
1435 // Tabulated 0.0625 * (sext (CArg & 0xf)).
1436 constexpr size_t ResValsSize = 16;
1437 static constexpr float ResVals[ResValsSize] = {
1438 0.0, 0.0625, 0.125, 0.1875, 0.25, 0.3125, 0.375, 0.4375,
1439 -0.5, -0.4375, -0.375, -0.3125, -0.25, -0.1875, -0.125, -0.0625};
1440 Constant *Res =
1441 ConstantFP::get(Ty, V: ResVals[CArg->getZExtValue() & (ResValsSize - 1)]);
1442 return IC.replaceInstUsesWith(I&: II, V: Res);
1443 }
1444 case Intrinsic::amdgcn_ubfe:
1445 case Intrinsic::amdgcn_sbfe: {
1446 // Decompose simple cases into standard shifts.
1447 Value *Src = II.getArgOperand(i: 0);
1448 if (isa<UndefValue>(Val: Src)) {
1449 return IC.replaceInstUsesWith(I&: II, V: Src);
1450 }
1451
1452 unsigned Width;
1453 Type *Ty = II.getType();
1454 unsigned IntSize = Ty->getIntegerBitWidth();
1455
1456 ConstantInt *CWidth = dyn_cast<ConstantInt>(Val: II.getArgOperand(i: 2));
1457 if (CWidth) {
1458 Width = CWidth->getZExtValue();
1459 if ((Width & (IntSize - 1)) == 0) {
1460 return IC.replaceInstUsesWith(I&: II, V: ConstantInt::getNullValue(Ty));
1461 }
1462
1463 // Hardware ignores high bits, so remove those.
1464 if (Width >= IntSize) {
1465 return IC.replaceOperand(
1466 I&: II, OpNum: 2, V: ConstantInt::get(Ty: CWidth->getType(), V: Width & (IntSize - 1)));
1467 }
1468 }
1469
1470 unsigned Offset;
1471 ConstantInt *COffset = dyn_cast<ConstantInt>(Val: II.getArgOperand(i: 1));
1472 if (COffset) {
1473 Offset = COffset->getZExtValue();
1474 if (Offset >= IntSize) {
1475 return IC.replaceOperand(
1476 I&: II, OpNum: 1,
1477 V: ConstantInt::get(Ty: COffset->getType(), V: Offset & (IntSize - 1)));
1478 }
1479 }
1480
1481 bool Signed = IID == Intrinsic::amdgcn_sbfe;
1482
1483 if (!CWidth || !COffset)
1484 break;
1485
1486 // The case of Width == 0 is handled above, which makes this transformation
1487 // safe. If Width == 0, then the ashr and lshr instructions become poison
1488 // value since the shift amount would be equal to the bit size.
1489 assert(Width != 0);
1490
1491 // TODO: This allows folding to undef when the hardware has specific
1492 // behavior?
1493 if (Offset + Width < IntSize) {
1494 Value *Shl = IC.Builder.CreateShl(LHS: Src, RHS: IntSize - Offset - Width);
1495 Value *RightShift = Signed ? IC.Builder.CreateAShr(LHS: Shl, RHS: IntSize - Width)
1496 : IC.Builder.CreateLShr(LHS: Shl, RHS: IntSize - Width);
1497 RightShift->takeName(V: &II);
1498 return IC.replaceInstUsesWith(I&: II, V: RightShift);
1499 }
1500
1501 Value *RightShift = Signed ? IC.Builder.CreateAShr(LHS: Src, RHS: Offset)
1502 : IC.Builder.CreateLShr(LHS: Src, RHS: Offset);
1503
1504 RightShift->takeName(V: &II);
1505 return IC.replaceInstUsesWith(I&: II, V: RightShift);
1506 }
1507 case Intrinsic::amdgcn_exp:
1508 case Intrinsic::amdgcn_exp_row:
1509 case Intrinsic::amdgcn_exp_compr: {
1510 ConstantInt *En = cast<ConstantInt>(Val: II.getArgOperand(i: 1));
1511 unsigned EnBits = En->getZExtValue();
1512 if (EnBits == 0xf)
1513 break; // All inputs enabled.
1514
1515 bool IsCompr = IID == Intrinsic::amdgcn_exp_compr;
1516 bool Changed = false;
1517 for (int I = 0; I < (IsCompr ? 2 : 4); ++I) {
1518 if ((!IsCompr && (EnBits & (1 << I)) == 0) ||
1519 (IsCompr && ((EnBits & (0x3 << (2 * I))) == 0))) {
1520 Value *Src = II.getArgOperand(i: I + 2);
1521 if (!isa<PoisonValue>(Val: Src)) {
1522 IC.replaceOperand(I&: II, OpNum: I + 2, V: PoisonValue::get(T: Src->getType()));
1523 Changed = true;
1524 }
1525 }
1526 }
1527
1528 if (Changed) {
1529 return &II;
1530 }
1531
1532 break;
1533 }
1534 case Intrinsic::amdgcn_fmed3: {
1535 Value *Src0 = II.getArgOperand(i: 0);
1536 Value *Src1 = II.getArgOperand(i: 1);
1537 Value *Src2 = II.getArgOperand(i: 2);
1538
1539 for (Value *Src : {Src0, Src1, Src2}) {
1540 if (isa<PoisonValue>(Val: Src))
1541 return IC.replaceInstUsesWith(I&: II, V: Src);
1542 }
1543
1544 if (II.isStrictFP())
1545 break;
1546
1547 // med3 with a nan input acts like
1548 // v_min_f32(v_min_f32(s0, s1), s2)
1549 //
1550 // Signalingness is ignored with ieee=0, so we fold to
1551 // minimumnum/maximumnum. With ieee=1, the v_min_f32 acts like llvm.minnum
1552 // with signaling nan handling. With ieee=0, like llvm.minimumnum except a
1553 // returned signaling nan will not be quieted.
1554
1555 // ieee=1
1556 // s0 snan: s2
1557 // s1 snan: s2
1558 // s2 snan: qnan
1559
1560 // s0 qnan: min(s1, s2)
1561 // s1 qnan: min(s0, s2)
1562 // s2 qnan: min(s0, s1)
1563
1564 // ieee=0
1565 // s0 _nan: min(s1, s2)
1566 // s1 _nan: min(s0, s2)
1567 // s2 _nan: min(s0, s1)
1568
1569 // med3 behavior with infinity
1570 // s0 +inf: max(s1, s2)
1571 // s1 +inf: max(s0, s2)
1572 // s2 +inf: max(s0, s1)
1573 // s0 -inf: min(s1, s2)
1574 // s1 -inf: min(s0, s2)
1575 // s2 -inf: min(s0, s1)
1576
1577 // Checking for NaN before canonicalization provides better fidelity when
1578 // mapping other operations onto fmed3 since the order of operands is
1579 // unchanged.
1580 Value *V = nullptr;
1581 const APFloat *ConstSrc0 = nullptr;
1582 const APFloat *ConstSrc1 = nullptr;
1583 const APFloat *ConstSrc2 = nullptr;
1584
1585 if ((match(V: Src0, P: m_APFloat(Res&: ConstSrc0)) &&
1586 (ConstSrc0->isNaN() || ConstSrc0->isInfinity())) ||
1587 isa<UndefValue>(Val: Src0)) {
1588 const bool IsPosInfinity = ConstSrc0 && ConstSrc0->isPosInfinity();
1589 switch (fpenvIEEEMode(I: II)) {
1590 case KnownIEEEMode::On:
1591 // TODO: If Src2 is snan, does it need quieting?
1592 if (ConstSrc0 && ConstSrc0->isNaN() && ConstSrc0->isSignaling())
1593 return IC.replaceInstUsesWith(I&: II, V: Src2);
1594
1595 V = IsPosInfinity ? IC.Builder.CreateMaxNum(LHS: Src1, RHS: Src2)
1596 : IC.Builder.CreateMinNum(LHS: Src1, RHS: Src2);
1597 break;
1598 case KnownIEEEMode::Off:
1599 V = IsPosInfinity ? IC.Builder.CreateMaximumNum(LHS: Src1, RHS: Src2)
1600 : IC.Builder.CreateMinimumNum(LHS: Src1, RHS: Src2);
1601 break;
1602 case KnownIEEEMode::Unknown:
1603 break;
1604 }
1605 } else if ((match(V: Src1, P: m_APFloat(Res&: ConstSrc1)) &&
1606 (ConstSrc1->isNaN() || ConstSrc1->isInfinity())) ||
1607 isa<UndefValue>(Val: Src1)) {
1608 const bool IsPosInfinity = ConstSrc1 && ConstSrc1->isPosInfinity();
1609 switch (fpenvIEEEMode(I: II)) {
1610 case KnownIEEEMode::On:
1611 // TODO: If Src2 is snan, does it need quieting?
1612 if (ConstSrc1 && ConstSrc1->isNaN() && ConstSrc1->isSignaling())
1613 return IC.replaceInstUsesWith(I&: II, V: Src2);
1614
1615 V = IsPosInfinity ? IC.Builder.CreateMaxNum(LHS: Src0, RHS: Src2)
1616 : IC.Builder.CreateMinNum(LHS: Src0, RHS: Src2);
1617 break;
1618 case KnownIEEEMode::Off:
1619 V = IsPosInfinity ? IC.Builder.CreateMaximumNum(LHS: Src0, RHS: Src2)
1620 : IC.Builder.CreateMinimumNum(LHS: Src0, RHS: Src2);
1621 break;
1622 case KnownIEEEMode::Unknown:
1623 break;
1624 }
1625 } else if ((match(V: Src2, P: m_APFloat(Res&: ConstSrc2)) &&
1626 (ConstSrc2->isNaN() || ConstSrc2->isInfinity())) ||
1627 isa<UndefValue>(Val: Src2)) {
1628 switch (fpenvIEEEMode(I: II)) {
1629 case KnownIEEEMode::On:
1630 if (ConstSrc2 && ConstSrc2->isNaN() && ConstSrc2->isSignaling()) {
1631 auto *Quieted = ConstantFP::get(Ty: II.getType(), V: ConstSrc2->makeQuiet());
1632 return IC.replaceInstUsesWith(I&: II, V: Quieted);
1633 }
1634
1635 V = (ConstSrc2 && ConstSrc2->isPosInfinity())
1636 ? IC.Builder.CreateMaxNum(LHS: Src0, RHS: Src1)
1637 : IC.Builder.CreateMinNum(LHS: Src0, RHS: Src1);
1638 break;
1639 case KnownIEEEMode::Off:
1640 V = (ConstSrc2 && ConstSrc2->isPosInfinity())
1641 ? IC.Builder.CreateMaximumNum(LHS: Src0, RHS: Src1)
1642 : IC.Builder.CreateMinimumNum(LHS: Src0, RHS: Src1);
1643 break;
1644 case KnownIEEEMode::Unknown:
1645 break;
1646 }
1647 }
1648
1649 if (V) {
1650 if (auto *CI = dyn_cast<CallInst>(Val: V)) {
1651 CI->copyFastMathFlags(I: &II);
1652 CI->takeName(V: &II);
1653 }
1654 return IC.replaceInstUsesWith(I&: II, V);
1655 }
1656
1657 bool Swap = false;
1658 // Canonicalize constants to RHS operands.
1659 //
1660 // fmed3(c0, x, c1) -> fmed3(x, c0, c1)
1661 if (isa<Constant>(Val: Src0) && !isa<Constant>(Val: Src1)) {
1662 std::swap(a&: Src0, b&: Src1);
1663 Swap = true;
1664 }
1665
1666 if (isa<Constant>(Val: Src1) && !isa<Constant>(Val: Src2)) {
1667 std::swap(a&: Src1, b&: Src2);
1668 Swap = true;
1669 }
1670
1671 if (isa<Constant>(Val: Src0) && !isa<Constant>(Val: Src1)) {
1672 std::swap(a&: Src0, b&: Src1);
1673 Swap = true;
1674 }
1675
1676 if (Swap) {
1677 II.setArgOperand(i: 0, v: Src0);
1678 II.setArgOperand(i: 1, v: Src1);
1679 II.setArgOperand(i: 2, v: Src2);
1680 return &II;
1681 }
1682
1683 if (const ConstantFP *C0 = dyn_cast<ConstantFP>(Val: Src0)) {
1684 if (const ConstantFP *C1 = dyn_cast<ConstantFP>(Val: Src1)) {
1685 if (const ConstantFP *C2 = dyn_cast<ConstantFP>(Val: Src2)) {
1686 APFloat Result = fmed3AMDGCN(Src0: C0->getValueAPF(), Src1: C1->getValueAPF(),
1687 Src2: C2->getValueAPF());
1688 return IC.replaceInstUsesWith(I&: II,
1689 V: ConstantFP::get(Ty: II.getType(), V: Result));
1690 }
1691 }
1692 }
1693
1694 if (!ST->hasMed3_16())
1695 break;
1696
1697 // Repeat floating-point width reduction done for minnum/maxnum.
1698 // fmed3((fpext X), (fpext Y), (fpext Z)) -> fpext (fmed3(X, Y, Z))
1699 if (Value *X = matchFPExtFromF16(Arg: Src0)) {
1700 if (Value *Y = matchFPExtFromF16(Arg: Src1)) {
1701 if (Value *Z = matchFPExtFromF16(Arg: Src2)) {
1702 Value *NewCall = IC.Builder.CreateIntrinsic(
1703 ID: IID, OverloadTypes: {X->getType()}, Args: {X, Y, Z}, FMFSource: &II, Name: II.getName());
1704 return new FPExtInst(NewCall, II.getType());
1705 }
1706 }
1707 }
1708
1709 break;
1710 }
1711 case Intrinsic::amdgcn_mbcnt_hi:
1712 // exec_hi is all 0, so this is just a copy.
1713 if (ST->isWave32())
1714 return IC.replaceInstUsesWith(I&: II, V: II.getArgOperand(i: 1));
1715 [[fallthrough]];
1716 case Intrinsic::amdgcn_mbcnt_lo: {
1717 ConstantRange AccRange =
1718 computeConstantRange(V: II.getArgOperand(i: 1),
1719 /*ForSigned=*/false, SQ: IC.getSimplifyQuery());
1720 if (AccRange.isFullSet())
1721 return nullptr;
1722
1723 // TODO: Can raise lower bound by inspecting first argument.
1724 ConstantRange MbcntRange(APInt(32, 0), APInt(32, 32 + 1));
1725 ConstantRange ComputedRange = AccRange.add(Other: MbcntRange);
1726 if (ComputedRange.isFullSet())
1727 return nullptr;
1728
1729 if (std::optional<ConstantRange> ExistingRange = II.getRange()) {
1730 ComputedRange = ComputedRange.intersectWith(CR: *ExistingRange);
1731 if (ComputedRange == *ExistingRange)
1732 return nullptr;
1733 }
1734
1735 II.addRangeRetAttr(CR: ComputedRange);
1736 return nullptr;
1737 }
1738 case Intrinsic::amdgcn_ballot: {
1739 Value *Arg = II.getArgOperand(i: 0);
1740 if (isa<PoisonValue>(Val: Arg))
1741 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1742
1743 if (auto *Src = dyn_cast<ConstantInt>(Val: Arg)) {
1744 if (Src->isZero()) {
1745 // amdgcn.ballot(i1 0) is zero.
1746 return IC.replaceInstUsesWith(I&: II, V: Constant::getNullValue(Ty: II.getType()));
1747 }
1748 }
1749 if (ST->isWave32() && II.getType()->getIntegerBitWidth() == 64) {
1750 // %b64 = call i64 ballot.i64(...)
1751 // =>
1752 // %b32 = call i32 ballot.i32(...)
1753 // %b64 = zext i32 %b32 to i64
1754 Value *Call = IC.Builder.CreateZExt(
1755 V: IC.Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_ballot,
1756 OverloadTypes: {IC.Builder.getInt32Ty()},
1757 Args: {II.getArgOperand(i: 0)}),
1758 DestTy: II.getType());
1759 Call->takeName(V: &II);
1760 return IC.replaceInstUsesWith(I&: II, V: Call);
1761 }
1762 break;
1763 }
1764 case Intrinsic::amdgcn_wavefrontsize: {
1765 if (ST->isWaveSizeKnown())
1766 return IC.replaceInstUsesWith(
1767 I&: II, V: ConstantInt::get(Ty: II.getType(), V: ST->getWavefrontSize()));
1768 break;
1769 }
1770 case Intrinsic::amdgcn_wqm_vote: {
1771 // wqm_vote is identity when the argument is constant.
1772 if (!isa<Constant>(Val: II.getArgOperand(i: 0)))
1773 break;
1774
1775 return IC.replaceInstUsesWith(I&: II, V: II.getArgOperand(i: 0));
1776 }
1777 case Intrinsic::amdgcn_kill: {
1778 const ConstantInt *C = dyn_cast<ConstantInt>(Val: II.getArgOperand(i: 0));
1779 if (!C || !C->getZExtValue())
1780 break;
1781
1782 // amdgcn.kill(i1 1) is a no-op
1783 return IC.eraseInstFromFunction(I&: II);
1784 }
1785 case Intrinsic::amdgcn_s_sendmsg:
1786 case Intrinsic::amdgcn_s_sendmsghalt: {
1787 // The second operand is copied to m0, but is only actually used for
1788 // certain message types. For message types that are known to not use m0,
1789 // fold it to poison.
1790 using namespace AMDGPU::SendMsg;
1791
1792 Value *M0Val = II.getArgOperand(i: 1);
1793 if (isa<PoisonValue>(Val: M0Val))
1794 break;
1795
1796 auto *MsgImm = cast<ConstantInt>(Val: II.getArgOperand(i: 0));
1797 uint16_t MsgId, OpId, StreamId;
1798 decodeMsg(Val: MsgImm->getZExtValue(), MsgId, OpId, StreamId, STI: *ST);
1799
1800 if (!msgDoesNotUseM0(MsgId, STI: *ST))
1801 break;
1802
1803 // Drop UB-implying attributes since we're replacing with poison.
1804 II.dropUBImplyingAttrsAndMetadata();
1805 IC.replaceOperand(I&: II, OpNum: 1, V: PoisonValue::get(T: M0Val->getType()));
1806 return nullptr;
1807 }
1808 case Intrinsic::amdgcn_update_dpp: {
1809 Value *Old = II.getArgOperand(i: 0);
1810
1811 auto *BC = cast<ConstantInt>(Val: II.getArgOperand(i: 5));
1812 auto *RM = cast<ConstantInt>(Val: II.getArgOperand(i: 3));
1813 auto *BM = cast<ConstantInt>(Val: II.getArgOperand(i: 4));
1814 if (BC->isNullValue() || RM->getZExtValue() != 0xF ||
1815 BM->getZExtValue() != 0xF || isa<PoisonValue>(Val: Old))
1816 break;
1817
1818 // If bound_ctrl = 1, row mask = bank mask = 0xf we can omit old value.
1819 return IC.replaceOperand(I&: II, OpNum: 0, V: PoisonValue::get(T: Old->getType()));
1820 }
1821 case Intrinsic::amdgcn_permlane16:
1822 case Intrinsic::amdgcn_permlane16_var:
1823 case Intrinsic::amdgcn_permlanex16:
1824 case Intrinsic::amdgcn_permlanex16_var: {
1825 // Discard vdst_in if it's not going to be read.
1826 Value *VDstIn = II.getArgOperand(i: 0);
1827 if (isa<PoisonValue>(Val: VDstIn))
1828 break;
1829
1830 // FetchInvalid operand idx.
1831 unsigned int FiIdx = (IID == Intrinsic::amdgcn_permlane16 ||
1832 IID == Intrinsic::amdgcn_permlanex16)
1833 ? 4 /* for permlane16 and permlanex16 */
1834 : 3; /* for permlane16_var and permlanex16_var */
1835
1836 // BoundCtrl operand idx.
1837 // For permlane16 and permlanex16 it should be 5
1838 // For Permlane16_var and permlanex16_var it should be 4
1839 unsigned int BcIdx = FiIdx + 1;
1840
1841 ConstantInt *FetchInvalid = cast<ConstantInt>(Val: II.getArgOperand(i: FiIdx));
1842 ConstantInt *BoundCtrl = cast<ConstantInt>(Val: II.getArgOperand(i: BcIdx));
1843 if (!FetchInvalid->getZExtValue() && !BoundCtrl->getZExtValue())
1844 break;
1845
1846 return IC.replaceOperand(I&: II, OpNum: 0, V: PoisonValue::get(T: VDstIn->getType()));
1847 }
1848 case Intrinsic::amdgcn_wave_shuffle:
1849 return tryOptimizeShufflePattern(IC, II, ST: *ST);
1850 case Intrinsic::amdgcn_permlane64:
1851 case Intrinsic::amdgcn_readfirstlane:
1852 case Intrinsic::amdgcn_readlane:
1853 case Intrinsic::amdgcn_ds_bpermute: {
1854 // If the data argument is uniform these intrinsics return it unchanged.
1855 unsigned SrcIdx = IID == Intrinsic::amdgcn_ds_bpermute ? 1 : 0;
1856 const Use &Src = II.getArgOperandUse(i: SrcIdx);
1857 if (isTriviallyUniform(U: Src))
1858 return IC.replaceInstUsesWith(I&: II, V: Src.get());
1859
1860 if (IID == Intrinsic::amdgcn_readlane &&
1861 simplifyDemandedLaneMaskArg(IC, II, LaneArgIdx: 1))
1862 return &II;
1863
1864 // If the lane argument of bpermute is uniform, change it to readlane. This
1865 // generates better code and can enable further optimizations because
1866 // readlane is AlwaysUniform.
1867 if (IID == Intrinsic::amdgcn_ds_bpermute) {
1868 const Use &Lane = II.getArgOperandUse(i: 0);
1869 if (isTriviallyUniform(U: Lane)) {
1870 Value *NewLane = IC.Builder.CreateLShr(LHS: Lane, RHS: 2);
1871 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
1872 M: II.getModule(), id: Intrinsic::amdgcn_readlane, OverloadTys: II.getType());
1873 II.setCalledFunction(NewDecl);
1874 II.setOperand(i_nocapture: 0, Val_nocapture: Src);
1875 II.setOperand(i_nocapture: 1, Val_nocapture: NewLane);
1876 return &II;
1877 }
1878 }
1879
1880 if (IID == Intrinsic::amdgcn_ds_bpermute)
1881 return tryOptimizeShufflePattern(IC, II, ST: *ST);
1882
1883 if (Instruction *Res = hoistLaneIntrinsicThroughOperand(IC, II))
1884 return Res;
1885
1886 return std::nullopt;
1887 }
1888 case Intrinsic::amdgcn_writelane: {
1889 // TODO: Fold bitcast like readlane.
1890 if (simplifyDemandedLaneMaskArg(IC, II, LaneArgIdx: 1))
1891 return &II;
1892 return std::nullopt;
1893 }
1894 case Intrinsic::amdgcn_trig_preop: {
1895 // The intrinsic is declared with name mangling, but currently the
1896 // instruction only exists for f64
1897 if (!II.getType()->isDoubleTy())
1898 break;
1899
1900 Value *Src = II.getArgOperand(i: 0);
1901 Value *Segment = II.getArgOperand(i: 1);
1902 if (isa<PoisonValue>(Val: Src) || isa<PoisonValue>(Val: Segment))
1903 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
1904
1905 if (isa<UndefValue>(Val: Segment))
1906 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getZero(Ty: II.getType()));
1907
1908 // Sign bit is not used.
1909 Value *StrippedSign = InstCombiner::stripSignOnlyFPOps(Val: Src);
1910 if (StrippedSign != Src)
1911 return IC.replaceOperand(I&: II, OpNum: 0, V: StrippedSign);
1912
1913 if (II.isStrictFP())
1914 break;
1915
1916 const ConstantFP *CSrc = dyn_cast<ConstantFP>(Val: Src);
1917 if (!CSrc && !isa<UndefValue>(Val: Src))
1918 break;
1919
1920 // The instruction ignores special cases, and literally just extracts the
1921 // exponents. Fold undef to nan, and index the table as normal.
1922 APInt FSrcInt = CSrc ? CSrc->getValueAPF().bitcastToAPInt()
1923 : APFloat::getQNaN(Sem: II.getType()->getFltSemantics())
1924 .bitcastToAPInt();
1925
1926 const ConstantInt *Cseg = dyn_cast<ConstantInt>(Val: Segment);
1927 if (!Cseg) {
1928 if (isa<UndefValue>(Val: Src))
1929 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getZero(Ty: II.getType()));
1930 break;
1931 }
1932
1933 unsigned Exponent = FSrcInt.extractBitsAsZExtValue(numBits: 11, bitPosition: 52);
1934 unsigned SegmentVal = Cseg->getValue().trunc(width: 5).getZExtValue();
1935 unsigned Shift = SegmentVal * 53;
1936 if (Exponent > 1077)
1937 Shift += Exponent - 1077;
1938
1939 // 2.0/PI table.
1940 static const uint32_t TwoByPi[] = {
1941 0xa2f9836e, 0x4e441529, 0xfc2757d1, 0xf534ddc0, 0xdb629599, 0x3c439041,
1942 0xfe5163ab, 0xdebbc561, 0xb7246e3a, 0x424dd2e0, 0x06492eea, 0x09d1921c,
1943 0xfe1deb1c, 0xb129a73e, 0xe88235f5, 0x2ebb4484, 0xe99c7026, 0xb45f7e41,
1944 0x3991d639, 0x835339f4, 0x9c845f8b, 0xbdf9283b, 0x1ff897ff, 0xde05980f,
1945 0xef2f118b, 0x5a0a6d1f, 0x6d367ecf, 0x27cb09b7, 0x4f463f66, 0x9e5fea2d,
1946 0x7527bac7, 0xebe5f17b, 0x3d0739f7, 0x8a5292ea, 0x6bfb5fb1, 0x1f8d5d08,
1947 0x56033046};
1948
1949 // Return 0 for outbound segment (hardware behavior).
1950 unsigned Idx = Shift >> 5;
1951 if (Idx + 2 >= std::size(TwoByPi)) {
1952 APFloat Zero = APFloat::getZero(Sem: II.getType()->getFltSemantics());
1953 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::get(Ty: II.getType(), V: Zero));
1954 }
1955
1956 unsigned BShift = Shift & 0x1f;
1957 uint64_t Thi = Make_64(High: TwoByPi[Idx], Low: TwoByPi[Idx + 1]);
1958 uint64_t Tlo = Make_64(High: TwoByPi[Idx + 2], Low: 0);
1959 if (BShift)
1960 Thi = (Thi << BShift) | (Tlo >> (64 - BShift));
1961 Thi = Thi >> 11;
1962 APFloat Result = APFloat((double)Thi);
1963
1964 int Scale = -53 - Shift;
1965 if (Exponent >= 1968)
1966 Scale += 128;
1967
1968 Result = scalbn(X: Result, Exp: Scale, RM: RoundingMode::NearestTiesToEven);
1969 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::get(Ty: Src->getType(), V: Result));
1970 }
1971 case Intrinsic::amdgcn_fmul_legacy: {
1972 Value *Op0 = II.getArgOperand(i: 0);
1973 Value *Op1 = II.getArgOperand(i: 1);
1974
1975 for (Value *Src : {Op0, Op1}) {
1976 if (isa<PoisonValue>(Val: Src))
1977 return IC.replaceInstUsesWith(I&: II, V: Src);
1978 }
1979
1980 // The legacy behaviour is that multiplying +/-0.0 by anything, even NaN or
1981 // infinity, gives +0.0.
1982 // TODO: Move to InstSimplify?
1983 if (match(V: Op0, P: PatternMatch::m_AnyZeroFP()) ||
1984 match(V: Op1, P: PatternMatch::m_AnyZeroFP()))
1985 return IC.replaceInstUsesWith(I&: II, V: ConstantFP::getZero(Ty: II.getType()));
1986
1987 // If we can prove we don't have one of the special cases then we can use a
1988 // normal fmul instruction instead.
1989 if (canSimplifyLegacyMulToMul(I: II, Op0, Op1, IC)) {
1990 auto *FMul = IC.Builder.CreateFMulFMF(L: Op0, R: Op1, FMFSource: &II);
1991 FMul->takeName(V: &II);
1992 return IC.replaceInstUsesWith(I&: II, V: FMul);
1993 }
1994 break;
1995 }
1996 case Intrinsic::amdgcn_fma_legacy: {
1997 Value *Op0 = II.getArgOperand(i: 0);
1998 Value *Op1 = II.getArgOperand(i: 1);
1999 Value *Op2 = II.getArgOperand(i: 2);
2000
2001 for (Value *Src : {Op0, Op1, Op2}) {
2002 if (isa<PoisonValue>(Val: Src))
2003 return IC.replaceInstUsesWith(I&: II, V: Src);
2004 }
2005
2006 // The legacy behaviour is that multiplying +/-0.0 by anything, even NaN or
2007 // infinity, gives +0.0.
2008 // TODO: Move to InstSimplify?
2009 if (match(V: Op0, P: PatternMatch::m_AnyZeroFP()) ||
2010 match(V: Op1, P: PatternMatch::m_AnyZeroFP())) {
2011 // It's tempting to just return Op2 here, but that would give the wrong
2012 // result if Op2 was -0.0.
2013 auto *Zero = ConstantFP::getZero(Ty: II.getType());
2014 auto *FAdd = IC.Builder.CreateFAddFMF(L: Zero, R: Op2, FMFSource: &II);
2015 FAdd->takeName(V: &II);
2016 return IC.replaceInstUsesWith(I&: II, V: FAdd);
2017 }
2018
2019 // If we can prove we don't have one of the special cases then we can use a
2020 // normal fma instead.
2021 if (canSimplifyLegacyMulToMul(I: II, Op0, Op1, IC)) {
2022 II.setCalledOperand(Intrinsic::getOrInsertDeclaration(
2023 M: II.getModule(), id: Intrinsic::fma, OverloadTys: II.getType()));
2024 return &II;
2025 }
2026 break;
2027 }
2028 case Intrinsic::amdgcn_is_shared:
2029 case Intrinsic::amdgcn_is_private: {
2030 Value *Src = II.getArgOperand(i: 0);
2031 if (isa<PoisonValue>(Val: Src))
2032 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
2033 if (isa<UndefValue>(Val: Src))
2034 return IC.replaceInstUsesWith(I&: II, V: UndefValue::get(T: II.getType()));
2035
2036 if (isa<ConstantPointerNull>(Val: II.getArgOperand(i: 0)))
2037 return IC.replaceInstUsesWith(I&: II, V: ConstantInt::getFalse(Ty: II.getType()));
2038 break;
2039 }
2040 case Intrinsic::amdgcn_make_buffer_rsrc: {
2041 Value *Src = II.getArgOperand(i: 0);
2042 if (isa<PoisonValue>(Val: Src))
2043 return IC.replaceInstUsesWith(I&: II, V: PoisonValue::get(T: II.getType()));
2044 return std::nullopt;
2045 }
2046 case Intrinsic::amdgcn_raw_buffer_store_format:
2047 case Intrinsic::amdgcn_struct_buffer_store_format:
2048 case Intrinsic::amdgcn_raw_tbuffer_store:
2049 case Intrinsic::amdgcn_struct_tbuffer_store:
2050 case Intrinsic::amdgcn_image_store_1d:
2051 case Intrinsic::amdgcn_image_store_1darray:
2052 case Intrinsic::amdgcn_image_store_2d:
2053 case Intrinsic::amdgcn_image_store_2darray:
2054 case Intrinsic::amdgcn_image_store_2darraymsaa:
2055 case Intrinsic::amdgcn_image_store_2dmsaa:
2056 case Intrinsic::amdgcn_image_store_3d:
2057 case Intrinsic::amdgcn_image_store_cube:
2058 case Intrinsic::amdgcn_image_store_mip_1d:
2059 case Intrinsic::amdgcn_image_store_mip_1darray:
2060 case Intrinsic::amdgcn_image_store_mip_2d:
2061 case Intrinsic::amdgcn_image_store_mip_2darray:
2062 case Intrinsic::amdgcn_image_store_mip_3d:
2063 case Intrinsic::amdgcn_image_store_mip_cube: {
2064 if (!isa<FixedVectorType>(Val: II.getArgOperand(i: 0)->getType()))
2065 break;
2066
2067 APInt DemandedElts;
2068 if (ST->hasDefaultComponentBroadcast())
2069 DemandedElts = defaultComponentBroadcast(V: II.getArgOperand(i: 0));
2070 else if (ST->hasDefaultComponentZero())
2071 DemandedElts = trimTrailingZerosInVector(IC, UseV: II.getArgOperand(i: 0), I: &II);
2072 else
2073 break;
2074
2075 int DMaskIdx = getAMDGPUImageDMaskIntrinsic(Intr: II.getIntrinsicID()) ? 1 : -1;
2076 if (simplifyAMDGCNMemoryIntrinsicDemanded(IC, II, DemandedElts, DMaskIdx,
2077 IsLoad: false)) {
2078 return IC.eraseInstFromFunction(I&: II);
2079 }
2080
2081 break;
2082 }
2083 case Intrinsic::amdgcn_prng_b32: {
2084 auto *Src = II.getArgOperand(i: 0);
2085 if (isa<UndefValue>(Val: Src)) {
2086 return IC.replaceInstUsesWith(I&: II, V: Src);
2087 }
2088 return std::nullopt;
2089 }
2090 case Intrinsic::amdgcn_mfma_scale_f32_16x16x128_f8f6f4:
2091 case Intrinsic::amdgcn_mfma_scale_f32_32x32x64_f8f6f4: {
2092 Value *Src0 = II.getArgOperand(i: 0);
2093 Value *Src1 = II.getArgOperand(i: 1);
2094 uint64_t CBSZ = cast<ConstantInt>(Val: II.getArgOperand(i: 3))->getZExtValue();
2095 uint64_t BLGP = cast<ConstantInt>(Val: II.getArgOperand(i: 4))->getZExtValue();
2096 auto *Src0Ty = cast<FixedVectorType>(Val: Src0->getType());
2097 auto *Src1Ty = cast<FixedVectorType>(Val: Src1->getType());
2098
2099 auto getFormatNumRegs = [](unsigned FormatVal) {
2100 switch (FormatVal) {
2101 case AMDGPU::MFMAScaleFormats::FP6_E2M3:
2102 case AMDGPU::MFMAScaleFormats::FP6_E3M2:
2103 return 6u;
2104 case AMDGPU::MFMAScaleFormats::FP4_E2M1:
2105 return 4u;
2106 case AMDGPU::MFMAScaleFormats::FP8_E4M3:
2107 case AMDGPU::MFMAScaleFormats::FP8_E5M2:
2108 return 8u;
2109 default:
2110 llvm_unreachable("invalid format value");
2111 }
2112 };
2113
2114 bool MadeChange = false;
2115 unsigned Src0NumElts = getFormatNumRegs(CBSZ);
2116 unsigned Src1NumElts = getFormatNumRegs(BLGP);
2117
2118 // Depending on the used format, fewer registers are required so shrink the
2119 // vector type.
2120 if (Src0Ty->getNumElements() > Src0NumElts) {
2121 Src0 = IC.Builder.CreateExtractVector(
2122 DstType: FixedVectorType::get(ElementType: Src0Ty->getElementType(), NumElts: Src0NumElts), SrcVec: Src0,
2123 Idx: uint64_t(0));
2124 MadeChange = true;
2125 }
2126
2127 if (Src1Ty->getNumElements() > Src1NumElts) {
2128 Src1 = IC.Builder.CreateExtractVector(
2129 DstType: FixedVectorType::get(ElementType: Src1Ty->getElementType(), NumElts: Src1NumElts), SrcVec: Src1,
2130 Idx: uint64_t(0));
2131 MadeChange = true;
2132 }
2133
2134 if (!MadeChange)
2135 return std::nullopt;
2136
2137 SmallVector<Value *, 10> Args(II.args());
2138 Args[0] = Src0;
2139 Args[1] = Src1;
2140
2141 Value *NewII = IC.Builder.CreateIntrinsic(
2142 ID: IID, OverloadTypes: {Src0->getType(), Src1->getType()}, Args, FMFSource: &II);
2143 NewII->takeName(V: &II);
2144 return IC.replaceInstUsesWith(I&: II, V: NewII);
2145 }
2146 case Intrinsic::amdgcn_wmma_f32_16x16x128_f8f6f4:
2147 case Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
2148 case Intrinsic::amdgcn_wmma_scale16_f32_16x16x128_f8f6f4: {
2149 Value *Src0 = II.getArgOperand(i: 1);
2150 Value *Src1 = II.getArgOperand(i: 3);
2151 unsigned FmtA = cast<ConstantInt>(Val: II.getArgOperand(i: 0))->getZExtValue();
2152 uint64_t FmtB = cast<ConstantInt>(Val: II.getArgOperand(i: 2))->getZExtValue();
2153 auto *Src0Ty = cast<FixedVectorType>(Val: Src0->getType());
2154 auto *Src1Ty = cast<FixedVectorType>(Val: Src1->getType());
2155
2156 bool MadeChange = false;
2157 unsigned Src0NumElts = AMDGPU::wmmaScaleF8F6F4FormatToNumRegs(Fmt: FmtA);
2158 unsigned Src1NumElts = AMDGPU::wmmaScaleF8F6F4FormatToNumRegs(Fmt: FmtB);
2159
2160 // Depending on the used format, fewer registers are required so shrink the
2161 // vector type.
2162 if (Src0Ty->getNumElements() > Src0NumElts) {
2163 Src0 = IC.Builder.CreateExtractVector(
2164 DstType: FixedVectorType::get(ElementType: Src0Ty->getElementType(), NumElts: Src0NumElts), SrcVec: Src0,
2165 Idx: IC.Builder.getInt64(C: 0));
2166 MadeChange = true;
2167 }
2168
2169 if (Src1Ty->getNumElements() > Src1NumElts) {
2170 Src1 = IC.Builder.CreateExtractVector(
2171 DstType: FixedVectorType::get(ElementType: Src1Ty->getElementType(), NumElts: Src1NumElts), SrcVec: Src1,
2172 Idx: IC.Builder.getInt64(C: 0));
2173 MadeChange = true;
2174 }
2175
2176 if (!MadeChange)
2177 return std::nullopt;
2178
2179 SmallVector<Value *, 13> Args(II.args());
2180 Args[1] = Src0;
2181 Args[3] = Src1;
2182
2183 Value *NewII = IC.Builder.CreateIntrinsic(
2184 ID: IID, OverloadTypes: {II.getArgOperand(i: 5)->getType(), Src0->getType(), Src1->getType()},
2185 Args, FMFSource: &II);
2186 NewII->takeName(V: &II);
2187 return IC.replaceInstUsesWith(I&: II, V: NewII);
2188 }
2189 }
2190 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
2191 AMDGPU::getImageDimIntrinsicInfo(Intr: II.getIntrinsicID())) {
2192 return simplifyAMDGCNImageIntrinsic(ST, ImageDimIntr, II, IC);
2193 }
2194 return std::nullopt;
2195}
2196
2197/// Implement SimplifyDemandedVectorElts for amdgcn buffer and image intrinsics.
2198///
2199/// The result of simplifying amdgcn image and buffer store intrinsics is updating
2200/// definitions of the intrinsics vector argument, not Uses of the result like
2201/// image and buffer loads.
2202/// Note: This only supports non-TFE/LWE image intrinsic calls; those have
2203/// struct returns.
2204static Value *simplifyAMDGCNMemoryIntrinsicDemanded(InstCombiner &IC,
2205 IntrinsicInst &II,
2206 APInt DemandedElts,
2207 int DMaskIdx, bool IsLoad) {
2208
2209 auto *IIVTy = cast<FixedVectorType>(Val: IsLoad ? II.getType()
2210 : II.getOperand(i_nocapture: 0)->getType());
2211 unsigned VWidth = IIVTy->getNumElements();
2212 if (VWidth == 1)
2213 return nullptr;
2214 Type *EltTy = IIVTy->getElementType();
2215
2216 IRBuilderBase::InsertPointGuard Guard(IC.Builder);
2217 IC.Builder.SetInsertPoint(&II);
2218
2219 // Assume the arguments are unchanged and later override them, if needed.
2220 SmallVector<Value *, 16> Args(II.args());
2221
2222 if (DMaskIdx < 0) {
2223 // Buffer case.
2224
2225 const unsigned ActiveBits = DemandedElts.getActiveBits();
2226 const unsigned UnusedComponentsAtFront = DemandedElts.countr_zero();
2227
2228 // Start assuming the prefix of elements is demanded, but possibly clear
2229 // some other bits if there are trailing zeros (unused components at front)
2230 // and update offset.
2231 DemandedElts = (1 << ActiveBits) - 1;
2232
2233 if (UnusedComponentsAtFront > 0) {
2234 static const unsigned InvalidOffsetIdx = 0xf;
2235
2236 unsigned OffsetIdx;
2237 switch (II.getIntrinsicID()) {
2238 case Intrinsic::amdgcn_raw_buffer_load:
2239 case Intrinsic::amdgcn_raw_ptr_buffer_load:
2240 OffsetIdx = 1;
2241 break;
2242 case Intrinsic::amdgcn_s_buffer_load:
2243 case Intrinsic::amdgcn_ptr_s_buffer_load:
2244 // If resulting type is vec3, there is no point in trimming the
2245 // load with updated offset, as the vec3 would most likely be widened to
2246 // vec4 anyway during lowering.
2247 if (ActiveBits == 4 && UnusedComponentsAtFront == 1)
2248 OffsetIdx = InvalidOffsetIdx;
2249 else
2250 OffsetIdx = 1;
2251 break;
2252 case Intrinsic::amdgcn_struct_buffer_load:
2253 case Intrinsic::amdgcn_struct_ptr_buffer_load:
2254 OffsetIdx = 2;
2255 break;
2256 default:
2257 // TODO: handle tbuffer* intrinsics.
2258 OffsetIdx = InvalidOffsetIdx;
2259 break;
2260 }
2261
2262 if (OffsetIdx != InvalidOffsetIdx) {
2263 // Clear demanded bits and update the offset.
2264 DemandedElts &= ~((1 << UnusedComponentsAtFront) - 1);
2265 auto *Offset = Args[OffsetIdx];
2266 unsigned SingleComponentSizeInBits =
2267 IC.getDataLayout().getTypeSizeInBits(Ty: EltTy);
2268 unsigned OffsetAdd =
2269 UnusedComponentsAtFront * SingleComponentSizeInBits / 8;
2270 auto *OffsetAddVal = ConstantInt::get(Ty: Offset->getType(), V: OffsetAdd);
2271 Args[OffsetIdx] = IC.Builder.CreateAdd(LHS: Offset, RHS: OffsetAddVal);
2272 }
2273 }
2274 } else {
2275 // Image case.
2276
2277 ConstantInt *DMask = cast<ConstantInt>(Val: Args[DMaskIdx]);
2278 unsigned DMaskVal = DMask->getZExtValue() & 0xf;
2279
2280 // dmask 0 has special semantics, do not simplify.
2281 if (DMaskVal == 0)
2282 return nullptr;
2283
2284 if (!IsLoad && !isMask_32(Value: DMaskVal))
2285 return nullptr;
2286
2287 // Mask off values that are undefined because the dmask doesn't cover them
2288 DemandedElts &= (1 << llvm::popcount(Value: DMaskVal)) - 1;
2289
2290 unsigned NewDMaskVal = 0;
2291 unsigned OrigLdStIdx = 0;
2292 for (unsigned SrcIdx = 0; SrcIdx < 4; ++SrcIdx) {
2293 const unsigned Bit = 1 << SrcIdx;
2294 if (!!(DMaskVal & Bit)) {
2295 if (!!DemandedElts[OrigLdStIdx])
2296 NewDMaskVal |= Bit;
2297 OrigLdStIdx++;
2298 }
2299 }
2300
2301 if (DMaskVal != NewDMaskVal)
2302 Args[DMaskIdx] = ConstantInt::get(Ty: DMask->getType(), V: NewDMaskVal);
2303 }
2304
2305 unsigned NewNumElts = DemandedElts.popcount();
2306 if (!NewNumElts)
2307 return PoisonValue::get(T: IIVTy);
2308
2309 if (NewNumElts >= VWidth && DemandedElts.isMask()) {
2310 if (DMaskIdx >= 0)
2311 II.setArgOperand(i: DMaskIdx, v: Args[DMaskIdx]);
2312 return nullptr;
2313 }
2314
2315 // Validate function argument and return types, extracting overloaded types
2316 // along the way.
2317 SmallVector<Type *, 6> OverloadTys;
2318 if (!Intrinsic::isSignatureValid(F: II.getCalledFunction(), OverloadTys))
2319 return nullptr;
2320
2321 Type *NewTy =
2322 (NewNumElts == 1) ? EltTy : FixedVectorType::get(ElementType: EltTy, NumElts: NewNumElts);
2323 OverloadTys[0] = NewTy;
2324
2325 if (!IsLoad) {
2326 SmallVector<int, 8> EltMask;
2327 for (unsigned OrigStoreIdx = 0; OrigStoreIdx < VWidth; ++OrigStoreIdx)
2328 if (DemandedElts[OrigStoreIdx])
2329 EltMask.push_back(Elt: OrigStoreIdx);
2330
2331 if (NewNumElts == 1)
2332 Args[0] = IC.Builder.CreateExtractElement(Vec: II.getOperand(i_nocapture: 0), Idx: EltMask[0]);
2333 else
2334 Args[0] = IC.Builder.CreateShuffleVector(V: II.getOperand(i_nocapture: 0), Mask: EltMask);
2335 }
2336
2337 CallInst *NewCall = IC.Builder.CreateIntrinsicWithoutFolding(
2338 ID: II.getIntrinsicID(), OverloadTypes: OverloadTys, Args);
2339 NewCall->takeName(V: &II);
2340 NewCall->copyMetadata(SrcInst: II);
2341 AttributeList OldAttrList = II.getAttributes();
2342 NewCall->setAttributes(OldAttrList);
2343
2344 if (IsLoad) {
2345 if (NewNumElts == 1) {
2346 return IC.Builder.CreateInsertElement(Vec: PoisonValue::get(T: IIVTy), NewElt: NewCall,
2347 Idx: DemandedElts.countr_zero());
2348 }
2349
2350 SmallVector<int, 8> EltMask;
2351 unsigned NewLoadIdx = 0;
2352 for (unsigned OrigLoadIdx = 0; OrigLoadIdx < VWidth; ++OrigLoadIdx) {
2353 if (!!DemandedElts[OrigLoadIdx])
2354 EltMask.push_back(Elt: NewLoadIdx++);
2355 else
2356 EltMask.push_back(Elt: NewNumElts);
2357 }
2358
2359 auto *Shuffle = IC.Builder.CreateShuffleVector(V: NewCall, Mask: EltMask);
2360
2361 return Shuffle;
2362 }
2363
2364 return NewCall;
2365}
2366
2367Value *GCNTTIImpl::simplifyAMDGCNLaneIntrinsicDemanded(
2368 InstCombiner &IC, IntrinsicInst &II, const APInt &DemandedElts,
2369 APInt &UndefElts) const {
2370 auto *VT = dyn_cast<FixedVectorType>(Val: II.getType());
2371 if (!VT)
2372 return nullptr;
2373
2374 const unsigned FirstElt = DemandedElts.countr_zero();
2375 const unsigned LastElt = DemandedElts.getActiveBits() - 1;
2376 const unsigned MaskLen = LastElt - FirstElt + 1;
2377
2378 unsigned OldNumElts = VT->getNumElements();
2379 if (MaskLen == OldNumElts && MaskLen != 1)
2380 return nullptr;
2381
2382 Type *EltTy = VT->getElementType();
2383 Type *NewVT = MaskLen == 1 ? EltTy : FixedVectorType::get(ElementType: EltTy, NumElts: MaskLen);
2384
2385 // Theoretically we should support these intrinsics for any legal type. Avoid
2386 // introducing cases that aren't direct register types like v3i16.
2387 if (!isTypeLegal(Ty: NewVT))
2388 return nullptr;
2389
2390 Value *Src = II.getArgOperand(i: 0);
2391
2392 // Make sure convergence tokens are preserved.
2393 // TODO: CreateIntrinsic should allow directly copying bundles
2394 SmallVector<OperandBundleDef, 2> OpBundles;
2395 II.getOperandBundlesAsDefs(Defs&: OpBundles);
2396
2397 Module *M = IC.Builder.GetInsertBlock()->getModule();
2398 Function *Remangled =
2399 Intrinsic::getOrInsertDeclaration(M, id: II.getIntrinsicID(), OverloadTys: {NewVT});
2400
2401 if (MaskLen == 1) {
2402 Value *Extract = IC.Builder.CreateExtractElement(Vec: Src, Idx: FirstElt);
2403
2404 // TODO: Preserve callsite attributes?
2405 CallInst *NewCall = IC.Builder.CreateCall(Callee: Remangled, Args: {Extract}, OpBundles);
2406
2407 return IC.Builder.CreateInsertElement(Vec: PoisonValue::get(T: II.getType()),
2408 NewElt: NewCall, Idx: FirstElt);
2409 }
2410
2411 SmallVector<int> ExtractMask(MaskLen, -1);
2412 for (unsigned I = 0; I != MaskLen; ++I) {
2413 if (DemandedElts[FirstElt + I])
2414 ExtractMask[I] = FirstElt + I;
2415 }
2416
2417 Value *Extract = IC.Builder.CreateShuffleVector(V: Src, Mask: ExtractMask);
2418
2419 // TODO: Preserve callsite attributes?
2420 CallInst *NewCall = IC.Builder.CreateCall(Callee: Remangled, Args: {Extract}, OpBundles);
2421
2422 SmallVector<int> InsertMask(OldNumElts, -1);
2423 for (unsigned I = 0; I != MaskLen; ++I) {
2424 if (DemandedElts[FirstElt + I])
2425 InsertMask[FirstElt + I] = I;
2426 }
2427
2428 // FIXME: If the call has a convergence bundle, we end up leaving the dead
2429 // call behind.
2430 return IC.Builder.CreateShuffleVector(V: NewCall, Mask: InsertMask);
2431}
2432
2433std::optional<Value *> GCNTTIImpl::simplifyDemandedVectorEltsIntrinsic(
2434 InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
2435 APInt &UndefElts2, APInt &UndefElts3,
2436 std::function<void(Instruction *, unsigned, APInt, APInt &)>
2437 SimplifyAndSetOp) const {
2438 switch (II.getIntrinsicID()) {
2439 case Intrinsic::amdgcn_readfirstlane:
2440 SimplifyAndSetOp(&II, 0, DemandedElts, UndefElts);
2441 return simplifyAMDGCNLaneIntrinsicDemanded(IC, II, DemandedElts, UndefElts);
2442 case Intrinsic::amdgcn_raw_buffer_load:
2443 case Intrinsic::amdgcn_raw_ptr_buffer_load:
2444 case Intrinsic::amdgcn_raw_buffer_load_format:
2445 case Intrinsic::amdgcn_raw_ptr_buffer_load_format:
2446 case Intrinsic::amdgcn_raw_tbuffer_load:
2447 case Intrinsic::amdgcn_raw_ptr_tbuffer_load:
2448 case Intrinsic::amdgcn_s_buffer_load:
2449 case Intrinsic::amdgcn_ptr_s_buffer_load:
2450 case Intrinsic::amdgcn_struct_buffer_load:
2451 case Intrinsic::amdgcn_struct_ptr_buffer_load:
2452 case Intrinsic::amdgcn_struct_buffer_load_format:
2453 case Intrinsic::amdgcn_struct_ptr_buffer_load_format:
2454 case Intrinsic::amdgcn_struct_tbuffer_load:
2455 case Intrinsic::amdgcn_struct_ptr_tbuffer_load:
2456 return simplifyAMDGCNMemoryIntrinsicDemanded(IC, II, DemandedElts);
2457 default: {
2458 if (getAMDGPUImageDMaskIntrinsic(Intr: II.getIntrinsicID())) {
2459 return simplifyAMDGCNMemoryIntrinsicDemanded(IC, II, DemandedElts, DMaskIdx: 0);
2460 }
2461 break;
2462 }
2463 }
2464 return std::nullopt;
2465}
2466