1//===- AMDGPULegalizerInfo.cpp -----------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the Machinelegalizer class for
10/// AMDGPU.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPULegalizerInfo.h"
15
16#include "AMDGPU.h"
17#include "AMDGPUGlobalISelUtils.h"
18#include "AMDGPUInstrInfo.h"
19#include "AMDGPUMemoryUtils.h"
20#include "AMDGPUTargetMachine.h"
21#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
22#include "SIInstrInfo.h"
23#include "SIMachineFunctionInfo.h"
24#include "SIRegisterInfo.h"
25#include "Utils/AMDGPUBaseInfo.h"
26#include "llvm/ADT/ScopeExit.h"
27#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
28#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
29#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
30#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
31#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
32#include "llvm/CodeGen/GlobalISel/Utils.h"
33#include "llvm/CodeGen/MachineFrameInfo.h"
34#include "llvm/CodeGen/PseudoSourceValueManager.h"
35#include "llvm/CodeGen/TargetOpcodes.h"
36#include "llvm/IR/DiagnosticInfo.h"
37#include "llvm/IR/IntrinsicsAMDGPU.h"
38#include "llvm/IR/IntrinsicsR600.h"
39
40#define DEBUG_TYPE "amdgpu-legalinfo"
41
42using namespace llvm;
43using namespace LegalizeActions;
44using namespace LegalizeMutations;
45using namespace LegalityPredicates;
46using namespace MIPatternMatch;
47
48// Hack until load/store selection patterns support any tuple of legal types.
49static cl::opt<bool> EnableNewLegality(
50 "amdgpu-global-isel-new-legality",
51 cl::desc("Use GlobalISel desired legality, rather than try to use"
52 "rules compatible with selection patterns"),
53 cl::init(Val: false),
54 cl::ReallyHidden);
55
56static constexpr unsigned MaxRegisterSize = 1024;
57
58// Round the number of elements to the next power of two elements
59static LLT getPow2VectorType(LLT Ty) {
60 unsigned NElts = Ty.getNumElements();
61 unsigned Pow2NElts = 1 << Log2_32_Ceil(Value: NElts);
62 return Ty.changeElementCount(EC: ElementCount::getFixed(MinVal: Pow2NElts));
63}
64
65// Round the number of bits to the next power of two bits
66static LLT getPow2ScalarType(LLT Ty) {
67 unsigned Bits = Ty.getSizeInBits();
68 unsigned Pow2Bits = 1 << Log2_32_Ceil(Value: Bits);
69 return LLT::scalar(SizeInBits: Pow2Bits);
70}
71
72/// \returns true if this is an odd sized vector which should widen by adding an
73/// additional element. This is mostly to handle <3 x s16> -> <4 x s16>. This
74/// excludes s1 vectors, which should always be scalarized.
75static LegalityPredicate isSmallOddVector(unsigned TypeIdx) {
76 return [=](const LegalityQuery &Query) {
77 const LLT Ty = Query.Types[TypeIdx];
78 if (!Ty.isVector())
79 return false;
80
81 const LLT EltTy = Ty.getElementType();
82 const unsigned EltSize = EltTy.getSizeInBits();
83 return Ty.getNumElements() % 2 != 0 &&
84 EltSize > 1 && EltSize < 32 &&
85 Ty.getSizeInBits() % 32 != 0;
86 };
87}
88
89static LegalityPredicate sizeIsMultipleOf32(unsigned TypeIdx) {
90 return [=](const LegalityQuery &Query) {
91 const LLT Ty = Query.Types[TypeIdx];
92 return Ty.getSizeInBits() % 32 == 0;
93 };
94}
95
96static LegalityPredicate isWideVec16(unsigned TypeIdx) {
97 return [=](const LegalityQuery &Query) {
98 const LLT Ty = Query.Types[TypeIdx];
99 const LLT EltTy = Ty.getScalarType();
100 return EltTy.getSizeInBits() == 16 && Ty.getNumElements() > 2;
101 };
102}
103
104static LegalizeMutation oneMoreElement(unsigned TypeIdx) {
105 return [=](const LegalityQuery &Query) {
106 const LLT Ty = Query.Types[TypeIdx];
107 const LLT EltTy = Ty.getElementType();
108 return std::pair(TypeIdx,
109 LLT::fixed_vector(NumElements: Ty.getNumElements() + 1, ScalarTy: EltTy));
110 };
111}
112
113static LegalizeMutation fewerEltsToSize64Vector(unsigned TypeIdx) {
114 return [=](const LegalityQuery &Query) {
115 const LLT Ty = Query.Types[TypeIdx];
116 const LLT EltTy = Ty.getElementType();
117 unsigned Size = Ty.getSizeInBits();
118 unsigned Pieces = (Size + 63) / 64;
119 unsigned NewNumElts = (Ty.getNumElements() + 1) / Pieces;
120 return std::pair(TypeIdx, LLT::scalarOrVector(
121 EC: ElementCount::getFixed(MinVal: NewNumElts), ScalarTy: EltTy));
122 };
123}
124
125// Increase the number of vector elements to reach the next multiple of 32-bit
126// type.
127static LegalizeMutation moreEltsToNext32Bit(unsigned TypeIdx) {
128 return [=](const LegalityQuery &Query) {
129 const LLT Ty = Query.Types[TypeIdx];
130
131 const LLT EltTy = Ty.getElementType();
132 const int Size = Ty.getSizeInBits();
133 const int EltSize = EltTy.getSizeInBits();
134 const int NextMul32 = (Size + 31) / 32;
135
136 assert(EltSize < 32);
137
138 const int NewNumElts = (32 * NextMul32 + EltSize - 1) / EltSize;
139 return std::pair(TypeIdx, LLT::fixed_vector(NumElements: NewNumElts, ScalarTy: EltTy));
140 };
141}
142
143// Retrieves the scalar type that's the same size as the mem desc
144static LegalizeMutation getScalarTypeFromMemDesc(unsigned TypeIdx) {
145 return [=](const LegalityQuery &Query) {
146 unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
147 return std::make_pair(x: TypeIdx, y: LLT::integer(SizeInBits: MemSize));
148 };
149}
150
151// Increase the number of vector elements to reach the next legal RegClass.
152static LegalizeMutation moreElementsToNextExistingRegClass(unsigned TypeIdx) {
153 return [=](const LegalityQuery &Query) {
154 const LLT Ty = Query.Types[TypeIdx];
155 const unsigned NumElts = Ty.getNumElements();
156 const unsigned EltSize = Ty.getElementType().getSizeInBits();
157 const unsigned MaxNumElts = MaxRegisterSize / EltSize;
158
159 assert(EltSize == 32 || EltSize == 64);
160 assert(Ty.getSizeInBits() < MaxRegisterSize);
161
162 unsigned NewNumElts;
163 // Find the nearest legal RegClass that is larger than the current type.
164 for (NewNumElts = NumElts; NewNumElts < MaxNumElts; ++NewNumElts) {
165 if (SIRegisterInfo::getSGPRClassForBitWidth(BitWidth: NewNumElts * EltSize))
166 break;
167 }
168 return std::pair(TypeIdx,
169 LLT::fixed_vector(NumElements: NewNumElts, ScalarTy: Ty.getElementType()));
170 };
171}
172
173static LLT getBufferRsrcScalarType(const LLT Ty) {
174 if (!Ty.isVector())
175 return LLT::scalar(SizeInBits: 128);
176 const ElementCount NumElems = Ty.getElementCount();
177 return LLT::vector(EC: NumElems, ScalarTy: LLT::scalar(SizeInBits: 128));
178}
179
180static LLT getBufferRsrcRegisterType(const LLT Ty) {
181 if (!Ty.isVector())
182 return LLT::fixed_vector(NumElements: 4, ScalarTy: LLT::integer(SizeInBits: 32));
183 const unsigned NumElems = Ty.getElementCount().getFixedValue();
184 return LLT::fixed_vector(NumElements: NumElems * 4, ScalarTy: LLT::integer(SizeInBits: 32));
185}
186
187static LLT getBitcastRegisterType(const LLT Ty) {
188 const unsigned Size = Ty.getSizeInBits();
189
190 if (Size <= 32) {
191 // <2 x i8> -> i16
192 // <4 x i8> -> i32
193 return LLT::integer(SizeInBits: Size);
194 }
195
196 return LLT::fixed_vector(NumElements: Size / 32, ScalarTy: LLT::integer(SizeInBits: 32));
197}
198
199static LegalizeMutation bitcastToRegisterType(unsigned TypeIdx) {
200 return [=](const LegalityQuery &Query) {
201 const LLT Ty = Query.Types[TypeIdx];
202 return std::pair(TypeIdx, getBitcastRegisterType(Ty));
203 };
204}
205
206static LegalizeMutation bitcastToVectorElement32(unsigned TypeIdx) {
207 return [=](const LegalityQuery &Query) {
208 const LLT Ty = Query.Types[TypeIdx];
209 unsigned Size = Ty.getSizeInBits();
210 assert(Size % 32 == 0);
211 return std::pair(TypeIdx,
212 LLT::scalarOrVector(EC: ElementCount::getFixed(MinVal: Size / 32),
213 ScalarTy: LLT::integer(SizeInBits: 32)));
214 };
215}
216
217static LegalityPredicate vectorSmallerThan(unsigned TypeIdx, unsigned Size) {
218 return [=](const LegalityQuery &Query) {
219 const LLT QueryTy = Query.Types[TypeIdx];
220 return QueryTy.isVector() && QueryTy.getSizeInBits() < Size;
221 };
222}
223
224static LegalityPredicate vectorWiderThan(unsigned TypeIdx, unsigned Size) {
225 return [=](const LegalityQuery &Query) {
226 const LLT QueryTy = Query.Types[TypeIdx];
227 return QueryTy.isVector() && QueryTy.getSizeInBits() > Size;
228 };
229}
230
231static LegalityPredicate numElementsNotEven(unsigned TypeIdx) {
232 return [=](const LegalityQuery &Query) {
233 const LLT QueryTy = Query.Types[TypeIdx];
234 return QueryTy.isVector() && QueryTy.getNumElements() % 2 != 0;
235 };
236}
237
238static bool isRegisterSize(const GCNSubtarget &ST, unsigned Size) {
239 return ((ST.useRealTrue16Insts() && Size == 16) || Size % 32 == 0) &&
240 Size <= MaxRegisterSize;
241}
242
243static bool isRegisterVectorElementType(LLT EltTy) {
244 const int EltSize = EltTy.getSizeInBits();
245 return EltSize == 16 || EltSize % 32 == 0;
246}
247
248static bool isRegisterVectorType(LLT Ty) {
249 const int EltSize = Ty.getElementType().getSizeInBits();
250 return EltSize == 32 || EltSize == 64 ||
251 (EltSize == 16 && Ty.getNumElements() % 2 == 0) ||
252 EltSize == 128 || EltSize == 256;
253}
254
255// TODO: replace all uses of isRegisterType with isRegisterClassType
256static bool isRegisterType(const GCNSubtarget &ST, LLT Ty) {
257 if (!isRegisterSize(ST, Size: Ty.getSizeInBits()))
258 return false;
259
260 if (Ty.isVector())
261 return isRegisterVectorType(Ty);
262
263 return true;
264}
265
266// Any combination of 32 or 64-bit elements up the maximum register size, and
267// multiples of v2s16.
268static LegalityPredicate isRegisterType(const GCNSubtarget &ST,
269 unsigned TypeIdx) {
270 return [=, &ST](const LegalityQuery &Query) {
271 return isRegisterType(ST, Ty: Query.Types[TypeIdx]);
272 };
273}
274
275// RegisterType that doesn't have a corresponding RegClass.
276// TODO: Once `isRegisterType` is replaced with `isRegisterClassType` this
277// should be removed.
278static LegalityPredicate isIllegalRegisterType(const GCNSubtarget &ST,
279 unsigned TypeIdx) {
280 return [=, &ST](const LegalityQuery &Query) {
281 LLT Ty = Query.Types[TypeIdx];
282 return isRegisterType(ST, Ty) &&
283 !SIRegisterInfo::getSGPRClassForBitWidth(BitWidth: Ty.getSizeInBits());
284 };
285}
286
287static LegalityPredicate elementTypeIsLegal(unsigned TypeIdx) {
288 return [=](const LegalityQuery &Query) {
289 const LLT QueryTy = Query.Types[TypeIdx];
290 if (!QueryTy.isVector())
291 return false;
292 const LLT EltTy = QueryTy.getElementType();
293 return EltTy == LLT::scalar(SizeInBits: 16) || EltTy.getSizeInBits() >= 32;
294 };
295}
296
297const LLT I16 = LLT::integer(SizeInBits: 16);
298constexpr LLT F16 = LLT::float16();
299constexpr LLT BF16 = LLT::bfloat16();
300
301constexpr LLT S1 = LLT::scalar(SizeInBits: 1);
302constexpr LLT S8 = LLT::scalar(SizeInBits: 8);
303constexpr LLT S16 = LLT::scalar(SizeInBits: 16);
304constexpr LLT S32 = LLT::scalar(SizeInBits: 32);
305constexpr LLT F32 = LLT::scalar(SizeInBits: 32); // TODO: Expected float32
306constexpr LLT S64 = LLT::scalar(SizeInBits: 64);
307constexpr LLT F64 = LLT::scalar(SizeInBits: 64); // TODO: Expected float64
308constexpr LLT S96 = LLT::scalar(SizeInBits: 96);
309constexpr LLT S128 = LLT::scalar(SizeInBits: 128);
310constexpr LLT S160 = LLT::scalar(SizeInBits: 160);
311constexpr LLT S192 = LLT::scalar(SizeInBits: 192);
312constexpr LLT S224 = LLT::scalar(SizeInBits: 224);
313constexpr LLT S256 = LLT::scalar(SizeInBits: 256);
314constexpr LLT S512 = LLT::scalar(SizeInBits: 512);
315constexpr LLT S1024 = LLT::scalar(SizeInBits: 1024);
316constexpr LLT MaxScalar = LLT::scalar(SizeInBits: MaxRegisterSize);
317
318constexpr LLT V2S8 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 8);
319constexpr LLT V2S16 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16);
320constexpr LLT V4S16 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16);
321constexpr LLT V6S16 = LLT::fixed_vector(NumElements: 6, ScalarSizeInBits: 16);
322constexpr LLT V8S16 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16);
323constexpr LLT V10S16 = LLT::fixed_vector(NumElements: 10, ScalarSizeInBits: 16);
324constexpr LLT V12S16 = LLT::fixed_vector(NumElements: 12, ScalarSizeInBits: 16);
325constexpr LLT V16S16 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 16);
326
327// TODO: Expected LLT::fixed_vector(2, LLT::float16())
328constexpr LLT V2F16 = LLT::fixed_vector(NumElements: 2, ScalarTy: LLT::scalar(SizeInBits: 16));
329constexpr LLT V2BF16 = V2F16; // FIXME
330
331constexpr LLT V2S32 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32);
332constexpr LLT V3S32 = LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32);
333constexpr LLT V4S32 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32);
334constexpr LLT V5S32 = LLT::fixed_vector(NumElements: 5, ScalarSizeInBits: 32);
335constexpr LLT V6S32 = LLT::fixed_vector(NumElements: 6, ScalarSizeInBits: 32);
336constexpr LLT V7S32 = LLT::fixed_vector(NumElements: 7, ScalarSizeInBits: 32);
337constexpr LLT V8S32 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 32);
338constexpr LLT V9S32 = LLT::fixed_vector(NumElements: 9, ScalarSizeInBits: 32);
339constexpr LLT V10S32 = LLT::fixed_vector(NumElements: 10, ScalarSizeInBits: 32);
340constexpr LLT V11S32 = LLT::fixed_vector(NumElements: 11, ScalarSizeInBits: 32);
341constexpr LLT V12S32 = LLT::fixed_vector(NumElements: 12, ScalarSizeInBits: 32);
342constexpr LLT V16S32 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 32);
343constexpr LLT V32S32 = LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 32);
344
345constexpr LLT V2S64 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64);
346constexpr LLT V3S64 = LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 64);
347constexpr LLT V4S64 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 64);
348constexpr LLT V5S64 = LLT::fixed_vector(NumElements: 5, ScalarSizeInBits: 64);
349constexpr LLT V6S64 = LLT::fixed_vector(NumElements: 6, ScalarSizeInBits: 64);
350constexpr LLT V7S64 = LLT::fixed_vector(NumElements: 7, ScalarSizeInBits: 64);
351constexpr LLT V8S64 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 64);
352constexpr LLT V16S64 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 64);
353
354constexpr LLT V2S128 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 128);
355constexpr LLT V4S128 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 128);
356
357constexpr std::initializer_list<LLT> AllScalarTypes = {
358 S32, S64, S96, S128, S160, S192, S224, S256, S512, S1024};
359
360constexpr std::initializer_list<LLT> AllS16Vectors{
361 V2S16, V4S16, V6S16, V8S16, V10S16, V12S16, V16S16, V2S128, V4S128};
362
363constexpr std::initializer_list<LLT> AllS32Vectors = {
364 V2S32, V3S32, V4S32, V5S32, V6S32, V7S32, V8S32,
365 V9S32, V10S32, V11S32, V12S32, V16S32, V32S32};
366
367constexpr std::initializer_list<LLT> AllS64Vectors = {
368 V2S64, V3S64, V4S64, V5S64, V6S64, V7S64, V8S64, V16S64};
369
370constexpr std::initializer_list<LLT> AllVectors{
371 V2S16, V4S16, V6S16, V8S16, V10S16, V12S16, V16S16, V2S128,
372 V4S128, V2S32, V3S32, V4S32, V5S32, V6S32, V7S32, V8S32,
373 V9S32, V10S32, V11S32, V12S32, V16S32, V32S32, V2S64, V3S64,
374 V4S64, V5S64, V6S64, V7S64, V8S64, V16S64};
375
376// Checks whether a type is in the list of legal register types.
377static bool isRegisterClassType(const GCNSubtarget &ST, LLT Ty) {
378 if (Ty.isPointerOrPointerVector())
379 Ty = Ty.changeElementType(NewEltTy: LLT::scalar(SizeInBits: Ty.getScalarSizeInBits()));
380
381 return is_contained(Set: AllS32Vectors, Element: Ty) || is_contained(Set: AllS64Vectors, Element: Ty) ||
382 is_contained(Set: AllScalarTypes, Element: Ty) ||
383 (ST.useRealTrue16Insts() && Ty == S16) ||
384 is_contained(Set: AllS16Vectors, Element: Ty);
385}
386
387static LegalityPredicate isRegisterClassType(const GCNSubtarget &ST,
388 unsigned TypeIdx) {
389 return [&ST, TypeIdx](const LegalityQuery &Query) {
390 return isRegisterClassType(ST, Ty: Query.Types[TypeIdx]);
391 };
392}
393
394// If we have a truncating store or an extending load with a data size larger
395// than 32-bits, we need to reduce to a 32-bit type.
396static LegalityPredicate isWideScalarExtLoadTruncStore(unsigned TypeIdx) {
397 return [=](const LegalityQuery &Query) {
398 const LLT Ty = Query.Types[TypeIdx];
399 return !Ty.isVector() && Ty.getSizeInBits() > 32 &&
400 Query.MMODescrs[0].MemoryTy.getSizeInBits() < Ty.getSizeInBits();
401 };
402}
403
404// If we have a truncating store or an extending load with a data size larger
405// than 32-bits and mem location is a power of 2
406static LegalityPredicate isTruncStoreToSizePowerOf2(unsigned TypeIdx) {
407 return [=](const LegalityQuery &Query) {
408 unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
409 return isWideScalarExtLoadTruncStore(TypeIdx)(Query) &&
410 isPowerOf2_64(Value: MemSize);
411 };
412}
413
414// TODO: Should load to s16 be legal? Most loads extend to 32-bits, but we
415// handle some operations by just promoting the register during
416// selection. There are also d16 loads on GFX9+ which preserve the high bits.
417static unsigned maxSizeForAddrSpace(const GCNSubtarget &ST, unsigned AS,
418 bool IsLoad, bool IsAtomic) {
419 switch (AS) {
420 case AMDGPUAS::PRIVATE_ADDRESS:
421 // FIXME: Private element size.
422 return ST.hasFlatScratchEnabled() ? 128 : 32;
423 case AMDGPUAS::LOCAL_ADDRESS:
424 return ST.useDS128() ? 128 : 64;
425 case AMDGPUAS::GLOBAL_ADDRESS:
426 case AMDGPUAS::CONSTANT_ADDRESS:
427 case AMDGPUAS::CONSTANT_ADDRESS_32BIT:
428 case AMDGPUAS::BUFFER_RESOURCE:
429 // Treat constant and global as identical. SMRD loads are sometimes usable for
430 // global loads (ideally constant address space should be eliminated)
431 // depending on the context. Legality cannot be context dependent, but
432 // RegBankSelect can split the load as necessary depending on the pointer
433 // register bank/uniformity and if the memory is invariant or not written in a
434 // kernel.
435 return IsLoad ? 512 : 128;
436 default:
437 // FIXME: Flat addresses may contextually need to be split to 32-bit parts
438 // if they may alias scratch depending on the subtarget. This needs to be
439 // moved to custom handling to use addressMayBeAccessedAsPrivate
440 return ST.hasMultiDwordFlatScratchAddressing() || IsAtomic ? 128 : 32;
441 }
442}
443
444static bool isLoadStoreSizeLegal(const GCNSubtarget &ST,
445 const LegalityQuery &Query) {
446 const LLT Ty = Query.Types[0];
447
448 // Handle G_LOAD, G_ZEXTLOAD, G_SEXTLOAD
449 const bool IsLoad = Query.Opcode != AMDGPU::G_STORE;
450
451 unsigned RegSize = Ty.getSizeInBits();
452 uint64_t MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
453 uint64_t AlignBits = Query.MMODescrs[0].AlignInBits;
454 unsigned AS = Query.Types[1].getAddressSpace();
455
456 // All of these need to be custom lowered to cast the pointer operand.
457 if (AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT)
458 return false;
459
460 // Do not handle extending vector loads.
461 if (Ty.isVector() && MemSize != RegSize)
462 return false;
463
464 // TODO: We should be able to widen loads if the alignment is high enough, but
465 // we also need to modify the memory access size.
466#if 0
467 // Accept widening loads based on alignment.
468 if (IsLoad && MemSize < Size)
469 MemSize = std::max(MemSize, Align);
470#endif
471
472 // Only 1-byte and 2-byte to 32-bit extloads are valid.
473 if (MemSize != RegSize && RegSize != 32)
474 return false;
475
476 if (MemSize > maxSizeForAddrSpace(ST, AS, IsLoad,
477 IsAtomic: Query.MMODescrs[0].Ordering !=
478 AtomicOrdering::NotAtomic))
479 return false;
480
481 switch (MemSize) {
482 case 8:
483 case 16:
484 case 32:
485 case 64:
486 case 128:
487 break;
488 case 96:
489 if (!ST.hasDwordx3LoadStores())
490 return false;
491 break;
492 case 256:
493 case 512:
494 // These may contextually need to be broken down.
495 break;
496 default:
497 return false;
498 }
499
500 assert(RegSize >= MemSize);
501
502 if (AlignBits < MemSize) {
503 const SITargetLowering *TLI = ST.getTargetLowering();
504 if (!TLI->allowsMisalignedMemoryAccessesImpl(Size: MemSize, AddrSpace: AS,
505 Alignment: Align(AlignBits / 8)))
506 return false;
507 }
508
509 return true;
510}
511
512// The newer buffer intrinsic forms take their resource arguments as
513// pointers in address space 8, aka s128 values. However, in order to not break
514// SelectionDAG, the underlying operations have to continue to take v4i32
515// arguments. Therefore, we convert resource pointers - or vectors of them
516// to integer values here.
517static bool hasBufferRsrcWorkaround(const LLT Ty) {
518 if (Ty.isPointer() && Ty.getAddressSpace() == AMDGPUAS::BUFFER_RESOURCE)
519 return true;
520 if (Ty.isVector()) {
521 const LLT ElemTy = Ty.getElementType();
522 return hasBufferRsrcWorkaround(Ty: ElemTy);
523 }
524 return false;
525}
526
527// The current selector can't handle <6 x s16>, <8 x s16>, s96, s128 etc, so
528// workaround this. Eventually it should ignore the type for loads and only care
529// about the size. Return true in cases where we will workaround this for now by
530// bitcasting.
531static bool loadStoreBitcastWorkaround(const LLT Ty) {
532 if (EnableNewLegality)
533 return false;
534
535 const unsigned Size = Ty.getSizeInBits();
536 if (Ty.isPointerVector())
537 return true;
538 if (Size <= 64)
539 return false;
540 // Address space 8 pointers get their own workaround.
541 if (hasBufferRsrcWorkaround(Ty))
542 return false;
543 if (!Ty.isVector())
544 return true;
545
546 unsigned EltSize = Ty.getScalarSizeInBits();
547 return EltSize != 32 && EltSize != 64;
548}
549
550static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query) {
551 const LLT Ty = Query.Types[0];
552 return isRegisterType(ST, Ty) && isLoadStoreSizeLegal(ST, Query) &&
553 !hasBufferRsrcWorkaround(Ty) && !loadStoreBitcastWorkaround(Ty);
554}
555
556/// Return true if a load or store of the type should be lowered with a bitcast
557/// to a different type.
558static bool shouldBitcastLoadStoreType(const GCNSubtarget &ST, const LLT Ty,
559 const LLT MemTy) {
560 const unsigned MemSizeInBits = MemTy.getSizeInBits();
561 const unsigned Size = Ty.getSizeInBits();
562 if (Size != MemSizeInBits)
563 return Size <= 32 && Ty.isVector();
564
565 if (loadStoreBitcastWorkaround(Ty) && isRegisterType(ST, Ty))
566 return true;
567
568 // Don't try to handle bitcasting vector ext loads for now.
569 return Ty.isVector() && (!MemTy.isVector() || MemTy == Ty) &&
570 (Size <= 32 || isRegisterSize(ST, Size)) &&
571 !isRegisterVectorElementType(EltTy: Ty.getElementType());
572}
573
574/// Return true if we should legalize a load by widening an odd sized memory
575/// access up to the alignment. Note this case when the memory access itself
576/// changes, not the size of the result register.
577static bool shouldWidenLoad(const GCNSubtarget &ST, LLT MemoryTy,
578 uint64_t AlignInBits, unsigned AddrSpace,
579 unsigned Opcode) {
580 unsigned SizeInBits = MemoryTy.getSizeInBits();
581 // We don't want to widen cases that are naturally legal.
582 if (isPowerOf2_32(Value: SizeInBits))
583 return false;
584
585 // If we have 96-bit memory operations, we shouldn't touch them. Note we may
586 // end up widening these for a scalar load during RegBankSelect, if we don't
587 // have 96-bit scalar loads.
588 if (SizeInBits == 96 && ST.hasDwordx3LoadStores())
589 return false;
590
591 if (SizeInBits >= maxSizeForAddrSpace(ST, AS: AddrSpace, IsLoad: Opcode, IsAtomic: false))
592 return false;
593
594 // A load is known dereferenceable up to the alignment, so it's legal to widen
595 // to it.
596 //
597 // TODO: Could check dereferenceable for less aligned cases.
598 unsigned RoundedSize = NextPowerOf2(A: SizeInBits);
599 if (AlignInBits < RoundedSize)
600 return false;
601
602 // Do not widen if it would introduce a slow unaligned load.
603 const SITargetLowering *TLI = ST.getTargetLowering();
604 unsigned Fast = 0;
605 return TLI->allowsMisalignedMemoryAccessesImpl(
606 Size: RoundedSize, AddrSpace, Alignment: Align(AlignInBits / 8),
607 Flags: MachineMemOperand::MOLoad, IsFast: &Fast) &&
608 Fast;
609}
610
611static bool shouldWidenLoad(const GCNSubtarget &ST, const LegalityQuery &Query,
612 unsigned Opcode) {
613 if (Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic)
614 return false;
615
616 return shouldWidenLoad(ST, MemoryTy: Query.MMODescrs[0].MemoryTy,
617 AlignInBits: Query.MMODescrs[0].AlignInBits,
618 AddrSpace: Query.Types[1].getAddressSpace(), Opcode);
619}
620
621/// Mutates IR (typicaly a load instruction) to use a <4 x s32> as the initial
622/// type of the operand `idx` and then to transform it to a `p8` via bitcasts
623/// and inttoptr. In addition, handle vectors of p8. Returns the new type.
624static LLT castBufferRsrcFromV4I32(MachineInstr &MI, MachineIRBuilder &B,
625 MachineRegisterInfo &MRI, unsigned Idx) {
626 MachineOperand &MO = MI.getOperand(i: Idx);
627
628 const LLT PointerTy = MRI.getType(Reg: MO.getReg());
629
630 // Paranoidly prevent us from doing this multiple times.
631 if (!hasBufferRsrcWorkaround(Ty: PointerTy))
632 return PointerTy;
633
634 const LLT ScalarTy = getBufferRsrcScalarType(Ty: PointerTy);
635 const LLT VectorTy = getBufferRsrcRegisterType(Ty: PointerTy);
636 if (!PointerTy.isVector()) {
637 // Happy path: (4 x s32) -> (s32, s32, s32, s32) -> (p8)
638 const unsigned NumParts = PointerTy.getSizeInBits() / 32;
639 const LLT I32 = LLT::integer(SizeInBits: 32);
640
641 Register VectorReg = MRI.createGenericVirtualRegister(Ty: VectorTy);
642 std::array<Register, 4> VectorElems;
643 B.setInsertPt(MBB&: B.getMBB(), II: ++B.getInsertPt());
644 for (unsigned I = 0; I < NumParts; ++I)
645 VectorElems[I] =
646 B.buildExtractVectorElementConstant(Res: I32, Val: VectorReg, Idx: I).getReg(Idx: 0);
647 B.buildMergeValues(Res: MO, Ops: VectorElems);
648 MO.setReg(VectorReg);
649 return VectorTy;
650 }
651 Register BitcastReg = MRI.createGenericVirtualRegister(Ty: VectorTy);
652 B.setInsertPt(MBB&: B.getMBB(), II: ++B.getInsertPt());
653 auto Scalar = B.buildBitcast(Dst: ScalarTy, Src: BitcastReg);
654 B.buildIntToPtr(Dst: MO, Src: Scalar);
655 MO.setReg(BitcastReg);
656
657 return VectorTy;
658}
659
660/// Cast a buffer resource (an address space 8 pointer) into a 4xi32, which is
661/// the form in which the value must be in order to be passed to the low-level
662/// representations used for MUBUF/MTBUF intrinsics. This is a hack, which is
663/// needed in order to account for the fact that we can't define a register
664/// class for s128 without breaking SelectionDAG.
665static Register castBufferRsrcToV4I32(Register Pointer, MachineIRBuilder &B) {
666 MachineRegisterInfo &MRI = *B.getMRI();
667 const LLT PointerTy = MRI.getType(Reg: Pointer);
668 const LLT ScalarTy = getBufferRsrcScalarType(Ty: PointerTy);
669 const LLT VectorTy = getBufferRsrcRegisterType(Ty: PointerTy);
670
671 if (!PointerTy.isVector()) {
672 // Special case: p8 -> (s32, s32, s32, s32) -> (4xs32)
673 SmallVector<Register, 4> PointerParts;
674 const unsigned NumParts = PointerTy.getSizeInBits() / 32;
675 auto Unmerged = B.buildUnmerge(Res: LLT::integer(SizeInBits: 32), Op: Pointer);
676 for (unsigned I = 0; I < NumParts; ++I)
677 PointerParts.push_back(Elt: Unmerged.getReg(Idx: I));
678 return B.buildBuildVector(Res: VectorTy, Ops: PointerParts).getReg(Idx: 0);
679 }
680 Register Scalar = B.buildPtrToInt(Dst: ScalarTy, Src: Pointer).getReg(Idx: 0);
681 return B.buildBitcast(Dst: VectorTy, Src: Scalar).getReg(Idx: 0);
682}
683
684static void castBufferRsrcArgToV4I32(MachineInstr &MI, MachineIRBuilder &B,
685 unsigned Idx) {
686 MachineOperand &MO = MI.getOperand(i: Idx);
687
688 const LLT PointerTy = B.getMRI()->getType(Reg: MO.getReg());
689 // Paranoidly prevent us from doing this multiple times.
690 if (!hasBufferRsrcWorkaround(Ty: PointerTy))
691 return;
692 MO.setReg(castBufferRsrcToV4I32(Pointer: MO.getReg(), B));
693}
694
695AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
696 const GCNTargetMachine &TM)
697 : ST(ST_) {
698 using namespace TargetOpcode;
699
700 auto GetAddrSpacePtr = [&TM](unsigned AS) {
701 return LLT::pointer(AddressSpace: AS, SizeInBits: TM.getPointerSizeInBits(AS));
702 };
703
704 const LLT GlobalPtr = GetAddrSpacePtr(AMDGPUAS::GLOBAL_ADDRESS);
705 const LLT ConstantPtr = GetAddrSpacePtr(AMDGPUAS::CONSTANT_ADDRESS);
706 const LLT Constant32Ptr = GetAddrSpacePtr(AMDGPUAS::CONSTANT_ADDRESS_32BIT);
707 const LLT LocalPtr = GetAddrSpacePtr(AMDGPUAS::LOCAL_ADDRESS);
708 const LLT RegionPtr = GetAddrSpacePtr(AMDGPUAS::REGION_ADDRESS);
709 const LLT FlatPtr = GetAddrSpacePtr(AMDGPUAS::FLAT_ADDRESS);
710 const LLT PrivatePtr = GetAddrSpacePtr(AMDGPUAS::PRIVATE_ADDRESS);
711 const LLT BufferFatPtr = GetAddrSpacePtr(AMDGPUAS::BUFFER_FAT_POINTER);
712 const LLT RsrcPtr = GetAddrSpacePtr(AMDGPUAS::BUFFER_RESOURCE);
713 const LLT BufferStridedPtr =
714 GetAddrSpacePtr(AMDGPUAS::BUFFER_STRIDED_POINTER);
715
716 const LLT CodePtr = FlatPtr;
717
718 const std::initializer_list<LLT> AddrSpaces64 = {
719 GlobalPtr, ConstantPtr, FlatPtr
720 };
721
722 const std::initializer_list<LLT> AddrSpaces32 = {
723 LocalPtr, PrivatePtr, Constant32Ptr, RegionPtr
724 };
725
726 const std::initializer_list<LLT> AddrSpaces128 = {RsrcPtr};
727
728 const std::initializer_list<LLT> FPTypesBase = {
729 S32, S64
730 };
731
732 const std::initializer_list<LLT> FPTypes16 = {
733 S32, S64, S16
734 };
735
736 const std::initializer_list<LLT> FPTypesPK16 = {
737 S32, S64, S16, V2S16
738 };
739
740 const std::initializer_list<LLT> FPTypesPK16_64 = {S32, S64, S16, V2S16,
741 V2S64};
742
743 const LLT MinScalarFPTy = ST.has16BitInsts() ? S16 : S32;
744
745 getActionDefinitionsBuilder(Opcode: G_BR).alwaysLegal();
746
747 // s1 for VCC branches, s32 for SCC branches.
748 getActionDefinitionsBuilder(Opcode: G_BRCOND).legalFor(Types: {S1, S32});
749
750 // TODO: All multiples of 32, vectors of pointers, all v2s16 pairs, more
751 // elements for v3s16
752 getActionDefinitionsBuilder(Opcode: G_PHI)
753 .legalFor(Types: {S32, S64, V2S16, S16, V4S16, S1, S128, S256})
754 .legalFor(Types: AllS32Vectors)
755 .legalFor(Types: AllS64Vectors)
756 .legalFor(Types: AddrSpaces64)
757 .legalFor(Types: AddrSpaces32)
758 .legalFor(Types: AddrSpaces128)
759 .legalIf(Predicate: isPointer(TypeIdx: 0))
760 .clampScalar(TypeIdx: 0, MinTy: S16, MaxTy: S256)
761 .widenScalarToNextPow2(TypeIdx: 0, MinSize: 32)
762 .clampMaxNumElements(TypeIdx: 0, EltTy: S32, MaxElements: 16)
763 .moreElementsIf(Predicate: isSmallOddVector(TypeIdx: 0), Mutation: oneMoreElement(TypeIdx: 0))
764 .scalarize(TypeIdx: 0);
765
766 if (ST.hasVOP3PInsts() && ST.hasAddNoCarryInsts() && ST.hasIntClamp()) {
767 // Full set of gfx9 features.
768 if (ST.hasPackedU64Ops()) {
769 getActionDefinitionsBuilder(Opcodes: {G_ADD, G_SUB})
770 .legalFor(Types: {S64, S32, S16, V2S16, V2S64})
771 .clampMaxNumElementsStrict(TypeIdx: 0, EltTy: S16, NumElts: 2)
772 .clampMaxNumElementsStrict(TypeIdx: 0, EltTy: S64, NumElts: 2)
773 .scalarize(TypeIdx: 0)
774 .minScalar(TypeIdx: 0, Ty: S16)
775 .widenScalarToNextMultipleOf(TypeIdx: 0, Size: 32)
776 .maxScalar(TypeIdx: 0, Ty: S32);
777 } else if (ST.hasScalarAddSub64()) {
778 getActionDefinitionsBuilder(Opcodes: {G_ADD, G_SUB})
779 .legalFor(Types: {S64, S32, S16, V2S16})
780 .clampMaxNumElementsStrict(TypeIdx: 0, EltTy: S16, NumElts: 2)
781 .scalarize(TypeIdx: 0)
782 .minScalar(TypeIdx: 0, Ty: S16)
783 .widenScalarToNextMultipleOf(TypeIdx: 0, Size: 32)
784 .maxScalar(TypeIdx: 0, Ty: S32);
785 } else {
786 getActionDefinitionsBuilder(Opcodes: {G_ADD, G_SUB})
787 .legalFor(Types: {S32, S16, V2S16})
788 .clampMaxNumElementsStrict(TypeIdx: 0, EltTy: S16, NumElts: 2)
789 .scalarize(TypeIdx: 0)
790 .minScalar(TypeIdx: 0, Ty: S16)
791 .widenScalarToNextMultipleOf(TypeIdx: 0, Size: 32)
792 .maxScalar(TypeIdx: 0, Ty: S32);
793 }
794
795 if (ST.hasScalarSMulU64()) {
796 getActionDefinitionsBuilder(Opcode: G_MUL)
797 .legalFor(Types: {S64, S32, S16, V2S16})
798 .clampMaxNumElementsStrict(TypeIdx: 0, EltTy: S16, NumElts: 2)
799 .scalarize(TypeIdx: 0)
800 .minScalar(TypeIdx: 0, Ty: S16)
801 .widenScalarToNextMultipleOf(TypeIdx: 0, Size: 32)
802 .custom();
803 } else {
804 getActionDefinitionsBuilder(Opcode: G_MUL)
805 .legalFor(Types: {S32, S16, V2S16})
806 .clampMaxNumElementsStrict(TypeIdx: 0, EltTy: S16, NumElts: 2)
807 .scalarize(TypeIdx: 0)
808 .minScalar(TypeIdx: 0, Ty: S16)
809 .widenScalarToNextMultipleOf(TypeIdx: 0, Size: 32)
810 .custom();
811 }
812 assert(ST.hasMad64_32());
813
814 getActionDefinitionsBuilder(Opcodes: {G_UADDSAT, G_USUBSAT, G_SADDSAT, G_SSUBSAT})
815 .legalFor(Types: {S32, S16, V2S16}) // Clamp modifier
816 .minScalarOrElt(TypeIdx: 0, Ty: S16)
817 .clampMaxNumElementsStrict(TypeIdx: 0, EltTy: S16, NumElts: 2)
818 .scalarize(TypeIdx: 0)
819 .widenScalarToNextPow2(TypeIdx: 0, MinSize: 32)
820 .lower();
821 } else if (ST.has16BitInsts()) {
822 getActionDefinitionsBuilder(Opcodes: {G_ADD, G_SUB})
823 .legalFor(Types: {S32, S16})
824 .minScalar(TypeIdx: 0, Ty: S16)
825 .widenScalarToNextMultipleOf(TypeIdx: 0, Size: 32)
826 .maxScalar(TypeIdx: 0, Ty: S32)
827 .scalarize(TypeIdx: 0);
828
829 getActionDefinitionsBuilder(Opcode: G_MUL)
830 .legalFor(Types: {S32, S16})
831 .scalarize(TypeIdx: 0)
832 .minScalar(TypeIdx: 0, Ty: S16)
833 .widenScalarToNextMultipleOf(TypeIdx: 0, Size: 32)
834 .custom();
835 assert(ST.hasMad64_32());
836
837 // Technically the saturating operations require clamp bit support, but this
838 // was introduced at the same time as 16-bit operations.
839 getActionDefinitionsBuilder(Opcodes: {G_UADDSAT, G_USUBSAT})
840 .legalFor(Types: {S32, S16}) // Clamp modifier
841 .minScalar(TypeIdx: 0, Ty: S16)
842 .scalarize(TypeIdx: 0)
843 .widenScalarToNextPow2(TypeIdx: 0, MinSize: 16)
844 .lower();
845
846 // We're just lowering this, but it helps get a better result to try to
847 // coerce to the desired type first.
848 getActionDefinitionsBuilder(Opcodes: {G_SADDSAT, G_SSUBSAT})
849 .minScalar(TypeIdx: 0, Ty: S16)
850 .scalarize(TypeIdx: 0)
851 .lower();
852 } else {
853 getActionDefinitionsBuilder(Opcodes: {G_ADD, G_SUB})
854 .legalFor(Types: {S32})
855 .widenScalarToNextMultipleOf(TypeIdx: 0, Size: 32)
856 .clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S32)
857 .scalarize(TypeIdx: 0);
858
859 auto &Mul = getActionDefinitionsBuilder(Opcode: G_MUL)
860 .legalFor(Types: {S32})
861 .scalarize(TypeIdx: 0)
862 .minScalar(TypeIdx: 0, Ty: S32)
863 .widenScalarToNextMultipleOf(TypeIdx: 0, Size: 32);
864
865 if (ST.hasMad64_32())
866 Mul.custom();
867 else
868 Mul.maxScalar(TypeIdx: 0, Ty: S32);
869
870 if (ST.hasIntClamp()) {
871 getActionDefinitionsBuilder(Opcodes: {G_UADDSAT, G_USUBSAT})
872 .legalFor(Types: {S32}) // Clamp modifier.
873 .scalarize(TypeIdx: 0)
874 .minScalarOrElt(TypeIdx: 0, Ty: S32)
875 .lower();
876 } else {
877 // Clamp bit support was added in VI, along with 16-bit operations.
878 getActionDefinitionsBuilder(Opcodes: {G_UADDSAT, G_USUBSAT})
879 .minScalar(TypeIdx: 0, Ty: S32)
880 .scalarize(TypeIdx: 0)
881 .lower();
882 }
883
884 // FIXME: DAG expansion gets better results. The widening uses the smaller
885 // range values and goes for the min/max lowering directly.
886 getActionDefinitionsBuilder(Opcodes: {G_SADDSAT, G_SSUBSAT})
887 .minScalar(TypeIdx: 0, Ty: S32)
888 .scalarize(TypeIdx: 0)
889 .lower();
890 }
891
892 getActionDefinitionsBuilder(
893 Opcodes: {G_SDIV, G_UDIV, G_SREM, G_UREM, G_SDIVREM, G_UDIVREM})
894 .customFor(Types: {S32, S64})
895 .clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S64)
896 .widenScalarToNextPow2(TypeIdx: 0, MinSize: 32)
897 .scalarize(TypeIdx: 0);
898
899 auto &Mulh = getActionDefinitionsBuilder(Opcodes: {G_UMULH, G_SMULH})
900 .legalFor(Types: {S32})
901 .maxScalar(TypeIdx: 0, Ty: S32);
902
903 if (ST.hasVOP3PInsts()) {
904 Mulh
905 .clampMaxNumElements(TypeIdx: 0, EltTy: S8, MaxElements: 2)
906 .lowerFor(Types: {V2S8});
907 }
908
909 Mulh
910 .scalarize(TypeIdx: 0)
911 .lower();
912
913 // Report legal for any types we can handle anywhere. For the cases only legal
914 // on the SALU, RegBankSelect will be able to re-legalize.
915 getActionDefinitionsBuilder(Opcodes: {G_AND, G_OR, G_XOR})
916 .legalFor(Types: {S32, S1, S64, V2S32, S16, V2S16, V4S16})
917 .clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S64)
918 .moreElementsIf(Predicate: isSmallOddVector(TypeIdx: 0), Mutation: oneMoreElement(TypeIdx: 0))
919 .fewerElementsIf(
920 Predicate: all(P0: vectorWiderThan(TypeIdx: 0, Size: 64), P1: scalarOrEltNarrowerThan(TypeIdx: 0, Size: 64)),
921 Mutation: fewerEltsToSize64Vector(TypeIdx: 0))
922 .widenScalarToNextPow2(TypeIdx: 0)
923 .scalarize(TypeIdx: 0);
924
925 getActionDefinitionsBuilder(
926 Opcodes: {G_UADDO, G_USUBO, G_UADDE, G_SADDE, G_USUBE, G_SSUBE})
927 .legalFor(Types: {{S32, S1}, {S32, S32}})
928 .clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S32)
929 .scalarize(TypeIdx: 0);
930
931 getActionDefinitionsBuilder(Opcode: G_BITCAST)
932 // Don't worry about the size constraint.
933 .legalIf(Predicate: all(P0: isRegisterClassType(ST, TypeIdx: 0), P1: isRegisterClassType(ST, TypeIdx: 1)))
934 .widenScalarIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {I16, F16, BF16}), P1: isScalar(TypeIdx: 1)),
935 Mutation: changeTo(TypeIdx: 0, Ty: LLT::integer(SizeInBits: 32)))
936 .widenScalarIf(Predicate: all(P0: isScalar(TypeIdx: 0), P1: typeInSet(TypeIdx: 1, TypesInit: {I16, F16, BF16})),
937 Mutation: changeTo(TypeIdx: 1, Ty: LLT::integer(SizeInBits: 32)))
938 .lower();
939
940 getActionDefinitionsBuilder(Opcode: G_CONSTANT)
941 .legalFor(Types: {S1, S32, S64, S16, GlobalPtr,
942 LocalPtr, ConstantPtr, PrivatePtr, FlatPtr })
943 .legalIf(Predicate: isPointer(TypeIdx: 0))
944 .clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S64)
945 .widenScalarToNextPow2(TypeIdx: 0);
946
947 getActionDefinitionsBuilder(Opcode: G_FCONSTANT)
948 .legalFor(Types: {S32, S64, S16})
949 .clampScalar(TypeIdx: 0, MinTy: S16, MaxTy: S64);
950
951 getActionDefinitionsBuilder(Opcodes: {G_IMPLICIT_DEF, G_FREEZE})
952 .legalIf(Predicate: isRegisterClassType(ST, TypeIdx: 0))
953 // s1 and s16 are special cases because they have legal operations on
954 // them, but don't really occupy registers in the normal way.
955 .legalFor(Types: {S1, S16})
956 .clampNumElements(TypeIdx: 0, MinTy: V16S32, MaxTy: V32S32)
957 .moreElementsIf(Predicate: isSmallOddVector(TypeIdx: 0), Mutation: oneMoreElement(TypeIdx: 0))
958 .clampScalarOrElt(TypeIdx: 0, MinTy: S32, MaxTy: MaxScalar)
959 .widenScalarToNextPow2(TypeIdx: 0, MinSize: 32)
960 .clampMaxNumElements(TypeIdx: 0, EltTy: S32, MaxElements: 16);
961
962 getActionDefinitionsBuilder(Opcode: G_FRAME_INDEX).legalFor(Types: {PrivatePtr});
963
964 // If the amount is divergent, we have to do a wave reduction to get the
965 // maximum value, so this is expanded during RegBankSelect.
966 getActionDefinitionsBuilder(Opcode: G_DYN_STACKALLOC)
967 .legalFor(Types: {{PrivatePtr, S32}});
968
969 getActionDefinitionsBuilder(Opcode: G_STACKSAVE)
970 .customFor(Types: {PrivatePtr});
971 getActionDefinitionsBuilder(Opcode: G_STACKRESTORE)
972 .legalFor(Types: {PrivatePtr});
973
974 getActionDefinitionsBuilder(Opcodes: {G_GET_FPENV, G_SET_FPENV}).customFor(Types: {S64});
975
976 getActionDefinitionsBuilder(Opcodes: {G_GET_ROUNDING, G_SET_ROUNDING}).legalFor(Types: {S32});
977
978 getActionDefinitionsBuilder(Opcode: G_GLOBAL_VALUE)
979 .customIf(Predicate: typeIsNot(TypeIdx: 0, Type: PrivatePtr));
980
981 getActionDefinitionsBuilder(Opcode: G_BLOCK_ADDR).legalFor(Types: {CodePtr});
982
983 auto &FPOpActions = getActionDefinitionsBuilder(
984 Opcodes: { G_FADD, G_FMUL, G_FMA, G_FCANONICALIZE,
985 G_STRICT_FADD, G_STRICT_FMUL, G_STRICT_FMA})
986 .legalFor(Types: {S32, S64});
987 auto &TrigActions = getActionDefinitionsBuilder(Opcodes: {G_FSIN, G_FCOS})
988 .customFor(Types: {S32, S64});
989 auto &FDIVActions = getActionDefinitionsBuilder(Opcode: G_FDIV)
990 .customFor(Types: {S32, S64});
991
992 if (ST.has16BitInsts()) {
993 if (ST.hasVOP3PInsts())
994 FPOpActions.legalFor(Types: {S16, V2S16});
995 else
996 FPOpActions.legalFor(Types: {S16});
997
998 TrigActions.customFor(Types: {S16});
999 FDIVActions.customFor(Types: {S16});
1000 }
1001
1002 if (ST.hasPackedFP32Ops()) {
1003 FPOpActions.legalFor(Types: {V2S32});
1004 FPOpActions.clampMaxNumElementsStrict(TypeIdx: 0, EltTy: S32, NumElts: 2);
1005 }
1006
1007 if (ST.hasPackedFP64Ops()) {
1008 FPOpActions.legalFor(Types: {V2S64});
1009 FPOpActions.clampMaxNumElementsStrict(TypeIdx: 0, EltTy: S64, NumElts: 2);
1010 }
1011
1012 if (ST.hasPackedFP64Ops()) {
1013 FPOpActions.legalFor(Types: {V2S64});
1014 FPOpActions.clampMaxNumElementsStrict(TypeIdx: 0, EltTy: S64, NumElts: 2);
1015 }
1016
1017 auto &MinNumMaxNumIeee =
1018 getActionDefinitionsBuilder(Opcodes: {G_FMINNUM_IEEE, G_FMAXNUM_IEEE});
1019
1020 if (ST.hasVOP3PInsts()) {
1021 MinNumMaxNumIeee.legalFor(Types: FPTypesPK16)
1022 .moreElementsIf(Predicate: isSmallOddVector(TypeIdx: 0), Mutation: oneMoreElement(TypeIdx: 0))
1023 .clampMaxNumElements(TypeIdx: 0, EltTy: S16, MaxElements: 2)
1024 .clampScalar(TypeIdx: 0, MinTy: S16, MaxTy: S64)
1025 .scalarize(TypeIdx: 0);
1026 } else if (ST.has16BitInsts()) {
1027 MinNumMaxNumIeee.legalFor(Types: FPTypes16).clampScalar(TypeIdx: 0, MinTy: S16, MaxTy: S64).scalarize(TypeIdx: 0);
1028 } else {
1029 MinNumMaxNumIeee.legalFor(Types: FPTypesBase)
1030 .clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S64)
1031 .scalarize(TypeIdx: 0);
1032 }
1033
1034 auto &MinNumMaxNum = getActionDefinitionsBuilder(
1035 Opcodes: {G_FMINNUM, G_FMAXNUM, G_FMINIMUMNUM, G_FMAXIMUMNUM});
1036
1037 if (ST.hasPackedFP64Ops()) {
1038 MinNumMaxNum.customFor(Types: FPTypesPK16_64)
1039 .moreElementsIf(Predicate: isSmallOddVector(TypeIdx: 0), Mutation: oneMoreElement(TypeIdx: 0))
1040 .clampMaxNumElements(TypeIdx: 0, EltTy: S16, MaxElements: 2)
1041 .clampMaxNumElements(TypeIdx: 0, EltTy: S64, MaxElements: 2)
1042 .clampScalar(TypeIdx: 0, MinTy: S16, MaxTy: S64)
1043 .scalarize(TypeIdx: 0);
1044 } else if (ST.hasVOP3PInsts()) {
1045 MinNumMaxNum.customFor(Types: FPTypesPK16)
1046 .moreElementsIf(Predicate: isSmallOddVector(TypeIdx: 0), Mutation: oneMoreElement(TypeIdx: 0))
1047 .clampMaxNumElements(TypeIdx: 0, EltTy: S16, MaxElements: 2)
1048 .clampScalar(TypeIdx: 0, MinTy: S16, MaxTy: S64)
1049 .scalarize(TypeIdx: 0);
1050 } else if (ST.has16BitInsts()) {
1051 MinNumMaxNum.customFor(Types: FPTypes16)
1052 .clampScalar(TypeIdx: 0, MinTy: S16, MaxTy: S64)
1053 .scalarize(TypeIdx: 0);
1054 } else {
1055 MinNumMaxNum.customFor(Types: FPTypesBase)
1056 .clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S64)
1057 .scalarize(TypeIdx: 0);
1058 }
1059
1060 if (ST.hasVOP3PInsts())
1061 FPOpActions.clampMaxNumElementsStrict(TypeIdx: 0, EltTy: S16, NumElts: 2);
1062
1063 FPOpActions
1064 .scalarize(TypeIdx: 0)
1065 .clampScalar(TypeIdx: 0, MinTy: ST.has16BitInsts() ? S16 : S32, MaxTy: S64);
1066
1067 TrigActions
1068 .scalarize(TypeIdx: 0)
1069 .clampScalar(TypeIdx: 0, MinTy: ST.has16BitInsts() ? S16 : S32, MaxTy: S64);
1070
1071 FDIVActions
1072 .scalarize(TypeIdx: 0)
1073 .clampScalar(TypeIdx: 0, MinTy: ST.has16BitInsts() ? S16 : S32, MaxTy: S64);
1074
1075 auto &FNegAbs = getActionDefinitionsBuilder(Opcodes: {G_FNEG, G_FABS});
1076 FNegAbs.legalFor(Types: FPTypesPK16)
1077 .legalFor(Pred: ST.hasPackedFP32Ops(), Types: {V2S32})
1078 .clampMaxNumElementsStrict(TypeIdx: 0, EltTy: S16, NumElts: 2);
1079 if (ST.hasPackedFP32Ops())
1080 FNegAbs.clampMaxNumElementsStrict(TypeIdx: 0, EltTy: S32, NumElts: 2);
1081 FNegAbs.scalarize(TypeIdx: 0).clampScalar(TypeIdx: 0, MinTy: S16, MaxTy: S64);
1082
1083 if (ST.has16BitInsts()) {
1084 getActionDefinitionsBuilder(Opcode: G_FSQRT)
1085 .legalFor(Types: {S16})
1086 .customFor(Types: {S32, S64})
1087 .scalarize(TypeIdx: 0)
1088 .unsupported();
1089 getActionDefinitionsBuilder(Opcode: G_FFLOOR)
1090 .legalFor(Types: {S32, S64, S16})
1091 .scalarize(TypeIdx: 0)
1092 .clampScalar(TypeIdx: 0, MinTy: S16, MaxTy: S64);
1093
1094 getActionDefinitionsBuilder(Opcodes: {G_FLDEXP, G_STRICT_FLDEXP})
1095 .legalFor(Types: {{S32, S32}, {S64, S32}, {S16, S16}})
1096 .scalarize(TypeIdx: 0)
1097 .maxScalarIf(Predicate: typeIs(TypeIdx: 0, TypesInit: S16), TypeIdx: 1, Ty: S16)
1098 .clampScalar(TypeIdx: 1, MinTy: S32, MaxTy: S32)
1099 .lower();
1100
1101 getActionDefinitionsBuilder(Opcode: G_FFREXP)
1102 .customFor(Types: {{S32, S32}, {S64, S32}, {S16, S16}, {S16, S32}})
1103 .scalarize(TypeIdx: 0)
1104 .lower();
1105
1106 getActionDefinitionsBuilder(Opcode: G_FMODF)
1107 .lowerFor(Types: {S16, S32, S64})
1108 .scalarize(TypeIdx: 0)
1109 .lower();
1110 } else {
1111 getActionDefinitionsBuilder(Opcode: G_FSQRT)
1112 .customFor(Types: {S32, S64, S16})
1113 .scalarize(TypeIdx: 0)
1114 .unsupported();
1115
1116
1117 if (ST.hasFractBug()) {
1118 getActionDefinitionsBuilder(Opcode: G_FFLOOR)
1119 .customFor(Types: {S64})
1120 .legalFor(Types: {S32, S64})
1121 .scalarize(TypeIdx: 0)
1122 .clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S64);
1123 } else {
1124 getActionDefinitionsBuilder(Opcode: G_FFLOOR)
1125 .legalFor(Types: {S32, S64})
1126 .scalarize(TypeIdx: 0)
1127 .clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S64);
1128 }
1129
1130 getActionDefinitionsBuilder(Opcodes: {G_FLDEXP, G_STRICT_FLDEXP})
1131 .legalFor(Types: {{S32, S32}, {S64, S32}})
1132 .scalarize(TypeIdx: 0)
1133 .clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S64)
1134 .clampScalar(TypeIdx: 1, MinTy: S32, MaxTy: S32)
1135 .lower();
1136
1137 getActionDefinitionsBuilder(Opcode: G_FFREXP)
1138 .customFor(Types: {{S32, S32}, {S64, S32}})
1139 .scalarize(TypeIdx: 0)
1140 .minScalar(TypeIdx: 0, Ty: S32)
1141 .clampScalar(TypeIdx: 1, MinTy: S32, MaxTy: S32)
1142 .lower();
1143
1144 getActionDefinitionsBuilder(Opcode: G_FMODF)
1145 .lowerFor(Types: {S32, S64})
1146 .scalarize(TypeIdx: 0)
1147 .lower();
1148 }
1149
1150 auto &FPTruncActions = getActionDefinitionsBuilder(Opcode: G_FPTRUNC);
1151 if (ST.hasCvtPkF16F32Inst()) {
1152 FPTruncActions.legalFor(Types: {{S32, S64}, {S16, S32}, {V2S16, V2S32}})
1153 .clampMaxNumElements(TypeIdx: 0, EltTy: S16, MaxElements: 2);
1154 } else {
1155 FPTruncActions.legalFor(Types: {{S32, S64}, {S16, S32}});
1156 }
1157 FPTruncActions.scalarize(TypeIdx: 0).lower();
1158
1159 getActionDefinitionsBuilder(Opcode: G_FPEXT)
1160 .legalFor(Types: {{S64, S32}, {S32, S16}})
1161 .narrowScalarFor(Types: {{S64, S16}}, Mutation: changeElementSizeTo(TypeIdx: 0, NewTy: S32))
1162 .scalarize(TypeIdx: 0);
1163
1164 auto &FSubActions = getActionDefinitionsBuilder(Opcodes: {G_FSUB, G_STRICT_FSUB});
1165 if (ST.has16BitInsts()) {
1166 FSubActions
1167 // Use actual fsub instruction
1168 .legalFor(Types: {S32, S16})
1169 // Must use fadd + fneg
1170 .lowerFor(Types: {S64, V2S16});
1171 } else {
1172 FSubActions
1173 // Use actual fsub instruction
1174 .legalFor(Types: {S32})
1175 // Must use fadd + fneg
1176 .lowerFor(Types: {S64, S16, V2S16});
1177 }
1178
1179 if (ST.hasPackedFP32Ops())
1180 FSubActions.lowerFor(Types: {V2S32}).clampMaxNumElements(TypeIdx: 0, EltTy: S32, MaxElements: 2);
1181
1182 FSubActions
1183 .clampMaxNumElements(TypeIdx: 0, EltTy: S16, MaxElements: 2)
1184 .scalarize(TypeIdx: 0)
1185 .clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S64);
1186
1187 // Whether this is legal depends on the floating point mode for the function.
1188 auto &FMad = getActionDefinitionsBuilder(Opcode: G_FMAD);
1189 if (ST.hasMadF16() && ST.hasMadMacF32Insts())
1190 FMad.customFor(Types: {S32, S16});
1191 else if (ST.hasMadMacF32Insts())
1192 FMad.customFor(Types: {S32});
1193 else if (ST.hasMadF16())
1194 FMad.customFor(Types: {S16});
1195 FMad.scalarize(TypeIdx: 0)
1196 .lower();
1197
1198 auto &FRem = getActionDefinitionsBuilder(Opcode: G_FREM);
1199 if (ST.has16BitInsts()) {
1200 FRem.customFor(Types: {S16, S32, S64});
1201 } else {
1202 FRem.minScalar(TypeIdx: 0, Ty: S32)
1203 .customFor(Types: {S32, S64});
1204 }
1205 FRem.scalarize(TypeIdx: 0);
1206
1207 // TODO: Do we need to clamp maximum bitwidth?
1208 getActionDefinitionsBuilder(Opcode: G_TRUNC)
1209 .legalIf(Predicate: isScalar(TypeIdx: 0))
1210 .legalFor(Types: {{V2S16, V2S32}})
1211 .clampMaxNumElements(TypeIdx: 0, EltTy: S16, MaxElements: 2)
1212 // Avoid scalarizing in cases that should be truly illegal. In unresolvable
1213 // situations (like an invalid implicit use), we don't want to infinite loop
1214 // in the legalizer.
1215 .fewerElementsIf(Predicate: elementTypeIsLegal(TypeIdx: 0), Mutation: LegalizeMutations::scalarize(TypeIdx: 0))
1216 .alwaysLegal();
1217
1218 getActionDefinitionsBuilder(Opcodes: {G_SEXT, G_ZEXT, G_ANYEXT})
1219 .legalFor(Types: {{S64, S32}, {S32, S16}, {S64, S16},
1220 {S32, S1}, {S64, S1}, {S16, S1}})
1221 .scalarize(TypeIdx: 0)
1222 .clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S64)
1223 .widenScalarToNextPow2(TypeIdx: 1, MinSize: 32);
1224
1225 // TODO: Split s1->s64 during regbankselect for VALU.
1226 auto &IToFP = getActionDefinitionsBuilder(Opcodes: {G_SITOFP, G_UITOFP})
1227 .legalFor(Types: {{S32, S32}, {S64, S32}})
1228 .widenScalarFor(Types: {{S16, S32}}, Mutation: changeElementSizeTo(TypeIdx: 0, NewTy: S32))
1229 .lowerIf(Predicate: typeIs(TypeIdx: 1, TypesInit: S1))
1230 .customFor(Types: {{S32, S64}, {S64, S64}});
1231 if (ST.has16BitInsts())
1232 IToFP.legalFor(Types: {{S16, S16}});
1233 IToFP.clampScalar(TypeIdx: 1, MinTy: S32, MaxTy: S64)
1234 .minScalar(TypeIdx: 0, Ty: S32)
1235 .scalarize(TypeIdx: 0)
1236 .widenScalarToNextPow2(TypeIdx: 1);
1237
1238 auto &FPToI = getActionDefinitionsBuilder(Opcodes: {G_FPTOSI, G_FPTOUI})
1239 .legalFor(Types: {{S32, S32}, {S32, S64}})
1240 .customFor(Types: {{S64, S32}, {S64, S64}})
1241 .widenScalarFor(Types: {{S32, S16}}, Mutation: changeElementSizeTo(TypeIdx: 1, NewTy: S32))
1242 .narrowScalarFor(Types: {{S64, S16}}, Mutation: changeElementSizeTo(TypeIdx: 0, NewTy: S32));
1243 if (ST.has16BitInsts())
1244 FPToI.legalFor(Types: {{S16, S16}});
1245 else
1246 FPToI.minScalar(TypeIdx: 1, Ty: S32);
1247
1248 FPToI.minScalar(TypeIdx: 0, Ty: S32)
1249 .widenScalarToNextPow2(TypeIdx: 0, MinSize: 32)
1250 .scalarize(TypeIdx: 0)
1251 .lower();
1252
1253 // clang-format off
1254 auto &FPToISat = getActionDefinitionsBuilder(Opcodes: {G_FPTOSI_SAT, G_FPTOUI_SAT})
1255 .legalFor(Types: {{S32, S32}, {S32, S64}, {S16, S32}})
1256 .legalFor(Pred: ST.has16BitInsts(), Types: {{S16, S16}})
1257 .legalFor(Pred: ST.hasVCvtPkIU16F32(), Types: {{V2S16, V2S32}})
1258 .narrowScalarFor(Types: {{S64, S16}}, Mutation: changeElementSizeTo(TypeIdx: 0, NewTy: S32));
1259
1260 // If available, widen width <16 to i16, intead of i32 so v_cvt_i16/u16_f16 can be used.
1261 if (ST.has16BitInsts())
1262 FPToISat.minScalarIf(Predicate: typeIs(TypeIdx: 1, TypesInit: S16), TypeIdx: 0, Ty: S16);
1263
1264 if (ST.hasVCvtPkIU16F32())
1265 FPToISat.clampMaxNumElements(TypeIdx: 0, EltTy: S16, MaxElements: 2);
1266
1267 FPToISat.minScalar(TypeIdx: 1, Ty: S32);
1268 FPToISat.minScalar(TypeIdx: 0, Ty: S32)
1269 .widenScalarToNextPow2(TypeIdx: 0, MinSize: 32)
1270 .scalarize(TypeIdx: 0)
1271 .lower();
1272 // clang-format on
1273
1274 getActionDefinitionsBuilder(Opcodes: {G_LROUND, G_LLROUND})
1275 .clampScalar(TypeIdx: 0, MinTy: S16, MaxTy: S64)
1276 .scalarize(TypeIdx: 0)
1277 .lower();
1278
1279 getActionDefinitionsBuilder(Opcode: G_INTRINSIC_FPTRUNC_ROUND)
1280 .legalFor(Types: {S16, S32})
1281 .scalarize(TypeIdx: 0)
1282 .lower();
1283
1284 // Lower G_FNEARBYINT and G_FRINT into G_INTRINSIC_ROUNDEVEN
1285 getActionDefinitionsBuilder(Opcodes: {G_INTRINSIC_ROUND, G_FRINT, G_FNEARBYINT})
1286 .scalarize(TypeIdx: 0)
1287 .lower();
1288
1289 getActionDefinitionsBuilder(Opcodes: {G_INTRINSIC_LRINT, G_INTRINSIC_LLRINT})
1290 .clampScalar(TypeIdx: 0, MinTy: S16, MaxTy: S64)
1291 .scalarize(TypeIdx: 0)
1292 .lower();
1293
1294 if (ST.has16BitInsts()) {
1295 getActionDefinitionsBuilder(
1296 Opcodes: {G_INTRINSIC_TRUNC, G_FCEIL, G_INTRINSIC_ROUNDEVEN})
1297 .legalFor(Types: {S16, S32, S64})
1298 .clampScalar(TypeIdx: 0, MinTy: S16, MaxTy: S64)
1299 .scalarize(TypeIdx: 0);
1300 } else if (ST.getGeneration() >= AMDGPUSubtarget::SEA_ISLANDS) {
1301 getActionDefinitionsBuilder(
1302 Opcodes: {G_INTRINSIC_TRUNC, G_FCEIL, G_INTRINSIC_ROUNDEVEN})
1303 .legalFor(Types: {S32, S64})
1304 .clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S64)
1305 .scalarize(TypeIdx: 0);
1306 } else {
1307 getActionDefinitionsBuilder(
1308 Opcodes: {G_INTRINSIC_TRUNC, G_FCEIL, G_INTRINSIC_ROUNDEVEN})
1309 .legalFor(Types: {S32})
1310 .customFor(Types: {S64})
1311 .clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S64)
1312 .scalarize(TypeIdx: 0);
1313 }
1314
1315 getActionDefinitionsBuilder(Opcode: G_PTR_ADD)
1316 .unsupportedFor(Types: {BufferFatPtr, BufferStridedPtr, RsrcPtr})
1317 .legalIf(Predicate: all(P0: isPointer(TypeIdx: 0), P1: sameSize(TypeIdx0: 0, TypeIdx1: 1)))
1318 .scalarize(TypeIdx: 0)
1319 .scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
1320
1321 getActionDefinitionsBuilder(Opcode: G_PTRMASK)
1322 .legalIf(Predicate: all(P0: sameSize(TypeIdx0: 0, TypeIdx1: 1), P1: typeInSet(TypeIdx: 1, TypesInit: {S64, S32})))
1323 .scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0)
1324 .scalarize(TypeIdx: 0);
1325
1326 auto &CmpBuilder =
1327 getActionDefinitionsBuilder(Opcode: G_ICMP)
1328 // The compare output type differs based on the register bank of the output,
1329 // so make both s1 and s32 legal.
1330 //
1331 // Scalar compares producing output in scc will be promoted to s32, as that
1332 // is the allocatable register type that will be needed for the copy from
1333 // scc. This will be promoted during RegBankSelect, and we assume something
1334 // before that won't try to use s32 result types.
1335 //
1336 // Vector compares producing an output in vcc/SGPR will use s1 in VCC reg
1337 // bank.
1338 .legalForCartesianProduct(
1339 Types0: {S1}, Types1: {S32, S64, GlobalPtr, LocalPtr, ConstantPtr, PrivatePtr, FlatPtr})
1340 .legalForCartesianProduct(
1341 Types0: {S32}, Types1: {S32, S64, GlobalPtr, LocalPtr, ConstantPtr, PrivatePtr, FlatPtr});
1342 if (ST.has16BitInsts()) {
1343 CmpBuilder.legalFor(Types: {{S1, S16}});
1344 }
1345
1346 CmpBuilder
1347 .widenScalarToNextPow2(TypeIdx: 1)
1348 .clampScalar(TypeIdx: 1, MinTy: S32, MaxTy: S64)
1349 .scalarize(TypeIdx: 0)
1350 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {S1, S32}), P1: isPointer(TypeIdx: 1)));
1351
1352 auto &FCmpBuilder =
1353 getActionDefinitionsBuilder(Opcode: G_FCMP).legalForCartesianProduct(
1354 Types0: {S1}, Types1: ST.has16BitInsts() ? FPTypes16 : FPTypesBase);
1355
1356 if (ST.hasSALUFloatInsts())
1357 FCmpBuilder.legalForCartesianProduct(Types0: {S32}, Types1: {S16, S32});
1358
1359 FCmpBuilder
1360 .widenScalarToNextPow2(TypeIdx: 1)
1361 .clampScalar(TypeIdx: 1, MinTy: S32, MaxTy: S64)
1362 .scalarize(TypeIdx: 0);
1363
1364 // FIXME: fpow has a selection pattern that should move to custom lowering.
1365 auto &ExpOps = getActionDefinitionsBuilder(Opcode: G_FPOW);
1366 if (ST.has16BitInsts())
1367 ExpOps.customFor(Types: {{S32}, {S16}});
1368 else
1369 ExpOps.customFor(Types: {S32});
1370 ExpOps.clampScalar(TypeIdx: 0, MinTy: MinScalarFPTy, MaxTy: S32)
1371 .scalarize(TypeIdx: 0);
1372
1373 getActionDefinitionsBuilder(Opcode: G_FPOWI)
1374 .clampScalar(TypeIdx: 0, MinTy: MinScalarFPTy, MaxTy: S32)
1375 .lower();
1376
1377 getActionDefinitionsBuilder(Opcode: G_FLOG2)
1378 .legalFor(Pred: ST.has16BitInsts(), Types: {S16})
1379 .customFor(Types: {S32, S16})
1380 .scalarize(TypeIdx: 0)
1381 .lower();
1382
1383 getActionDefinitionsBuilder(Opcode: G_FEXP2)
1384 .legalFor(Pred: ST.has16BitInsts(), Types: {S16})
1385 .customFor(Types: {S32, S64, S16})
1386 .scalarize(TypeIdx: 0)
1387 .lower();
1388
1389 auto &LogOps =
1390 getActionDefinitionsBuilder(Opcodes: {G_FLOG, G_FLOG10, G_FEXP, G_FEXP10});
1391 LogOps.customFor(Types: {S32, S16, S64});
1392 LogOps.clampScalar(TypeIdx: 0, MinTy: MinScalarFPTy, MaxTy: S32)
1393 .scalarize(TypeIdx: 0);
1394
1395 // The 64-bit versions produce 32-bit results, but only on the SALU.
1396 getActionDefinitionsBuilder(Opcode: G_CTPOP)
1397 .legalFor(Types: {{S32, S32}, {S32, S64}})
1398 .clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S32)
1399 .widenScalarToNextPow2(TypeIdx: 1, MinSize: 32)
1400 .clampScalar(TypeIdx: 1, MinTy: S32, MaxTy: S64)
1401 .scalarize(TypeIdx: 0)
1402 .widenScalarToNextPow2(TypeIdx: 0, MinSize: 32);
1403
1404 // If no 16 bit instr is available, lower into different instructions.
1405 if (ST.has16BitInsts())
1406 getActionDefinitionsBuilder(Opcode: G_IS_FPCLASS)
1407 .legalForCartesianProduct(Types0: {S1}, Types1: FPTypes16)
1408 .widenScalarToNextPow2(TypeIdx: 1)
1409 .scalarize(TypeIdx: 0)
1410 .lower();
1411 else
1412 getActionDefinitionsBuilder(Opcode: G_IS_FPCLASS)
1413 .legalForCartesianProduct(Types0: {S1}, Types1: FPTypesBase)
1414 .lowerFor(Types: {S1, S16})
1415 .widenScalarToNextPow2(TypeIdx: 1)
1416 .scalarize(TypeIdx: 0)
1417 .lower();
1418
1419 // The hardware instructions return a different result on 0 than the generic
1420 // instructions expect. The hardware produces -1, but these produce the
1421 // bitwidth.
1422 getActionDefinitionsBuilder(Opcodes: {G_CTLZ, G_CTTZ})
1423 .scalarize(TypeIdx: 0)
1424 .clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S32)
1425 .clampScalar(TypeIdx: 1, MinTy: S32, MaxTy: S64)
1426 .widenScalarToNextPow2(TypeIdx: 0, MinSize: 32)
1427 .widenScalarToNextPow2(TypeIdx: 1, MinSize: 32)
1428 .custom();
1429
1430 // The 64-bit versions produce 32-bit results, but only on the SALU.
1431 getActionDefinitionsBuilder(Opcode: G_CTLZ_ZERO_POISON)
1432 .legalFor(Types: {{S32, S32}, {S32, S64}})
1433 .customIf(Predicate: scalarNarrowerThan(TypeIdx: 1, Size: 32))
1434 .clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S32)
1435 .clampScalar(TypeIdx: 1, MinTy: S32, MaxTy: S64)
1436 .scalarize(TypeIdx: 0)
1437 .widenScalarToNextPow2(TypeIdx: 0, MinSize: 32)
1438 .widenScalarToNextPow2(TypeIdx: 1, MinSize: 32);
1439
1440 getActionDefinitionsBuilder(Opcode: G_CTTZ_ZERO_POISON)
1441 .legalFor(Types: {{S32, S32}, {S32, S64}})
1442 .clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S32)
1443 .clampScalar(TypeIdx: 1, MinTy: S32, MaxTy: S64)
1444 .scalarize(TypeIdx: 0)
1445 .widenScalarToNextPow2(TypeIdx: 0, MinSize: 32)
1446 .widenScalarToNextPow2(TypeIdx: 1, MinSize: 32);
1447
1448 getActionDefinitionsBuilder(Opcode: G_CTLS)
1449 .customFor(Types: {{S32, S32}})
1450 .scalarize(TypeIdx: 0)
1451 .clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S32)
1452 .clampScalar(TypeIdx: 1, MinTy: S32, MaxTy: S32);
1453
1454 // S64 is only legal on SALU, and needs to be broken into 32-bit elements in
1455 // RegBankSelect.
1456 getActionDefinitionsBuilder(Opcode: G_BITREVERSE)
1457 .legalFor(Types: {S32, S64})
1458 .clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S64)
1459 .scalarize(TypeIdx: 0)
1460 .widenScalarToNextPow2(TypeIdx: 0);
1461
1462 if (ST.has16BitInsts()) {
1463 getActionDefinitionsBuilder(Opcode: G_BSWAP)
1464 .legalFor(Types: {S16, S32, V2S16})
1465 .clampMaxNumElementsStrict(TypeIdx: 0, EltTy: S16, NumElts: 2)
1466 // FIXME: Fixing non-power-of-2 before clamp is workaround for
1467 // narrowScalar limitation.
1468 .widenScalarToNextPow2(TypeIdx: 0)
1469 .clampScalar(TypeIdx: 0, MinTy: S16, MaxTy: S32)
1470 .scalarize(TypeIdx: 0);
1471
1472 if (ST.hasVOP3PInsts()) {
1473 getActionDefinitionsBuilder(Opcode: G_ABS)
1474 .legalFor(Types: {S32, S16, V2S16})
1475 .clampMaxNumElements(TypeIdx: 0, EltTy: S16, MaxElements: 2)
1476 .minScalar(TypeIdx: 0, Ty: S16)
1477 .widenScalarToNextPow2(TypeIdx: 0)
1478 .scalarize(TypeIdx: 0)
1479 .lower();
1480 if (ST.hasMinMaxI64Insts()) {
1481 getActionDefinitionsBuilder(Opcodes: {G_SMIN, G_SMAX, G_UMIN, G_UMAX})
1482 .legalFor(Types: {S32, S16, S64, V2S16})
1483 .clampMaxNumElements(TypeIdx: 0, EltTy: S16, MaxElements: 2)
1484 .minScalar(TypeIdx: 0, Ty: S16)
1485 .widenScalarToNextPow2(TypeIdx: 0)
1486 .scalarize(TypeIdx: 0)
1487 .lower();
1488 } else {
1489 getActionDefinitionsBuilder(Opcodes: {G_SMIN, G_SMAX, G_UMIN, G_UMAX})
1490 .legalFor(Types: {S32, S16, V2S16})
1491 .clampMaxNumElements(TypeIdx: 0, EltTy: S16, MaxElements: 2)
1492 .minScalar(TypeIdx: 0, Ty: S16)
1493 .widenScalarToNextPow2(TypeIdx: 0)
1494 .scalarize(TypeIdx: 0)
1495 .lower();
1496 }
1497 } else {
1498 getActionDefinitionsBuilder(Opcodes: {G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS})
1499 .legalFor(Types: {S32, S16})
1500 .widenScalarToNextPow2(TypeIdx: 0)
1501 .minScalar(TypeIdx: 0, Ty: S16)
1502 .scalarize(TypeIdx: 0)
1503 .lower();
1504 }
1505 } else {
1506 // TODO: Should have same legality without v_perm_b32
1507 getActionDefinitionsBuilder(Opcode: G_BSWAP)
1508 .legalFor(Types: {S32})
1509 .lowerIf(Predicate: scalarNarrowerThan(TypeIdx: 0, Size: 32))
1510 // FIXME: Fixing non-power-of-2 before clamp is workaround for
1511 // narrowScalar limitation.
1512 .widenScalarToNextPow2(TypeIdx: 0)
1513 .maxScalar(TypeIdx: 0, Ty: S32)
1514 .scalarize(TypeIdx: 0)
1515 .lower();
1516
1517 getActionDefinitionsBuilder(Opcodes: {G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS})
1518 .legalFor(Types: {S32})
1519 .minScalar(TypeIdx: 0, Ty: S32)
1520 .widenScalarToNextPow2(TypeIdx: 0)
1521 .scalarize(TypeIdx: 0)
1522 .lower();
1523 }
1524
1525 getActionDefinitionsBuilder(Opcode: G_INTTOPTR)
1526 // List the common cases
1527 .legalForCartesianProduct(Types0: AddrSpaces64, Types1: {S64})
1528 .legalForCartesianProduct(Types0: AddrSpaces32, Types1: {S32})
1529 .scalarize(TypeIdx: 0)
1530 // Accept any address space as long as the size matches
1531 .legalIf(Predicate: sameSize(TypeIdx0: 0, TypeIdx1: 1))
1532 .widenScalarIf(Predicate: smallerThan(TypeIdx0: 1, TypeIdx1: 0),
1533 Mutation: [](const LegalityQuery &Query) {
1534 return std::pair(
1535 1, LLT::scalar(SizeInBits: Query.Types[0].getSizeInBits()));
1536 })
1537 .narrowScalarIf(Predicate: largerThan(TypeIdx0: 1, TypeIdx1: 0), Mutation: [](const LegalityQuery &Query) {
1538 return std::pair(1, LLT::scalar(SizeInBits: Query.Types[0].getSizeInBits()));
1539 });
1540
1541 getActionDefinitionsBuilder(Opcode: G_PTRTOINT)
1542 // List the common cases
1543 .legalForCartesianProduct(Types0: AddrSpaces64, Types1: {S64})
1544 .legalForCartesianProduct(Types0: AddrSpaces32, Types1: {S32})
1545 .scalarize(TypeIdx: 0)
1546 // Accept any address space as long as the size matches
1547 .legalIf(Predicate: sameSize(TypeIdx0: 0, TypeIdx1: 1))
1548 .widenScalarIf(Predicate: smallerThan(TypeIdx0: 0, TypeIdx1: 1),
1549 Mutation: [](const LegalityQuery &Query) {
1550 return std::pair(
1551 0, LLT::scalar(SizeInBits: Query.Types[1].getSizeInBits()));
1552 })
1553 .narrowScalarIf(Predicate: largerThan(TypeIdx0: 0, TypeIdx1: 1), Mutation: [](const LegalityQuery &Query) {
1554 return std::pair(0, LLT::scalar(SizeInBits: Query.Types[1].getSizeInBits()));
1555 });
1556
1557 getActionDefinitionsBuilder(Opcode: G_ADDRSPACE_CAST)
1558 .scalarize(TypeIdx: 0)
1559 .custom();
1560
1561 const auto needToSplitMemOp = [=](const LegalityQuery &Query,
1562 bool IsLoad) -> bool {
1563 const LLT DstTy = Query.Types[0];
1564
1565 // Split vector extloads.
1566 unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
1567
1568 if (DstTy.isVector() && DstTy.getSizeInBits() > MemSize)
1569 return true;
1570
1571 const LLT PtrTy = Query.Types[1];
1572 unsigned AS = PtrTy.getAddressSpace();
1573 if (MemSize > maxSizeForAddrSpace(ST, AS, IsLoad,
1574 IsAtomic: Query.MMODescrs[0].Ordering !=
1575 AtomicOrdering::NotAtomic))
1576 return true;
1577
1578 // Catch weird sized loads that don't evenly divide into the access sizes
1579 // TODO: May be able to widen depending on alignment etc.
1580 unsigned NumRegs = (MemSize + 31) / 32;
1581 if (NumRegs == 3) {
1582 if (!ST.hasDwordx3LoadStores())
1583 return true;
1584 } else {
1585 // If the alignment allows, these should have been widened.
1586 if (!isPowerOf2_32(Value: NumRegs))
1587 return true;
1588 }
1589
1590 return false;
1591 };
1592
1593 unsigned GlobalAlign32 = ST.hasUnalignedBufferAccessEnabled() ? 0 : 32;
1594 unsigned GlobalAlign16 = ST.hasUnalignedBufferAccessEnabled() ? 0 : 16;
1595 unsigned GlobalAlign8 = ST.hasUnalignedBufferAccessEnabled() ? 0 : 8;
1596
1597 // TODO: Refine based on subtargets which support unaligned access or 128-bit
1598 // LDS
1599 // TODO: Unsupported flat for SI.
1600
1601 for (unsigned Op : {G_LOAD, G_STORE}) {
1602 const bool IsStore = Op == G_STORE;
1603
1604 auto &Actions = getActionDefinitionsBuilder(Opcode: Op);
1605 // Explicitly list some common cases.
1606 // TODO: Does this help compile time at all?
1607 Actions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: S32, .Type1: GlobalPtr, .MemTy: S32, .Align: GlobalAlign32},
1608 {.Type0: V2S32, .Type1: GlobalPtr, .MemTy: V2S32, .Align: GlobalAlign32},
1609 {.Type0: V4S32, .Type1: GlobalPtr, .MemTy: V4S32, .Align: GlobalAlign32},
1610 {.Type0: S64, .Type1: GlobalPtr, .MemTy: S64, .Align: GlobalAlign32},
1611 {.Type0: V2S64, .Type1: GlobalPtr, .MemTy: V2S64, .Align: GlobalAlign32},
1612 {.Type0: V2S16, .Type1: GlobalPtr, .MemTy: V2S16, .Align: GlobalAlign32},
1613 {.Type0: S32, .Type1: GlobalPtr, .MemTy: S8, .Align: GlobalAlign8},
1614 {.Type0: S32, .Type1: GlobalPtr, .MemTy: S16, .Align: GlobalAlign16},
1615
1616 {.Type0: S32, .Type1: LocalPtr, .MemTy: S32, .Align: 32},
1617 {.Type0: S64, .Type1: LocalPtr, .MemTy: S64, .Align: 32},
1618 {.Type0: V2S32, .Type1: LocalPtr, .MemTy: V2S32, .Align: 32},
1619 {.Type0: S32, .Type1: LocalPtr, .MemTy: S8, .Align: 8},
1620 {.Type0: S32, .Type1: LocalPtr, .MemTy: S16, .Align: 16},
1621 {.Type0: V2S16, .Type1: LocalPtr, .MemTy: S32, .Align: 32},
1622
1623 {.Type0: S32, .Type1: PrivatePtr, .MemTy: S32, .Align: 32},
1624 {.Type0: S32, .Type1: PrivatePtr, .MemTy: S8, .Align: 8},
1625 {.Type0: S32, .Type1: PrivatePtr, .MemTy: S16, .Align: 16},
1626 {.Type0: V2S16, .Type1: PrivatePtr, .MemTy: S32, .Align: 32},
1627
1628 {.Type0: S32, .Type1: ConstantPtr, .MemTy: S32, .Align: GlobalAlign32},
1629 {.Type0: V2S32, .Type1: ConstantPtr, .MemTy: V2S32, .Align: GlobalAlign32},
1630 {.Type0: V4S32, .Type1: ConstantPtr, .MemTy: V4S32, .Align: GlobalAlign32},
1631 {.Type0: S64, .Type1: ConstantPtr, .MemTy: S64, .Align: GlobalAlign32},
1632 {.Type0: V2S32, .Type1: ConstantPtr, .MemTy: V2S32, .Align: GlobalAlign32}});
1633
1634 Actions.legalForTypesWithMemDesc(Pred: ST.useRealTrue16Insts(), /* Pred */
1635 TypesAndMemDesc: {{.Type0: S16, .Type1: GlobalPtr, .MemTy: S8, .Align: GlobalAlign8},
1636 {.Type0: S16, .Type1: GlobalPtr, .MemTy: S16, .Align: GlobalAlign16},
1637 {.Type0: S16, .Type1: LocalPtr, .MemTy: S8, .Align: 8},
1638 {.Type0: S16, .Type1: LocalPtr, .MemTy: S16, .Align: 16},
1639 {.Type0: S16, .Type1: PrivatePtr, .MemTy: S8, .Align: 8},
1640 {.Type0: S16, .Type1: PrivatePtr, .MemTy: S16, .Align: 16}});
1641
1642 Actions.legalIf(
1643 Predicate: [=](const LegalityQuery &Query) -> bool {
1644 return isLoadStoreLegal(ST, Query);
1645 });
1646
1647 // The custom pointers (fat pointers, buffer resources) don't work with load
1648 // and store at this level. Fat pointers should have been lowered to
1649 // intrinsics before the translation to MIR.
1650 Actions.unsupportedIf(
1651 Predicate: typeInSet(TypeIdx: 1, TypesInit: {BufferFatPtr, BufferStridedPtr, RsrcPtr}));
1652
1653 // Address space 8 pointers are handled by a 4xs32 load, bitcast, and
1654 // ptrtoint. This is needed to account for the fact that we can't have i128
1655 // as a register class for SelectionDAG reasons.
1656 Actions.customIf(Predicate: [=](const LegalityQuery &Query) -> bool {
1657 return hasBufferRsrcWorkaround(Ty: Query.Types[0]);
1658 });
1659
1660 // Constant 32-bit is handled by addrspacecasting the 32-bit pointer to
1661 // 64-bits.
1662 //
1663 // TODO: Should generalize bitcast action into coerce, which will also cover
1664 // inserting addrspacecasts.
1665 Actions.customIf(Predicate: typeIs(TypeIdx: 1, TypesInit: Constant32Ptr));
1666
1667 // Turn any illegal element vectors into something easier to deal
1668 // with. These will ultimately produce 32-bit scalar shifts to extract the
1669 // parts anyway.
1670 //
1671 // For odd 16-bit element vectors, prefer to split those into pieces with
1672 // 16-bit vector parts.
1673 Actions.bitcastIf(
1674 Predicate: [=](const LegalityQuery &Query) -> bool {
1675 return shouldBitcastLoadStoreType(ST, Ty: Query.Types[0],
1676 MemTy: Query.MMODescrs[0].MemoryTy);
1677 }, Mutation: bitcastToRegisterType(TypeIdx: 0));
1678
1679 if (!IsStore) {
1680 // Widen suitably aligned loads by loading extra bytes. The standard
1681 // legalization actions can't properly express widening memory operands.
1682 Actions.customIf(Predicate: [=](const LegalityQuery &Query) -> bool {
1683 return shouldWidenLoad(ST, Query, Opcode: G_LOAD);
1684 });
1685 }
1686
1687 // FIXME: load/store narrowing should be moved to lower action
1688 Actions
1689 .narrowScalarIf(
1690 Predicate: [=](const LegalityQuery &Query) -> bool {
1691 return !Query.Types[0].isVector() &&
1692 needToSplitMemOp(Query, Op == G_LOAD);
1693 },
1694 Mutation: [=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
1695 const LLT DstTy = Query.Types[0];
1696 const LLT PtrTy = Query.Types[1];
1697
1698 const unsigned DstSize = DstTy.getSizeInBits();
1699 unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
1700
1701 // Split extloads.
1702 if (DstSize > MemSize)
1703 return std::pair(0, LLT::scalar(SizeInBits: MemSize));
1704
1705 unsigned MaxSize = maxSizeForAddrSpace(
1706 ST, AS: PtrTy.getAddressSpace(), IsLoad: Op == G_LOAD,
1707 IsAtomic: Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic);
1708 if (MemSize > MaxSize)
1709 return std::pair(0, LLT::scalar(SizeInBits: MaxSize));
1710
1711 uint64_t Align = Query.MMODescrs[0].AlignInBits;
1712 return std::pair(0, LLT::scalar(SizeInBits: Align));
1713 })
1714 .fewerElementsIf(
1715 Predicate: [=](const LegalityQuery &Query) -> bool {
1716 return Query.Types[0].isVector() &&
1717 needToSplitMemOp(Query, Op == G_LOAD);
1718 },
1719 Mutation: [=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
1720 const LLT DstTy = Query.Types[0];
1721 const LLT PtrTy = Query.Types[1];
1722
1723 LLT EltTy = DstTy.getElementType();
1724 unsigned MaxSize = maxSizeForAddrSpace(
1725 ST, AS: PtrTy.getAddressSpace(), IsLoad: Op == G_LOAD,
1726 IsAtomic: Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic);
1727
1728 // FIXME: Handle widened to power of 2 results better. This ends
1729 // up scalarizing.
1730 // FIXME: 3 element stores scalarized on SI
1731
1732 // Split if it's too large for the address space.
1733 unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
1734 if (MemSize > MaxSize) {
1735 unsigned NumElts = DstTy.getNumElements();
1736 unsigned EltSize = EltTy.getSizeInBits();
1737
1738 if (MaxSize % EltSize == 0) {
1739 return std::pair(
1740 0, LLT::scalarOrVector(
1741 EC: ElementCount::getFixed(MinVal: MaxSize / EltSize), ScalarTy: EltTy));
1742 }
1743
1744 unsigned NumPieces = MemSize / MaxSize;
1745
1746 // FIXME: Refine when odd breakdowns handled
1747 // The scalars will need to be re-legalized.
1748 if (NumPieces == 1 || NumPieces >= NumElts ||
1749 NumElts % NumPieces != 0)
1750 return std::pair(0, EltTy);
1751
1752 return std::pair(0,
1753 LLT::fixed_vector(NumElements: NumElts / NumPieces, ScalarTy: EltTy));
1754 }
1755
1756 // FIXME: We could probably handle weird extending loads better.
1757 if (DstTy.getSizeInBits() > MemSize)
1758 return std::pair(0, EltTy);
1759
1760 unsigned EltSize = EltTy.getSizeInBits();
1761 unsigned DstSize = DstTy.getSizeInBits();
1762 if (!isPowerOf2_32(Value: DstSize)) {
1763 // We're probably decomposing an odd sized store. Try to split
1764 // to the widest type. TODO: Account for alignment. As-is it
1765 // should be OK, since the new parts will be further legalized.
1766 unsigned FloorSize = llvm::bit_floor(Value: DstSize);
1767 return std::pair(
1768 0, LLT::scalarOrVector(
1769 EC: ElementCount::getFixed(MinVal: FloorSize / EltSize), ScalarTy: EltTy));
1770 }
1771
1772 // May need relegalization for the scalars.
1773 return std::pair(0, EltTy);
1774 })
1775 .widenScalarIf(Predicate: scalarNarrowerThan(TypeIdx: 0, Size: 32), Mutation: changeTo(TypeIdx: 0, Ty: LLT::integer(SizeInBits: 32)))
1776 .narrowScalarIf(Predicate: isTruncStoreToSizePowerOf2(TypeIdx: 0),
1777 Mutation: getScalarTypeFromMemDesc(TypeIdx: 0))
1778 .widenScalarToNextPow2(TypeIdx: 0)
1779 .moreElementsIf(Predicate: vectorSmallerThan(TypeIdx: 0, Size: 32), Mutation: moreEltsToNext32Bit(TypeIdx: 0))
1780 .lower();
1781 }
1782
1783 // FIXME: Unaligned accesses not lowered.
1784 auto &ExtLoads =
1785 getActionDefinitionsBuilder(Opcodes: {G_SEXTLOAD, G_ZEXTLOAD})
1786 .legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: S32, .Type1: GlobalPtr, .MemTy: S8, .Align: 8},
1787 {.Type0: S32, .Type1: GlobalPtr, .MemTy: S16, .Align: 2 * 8},
1788 {.Type0: S32, .Type1: LocalPtr, .MemTy: S8, .Align: 8},
1789 {.Type0: S32, .Type1: LocalPtr, .MemTy: S16, .Align: 16},
1790 {.Type0: S32, .Type1: PrivatePtr, .MemTy: S8, .Align: 8},
1791 {.Type0: S32, .Type1: PrivatePtr, .MemTy: S16, .Align: 16},
1792 {.Type0: S32, .Type1: ConstantPtr, .MemTy: S8, .Align: 8},
1793 {.Type0: S32, .Type1: ConstantPtr, .MemTy: S16, .Align: 2 * 8}})
1794 .legalForTypesWithMemDesc(Pred: ST.useRealTrue16Insts(),
1795 TypesAndMemDesc: {{.Type0: S16, .Type1: GlobalPtr, .MemTy: S8, .Align: GlobalAlign8},
1796 {.Type0: S16, .Type1: LocalPtr, .MemTy: S8, .Align: GlobalAlign8},
1797 {.Type0: S16, .Type1: PrivatePtr, .MemTy: S8, .Align: GlobalAlign8},
1798 {.Type0: S16, .Type1: ConstantPtr, .MemTy: S8, .Align: GlobalAlign8}})
1799 .legalIf(Predicate: [=](const LegalityQuery &Query) -> bool {
1800 return isLoadStoreLegal(ST, Query);
1801 });
1802
1803 if (ST.hasFlatAddressSpace()) {
1804 ExtLoads.legalForTypesWithMemDesc(
1805 TypesAndMemDesc: {{.Type0: S32, .Type1: FlatPtr, .MemTy: S8, .Align: 8}, {.Type0: S32, .Type1: FlatPtr, .MemTy: S16, .Align: 16}});
1806
1807 ExtLoads.legalForTypesWithMemDesc(Pred: ST.useRealTrue16Insts(),
1808 TypesAndMemDesc: {{.Type0: S16, .Type1: FlatPtr, .MemTy: S8, .Align: GlobalAlign8}});
1809 }
1810
1811 // Constant 32-bit is handled by addrspacecasting the 32-bit pointer to
1812 // 64-bits.
1813 //
1814 // TODO: Should generalize bitcast action into coerce, which will also cover
1815 // inserting addrspacecasts.
1816 ExtLoads.customIf(Predicate: typeIs(TypeIdx: 1, TypesInit: Constant32Ptr));
1817
1818 ExtLoads.narrowScalarIf(
1819 Predicate: [](const LegalityQuery &Query) {
1820 LLT MemTy = Query.MMODescrs[0].MemoryTy;
1821 return MemTy.isScalar() && MemTy.getSizeInBits() > 32 &&
1822 Query.Types[0].getSizeInBits() > MemTy.getSizeInBits();
1823 }, // For large MemSize, narrowscalar to MemSize (load MemSize + ext)
1824 Mutation: getScalarTypeFromMemDesc(TypeIdx: 0));
1825 ExtLoads.clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S32)
1826 .widenScalarToNextPow2(TypeIdx: 0)
1827 .lower();
1828
1829 auto &Atomics = getActionDefinitionsBuilder(
1830 Opcodes: {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB,
1831 G_ATOMICRMW_AND, G_ATOMICRMW_OR, G_ATOMICRMW_XOR,
1832 G_ATOMICRMW_MAX, G_ATOMICRMW_MIN, G_ATOMICRMW_UMAX,
1833 G_ATOMICRMW_UMIN, G_ATOMICRMW_UINC_WRAP, G_ATOMICRMW_UDEC_WRAP})
1834 .legalFor(Types: {{S32, GlobalPtr}, {S32, LocalPtr},
1835 {S64, GlobalPtr}, {S64, LocalPtr},
1836 {S32, RegionPtr}, {S64, RegionPtr}});
1837 if (ST.hasFlatAddressSpace()) {
1838 Atomics.legalFor(Types: {{S32, FlatPtr}, {S64, FlatPtr}});
1839 }
1840
1841 auto &Atomics32 =
1842 getActionDefinitionsBuilder(Opcodes: {G_ATOMICRMW_USUB_COND, G_ATOMICRMW_USUB_SAT})
1843 .legalFor(Types: {{S32, GlobalPtr}, {S32, LocalPtr}, {S32, RegionPtr}});
1844 if (ST.hasFlatAddressSpace()) {
1845 Atomics32.legalFor(Types: {{S32, FlatPtr}});
1846 }
1847
1848 // TODO: v2bf16 operations, and fat buffer pointer support.
1849 auto &Atomic = getActionDefinitionsBuilder(Opcode: G_ATOMICRMW_FADD);
1850 if (ST.hasLDSFPAtomicAddF32()) {
1851 Atomic.legalFor(Types: {{S32, LocalPtr}, {S32, RegionPtr}});
1852 if (ST.hasLdsAtomicAddF64())
1853 Atomic.legalFor(Types: {{S64, LocalPtr}});
1854 if (ST.hasAtomicDsPkAdd16Insts())
1855 Atomic.legalFor(Types: {{V2F16, LocalPtr}, {V2BF16, LocalPtr}});
1856 }
1857 if (ST.hasAtomicFaddInsts())
1858 Atomic.legalFor(Types: {{S32, GlobalPtr}});
1859 if (ST.hasFlatAtomicFaddF32Inst())
1860 Atomic.legalFor(Types: {{S32, FlatPtr}});
1861
1862 if (ST.hasGFX90AInsts() || ST.hasGFX1250Insts()) {
1863 // These are legal with some caveats, and should have undergone expansion in
1864 // the IR in most situations
1865 // TODO: Move atomic expansion into legalizer
1866 Atomic.legalFor(Types: {
1867 {S32, GlobalPtr},
1868 {S64, GlobalPtr},
1869 {S64, FlatPtr}
1870 });
1871 }
1872
1873 if (ST.hasAtomicBufferGlobalPkAddF16NoRtnInsts() ||
1874 ST.hasAtomicBufferGlobalPkAddF16Insts())
1875 Atomic.legalFor(Types: {{V2F16, GlobalPtr}, {V2F16, BufferFatPtr}});
1876 if (ST.hasAtomicGlobalPkAddBF16Inst())
1877 Atomic.legalFor(Types: {{V2BF16, GlobalPtr}});
1878 if (ST.hasAtomicFlatPkAdd16Insts())
1879 Atomic.legalFor(Types: {{V2F16, FlatPtr}, {V2BF16, FlatPtr}});
1880
1881
1882 // Most of the legalization work here is done by AtomicExpand. We could
1883 // probably use a simpler legality rule that just assumes anything is OK.
1884 auto &AtomicFMinFMax =
1885 getActionDefinitionsBuilder(Opcodes: {G_ATOMICRMW_FMIN, G_ATOMICRMW_FMAX})
1886 .legalFor(Types: {{F32, LocalPtr}, {F64, LocalPtr}});
1887
1888 if (ST.hasAtomicFMinFMaxF32GlobalInsts())
1889 AtomicFMinFMax.legalFor(Types: {{F32, GlobalPtr},{F32, BufferFatPtr}});
1890 if (ST.hasAtomicFMinFMaxF64GlobalInsts())
1891 AtomicFMinFMax.legalFor(Types: {{F64, GlobalPtr}, {F64, BufferFatPtr}});
1892 if (ST.hasAtomicFMinFMaxF32FlatInsts())
1893 AtomicFMinFMax.legalFor(Types: {F32, FlatPtr});
1894 if (ST.hasAtomicFMinFMaxF64FlatInsts())
1895 AtomicFMinFMax.legalFor(Types: {F64, FlatPtr});
1896
1897 // BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling, and output
1898 // demarshalling
1899 getActionDefinitionsBuilder(Opcode: G_ATOMIC_CMPXCHG)
1900 .customFor(Types: {{S32, GlobalPtr}, {S64, GlobalPtr},
1901 {S32, FlatPtr}, {S64, FlatPtr}})
1902 .legalFor(Types: {{S32, LocalPtr}, {S64, LocalPtr},
1903 {S32, RegionPtr}, {S64, RegionPtr}});
1904 // TODO: Pointer types, any 32-bit or 64-bit vector
1905
1906 // Condition should be s32 for scalar, s1 for vector.
1907 getActionDefinitionsBuilder(Opcode: G_SELECT)
1908 .legalForCartesianProduct(Types0: {S32, S64, S16, V2S32, V2S16, V4S16, GlobalPtr,
1909 LocalPtr, FlatPtr, PrivatePtr,
1910 LLT::fixed_vector(NumElements: 2, ScalarTy: LocalPtr),
1911 LLT::fixed_vector(NumElements: 2, ScalarTy: PrivatePtr)},
1912 Types1: {S1, S32})
1913 .clampScalar(TypeIdx: 0, MinTy: S16, MaxTy: S64)
1914 .scalarize(TypeIdx: 1)
1915 .moreElementsIf(Predicate: isSmallOddVector(TypeIdx: 0), Mutation: oneMoreElement(TypeIdx: 0))
1916 .fewerElementsIf(Predicate: numElementsNotEven(TypeIdx: 0), Mutation: scalarize(TypeIdx: 0))
1917 .clampMaxNumElements(TypeIdx: 0, EltTy: S32, MaxElements: 2)
1918 .clampMaxNumElements(TypeIdx: 0, EltTy: LocalPtr, MaxElements: 2)
1919 .clampMaxNumElements(TypeIdx: 0, EltTy: PrivatePtr, MaxElements: 2)
1920 .scalarize(TypeIdx: 0)
1921 .widenScalarToNextPow2(TypeIdx: 0)
1922 .legalIf(Predicate: all(P0: isPointer(TypeIdx: 0), P1: typeInSet(TypeIdx: 1, TypesInit: {S1, S32})));
1923
1924 // TODO: Only the low 4/5/6 bits of the shift amount are observed, so we can
1925 // be more flexible with the shift amount type.
1926 auto &Shifts = getActionDefinitionsBuilder(Opcodes: {G_SHL, G_LSHR, G_ASHR})
1927 .legalFor(Types: {{S32, S32}, {S64, S32}});
1928 if (ST.has16BitInsts()) {
1929 if (ST.hasVOP3PInsts()) {
1930 Shifts.legalFor(Types: {{S16, S16}, {V2S16, V2S16}})
1931 .clampMaxNumElements(TypeIdx: 0, EltTy: S16, MaxElements: 2);
1932 } else
1933 Shifts.legalFor(Types: {{S16, S16}});
1934
1935 // TODO: Support 16-bit shift amounts for all types
1936 Shifts.widenScalarIf(
1937 Predicate: [=](const LegalityQuery &Query) {
1938 // Use 16-bit shift amounts for any 16-bit shift. Otherwise we want a
1939 // 32-bit amount.
1940 const LLT ValTy = Query.Types[0];
1941 const LLT AmountTy = Query.Types[1];
1942 return ValTy.isScalar() && ValTy.getSizeInBits() <= 16 &&
1943 AmountTy.getSizeInBits() < 16;
1944 },
1945 Mutation: changeElementSizeTo(TypeIdx: 1, NewTy: S16));
1946 Shifts.maxScalarIf(Predicate: typeIs(TypeIdx: 0, TypesInit: S16), TypeIdx: 1, Ty: S16);
1947 Shifts.clampScalar(TypeIdx: 1, MinTy: S32, MaxTy: S32);
1948 Shifts.widenScalarToNextPow2(TypeIdx: 0, MinSize: 16);
1949 Shifts.clampScalar(TypeIdx: 0, MinTy: S16, MaxTy: S64);
1950
1951 getActionDefinitionsBuilder(Opcodes: {G_SSHLSAT, G_USHLSAT})
1952 .minScalar(TypeIdx: 0, Ty: S16)
1953 .scalarize(TypeIdx: 0)
1954 .lower();
1955 } else {
1956 // Make sure we legalize the shift amount type first, as the general
1957 // expansion for the shifted type will produce much worse code if it hasn't
1958 // been truncated already.
1959 Shifts.clampScalar(TypeIdx: 1, MinTy: S32, MaxTy: S32);
1960 Shifts.widenScalarToNextPow2(TypeIdx: 0, MinSize: 32);
1961 Shifts.clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S64);
1962
1963 getActionDefinitionsBuilder(Opcodes: {G_SSHLSAT, G_USHLSAT})
1964 .minScalar(TypeIdx: 0, Ty: S32)
1965 .scalarize(TypeIdx: 0)
1966 .lower();
1967 }
1968 Shifts.scalarize(TypeIdx: 0);
1969
1970 for (unsigned Op : {G_EXTRACT_VECTOR_ELT, G_INSERT_VECTOR_ELT}) {
1971 unsigned VecTypeIdx = Op == G_EXTRACT_VECTOR_ELT ? 1 : 0;
1972 unsigned EltTypeIdx = Op == G_EXTRACT_VECTOR_ELT ? 0 : 1;
1973 unsigned IdxTypeIdx = 2;
1974
1975 getActionDefinitionsBuilder(Opcode: Op)
1976 .customIf(Predicate: [=](const LegalityQuery &Query) {
1977 const LLT EltTy = Query.Types[EltTypeIdx];
1978 const LLT VecTy = Query.Types[VecTypeIdx];
1979 const LLT IdxTy = Query.Types[IdxTypeIdx];
1980 const unsigned EltSize = EltTy.getSizeInBits();
1981 const bool isLegalVecType =
1982 !!SIRegisterInfo::getSGPRClassForBitWidth(BitWidth: VecTy.getSizeInBits());
1983 // Address space 8 pointers are 128-bit wide values, but the logic
1984 // below will try to bitcast them to 2N x s64, which will fail.
1985 // Therefore, as an intermediate step, wrap extracts/insertions from a
1986 // ptrtoint-ing the vector and scalar arguments (or inttoptring the
1987 // extraction result) in order to produce a vector operation that can
1988 // be handled by the logic below.
1989 if (EltTy.isPointer() && EltSize > 64)
1990 return true;
1991 return (EltSize == 32 || EltSize == 64) &&
1992 VecTy.getSizeInBits() % 32 == 0 &&
1993 VecTy.getSizeInBits() <= MaxRegisterSize &&
1994 IdxTy.getSizeInBits() == 32 &&
1995 isLegalVecType;
1996 })
1997 .bitcastIf(Predicate: all(P0: sizeIsMultipleOf32(TypeIdx: VecTypeIdx),
1998 P1: scalarOrEltNarrowerThan(TypeIdx: VecTypeIdx, Size: 32)),
1999 Mutation: bitcastToVectorElement32(TypeIdx: VecTypeIdx))
2000 //.bitcastIf(vectorSmallerThan(1, 32), bitcastToScalar(1))
2001 .bitcastIf(Predicate: all(P0: sizeIsMultipleOf32(TypeIdx: VecTypeIdx),
2002 P1: scalarOrEltWiderThan(TypeIdx: VecTypeIdx, Size: 64)),
2003 Mutation: [=](const LegalityQuery &Query) {
2004 // For > 64-bit element types, try to turn this into a
2005 // 64-bit element vector since we may be able to do better
2006 // indexing if this is scalar. If not, fall back to 32.
2007 const LLT EltTy = Query.Types[EltTypeIdx];
2008 const LLT VecTy = Query.Types[VecTypeIdx];
2009 const unsigned DstEltSize = EltTy.getSizeInBits();
2010 const unsigned VecSize = VecTy.getSizeInBits();
2011
2012 const unsigned TargetEltSize =
2013 DstEltSize % 64 == 0 ? 64 : 32;
2014 return std::pair(VecTypeIdx,
2015 LLT::fixed_vector(NumElements: VecSize / TargetEltSize,
2016 ScalarSizeInBits: TargetEltSize));
2017 })
2018 .clampScalar(TypeIdx: EltTypeIdx, MinTy: S32, MaxTy: S64)
2019 .clampScalar(TypeIdx: VecTypeIdx, MinTy: S32, MaxTy: S64)
2020 .clampScalar(TypeIdx: IdxTypeIdx, MinTy: S32, MaxTy: S32)
2021 .clampMaxNumElements(TypeIdx: VecTypeIdx, EltTy: S32, MaxElements: 32)
2022 // TODO: Clamp elements for 64-bit vectors?
2023 .moreElementsIf(Predicate: isIllegalRegisterType(ST, TypeIdx: VecTypeIdx),
2024 Mutation: moreElementsToNextExistingRegClass(TypeIdx: VecTypeIdx))
2025 // It should only be necessary with variable indexes.
2026 // As a last resort, lower to the stack
2027 .lower();
2028 }
2029
2030 getActionDefinitionsBuilder(Opcode: G_EXTRACT_VECTOR_ELT)
2031 .unsupportedIf(Predicate: [=](const LegalityQuery &Query) {
2032 const LLT &EltTy = Query.Types[1].getElementType();
2033 return Query.Types[0] != EltTy;
2034 });
2035
2036 for (unsigned Op : {G_EXTRACT, G_INSERT}) {
2037 unsigned BigTyIdx = Op == G_EXTRACT ? 1 : 0;
2038 unsigned LitTyIdx = Op == G_EXTRACT ? 0 : 1;
2039 getActionDefinitionsBuilder(Opcode: Op)
2040 .widenScalarIf(
2041 Predicate: [=](const LegalityQuery &Query) {
2042 const LLT BigTy = Query.Types[BigTyIdx];
2043 return (BigTy.getScalarSizeInBits() < 16);
2044 },
2045 Mutation: LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx: BigTyIdx, Min: 16))
2046 .widenScalarIf(
2047 Predicate: [=](const LegalityQuery &Query) {
2048 const LLT LitTy = Query.Types[LitTyIdx];
2049 return (LitTy.getScalarSizeInBits() < 16);
2050 },
2051 Mutation: LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx: LitTyIdx, Min: 16))
2052 .moreElementsIf(Predicate: isSmallOddVector(TypeIdx: BigTyIdx), Mutation: oneMoreElement(TypeIdx: BigTyIdx))
2053 .widenScalarToNextPow2(TypeIdx: BigTyIdx, MinSize: 32)
2054 .customIf(Predicate: [=](const LegalityQuery &Query) {
2055 // Generic lower operates on the full-width value, producing
2056 // shift+trunc/mask sequences. For simple cases where extract/insert
2057 // values are 32-bit aligned, we can instead unmerge/merge and work on
2058 // the 32-bit components. However, we can't check the offset here so
2059 // custom lower function will have to call generic lowering if offset
2060 // is not 32-bit aligned.
2061 const LLT BigTy = Query.Types[BigTyIdx];
2062 const LLT LitTy = Query.Types[LitTyIdx];
2063 return !BigTy.isVector() && BigTy.getSizeInBits() % 32 == 0 &&
2064 LitTy.getSizeInBits() % 32 == 0;
2065 })
2066 .lower();
2067 }
2068
2069 auto &BuildVector =
2070 getActionDefinitionsBuilder(Opcode: G_BUILD_VECTOR)
2071 .legalForCartesianProduct(Types0: AllS32Vectors, Types1: {S32})
2072 .legalForCartesianProduct(Types0: AllS64Vectors, Types1: {S64})
2073 .clampNumElements(TypeIdx: 0, MinTy: V16S32, MaxTy: V32S32)
2074 .clampNumElements(TypeIdx: 0, MinTy: V2S64, MaxTy: V16S64)
2075 .fewerElementsIf(Predicate: isWideVec16(TypeIdx: 0),
2076 Mutation: changeElementCountTo(TypeIdx: 0, EC: ElementCount::getFixed(MinVal: 2)))
2077 .moreElementsIf(Predicate: isIllegalRegisterType(ST, TypeIdx: 0),
2078 Mutation: moreElementsToNextExistingRegClass(TypeIdx: 0));
2079
2080 if (ST.hasScalarPackInsts()) {
2081 BuildVector
2082 // FIXME: Should probably widen s1 vectors straight to s32
2083 .minScalarOrElt(TypeIdx: 0, Ty: S16)
2084 .minScalar(TypeIdx: 1, Ty: S16);
2085
2086 getActionDefinitionsBuilder(Opcode: G_BUILD_VECTOR_TRUNC)
2087 .legalFor(Types: {V2S16, S32})
2088 .lower();
2089 } else {
2090 BuildVector.customFor(Types: {V2S16, S16});
2091 BuildVector.minScalarOrElt(TypeIdx: 0, Ty: S32);
2092
2093 getActionDefinitionsBuilder(Opcode: G_BUILD_VECTOR_TRUNC)
2094 .customFor(Types: {V2S16, S32})
2095 .lower();
2096 }
2097
2098 BuildVector.legalIf(Predicate: isRegisterType(ST, TypeIdx: 0));
2099
2100 // FIXME: Clamp maximum size
2101 getActionDefinitionsBuilder(Opcode: G_CONCAT_VECTORS)
2102 .legalIf(Predicate: all(P0: isRegisterType(ST, TypeIdx: 0), P1: isRegisterType(ST, TypeIdx: 1)))
2103 .clampMaxNumElements(TypeIdx: 0, EltTy: S32, MaxElements: 32)
2104 .clampMaxNumElements(TypeIdx: 1, EltTy: S16, MaxElements: 2) // TODO: Make 4?
2105 .clampMaxNumElements(TypeIdx: 0, EltTy: S16, MaxElements: 64);
2106
2107 getActionDefinitionsBuilder(Opcode: G_SHUFFLE_VECTOR).lower();
2108
2109 // Merge/Unmerge
2110 for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
2111 unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
2112 unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
2113
2114 auto notValidElt = [=](const LegalityQuery &Query, unsigned TypeIdx) {
2115 const LLT Ty = Query.Types[TypeIdx];
2116 if (Ty.isVector()) {
2117 const LLT &EltTy = Ty.getElementType();
2118 if (EltTy.getSizeInBits() < 8 || EltTy.getSizeInBits() > 512)
2119 return true;
2120 if (!llvm::has_single_bit<uint32_t>(Value: EltTy.getSizeInBits()))
2121 return true;
2122 }
2123 return false;
2124 };
2125
2126 auto &Builder =
2127 getActionDefinitionsBuilder(Opcode: Op)
2128 .legalIf(Predicate: all(P0: isRegisterType(ST, TypeIdx: 0), P1: isRegisterType(ST, TypeIdx: 1)))
2129 .lowerFor(Types: {{S16, V2S16}})
2130 .lowerIf(Predicate: [=](const LegalityQuery &Query) {
2131 const LLT BigTy = Query.Types[BigTyIdx];
2132 return BigTy.getSizeInBits() == 32;
2133 })
2134 // Try to widen to s16 first for small types.
2135 // TODO: Only do this on targets with legal s16 shifts
2136 .minScalarOrEltIf(Predicate: scalarNarrowerThan(TypeIdx: LitTyIdx, Size: 16), TypeIdx: LitTyIdx, Ty: S16)
2137 .widenScalarToNextPow2(TypeIdx: LitTyIdx, /*Min*/ MinSize: 16)
2138 .moreElementsIf(Predicate: isSmallOddVector(TypeIdx: BigTyIdx),
2139 Mutation: oneMoreElement(TypeIdx: BigTyIdx))
2140 .fewerElementsIf(Predicate: all(P0: typeIs(TypeIdx: 0, TypesInit: S16), P1: vectorWiderThan(TypeIdx: 1, Size: 32),
2141 args: elementTypeIs(TypeIdx: 1, EltTy: S16)),
2142 Mutation: changeElementCountTo(TypeIdx: 1, EC: ElementCount::getFixed(MinVal: 2)))
2143 // Clamp the little scalar to s8-s256 and make it a power of 2. It's
2144 // not worth considering the multiples of 64 since 2*192 and 2*384
2145 // are not valid.
2146 .clampScalar(TypeIdx: LitTyIdx, MinTy: S32, MaxTy: S512)
2147 .widenScalarToNextPow2(TypeIdx: LitTyIdx, /*Min*/ MinSize: 32)
2148 // Break up vectors with weird elements into scalars
2149 .fewerElementsIf(
2150 Predicate: [=](const LegalityQuery &Query) {
2151 return notValidElt(Query, LitTyIdx);
2152 },
2153 Mutation: scalarize(TypeIdx: 0))
2154 .fewerElementsIf(
2155 Predicate: [=](const LegalityQuery &Query) {
2156 return notValidElt(Query, BigTyIdx);
2157 },
2158 Mutation: scalarize(TypeIdx: 1))
2159 .clampScalar(TypeIdx: BigTyIdx, MinTy: S32, MaxTy: MaxScalar);
2160
2161 if (Op == G_MERGE_VALUES) {
2162 Builder.widenScalarIf(
2163 // TODO: Use 16-bit shifts if legal for 8-bit values?
2164 Predicate: [=](const LegalityQuery &Query) {
2165 const LLT Ty = Query.Types[LitTyIdx];
2166 return Ty.getSizeInBits() < 32;
2167 },
2168 Mutation: changeElementSizeTo(TypeIdx: LitTyIdx, NewTy: S32));
2169 }
2170
2171 Builder.widenScalarIf(
2172 Predicate: [=](const LegalityQuery &Query) {
2173 const LLT Ty = Query.Types[BigTyIdx];
2174 return Ty.getSizeInBits() % 16 != 0;
2175 },
2176 Mutation: [=](const LegalityQuery &Query) {
2177 // Pick the next power of 2, or a multiple of 64 over 128.
2178 // Whichever is smaller.
2179 const LLT &Ty = Query.Types[BigTyIdx];
2180 unsigned NewSizeInBits = 1 << Log2_32_Ceil(Value: Ty.getSizeInBits() + 1);
2181 if (NewSizeInBits >= 256) {
2182 unsigned RoundedTo = alignTo<64>(Value: Ty.getSizeInBits() + 1);
2183 if (RoundedTo < NewSizeInBits)
2184 NewSizeInBits = RoundedTo;
2185 }
2186 return std::pair(BigTyIdx, LLT::scalar(SizeInBits: NewSizeInBits));
2187 })
2188 // Any vectors left are the wrong size. Scalarize them.
2189 .scalarize(TypeIdx: 0)
2190 .scalarize(TypeIdx: 1);
2191 }
2192
2193 // S64 is only legal on SALU, and needs to be broken into 32-bit elements in
2194 // RegBankSelect.
2195 auto &SextInReg = getActionDefinitionsBuilder(Opcode: G_SEXT_INREG)
2196 .legalFor(Types: {{S32}, {S64}})
2197 .clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S64);
2198
2199 if (ST.hasVOP3PInsts()) {
2200 SextInReg.lowerFor(Types: {{V2S16}})
2201 // Prefer to reduce vector widths for 16-bit vectors before lowering, to
2202 // get more vector shift opportunities, since we'll get those when
2203 // expanded.
2204 .clampMaxNumElementsStrict(TypeIdx: 0, EltTy: S16, NumElts: 2);
2205 } else if (ST.has16BitInsts()) {
2206 SextInReg.lowerFor(Types: {{S32}, {S64}, {S16}});
2207 } else {
2208 // Prefer to promote to s32 before lowering if we don't have 16-bit
2209 // shifts. This avoid a lot of intermediate truncate and extend operations.
2210 SextInReg.lowerFor(Types: {{S32}, {S64}});
2211 }
2212
2213 SextInReg
2214 .scalarize(TypeIdx: 0)
2215 .clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S64)
2216 .lower();
2217
2218 getActionDefinitionsBuilder(Opcodes: {G_ROTR, G_ROTL})
2219 .scalarize(TypeIdx: 0)
2220 .lower();
2221
2222 auto &FSHRActionDefs = getActionDefinitionsBuilder(Opcode: G_FSHR);
2223 FSHRActionDefs.legalFor(Types: {{S32, S32}})
2224 .clampMaxNumElementsStrict(TypeIdx: 0, EltTy: S16, NumElts: 2);
2225 if (ST.hasVOP3PInsts())
2226 FSHRActionDefs.lowerFor(Types: {{V2S16, V2S16}});
2227 FSHRActionDefs.scalarize(TypeIdx: 0).lower();
2228
2229 if (ST.hasVOP3PInsts()) {
2230 getActionDefinitionsBuilder(Opcode: G_FSHL)
2231 .lowerFor(Types: {{V2S16, V2S16}})
2232 .clampMaxNumElementsStrict(TypeIdx: 0, EltTy: S16, NumElts: 2)
2233 .scalarize(TypeIdx: 0)
2234 .lower();
2235 } else {
2236 getActionDefinitionsBuilder(Opcode: G_FSHL)
2237 .scalarize(TypeIdx: 0)
2238 .lower();
2239 }
2240
2241 getActionDefinitionsBuilder(Opcode: G_READCYCLECOUNTER)
2242 .legalFor(Types: {S64});
2243
2244 getActionDefinitionsBuilder(Opcode: G_READSTEADYCOUNTER).legalFor(Types: {S64});
2245
2246 getActionDefinitionsBuilder(Opcode: G_FENCE)
2247 .alwaysLegal();
2248
2249 getActionDefinitionsBuilder(Opcodes: {G_SMULO, G_UMULO})
2250 .scalarize(TypeIdx: 0)
2251 .minScalar(TypeIdx: 0, Ty: S32)
2252 .lower();
2253
2254 getActionDefinitionsBuilder(Opcodes: {G_SBFX, G_UBFX})
2255 .legalFor(Types: {{S32, S32}, {S64, S32}})
2256 .clampScalar(TypeIdx: 1, MinTy: S32, MaxTy: S32)
2257 .clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S64)
2258 .widenScalarToNextPow2(TypeIdx: 0)
2259 .scalarize(TypeIdx: 0);
2260
2261 getActionDefinitionsBuilder(
2262 Opcodes: {// TODO: Verify V_BFI_B32 is generated from expanded bit ops
2263 G_FCOPYSIGN,
2264
2265 G_ATOMIC_CMPXCHG_WITH_SUCCESS, G_ATOMICRMW_NAND, G_ATOMICRMW_FSUB,
2266 G_READ_REGISTER, G_WRITE_REGISTER,
2267
2268 G_SADDO, G_SSUBO})
2269 .lower();
2270
2271 if (ST.hasIEEEMinimumMaximumInsts()) {
2272 getActionDefinitionsBuilder(Opcodes: {G_FMINIMUM, G_FMAXIMUM})
2273 .legalFor(Types: FPTypesPK16)
2274 .clampMaxNumElements(TypeIdx: 0, EltTy: S16, MaxElements: 2)
2275 .scalarize(TypeIdx: 0);
2276 } else if (ST.hasVOP3PInsts()) {
2277 getActionDefinitionsBuilder(Opcodes: {G_FMINIMUM, G_FMAXIMUM})
2278 .lowerFor(Types: {V2S16})
2279 .clampMaxNumElementsStrict(TypeIdx: 0, EltTy: S16, NumElts: 2)
2280 .scalarize(TypeIdx: 0)
2281 .lower();
2282 } else {
2283 getActionDefinitionsBuilder(Opcodes: {G_FMINIMUM, G_FMAXIMUM})
2284 .scalarize(TypeIdx: 0)
2285 .clampScalar(TypeIdx: 0, MinTy: S32, MaxTy: S64)
2286 .lower();
2287 }
2288
2289 getActionDefinitionsBuilder(
2290 Opcodes: {G_MEMCPY, G_MEMCPY_INLINE, G_MEMMOVE, G_MEMSET, G_MEMSET_INLINE})
2291 .lower();
2292
2293 getActionDefinitionsBuilder(Opcodes: {G_TRAP, G_DEBUGTRAP}).custom();
2294
2295 getActionDefinitionsBuilder(Opcodes: {G_VASTART, G_VAARG, G_BRJT, G_JUMP_TABLE,
2296 G_INDEXED_LOAD, G_INDEXED_SEXTLOAD,
2297 G_INDEXED_ZEXTLOAD, G_INDEXED_STORE})
2298 .unsupported();
2299
2300 getActionDefinitionsBuilder(Opcode: G_PREFETCH).alwaysLegal();
2301
2302 getActionDefinitionsBuilder(
2303 Opcodes: {G_VECREDUCE_SMIN, G_VECREDUCE_SMAX, G_VECREDUCE_UMIN, G_VECREDUCE_UMAX,
2304 G_VECREDUCE_ADD, G_VECREDUCE_MUL, G_VECREDUCE_FMUL, G_VECREDUCE_FMIN,
2305 G_VECREDUCE_FMAX, G_VECREDUCE_FMINIMUM, G_VECREDUCE_FMAXIMUM,
2306 G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})
2307 .legalFor(Types: AllVectors)
2308 .scalarize(TypeIdx: 1)
2309 .lower();
2310
2311 getActionDefinitionsBuilder(Opcodes: {G_INTRINSIC, G_INTRINSIC_W_SIDE_EFFECTS,
2312 G_INTRINSIC_CONVERGENT,
2313 G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS})
2314 .alwaysLegal();
2315
2316 verify(MII: *ST.getInstrInfo());
2317}
2318
2319bool AMDGPULegalizerInfo::legalizeCustom(
2320 LegalizerHelper &Helper, MachineInstr &MI,
2321 LostDebugLocObserver &LocObserver) const {
2322 MachineIRBuilder &B = Helper.MIRBuilder;
2323 MachineRegisterInfo &MRI = *B.getMRI();
2324
2325 switch (MI.getOpcode()) {
2326 case TargetOpcode::G_ADDRSPACE_CAST:
2327 return legalizeAddrSpaceCast(MI, MRI, B);
2328 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
2329 return legalizeFroundeven(MI, MRI, B);
2330 case TargetOpcode::G_FCEIL:
2331 return legalizeFceil(MI, MRI, B);
2332 case TargetOpcode::G_FREM:
2333 return legalizeFrem(MI, MRI, B);
2334 case TargetOpcode::G_INTRINSIC_TRUNC:
2335 return legalizeIntrinsicTrunc(MI, MRI, B);
2336 case TargetOpcode::G_SITOFP:
2337 return legalizeITOFP(MI, MRI, B, Signed: true);
2338 case TargetOpcode::G_UITOFP:
2339 return legalizeITOFP(MI, MRI, B, Signed: false);
2340 case TargetOpcode::G_FPTOSI:
2341 return legalizeFPTOI(MI, MRI, B, Signed: true);
2342 case TargetOpcode::G_FPTOUI:
2343 return legalizeFPTOI(MI, MRI, B, Signed: false);
2344 case TargetOpcode::G_FMINNUM:
2345 case TargetOpcode::G_FMAXNUM:
2346 case TargetOpcode::G_FMINIMUMNUM:
2347 case TargetOpcode::G_FMAXIMUMNUM:
2348 return legalizeMinNumMaxNum(Helper, MI);
2349 case TargetOpcode::G_EXTRACT:
2350 return legalizeExtract(Helper, MI);
2351 case TargetOpcode::G_INSERT:
2352 return legalizeInsert(Helper, MI);
2353 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
2354 return legalizeExtractVectorElt(MI, MRI, B);
2355 case TargetOpcode::G_INSERT_VECTOR_ELT:
2356 return legalizeInsertVectorElt(MI, MRI, B);
2357 case TargetOpcode::G_FSIN:
2358 case TargetOpcode::G_FCOS:
2359 return legalizeSinCos(MI, MRI, B);
2360 case TargetOpcode::G_GLOBAL_VALUE:
2361 return legalizeGlobalValue(MI, MRI, B);
2362 case TargetOpcode::G_LOAD:
2363 case TargetOpcode::G_SEXTLOAD:
2364 case TargetOpcode::G_ZEXTLOAD:
2365 return legalizeLoad(Helper, MI);
2366 case TargetOpcode::G_STORE:
2367 return legalizeStore(Helper, MI);
2368 case TargetOpcode::G_FMAD:
2369 return legalizeFMad(MI, MRI, B);
2370 case TargetOpcode::G_FDIV:
2371 return legalizeFDIV(MI, MRI, B);
2372 case TargetOpcode::G_FFREXP:
2373 return legalizeFFREXP(MI, MRI, B);
2374 case TargetOpcode::G_FSQRT:
2375 return legalizeFSQRT(MI, MRI, B);
2376 case TargetOpcode::G_UDIV:
2377 case TargetOpcode::G_UREM:
2378 case TargetOpcode::G_UDIVREM:
2379 return legalizeUnsignedDIV_REM(MI, MRI, B);
2380 case TargetOpcode::G_SDIV:
2381 case TargetOpcode::G_SREM:
2382 case TargetOpcode::G_SDIVREM:
2383 return legalizeSignedDIV_REM(MI, MRI, B);
2384 case TargetOpcode::G_ATOMIC_CMPXCHG:
2385 return legalizeAtomicCmpXChg(MI, MRI, B);
2386 case TargetOpcode::G_FLOG2:
2387 return legalizeFlog2(MI, B);
2388 case TargetOpcode::G_FLOG:
2389 case TargetOpcode::G_FLOG10:
2390 return legalizeFlogCommon(MI, B);
2391 case TargetOpcode::G_FEXP2:
2392 return legalizeFExp2(MI, B);
2393 case TargetOpcode::G_FEXP:
2394 case TargetOpcode::G_FEXP10:
2395 return legalizeFExp(MI, B);
2396 case TargetOpcode::G_FPOW:
2397 return legalizeFPow(MI, B);
2398 case TargetOpcode::G_FFLOOR:
2399 return legalizeFFloor(MI, MRI, B);
2400 case TargetOpcode::G_BUILD_VECTOR:
2401 case TargetOpcode::G_BUILD_VECTOR_TRUNC:
2402 return legalizeBuildVector(MI, MRI, B);
2403 case TargetOpcode::G_MUL:
2404 return legalizeMul(Helper, MI);
2405 case TargetOpcode::G_CTLZ:
2406 case TargetOpcode::G_CTTZ:
2407 return legalizeCTLZ_CTTZ(MI, MRI, B);
2408 case TargetOpcode::G_CTLS:
2409 return legalizeCTLS(MI, MRI, B);
2410 case TargetOpcode::G_CTLZ_ZERO_POISON:
2411 return legalizeCTLZ_ZERO_POISON(MI, MRI, B);
2412 case TargetOpcode::G_STACKSAVE:
2413 return legalizeStackSave(MI, B);
2414 case TargetOpcode::G_GET_FPENV:
2415 return legalizeGetFPEnv(MI, MRI, B);
2416 case TargetOpcode::G_SET_FPENV:
2417 return legalizeSetFPEnv(MI, MRI, B);
2418 case TargetOpcode::G_TRAP:
2419 return legalizeTrap(MI, MRI, B);
2420 case TargetOpcode::G_DEBUGTRAP:
2421 return legalizeDebugTrap(MI, MRI, B);
2422 default:
2423 return false;
2424 }
2425
2426 llvm_unreachable("expected switch to return");
2427}
2428
2429Register AMDGPULegalizerInfo::getSegmentAperture(
2430 unsigned AS,
2431 MachineRegisterInfo &MRI,
2432 MachineIRBuilder &B) const {
2433 MachineFunction &MF = B.getMF();
2434 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2435 const LLT I32 = LLT::integer(SizeInBits: 32);
2436 const LLT I64 = LLT::integer(SizeInBits: 64);
2437
2438 assert(AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::PRIVATE_ADDRESS);
2439
2440 if (ST.hasApertureRegs()) {
2441 // Note: this register is somewhat broken. When used as a 32-bit operand,
2442 // it only returns zeroes. The real value is in the upper 32 bits.
2443 // Thus, we must emit extract the high 32 bits.
2444 const unsigned ApertureRegNo = (AS == AMDGPUAS::LOCAL_ADDRESS)
2445 ? AMDGPU::SRC_SHARED_BASE
2446 : AMDGPU::SRC_PRIVATE_BASE;
2447 assert((ApertureRegNo != AMDGPU::SRC_PRIVATE_BASE ||
2448 !ST.hasGloballyAddressableScratch()) &&
2449 "Cannot use src_private_base with globally addressable scratch!");
2450 Register Dst = MRI.createGenericVirtualRegister(Ty: I64);
2451 MRI.setRegClass(Reg: Dst, RC: &AMDGPU::SReg_64RegClass);
2452 B.buildCopy(Res: {Dst}, Op: {Register(ApertureRegNo)});
2453 return B.buildUnmerge(Res: I32, Op: Dst).getReg(Idx: 1);
2454 }
2455
2456 Register LoadAddr = MRI.createGenericVirtualRegister(
2457 Ty: LLT::pointer(AddressSpace: AMDGPUAS::CONSTANT_ADDRESS, SizeInBits: 64));
2458 // For code object version 5, private_base and shared_base are passed through
2459 // implicit kernargs.
2460 if (AMDGPU::getAMDHSACodeObjectVersion(M: *MF.getFunction().getParent()) >=
2461 AMDGPU::AMDHSA_COV5) {
2462 MachinePointerInfo PtrInfo = getKernargSegmentPtrInfo(MF&: B.getMF());
2463
2464 AMDGPUTargetLowering::ImplicitParameter Param =
2465 AS == AMDGPUAS::LOCAL_ADDRESS ? AMDGPUTargetLowering::SHARED_BASE
2466 : AMDGPUTargetLowering::PRIVATE_BASE;
2467 uint64_t Offset =
2468 ST.getTargetLowering()->getImplicitParameterOffset(MF: B.getMF(), Param);
2469
2470 Register KernargPtrReg = MRI.createGenericVirtualRegister(
2471 Ty: LLT::pointer(AddressSpace: AMDGPUAS::CONSTANT_ADDRESS, SizeInBits: 64));
2472
2473 if (!loadInputValue(DstReg: KernargPtrReg, B,
2474 ArgType: AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR))
2475 return Register();
2476
2477 MachineMemOperand *MMO = MF.getMachineMemOperand(
2478 PtrInfo: PtrInfo.getWithOffset(O: Offset),
2479 f: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
2480 MachineMemOperand::MOInvariant,
2481 MemTy: LLT::integer(SizeInBits: 32), base_alignment: commonAlignment(A: Align(64), Offset));
2482
2483 // Pointer address
2484 B.buildObjectPtrOffset(Res: LoadAddr, Op0: KernargPtrReg,
2485 Op1: B.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: Offset).getReg(Idx: 0));
2486 // Load address
2487 return B.buildLoad(Res: I32, Addr: LoadAddr, MMO&: *MMO).getReg(Idx: 0);
2488 }
2489
2490 Register QueuePtr = MRI.createGenericVirtualRegister(
2491 Ty: LLT::pointer(AddressSpace: AMDGPUAS::CONSTANT_ADDRESS, SizeInBits: 64));
2492
2493 if (!loadInputValue(DstReg: QueuePtr, B, ArgType: AMDGPUFunctionArgInfo::QUEUE_PTR))
2494 return Register();
2495
2496 // TODO: Use custom PseudoSourceValue
2497 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
2498
2499 // Offset into amd_queue_t for group_segment_aperture_base_hi /
2500 // private_segment_aperture_base_hi.
2501 uint32_t StructOffset = (AS == AMDGPUAS::LOCAL_ADDRESS) ? 0x40 : 0x44;
2502
2503 MachineMemOperand *MMO = MF.getMachineMemOperand(
2504 PtrInfo,
2505 f: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
2506 MachineMemOperand::MOInvariant,
2507 MemTy: LLT::integer(SizeInBits: 32), base_alignment: commonAlignment(A: Align(64), Offset: StructOffset));
2508
2509 B.buildObjectPtrOffset(
2510 Res: LoadAddr, Op0: QueuePtr,
2511 Op1: B.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: StructOffset).getReg(Idx: 0));
2512 return B.buildLoad(Res: I32, Addr: LoadAddr, MMO&: *MMO).getReg(Idx: 0);
2513}
2514
2515/// Return true if the value is a known valid address, such that a null check is
2516/// not necessary.
2517static bool isKnownNonNull(Register Val, MachineRegisterInfo &MRI,
2518 const AMDGPUTargetMachine &TM, unsigned AddrSpace) {
2519 MachineInstr *Def = MRI.getVRegDef(Reg: Val);
2520 switch (Def->getOpcode()) {
2521 case AMDGPU::G_FRAME_INDEX:
2522 case AMDGPU::G_GLOBAL_VALUE:
2523 case AMDGPU::G_BLOCK_ADDR:
2524 return true;
2525 case AMDGPU::G_CONSTANT: {
2526 const ConstantInt *CI = Def->getOperand(i: 1).getCImm();
2527 return CI->getSExtValue() != AMDGPU::getNullPointerValue(AS: AddrSpace);
2528 }
2529 default:
2530 return false;
2531 }
2532
2533 return false;
2534}
2535
2536bool AMDGPULegalizerInfo::legalizeAddrSpaceCast(
2537 MachineInstr &MI, MachineRegisterInfo &MRI,
2538 MachineIRBuilder &B) const {
2539 MachineFunction &MF = B.getMF();
2540
2541 // MI can either be a G_ADDRSPACE_CAST or a
2542 // G_INTRINSIC @llvm.amdgcn.addrspacecast.nonnull
2543 assert(MI.getOpcode() == TargetOpcode::G_ADDRSPACE_CAST ||
2544 (isa<GIntrinsic>(MI) && cast<GIntrinsic>(MI).getIntrinsicID() ==
2545 Intrinsic::amdgcn_addrspacecast_nonnull));
2546
2547 const LLT I32 = LLT::integer(SizeInBits: 32);
2548 const LLT I64 = LLT::integer(SizeInBits: 64);
2549 Register Dst = MI.getOperand(i: 0).getReg();
2550 Register Src = isa<GIntrinsic>(Val: MI) ? MI.getOperand(i: 2).getReg()
2551 : MI.getOperand(i: 1).getReg();
2552 LLT DstTy = MRI.getType(Reg: Dst);
2553 LLT SrcTy = MRI.getType(Reg: Src);
2554 unsigned DestAS = DstTy.getAddressSpace();
2555 unsigned SrcAS = SrcTy.getAddressSpace();
2556
2557 // TODO: Avoid reloading from the queue ptr for each cast, or at least each
2558 // vector element.
2559 assert(!DstTy.isVector());
2560
2561 const AMDGPUTargetMachine &TM
2562 = static_cast<const AMDGPUTargetMachine &>(MF.getTarget());
2563
2564 if (TM.isNoopAddrSpaceCast(SrcAS, DestAS)) {
2565 MI.setDesc(B.getTII().get(Opcode: TargetOpcode::G_BITCAST));
2566 return true;
2567 }
2568
2569 if (SrcAS == AMDGPUAS::FLAT_ADDRESS &&
2570 (DestAS == AMDGPUAS::LOCAL_ADDRESS ||
2571 DestAS == AMDGPUAS::PRIVATE_ADDRESS)) {
2572 auto castFlatToLocalOrPrivate = [&](const DstOp &Dst) -> Register {
2573 if (DestAS == AMDGPUAS::PRIVATE_ADDRESS &&
2574 ST.hasGloballyAddressableScratch()) {
2575 // flat -> private with globally addressable scratch: subtract
2576 // src_flat_scratch_base_lo.
2577 Register SrcLo = B.buildExtract(Res: I32, Src, Index: 0).getReg(Idx: 0);
2578 Register FlatScratchBaseLo =
2579 B.buildInstr(Opc: AMDGPU::S_MOV_B32, DstOps: {I32},
2580 SrcOps: {Register(AMDGPU::SRC_FLAT_SCRATCH_BASE_LO)})
2581 .getReg(Idx: 0);
2582 MRI.setRegClass(Reg: FlatScratchBaseLo, RC: &AMDGPU::SReg_32RegClass);
2583 Register Sub = B.buildSub(Dst: I32, Src0: SrcLo, Src1: FlatScratchBaseLo).getReg(Idx: 0);
2584 return B.buildIntToPtr(Dst, Src: Sub).getReg(Idx: 0);
2585 }
2586
2587 // Extract low 32-bits of the pointer.
2588 return B.buildExtract(Res: Dst, Src, Index: 0).getReg(Idx: 0);
2589 };
2590
2591 // For llvm.amdgcn.addrspacecast.nonnull we can always assume non-null, for
2592 // G_ADDRSPACE_CAST we need to guess.
2593 if (isa<GIntrinsic>(Val: MI) || isKnownNonNull(Val: Src, MRI, TM, AddrSpace: SrcAS)) {
2594 castFlatToLocalOrPrivate(Dst);
2595 MI.eraseFromParent();
2596 return true;
2597 }
2598
2599 unsigned NullVal = AMDGPU::getNullPointerValue(AS: DestAS);
2600
2601 auto SegmentNull = B.buildConstant(Res: DstTy, Val: NullVal);
2602 auto FlatNull = B.buildConstant(Res: SrcTy, Val: 0);
2603
2604 // Extract low 32-bits of the pointer.
2605 auto PtrLo32 = castFlatToLocalOrPrivate(DstTy);
2606
2607 auto CmpRes =
2608 B.buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::scalar(SizeInBits: 1), Op0: Src, Op1: FlatNull.getReg(Idx: 0));
2609 B.buildSelect(Res: Dst, Tst: CmpRes, Op0: PtrLo32, Op1: SegmentNull.getReg(Idx: 0));
2610
2611 MI.eraseFromParent();
2612 return true;
2613 }
2614
2615 if (DestAS == AMDGPUAS::FLAT_ADDRESS &&
2616 (SrcAS == AMDGPUAS::LOCAL_ADDRESS ||
2617 SrcAS == AMDGPUAS::PRIVATE_ADDRESS)) {
2618 auto castLocalOrPrivateToFlat = [&](const DstOp &Dst) -> Register {
2619 // Coerce the type of the low half of the result so we can use
2620 // merge_values.
2621 Register SrcAsInt = B.buildPtrToInt(Dst: I32, Src).getReg(Idx: 0);
2622
2623 if (SrcAS == AMDGPUAS::PRIVATE_ADDRESS &&
2624 ST.hasGloballyAddressableScratch()) {
2625 // For wave32: Addr = (TID[4:0] << 52) + FLAT_SCRATCH_BASE + privateAddr
2626 // For wave64: Addr = (TID[5:0] << 51) + FLAT_SCRATCH_BASE + privateAddr
2627 Register AllOnes = B.buildConstant(Res: I32, Val: -1).getReg(Idx: 0);
2628 Register ThreadID = B.buildConstant(Res: I32, Val: 0).getReg(Idx: 0);
2629 ThreadID = B.buildIntrinsic(ID: Intrinsic::amdgcn_mbcnt_lo, Res: {I32})
2630 .addUse(RegNo: AllOnes)
2631 .addUse(RegNo: ThreadID)
2632 .getReg(Idx: 0);
2633 if (ST.isWave64()) {
2634 ThreadID = B.buildIntrinsic(ID: Intrinsic::amdgcn_mbcnt_hi, Res: {I32})
2635 .addUse(RegNo: AllOnes)
2636 .addUse(RegNo: ThreadID)
2637 .getReg(Idx: 0);
2638 }
2639 Register ShAmt =
2640 B.buildConstant(Res: I32, Val: 57 - 32 - ST.getWavefrontSizeLog2()).getReg(Idx: 0);
2641 Register SrcHi = B.buildShl(Dst: I32, Src0: ThreadID, Src1: ShAmt).getReg(Idx: 0);
2642 Register CvtPtr =
2643 B.buildMergeLikeInstr(Res: DstTy, Ops: {SrcAsInt, SrcHi}).getReg(Idx: 0);
2644 // Accessing src_flat_scratch_base_lo as a 64-bit operand gives the full
2645 // 64-bit hi:lo value.
2646 Register FlatScratchBase =
2647 B.buildInstr(Opc: AMDGPU::S_MOV_B64, DstOps: {I64},
2648 SrcOps: {Register(AMDGPU::SRC_FLAT_SCRATCH_BASE)})
2649 .getReg(Idx: 0);
2650 MRI.setRegClass(Reg: FlatScratchBase, RC: &AMDGPU::SReg_64RegClass);
2651 return B.buildPtrAdd(Res: Dst, Op0: CvtPtr, Op1: FlatScratchBase).getReg(Idx: 0);
2652 }
2653
2654 Register ApertureReg = getSegmentAperture(AS: SrcAS, MRI, B);
2655 if (!ApertureReg.isValid())
2656 return false;
2657
2658 // TODO: Should we allow mismatched types but matching sizes in merges to
2659 // avoid the ptrtoint?
2660 return B.buildMergeLikeInstr(Res: Dst, Ops: {SrcAsInt, ApertureReg}).getReg(Idx: 0);
2661 };
2662
2663 // For llvm.amdgcn.addrspacecast.nonnull we can always assume non-null, for
2664 // G_ADDRSPACE_CAST we need to guess.
2665 if (isa<GIntrinsic>(Val: MI) || isKnownNonNull(Val: Src, MRI, TM, AddrSpace: SrcAS)) {
2666 castLocalOrPrivateToFlat(Dst);
2667 MI.eraseFromParent();
2668 return true;
2669 }
2670
2671 Register BuildPtr = castLocalOrPrivateToFlat(DstTy);
2672
2673 auto SegmentNull =
2674 B.buildConstant(Res: SrcTy, Val: AMDGPU::getNullPointerValue(AS: SrcAS));
2675 auto FlatNull = B.buildConstant(Res: DstTy, Val: AMDGPU::getNullPointerValue(AS: DestAS));
2676
2677 auto CmpRes = B.buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::scalar(SizeInBits: 1), Op0: Src,
2678 Op1: SegmentNull.getReg(Idx: 0));
2679
2680 B.buildSelect(Res: Dst, Tst: CmpRes, Op0: BuildPtr, Op1: FlatNull);
2681
2682 MI.eraseFromParent();
2683 return true;
2684 }
2685
2686 if (DestAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
2687 SrcTy.getSizeInBits() == 64) {
2688 // Truncate.
2689 B.buildExtract(Res: Dst, Src, Index: 0);
2690 MI.eraseFromParent();
2691 return true;
2692 }
2693
2694 if (SrcAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
2695 DstTy.getSizeInBits() == 64) {
2696 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
2697 uint32_t AddrHiVal = Info->get32BitAddressHighBits();
2698 auto PtrLo = B.buildPtrToInt(Dst: I32, Src);
2699 if (AddrHiVal == 0) {
2700 auto Zext = B.buildZExt(Res: I64, Op: PtrLo);
2701 B.buildIntToPtr(Dst, Src: Zext);
2702 } else {
2703 auto HighAddr = B.buildConstant(Res: I32, Val: AddrHiVal);
2704 B.buildMergeLikeInstr(Res: Dst, Ops: {PtrLo, HighAddr});
2705 }
2706
2707 MI.eraseFromParent();
2708 return true;
2709 }
2710
2711 // Invalid casts are poison.
2712 // TODO: Should return poison
2713 B.buildUndef(Res: Dst);
2714 MI.eraseFromParent();
2715 return true;
2716}
2717
2718bool AMDGPULegalizerInfo::legalizeFroundeven(MachineInstr &MI,
2719 MachineRegisterInfo &MRI,
2720 MachineIRBuilder &B) const {
2721 Register Src = MI.getOperand(i: 1).getReg();
2722 LLT Ty = MRI.getType(Reg: Src);
2723 assert(Ty.isScalar() && Ty.getSizeInBits() == 64);
2724
2725 APFloat C1Val(APFloat::IEEEdouble(), "0x1.0p+52");
2726 APFloat C2Val(APFloat::IEEEdouble(), "0x1.fffffffffffffp+51");
2727
2728 auto C1 = B.buildFConstant(Res: Ty, Val: C1Val);
2729 auto CopySign = B.buildFCopysign(Dst: Ty, Src0: C1, Src1: Src);
2730
2731 // TODO: Should this propagate fast-math-flags?
2732 auto Tmp1 = B.buildFAdd(Dst: Ty, Src0: Src, Src1: CopySign);
2733 auto Tmp2 = B.buildFSub(Dst: Ty, Src0: Tmp1, Src1: CopySign);
2734
2735 auto C2 = B.buildFConstant(Res: Ty, Val: C2Val);
2736 auto Fabs = B.buildFAbs(Dst: Ty, Src0: Src);
2737
2738 auto Cond = B.buildFCmp(Pred: CmpInst::FCMP_OGT, Res: LLT::scalar(SizeInBits: 1), Op0: Fabs, Op1: C2);
2739 B.buildSelect(Res: MI.getOperand(i: 0).getReg(), Tst: Cond, Op0: Src, Op1: Tmp2);
2740 MI.eraseFromParent();
2741 return true;
2742}
2743
2744bool AMDGPULegalizerInfo::legalizeFceil(
2745 MachineInstr &MI, MachineRegisterInfo &MRI,
2746 MachineIRBuilder &B) const {
2747
2748 const LLT S1 = LLT::scalar(SizeInBits: 1);
2749 const LLT F64 = LLT::float64();
2750
2751 Register Src = MI.getOperand(i: 1).getReg();
2752 assert(MRI.getType(Src) == F64);
2753
2754 // result = trunc(src)
2755 // if (src > 0.0 && src != result)
2756 // result += 1.0
2757
2758 auto Trunc = B.buildIntrinsicTrunc(Dst: F64, Src0: Src);
2759
2760 const auto Zero = B.buildFConstant(Res: F64, Val: 0.0);
2761 const auto One = B.buildFConstant(Res: F64, Val: 1.0);
2762 auto Lt0 = B.buildFCmp(Pred: CmpInst::FCMP_OGT, Res: S1, Op0: Src, Op1: Zero);
2763 auto NeTrunc = B.buildFCmp(Pred: CmpInst::FCMP_ONE, Res: S1, Op0: Src, Op1: Trunc);
2764 auto And = B.buildAnd(Dst: S1, Src0: Lt0, Src1: NeTrunc);
2765 auto Add = B.buildSelect(Res: F64, Tst: And, Op0: One, Op1: Zero);
2766
2767 // TODO: Should this propagate fast-math-flags?
2768 B.buildFAdd(Dst: MI.getOperand(i: 0).getReg(), Src0: Trunc, Src1: Add);
2769 MI.eraseFromParent();
2770 return true;
2771}
2772
2773bool AMDGPULegalizerInfo::legalizeFrem(
2774 MachineInstr &MI, MachineRegisterInfo &MRI,
2775 MachineIRBuilder &B) const {
2776 Register DstReg = MI.getOperand(i: 0).getReg();
2777 Register Src0Reg = MI.getOperand(i: 1).getReg();
2778 Register Src1Reg = MI.getOperand(i: 2).getReg();
2779 auto Flags = MI.getFlags();
2780 LLT Ty = MRI.getType(Reg: DstReg);
2781
2782 auto Div = B.buildFDiv(Dst: Ty, Src0: Src0Reg, Src1: Src1Reg, Flags);
2783 auto Trunc = B.buildIntrinsicTrunc(Dst: Ty, Src0: Div, Flags);
2784 auto Neg = B.buildFNeg(Dst: Ty, Src0: Trunc, Flags);
2785 B.buildFMA(Dst: DstReg, Src0: Neg, Src1: Src1Reg, Src2: Src0Reg, Flags);
2786 MI.eraseFromParent();
2787 return true;
2788}
2789
2790static MachineInstrBuilder extractF64Exponent(Register Hi,
2791 MachineIRBuilder &B) {
2792 const unsigned FractBits = 52;
2793 const unsigned ExpBits = 11;
2794 LLT I32 = LLT::integer(SizeInBits: 32);
2795
2796 auto Const0 = B.buildConstant(Res: I32, Val: FractBits - 32);
2797 auto Const1 = B.buildConstant(Res: I32, Val: ExpBits);
2798
2799 auto ExpPart = B.buildIntrinsic(ID: Intrinsic::amdgcn_ubfe, Res: {I32})
2800 .addUse(RegNo: Hi)
2801 .addUse(RegNo: Const0.getReg(Idx: 0))
2802 .addUse(RegNo: Const1.getReg(Idx: 0));
2803
2804 return B.buildSub(Dst: I32, Src0: ExpPart, Src1: B.buildConstant(Res: I32, Val: 1023));
2805}
2806
2807bool AMDGPULegalizerInfo::legalizeIntrinsicTrunc(
2808 MachineInstr &MI, MachineRegisterInfo &MRI,
2809 MachineIRBuilder &B) const {
2810 const LLT S1 = LLT::scalar(SizeInBits: 1);
2811 const LLT I32 = LLT::integer(SizeInBits: 32);
2812 const LLT I64 = LLT::integer(SizeInBits: 64);
2813
2814 Register Src = MI.getOperand(i: 1).getReg();
2815 assert(MRI.getType(Src) == LLT::float64());
2816
2817 auto SrcInt = B.buildBitcast(Dst: I64, Src);
2818
2819 // TODO: Should this use extract since the low half is unused?
2820 auto Unmerge = B.buildUnmerge(Res: {I32, I32}, Op: SrcInt);
2821 Register Hi = Unmerge.getReg(Idx: 1);
2822
2823 // Extract the upper half, since this is where we will find the sign and
2824 // exponent.
2825 auto Exp = extractF64Exponent(Hi, B);
2826
2827 const unsigned FractBits = 52;
2828
2829 // Extract the sign bit.
2830 const auto SignBitMask = B.buildConstant(Res: I32, UINT32_C(1) << 31);
2831 auto SignBit = B.buildAnd(Dst: I32, Src0: Hi, Src1: SignBitMask);
2832
2833 const auto FractMask = B.buildConstant(Res: I64, Val: (UINT64_C(1) << FractBits) - 1);
2834
2835 const auto Zero32 = B.buildConstant(Res: I32, Val: 0);
2836
2837 // Extend back to 64-bits.
2838 auto SignBit64 = B.buildMergeLikeInstr(Res: I64, Ops: {Zero32, SignBit});
2839
2840 auto Shr = B.buildAShr(Dst: I64, Src0: FractMask, Src1: Exp);
2841 auto Not = B.buildNot(Dst: I64, Src0: Shr);
2842 auto Tmp0 = B.buildAnd(Dst: I64, Src0: SrcInt, Src1: Not);
2843 auto FiftyOne = B.buildConstant(Res: I32, Val: FractBits - 1);
2844
2845 auto ExpLt0 = B.buildICmp(Pred: CmpInst::ICMP_SLT, Res: S1, Op0: Exp, Op1: Zero32);
2846 auto ExpGt51 = B.buildICmp(Pred: CmpInst::ICMP_SGT, Res: S1, Op0: Exp, Op1: FiftyOne);
2847
2848 auto Tmp1 = B.buildSelect(Res: I64, Tst: ExpLt0, Op0: SignBit64, Op1: Tmp0);
2849 auto Res = B.buildSelect(Res: I64, Tst: ExpGt51, Op0: SrcInt, Op1: Tmp1);
2850 B.buildBitcast(Dst: MI.getOperand(i: 0).getReg(), Src: Res);
2851 MI.eraseFromParent();
2852 return true;
2853}
2854
2855bool AMDGPULegalizerInfo::legalizeITOFP(
2856 MachineInstr &MI, MachineRegisterInfo &MRI,
2857 MachineIRBuilder &B, bool Signed) const {
2858
2859 Register Dst = MI.getOperand(i: 0).getReg();
2860 Register Src = MI.getOperand(i: 1).getReg();
2861
2862 const LLT I64 = LLT::integer(SizeInBits: 64);
2863 const LLT I32 = LLT::integer(SizeInBits: 32);
2864 const LLT F64 = LLT::float64();
2865 const LLT F32 = LLT::float32();
2866
2867 assert(MRI.getType(Src) == I64);
2868
2869 auto Unmerge = B.buildUnmerge(Res: {I32, I32}, Op: Src);
2870 auto ThirtyTwo = B.buildConstant(Res: I32, Val: 32);
2871
2872 if (MRI.getType(Reg: Dst) == F64) {
2873 auto CvtHi = Signed ? B.buildSITOFP(Dst: F64, Src0: Unmerge.getReg(Idx: 1))
2874 : B.buildUITOFP(Dst: F64, Src0: Unmerge.getReg(Idx: 1));
2875
2876 auto CvtLo = B.buildUITOFP(Dst: F64, Src0: Unmerge.getReg(Idx: 0));
2877 auto LdExp = B.buildFLdexp(Dst: F64, Src0: CvtHi, Src1: ThirtyTwo);
2878
2879 // TODO: Should this propagate fast-math-flags?
2880 B.buildFAdd(Dst, Src0: LdExp, Src1: CvtLo);
2881 MI.eraseFromParent();
2882 return true;
2883 }
2884
2885 assert(MRI.getType(Dst) == F32);
2886
2887 auto One = B.buildConstant(Res: I32, Val: 1);
2888
2889 MachineInstrBuilder ShAmt;
2890 if (Signed) {
2891 auto ThirtyOne = B.buildConstant(Res: I32, Val: 31);
2892 auto X = B.buildXor(Dst: I32, Src0: Unmerge.getReg(Idx: 0), Src1: Unmerge.getReg(Idx: 1));
2893 auto OppositeSign = B.buildAShr(Dst: I32, Src0: X, Src1: ThirtyOne);
2894 auto MaxShAmt = B.buildAdd(Dst: I32, Src0: ThirtyTwo, Src1: OppositeSign);
2895 auto LS = B.buildIntrinsic(ID: Intrinsic::amdgcn_sffbh, Res: {I32})
2896 .addUse(RegNo: Unmerge.getReg(Idx: 1));
2897 auto LS2 = B.buildSub(Dst: I32, Src0: LS, Src1: One);
2898 ShAmt = B.buildUMin(Dst: I32, Src0: LS2, Src1: MaxShAmt);
2899 } else
2900 ShAmt = B.buildCTLZ(Dst: I32, Src0: Unmerge.getReg(Idx: 1));
2901 auto Norm = B.buildShl(Dst: I64, Src0: Src, Src1: ShAmt);
2902 auto Unmerge2 = B.buildUnmerge(Res: {I32, I32}, Op: Norm);
2903 auto Adjust = B.buildUMin(Dst: I32, Src0: One, Src1: Unmerge2.getReg(Idx: 0));
2904 auto Norm2 = B.buildOr(Dst: I32, Src0: Unmerge2.getReg(Idx: 1), Src1: Adjust);
2905 auto FVal = Signed ? B.buildSITOFP(Dst: F32, Src0: Norm2) : B.buildUITOFP(Dst: F32, Src0: Norm2);
2906 auto Scale = B.buildSub(Dst: I32, Src0: ThirtyTwo, Src1: ShAmt);
2907 B.buildFLdexp(Dst, Src0: FVal, Src1: Scale);
2908 MI.eraseFromParent();
2909 return true;
2910}
2911
2912// TODO: Copied from DAG implementation. Verify logic and document how this
2913// actually works.
2914bool AMDGPULegalizerInfo::legalizeFPTOI(MachineInstr &MI,
2915 MachineRegisterInfo &MRI,
2916 MachineIRBuilder &B,
2917 bool Signed) const {
2918
2919 Register Dst = MI.getOperand(i: 0).getReg();
2920 Register Src = MI.getOperand(i: 1).getReg();
2921
2922 const LLT I64 = LLT::integer(SizeInBits: 64);
2923 const LLT I32 = LLT::integer(SizeInBits: 32);
2924 const LLT F64 = LLT::float64();
2925 const LLT F32 = LLT::float32();
2926
2927 const LLT SrcLT = MRI.getType(Reg: Src);
2928 assert((SrcLT == F32 || SrcLT == F64) && MRI.getType(Dst) == I64);
2929
2930 unsigned Flags = MI.getFlags();
2931
2932 // The basic idea of converting a floating point number into a pair of 32-bit
2933 // integers is illustrated as follows:
2934 //
2935 // tf := trunc(val);
2936 // hif := floor(tf * 2^-32);
2937 // lof := tf - hif * 2^32; // lof is always positive due to floor.
2938 // hi := fptoi(hif);
2939 // lo := fptoi(lof);
2940 //
2941 auto Trunc = B.buildIntrinsicTrunc(Dst: SrcLT, Src0: Src, Flags);
2942 MachineInstrBuilder Sign;
2943 if (Signed && SrcLT == F32) {
2944 // However, a 32-bit floating point number has only 23 bits mantissa and
2945 // it's not enough to hold all the significant bits of `lof` if val is
2946 // negative. To avoid the loss of precision, We need to take the absolute
2947 // value after truncating and flip the result back based on the original
2948 // signedness.
2949 auto SrcInt = B.buildBitcast(Dst: I32, Src);
2950 Sign = B.buildAShr(Dst: I32, Src0: SrcInt, Src1: B.buildConstant(Res: I32, Val: 31));
2951 Trunc = B.buildFAbs(Dst: F32, Src0: Trunc, Flags);
2952 }
2953 MachineInstrBuilder K0, K1;
2954 if (SrcLT == F64) {
2955 K0 = B.buildFConstant(
2956 Res: F64, Val: llvm::bit_cast<double>(UINT64_C(/*2^-32*/ 0x3df0000000000000)));
2957 K1 = B.buildFConstant(
2958 Res: F64, Val: llvm::bit_cast<double>(UINT64_C(/*-2^32*/ 0xc1f0000000000000)));
2959 } else {
2960 K0 = B.buildFConstant(
2961 Res: F32, Val: llvm::bit_cast<float>(UINT32_C(/*2^-32*/ 0x2f800000)));
2962 K1 = B.buildFConstant(
2963 Res: F32, Val: llvm::bit_cast<float>(UINT32_C(/*-2^32*/ 0xcf800000)));
2964 }
2965
2966 auto Mul = B.buildFMul(Dst: SrcLT, Src0: Trunc, Src1: K0, Flags);
2967 auto FloorMul = B.buildFFloor(Dst: SrcLT, Src0: Mul, Flags);
2968 auto Fma = B.buildFMA(Dst: SrcLT, Src0: FloorMul, Src1: K1, Src2: Trunc, Flags);
2969
2970 auto Hi = (Signed && SrcLT == F64) ? B.buildFPTOSI(Dst: I32, Src0: FloorMul)
2971 : B.buildFPTOUI(Dst: I32, Src0: FloorMul);
2972 auto Lo = B.buildFPTOUI(Dst: I32, Src0: Fma);
2973
2974 if (Signed && SrcLT == F32) {
2975 // Flip the result based on the signedness, which is either all 0s or 1s.
2976 Sign = B.buildMergeLikeInstr(Res: I64, Ops: {Sign, Sign});
2977 // r := xor({lo, hi}, sign) - sign;
2978 B.buildSub(Dst, Src0: B.buildXor(Dst: I64, Src0: B.buildMergeLikeInstr(Res: I64, Ops: {Lo, Hi}), Src1: Sign),
2979 Src1: Sign);
2980 } else
2981 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
2982 MI.eraseFromParent();
2983
2984 return true;
2985}
2986
2987bool AMDGPULegalizerInfo::legalizeMinNumMaxNum(LegalizerHelper &Helper,
2988 MachineInstr &MI) const {
2989 MachineFunction &MF = Helper.MIRBuilder.getMF();
2990 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2991
2992 // With ieee_mode disabled, the instructions have the correct behavior.
2993 if (!MFI->getMode().IEEE)
2994 return true;
2995
2996 return Helper.lowerFMinNumMaxNum(MI) == LegalizerHelper::Legalized;
2997}
2998
2999bool AMDGPULegalizerInfo::legalizeExtract(LegalizerHelper &Helper,
3000 MachineInstr &MI) const {
3001 MachineIRBuilder &B = Helper.MIRBuilder;
3002 MachineRegisterInfo &MRI = *B.getMRI();
3003 Register DstReg = MI.getOperand(i: 0).getReg();
3004 Register SrcReg = MI.getOperand(i: 1).getReg();
3005 uint64_t Offset = MI.getOperand(i: 2).getImm();
3006
3007 // Fall back to generic lowering for offset 0 (trivial trunc) and
3008 // non-32-bit-aligned cases which require shift+trunc sequences
3009 // that generic code handles correctly.
3010 if (Offset == 0 || Offset % 32 != 0)
3011 return Helper.lowerExtract(MI) == LegalizerHelper::Legalized;
3012
3013 const LLT DstTy = MRI.getType(Reg: DstReg);
3014 unsigned StartIdx = Offset / 32;
3015 unsigned DstCount = DstTy.getSizeInBits() / 32;
3016 auto Unmerge = B.buildUnmerge(Res: LLT::integer(SizeInBits: 32), Op: SrcReg);
3017
3018 if (DstCount == 1) {
3019 if (DstTy.isPointer())
3020 B.buildIntToPtr(Dst: DstReg, Src: Unmerge.getReg(Idx: StartIdx));
3021 else
3022 MRI.replaceRegWith(FromReg: DstReg, ToReg: Unmerge.getReg(Idx: StartIdx));
3023 } else {
3024 SmallVector<Register, 8> MergeVec;
3025 for (unsigned I = 0; I < DstCount; ++I)
3026 MergeVec.push_back(Elt: Unmerge.getReg(Idx: StartIdx + I));
3027 B.buildMergeLikeInstr(Res: DstReg, Ops: MergeVec);
3028 }
3029
3030 MI.eraseFromParent();
3031 return true;
3032}
3033
3034bool AMDGPULegalizerInfo::legalizeInsert(LegalizerHelper &Helper,
3035 MachineInstr &MI) const {
3036 MachineIRBuilder &B = Helper.MIRBuilder;
3037 MachineRegisterInfo &MRI = *B.getMRI();
3038 Register DstReg = MI.getOperand(i: 0).getReg();
3039 Register SrcReg = MI.getOperand(i: 1).getReg();
3040 Register InsertSrc = MI.getOperand(i: 2).getReg();
3041 uint64_t Offset = MI.getOperand(i: 3).getImm();
3042
3043 unsigned DstSize = MRI.getType(Reg: DstReg).getSizeInBits();
3044 const LLT InsertTy = MRI.getType(Reg: InsertSrc);
3045 unsigned InsertSize = InsertTy.getSizeInBits();
3046
3047 // Fall back to generic lowering for non-32-bit-aligned cases which
3048 // require shift+mask sequences that generic code handles correctly.
3049 if (Offset % 32 != 0 || DstSize % 32 != 0 || InsertSize % 32 != 0)
3050 return Helper.lowerInsert(MI) == LegalizerHelper::Legalized;
3051
3052 const LLT I32 = LLT::integer(SizeInBits: 32);
3053 unsigned DstCount = DstSize / 32;
3054 unsigned InsertCount = InsertSize / 32;
3055 unsigned StartIdx = Offset / 32;
3056
3057 auto SrcUnmerge = B.buildUnmerge(Res: I32, Op: SrcReg);
3058
3059 SmallVector<Register, 8> MergeVec;
3060 for (unsigned I = 0; I < StartIdx; ++I)
3061 MergeVec.push_back(Elt: SrcUnmerge.getReg(Idx: I));
3062
3063 if (InsertCount == 1) {
3064 // Merge-like instructions require same source types. Convert pointer
3065 // to scalar when inserting a pointer value into a scalar.
3066 if (InsertTy.isPointer())
3067 InsertSrc = B.buildPtrToInt(Dst: I32, Src: InsertSrc).getReg(Idx: 0);
3068 MergeVec.push_back(Elt: InsertSrc);
3069 } else {
3070 auto InsertUnmerge = B.buildUnmerge(Res: I32, Op: InsertSrc);
3071 for (unsigned I = 0; I < InsertCount; ++I)
3072 MergeVec.push_back(Elt: InsertUnmerge.getReg(Idx: I));
3073 }
3074
3075 for (unsigned I = StartIdx + InsertCount; I < DstCount; ++I)
3076 MergeVec.push_back(Elt: SrcUnmerge.getReg(Idx: I));
3077
3078 B.buildMergeLikeInstr(Res: DstReg, Ops: MergeVec);
3079
3080 MI.eraseFromParent();
3081 return true;
3082}
3083
3084bool AMDGPULegalizerInfo::legalizeExtractVectorElt(
3085 MachineInstr &MI, MachineRegisterInfo &MRI,
3086 MachineIRBuilder &B) const {
3087 // TODO: Should move some of this into LegalizerHelper.
3088
3089 // TODO: Promote dynamic indexing of i16/f16 to i32/f32
3090
3091 Register Dst = MI.getOperand(i: 0).getReg();
3092 Register Vec = MI.getOperand(i: 1).getReg();
3093
3094 LLT VecTy = MRI.getType(Reg: Vec);
3095 LLT EltTy = VecTy.getElementType();
3096 assert(EltTy == MRI.getType(Dst));
3097
3098 // Other legalization maps vector<? x [type bigger than 64 bits]> via bitcasts
3099 // but we can't go directly to that logic becasue you can't bitcast a vector
3100 // of pointers to a vector of integers. Therefore, introduce an intermediate
3101 // vector of integers using ptrtoint (and inttoptr on the output) in order to
3102 // drive the legalization forward.
3103 if (EltTy.isPointer() && EltTy.getSizeInBits() > 64) {
3104 LLT IntTy = LLT::integer(SizeInBits: EltTy.getSizeInBits());
3105 LLT IntVecTy = VecTy.changeElementType(NewEltTy: IntTy);
3106
3107 auto IntVec = B.buildPtrToInt(Dst: IntVecTy, Src: Vec);
3108 auto IntElt = B.buildExtractVectorElement(Res: IntTy, Val: IntVec, Idx: MI.getOperand(i: 2));
3109 B.buildIntToPtr(Dst, Src: IntElt);
3110
3111 MI.eraseFromParent();
3112 return true;
3113 }
3114
3115 // FIXME: Artifact combiner probably should have replaced the truncated
3116 // constant before this, so we shouldn't need
3117 // getIConstantVRegValWithLookThrough.
3118 std::optional<ValueAndVReg> MaybeIdxVal =
3119 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI);
3120 if (!MaybeIdxVal) // Dynamic case will be selected to register indexing.
3121 return true;
3122 const uint64_t IdxVal = MaybeIdxVal->Value.getZExtValue();
3123
3124 if (IdxVal < VecTy.getNumElements()) {
3125 auto Unmerge = B.buildUnmerge(Res: EltTy, Op: Vec);
3126 B.buildCopy(Res: Dst, Op: Unmerge.getReg(Idx: IdxVal));
3127 } else {
3128 B.buildUndef(Res: Dst);
3129 }
3130
3131 MI.eraseFromParent();
3132 return true;
3133}
3134
3135bool AMDGPULegalizerInfo::legalizeInsertVectorElt(
3136 MachineInstr &MI, MachineRegisterInfo &MRI,
3137 MachineIRBuilder &B) const {
3138 // TODO: Should move some of this into LegalizerHelper.
3139
3140 // TODO: Promote dynamic indexing of i16/f16 to i32/f32
3141
3142 Register Dst = MI.getOperand(i: 0).getReg();
3143 Register Vec = MI.getOperand(i: 1).getReg();
3144 Register Ins = MI.getOperand(i: 2).getReg();
3145
3146 LLT VecTy = MRI.getType(Reg: Vec);
3147 LLT EltTy = VecTy.getElementType();
3148 assert(EltTy == MRI.getType(Ins));
3149
3150 // Other legalization maps vector<? x [type bigger than 64 bits]> via bitcasts
3151 // but we can't go directly to that logic becasue you can't bitcast a vector
3152 // of pointers to a vector of integers. Therefore, make the pointer vector
3153 // into an equivalent vector of integers with ptrtoint, insert the ptrtoint'd
3154 // new value, and then inttoptr the result vector back. This will then allow
3155 // the rest of legalization to take over.
3156 if (EltTy.isPointer() && EltTy.getSizeInBits() > 64) {
3157 LLT IntTy = LLT::integer(SizeInBits: EltTy.getSizeInBits());
3158 LLT IntVecTy = VecTy.changeElementType(NewEltTy: IntTy);
3159
3160 auto IntVecSource = B.buildPtrToInt(Dst: IntVecTy, Src: Vec);
3161 auto IntIns = B.buildPtrToInt(Dst: IntTy, Src: Ins);
3162 auto IntVecDest = B.buildInsertVectorElement(Res: IntVecTy, Val: IntVecSource, Elt: IntIns,
3163 Idx: MI.getOperand(i: 3));
3164 B.buildIntToPtr(Dst, Src: IntVecDest);
3165 MI.eraseFromParent();
3166 return true;
3167 }
3168
3169 // FIXME: Artifact combiner probably should have replaced the truncated
3170 // constant before this, so we shouldn't need
3171 // getIConstantVRegValWithLookThrough.
3172 std::optional<ValueAndVReg> MaybeIdxVal =
3173 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 3).getReg(), MRI);
3174 if (!MaybeIdxVal) // Dynamic case will be selected to register indexing.
3175 return true;
3176
3177 const uint64_t IdxVal = MaybeIdxVal->Value.getZExtValue();
3178
3179 unsigned NumElts = VecTy.getNumElements();
3180 if (IdxVal < NumElts) {
3181 SmallVector<Register, 8> SrcRegs;
3182 for (unsigned i = 0; i < NumElts; ++i)
3183 SrcRegs.push_back(Elt: MRI.createGenericVirtualRegister(Ty: EltTy));
3184 B.buildUnmerge(Res: SrcRegs, Op: Vec);
3185
3186 SrcRegs[IdxVal] = MI.getOperand(i: 2).getReg();
3187 B.buildMergeLikeInstr(Res: Dst, Ops: SrcRegs);
3188 } else {
3189 B.buildUndef(Res: Dst);
3190 }
3191
3192 MI.eraseFromParent();
3193 return true;
3194}
3195
3196bool AMDGPULegalizerInfo::legalizeSinCos(
3197 MachineInstr &MI, MachineRegisterInfo &MRI,
3198 MachineIRBuilder &B) const {
3199
3200 Register DstReg = MI.getOperand(i: 0).getReg();
3201 Register SrcReg = MI.getOperand(i: 1).getReg();
3202 LLT Ty = MRI.getType(Reg: DstReg);
3203 unsigned Flags = MI.getFlags();
3204
3205 Register TrigVal;
3206 auto OneOver2Pi = B.buildFConstant(Res: Ty, Val: 0.5 * numbers::inv_pi);
3207 if (ST.hasTrigReducedRange()) {
3208 auto MulVal = B.buildFMul(Dst: Ty, Src0: SrcReg, Src1: OneOver2Pi, Flags);
3209 TrigVal = B.buildIntrinsic(ID: Intrinsic::amdgcn_fract, Res: {Ty})
3210 .addUse(RegNo: MulVal.getReg(Idx: 0))
3211 .setMIFlags(Flags)
3212 .getReg(Idx: 0);
3213 } else
3214 TrigVal = B.buildFMul(Dst: Ty, Src0: SrcReg, Src1: OneOver2Pi, Flags).getReg(Idx: 0);
3215
3216 Intrinsic::ID TrigIntrin = MI.getOpcode() == AMDGPU::G_FSIN ?
3217 Intrinsic::amdgcn_sin : Intrinsic::amdgcn_cos;
3218 B.buildIntrinsic(ID: TrigIntrin, Res: ArrayRef<Register>(DstReg))
3219 .addUse(RegNo: TrigVal)
3220 .setMIFlags(Flags);
3221 MI.eraseFromParent();
3222 return true;
3223}
3224
3225bool AMDGPULegalizerInfo::buildPCRelGlobalAddress(Register DstReg, LLT PtrTy,
3226 MachineIRBuilder &B,
3227 const GlobalValue *GV,
3228 int64_t Offset,
3229 unsigned GAFlags) const {
3230 assert(isInt<32>(Offset + 4) && "32-bit offset is expected!");
3231 // In order to support pc-relative addressing, SI_PC_ADD_REL_OFFSET is lowered
3232 // to the following code sequence:
3233 //
3234 // For constant address space:
3235 // s_getpc_b64 s[0:1]
3236 // s_add_u32 s0, s0, $symbol
3237 // s_addc_u32 s1, s1, 0
3238 //
3239 // s_getpc_b64 returns the address of the s_add_u32 instruction and then
3240 // a fixup or relocation is emitted to replace $symbol with a literal
3241 // constant, which is a pc-relative offset from the encoding of the $symbol
3242 // operand to the global variable.
3243 //
3244 // For global address space:
3245 // s_getpc_b64 s[0:1]
3246 // s_add_u32 s0, s0, $symbol@{gotpc}rel32@lo
3247 // s_addc_u32 s1, s1, $symbol@{gotpc}rel32@hi
3248 //
3249 // s_getpc_b64 returns the address of the s_add_u32 instruction and then
3250 // fixups or relocations are emitted to replace $symbol@*@lo and
3251 // $symbol@*@hi with lower 32 bits and higher 32 bits of a literal constant,
3252 // which is a 64-bit pc-relative offset from the encoding of the $symbol
3253 // operand to the global variable.
3254
3255 LLT ConstPtrTy = LLT::pointer(AddressSpace: AMDGPUAS::CONSTANT_ADDRESS, SizeInBits: 64);
3256
3257 Register PCReg = PtrTy.getSizeInBits() != 32 ? DstReg :
3258 B.getMRI()->createGenericVirtualRegister(Ty: ConstPtrTy);
3259
3260 if (ST.has64BitLiterals()) {
3261 assert(GAFlags != SIInstrInfo::MO_NONE);
3262
3263 MachineInstrBuilder MIB =
3264 B.buildInstr(Opcode: AMDGPU::SI_PC_ADD_REL_OFFSET64).addDef(RegNo: PCReg);
3265 MIB.addGlobalAddress(GV, Offset, TargetFlags: GAFlags + 2);
3266 } else {
3267 MachineInstrBuilder MIB =
3268 B.buildInstr(Opcode: AMDGPU::SI_PC_ADD_REL_OFFSET).addDef(RegNo: PCReg);
3269
3270 MIB.addGlobalAddress(GV, Offset, TargetFlags: GAFlags);
3271 if (GAFlags == SIInstrInfo::MO_NONE)
3272 MIB.addImm(Val: 0);
3273 else
3274 MIB.addGlobalAddress(GV, Offset, TargetFlags: GAFlags + 1);
3275 }
3276
3277 if (!B.getMRI()->getRegClassOrNull(Reg: PCReg))
3278 B.getMRI()->setRegClass(Reg: PCReg, RC: &AMDGPU::SReg_64RegClass);
3279
3280 if (PtrTy.getSizeInBits() == 32)
3281 B.buildExtract(Res: DstReg, Src: PCReg, Index: 0);
3282 return true;
3283}
3284
3285// Emit a ABS32_LO / ABS32_HI relocation stub.
3286void AMDGPULegalizerInfo::buildAbsGlobalAddress(
3287 Register DstReg, LLT PtrTy, MachineIRBuilder &B, const GlobalValue *GV,
3288 MachineRegisterInfo &MRI) const {
3289 bool RequiresHighHalf = PtrTy.getSizeInBits() != 32;
3290
3291 if (RequiresHighHalf && ST.has64BitLiterals()) {
3292 if (!MRI.getRegClassOrNull(Reg: DstReg))
3293 MRI.setRegClass(Reg: DstReg, RC: &AMDGPU::SReg_64RegClass);
3294 B.buildInstr(Opcode: AMDGPU::S_MOV_B64)
3295 .addDef(RegNo: DstReg)
3296 .addGlobalAddress(GV, Offset: 0, TargetFlags: SIInstrInfo::MO_ABS64);
3297 return;
3298 }
3299
3300 LLT I32 = LLT::integer(SizeInBits: 32);
3301
3302 // Use the destination directly, if and only if we store the lower address
3303 // part only and we don't have a register class being set.
3304 Register AddrLo = !RequiresHighHalf && !MRI.getRegClassOrNull(Reg: DstReg)
3305 ? DstReg
3306 : MRI.createGenericVirtualRegister(Ty: I32);
3307
3308 if (!MRI.getRegClassOrNull(Reg: AddrLo))
3309 MRI.setRegClass(Reg: AddrLo, RC: &AMDGPU::SReg_32RegClass);
3310
3311 // Write the lower half.
3312 B.buildInstr(Opcode: AMDGPU::S_MOV_B32)
3313 .addDef(RegNo: AddrLo)
3314 .addGlobalAddress(GV, Offset: 0, TargetFlags: SIInstrInfo::MO_ABS32_LO);
3315
3316 // If required, write the upper half as well.
3317 if (RequiresHighHalf) {
3318 assert(PtrTy.getSizeInBits() == 64 &&
3319 "Must provide a 64-bit pointer type!");
3320
3321 Register AddrHi = MRI.createGenericVirtualRegister(Ty: I32);
3322 MRI.setRegClass(Reg: AddrHi, RC: &AMDGPU::SReg_32RegClass);
3323
3324 B.buildInstr(Opcode: AMDGPU::S_MOV_B32)
3325 .addDef(RegNo: AddrHi)
3326 .addGlobalAddress(GV, Offset: 0, TargetFlags: SIInstrInfo::MO_ABS32_HI);
3327
3328 // Use the destination directly, if and only if we don't have a register
3329 // class being set.
3330 Register AddrDst = !MRI.getRegClassOrNull(Reg: DstReg)
3331 ? DstReg
3332 : MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: 64));
3333
3334 if (!MRI.getRegClassOrNull(Reg: AddrDst))
3335 MRI.setRegClass(Reg: AddrDst, RC: &AMDGPU::SReg_64RegClass);
3336
3337 B.buildMergeValues(Res: AddrDst, Ops: {AddrLo, AddrHi});
3338
3339 // If we created a new register for the destination, cast the result into
3340 // the final output.
3341 if (AddrDst != DstReg)
3342 B.buildCast(Dst: DstReg, Src: AddrDst);
3343 } else if (AddrLo != DstReg) {
3344 // If we created a new register for the destination, cast the result into
3345 // the final output.
3346 B.buildCast(Dst: DstReg, Src: AddrLo);
3347 }
3348}
3349
3350bool AMDGPULegalizerInfo::legalizeGlobalValue(
3351 MachineInstr &MI, MachineRegisterInfo &MRI,
3352 MachineIRBuilder &B) const {
3353 Register DstReg = MI.getOperand(i: 0).getReg();
3354 LLT Ty = MRI.getType(Reg: DstReg);
3355 unsigned AS = Ty.getAddressSpace();
3356
3357 const GlobalValue *GV = MI.getOperand(i: 1).getGlobal();
3358 MachineFunction &MF = B.getMF();
3359 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
3360
3361 if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
3362 if (!MFI->isModuleEntryFunction() &&
3363 GV->getName() != "llvm.amdgcn.module.lds" &&
3364 !AMDGPU::isNamedBarrier(GV: *cast<GlobalVariable>(Val: GV))) {
3365 const Function &Fn = MF.getFunction();
3366 Fn.getContext().diagnose(DI: DiagnosticInfoUnsupported(
3367 Fn, "local memory global used by non-kernel function",
3368 MI.getDebugLoc(), DS_Warning));
3369
3370 // We currently don't have a way to correctly allocate LDS objects that
3371 // aren't directly associated with a kernel. We do force inlining of
3372 // functions that use local objects. However, if these dead functions are
3373 // not eliminated, we don't want a compile time error. Just emit a warning
3374 // and a trap, since there should be no callable path here.
3375 B.buildTrap();
3376 B.buildUndef(Res: DstReg);
3377 MI.eraseFromParent();
3378 return true;
3379 }
3380
3381 // TODO: We could emit code to handle the initialization somewhere.
3382 // We ignore the initializer for now and legalize it to allow selection.
3383 // The initializer will anyway get errored out during assembly emission.
3384 const SITargetLowering *TLI = ST.getTargetLowering();
3385 if (!TLI->shouldUseLDSConstAddress(GV)) {
3386 MI.getOperand(i: 1).setTargetFlags(SIInstrInfo::MO_ABS32_LO);
3387 return true; // Leave in place;
3388 }
3389
3390 const GlobalVariable &GVar = *cast<GlobalVariable>(Val: GV);
3391 if (AS == AMDGPUAS::LOCAL_ADDRESS && GV->hasExternalLinkage()) {
3392 // HIP uses an unsized array `extern __shared__ T s[]` or similar
3393 // zero-sized type in other languages to declare the dynamic shared
3394 // memory which size is not known at the compile time. They will be
3395 // allocated by the runtime and placed directly after the static
3396 // allocated ones. They all share the same offset.
3397 if (GVar.getGlobalSize(DL: GVar.getDataLayout()) == 0) {
3398 // Adjust alignment for that dynamic shared memory array.
3399 MFI->setDynLDSAlign(F: MF.getFunction(), GV: GVar);
3400 LLT I32 = LLT::integer(SizeInBits: 32);
3401 auto Sz = B.buildIntrinsic(ID: Intrinsic::amdgcn_groupstaticsize, Res: {I32});
3402 B.buildIntToPtr(Dst: DstReg, Src: Sz);
3403 MI.eraseFromParent();
3404 return true;
3405 }
3406 }
3407
3408 B.buildConstant(Res: DstReg, Val: MFI->allocateLDSGlobal(DL: B.getDataLayout(), GV: GVar));
3409 MI.eraseFromParent();
3410 return true;
3411 }
3412
3413 if (ST.isAmdPalOS() || ST.isMesa3DOS()) {
3414 buildAbsGlobalAddress(DstReg, PtrTy: Ty, B, GV, MRI);
3415 MI.eraseFromParent();
3416 return true;
3417 }
3418
3419 const SITargetLowering *TLI = ST.getTargetLowering();
3420
3421 if (TLI->shouldEmitFixup(GV)) {
3422 buildPCRelGlobalAddress(DstReg, PtrTy: Ty, B, GV, Offset: 0);
3423 MI.eraseFromParent();
3424 return true;
3425 }
3426
3427 if (TLI->shouldEmitPCReloc(GV)) {
3428 buildPCRelGlobalAddress(DstReg, PtrTy: Ty, B, GV, Offset: 0, GAFlags: SIInstrInfo::MO_REL32);
3429 MI.eraseFromParent();
3430 return true;
3431 }
3432
3433 LLT PtrTy = LLT::pointer(AddressSpace: AMDGPUAS::CONSTANT_ADDRESS, SizeInBits: 64);
3434 Register GOTAddr = MRI.createGenericVirtualRegister(Ty: PtrTy);
3435
3436 LLT LoadTy = Ty.getSizeInBits() == 32 ? PtrTy : Ty;
3437 MachineMemOperand *GOTMMO = MF.getMachineMemOperand(
3438 PtrInfo: MachinePointerInfo::getGOT(MF),
3439 f: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
3440 MachineMemOperand::MOInvariant,
3441 MemTy: LoadTy, base_alignment: Align(8));
3442
3443 buildPCRelGlobalAddress(DstReg: GOTAddr, PtrTy, B, GV, Offset: 0, GAFlags: SIInstrInfo::MO_GOTPCREL32);
3444
3445 if (Ty.getSizeInBits() == 32) {
3446 // Truncate if this is a 32-bit constant address.
3447 auto Load = B.buildLoad(Res: PtrTy, Addr: GOTAddr, MMO&: *GOTMMO);
3448 B.buildExtract(Res: DstReg, Src: Load, Index: 0);
3449 } else
3450 B.buildLoad(Res: DstReg, Addr: GOTAddr, MMO&: *GOTMMO);
3451
3452 MI.eraseFromParent();
3453 return true;
3454}
3455
3456static LLT widenToNextPowerOf2(LLT Ty) {
3457 if (Ty.isVector())
3458 return Ty.changeElementCount(
3459 EC: ElementCount::getFixed(MinVal: PowerOf2Ceil(A: Ty.getNumElements())));
3460 return Ty.changeElementSize(NewEltSize: PowerOf2Ceil(A: Ty.getSizeInBits()));
3461}
3462
3463bool AMDGPULegalizerInfo::legalizeLoad(LegalizerHelper &Helper,
3464 MachineInstr &MI) const {
3465 MachineIRBuilder &B = Helper.MIRBuilder;
3466 MachineRegisterInfo &MRI = *B.getMRI();
3467 GISelChangeObserver &Observer = Helper.Observer;
3468
3469 Register PtrReg = MI.getOperand(i: 1).getReg();
3470 LLT PtrTy = MRI.getType(Reg: PtrReg);
3471 unsigned AddrSpace = PtrTy.getAddressSpace();
3472
3473 if (AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT) {
3474 LLT ConstPtr = LLT::pointer(AddressSpace: AMDGPUAS::CONSTANT_ADDRESS, SizeInBits: 64);
3475 auto Cast = B.buildAddrSpaceCast(Dst: ConstPtr, Src: PtrReg);
3476 Observer.changingInstr(MI);
3477 MI.getOperand(i: 1).setReg(Cast.getReg(Idx: 0));
3478 Observer.changedInstr(MI);
3479 return true;
3480 }
3481
3482 if (MI.getOpcode() != AMDGPU::G_LOAD)
3483 return false;
3484
3485 Register ValReg = MI.getOperand(i: 0).getReg();
3486 LLT ValTy = MRI.getType(Reg: ValReg);
3487
3488 if (hasBufferRsrcWorkaround(Ty: ValTy)) {
3489 Observer.changingInstr(MI);
3490 castBufferRsrcFromV4I32(MI, B, MRI, Idx: 0);
3491 Observer.changedInstr(MI);
3492 return true;
3493 }
3494
3495 MachineMemOperand *MMO = *MI.memoperands_begin();
3496 const unsigned ValSize = ValTy.getSizeInBits();
3497 const LLT MemTy = MMO->getMemoryType();
3498 const Align MemAlign = MMO->getAlign();
3499 const unsigned MemSize = MemTy.getSizeInBits();
3500 const uint64_t AlignInBits = 8 * MemAlign.value();
3501
3502 // Widen non-power-of-2 loads to the alignment if needed
3503 if (shouldWidenLoad(ST, MemoryTy: MemTy, AlignInBits, AddrSpace, Opcode: MI.getOpcode())) {
3504 const unsigned WideMemSize = PowerOf2Ceil(A: MemSize);
3505
3506 // This was already the correct extending load result type, so just adjust
3507 // the memory type.
3508 if (WideMemSize == ValSize) {
3509 MachineFunction &MF = B.getMF();
3510
3511 MachineMemOperand *WideMMO =
3512 MF.getMachineMemOperand(MMO, Offset: 0, Size: WideMemSize / 8);
3513 Observer.changingInstr(MI);
3514 MI.setMemRefs(MF, MemRefs: {WideMMO});
3515 Observer.changedInstr(MI);
3516 return true;
3517 }
3518
3519 // Don't bother handling edge case that should probably never be produced.
3520 if (ValSize > WideMemSize)
3521 return false;
3522
3523 LLT WideTy = widenToNextPowerOf2(Ty: ValTy);
3524
3525 Register WideLoad;
3526 if (!WideTy.isVector()) {
3527 WideLoad = B.buildLoadFromOffset(Dst: WideTy, BasePtr: PtrReg, BaseMMO&: *MMO, Offset: 0).getReg(Idx: 0);
3528 B.buildTrunc(Res: ValReg, Op: WideLoad).getReg(Idx: 0);
3529 } else {
3530 // Extract the subvector.
3531
3532 if (isRegisterType(ST, Ty: ValTy)) {
3533 // If this a case where G_EXTRACT is legal, use it.
3534 // (e.g. <3 x i32> -> <4 x i32>)
3535 WideLoad = B.buildLoadFromOffset(Dst: WideTy, BasePtr: PtrReg, BaseMMO&: *MMO, Offset: 0).getReg(Idx: 0);
3536 B.buildExtract(Res: ValReg, Src: WideLoad, Index: 0);
3537 } else {
3538 // For cases where the widened type isn't a nice register value, unmerge
3539 // from a widened register (e.g. <3 x i16> -> <4 x i16>)
3540 WideLoad = B.buildLoadFromOffset(Dst: WideTy, BasePtr: PtrReg, BaseMMO&: *MMO, Offset: 0).getReg(Idx: 0);
3541 B.buildDeleteTrailingVectorElements(Res: ValReg, Op0: WideLoad);
3542 }
3543 }
3544
3545 MI.eraseFromParent();
3546 return true;
3547 }
3548
3549 return false;
3550}
3551
3552bool AMDGPULegalizerInfo::legalizeStore(LegalizerHelper &Helper,
3553 MachineInstr &MI) const {
3554 MachineIRBuilder &B = Helper.MIRBuilder;
3555 MachineRegisterInfo &MRI = *B.getMRI();
3556 GISelChangeObserver &Observer = Helper.Observer;
3557
3558 Register DataReg = MI.getOperand(i: 0).getReg();
3559 LLT DataTy = MRI.getType(Reg: DataReg);
3560
3561 if (hasBufferRsrcWorkaround(Ty: DataTy)) {
3562 Observer.changingInstr(MI);
3563 castBufferRsrcArgToV4I32(MI, B, Idx: 0);
3564 Observer.changedInstr(MI);
3565 return true;
3566 }
3567 return false;
3568}
3569
3570bool AMDGPULegalizerInfo::legalizeFMad(
3571 MachineInstr &MI, MachineRegisterInfo &MRI,
3572 MachineIRBuilder &B) const {
3573 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3574 assert(Ty.isScalar());
3575
3576 MachineFunction &MF = B.getMF();
3577 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
3578
3579 // TODO: Always legal with future ftz flag.
3580 // TODO: Type is expected to be LLT::float32()/LLT::float16()
3581 // FIXME: Do we need just output?
3582 if (Ty == LLT::float32() &&
3583 MFI->getMode().FP32Denormals == DenormalMode::getPreserveSign())
3584 return true;
3585 if (Ty == LLT::float16() &&
3586 MFI->getMode().FP64FP16Denormals == DenormalMode::getPreserveSign())
3587 return true;
3588
3589 MachineIRBuilder HelperBuilder(MI);
3590 GISelObserverWrapper DummyObserver;
3591 LegalizerHelper Helper(MF, DummyObserver, HelperBuilder);
3592 return Helper.lowerFMad(MI) == LegalizerHelper::Legalized;
3593}
3594
3595bool AMDGPULegalizerInfo::legalizeAtomicCmpXChg(
3596 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
3597 Register DstReg = MI.getOperand(i: 0).getReg();
3598 Register PtrReg = MI.getOperand(i: 1).getReg();
3599 Register CmpVal = MI.getOperand(i: 2).getReg();
3600 Register NewVal = MI.getOperand(i: 3).getReg();
3601
3602 assert(AMDGPU::isFlatGlobalAddrSpace(MRI.getType(PtrReg).getAddressSpace()) &&
3603 "this should not have been custom lowered");
3604
3605 LLT ValTy = MRI.getType(Reg: CmpVal);
3606 LLT VecTy = LLT::fixed_vector(NumElements: 2, ScalarTy: ValTy);
3607
3608 Register PackedVal = B.buildBuildVector(Res: VecTy, Ops: { NewVal, CmpVal }).getReg(Idx: 0);
3609
3610 B.buildInstr(Opcode: AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG)
3611 .addDef(RegNo: DstReg)
3612 .addUse(RegNo: PtrReg)
3613 .addUse(RegNo: PackedVal)
3614 .setMemRefs(MI.memoperands());
3615
3616 MI.eraseFromParent();
3617 return true;
3618}
3619
3620/// Return true if it's known that \p Src can never be an f32 denormal value.
3621static bool valueIsKnownNeverF32Denorm(const MachineRegisterInfo &MRI,
3622 Register Src) {
3623 const MachineInstr *DefMI = MRI.getVRegDef(Reg: Src);
3624 switch (DefMI->getOpcode()) {
3625 case TargetOpcode::G_INTRINSIC: {
3626 switch (cast<GIntrinsic>(Val: DefMI)->getIntrinsicID()) {
3627 case Intrinsic::amdgcn_frexp_mant:
3628 case Intrinsic::amdgcn_log:
3629 case Intrinsic::amdgcn_log_clamp:
3630 case Intrinsic::amdgcn_exp2:
3631 case Intrinsic::amdgcn_sqrt:
3632 return true;
3633 default:
3634 break;
3635 }
3636
3637 break;
3638 }
3639 case TargetOpcode::G_FSQRT:
3640 return true;
3641 case TargetOpcode::G_FFREXP: {
3642 if (DefMI->getOperand(i: 0).getReg() == Src)
3643 return true;
3644 break;
3645 }
3646 case TargetOpcode::G_FPEXT: {
3647 return MRI.getType(Reg: DefMI->getOperand(i: 1).getReg()) == LLT::float16();
3648 }
3649 default:
3650 return false;
3651 }
3652
3653 return false;
3654}
3655
3656static bool allowApproxFunc(const MachineFunction &MF, unsigned Flags) {
3657 return Flags & MachineInstr::FmAfn;
3658}
3659
3660static bool needsDenormHandlingF32(const MachineFunction &MF, Register Src,
3661 unsigned Flags) {
3662 return !valueIsKnownNeverF32Denorm(MRI: MF.getRegInfo(), Src) &&
3663 MF.getDenormalMode(FPType: APFloat::IEEEsingle()).Input !=
3664 DenormalMode::PreserveSign;
3665}
3666
3667std::pair<Register, Register>
3668AMDGPULegalizerInfo::getScaledLogInput(MachineIRBuilder &B, Register Src,
3669 unsigned Flags) const {
3670 if (!needsDenormHandlingF32(MF: B.getMF(), Src, Flags))
3671 return {};
3672
3673 const LLT F32 = LLT::float32();
3674 auto SmallestNormal = B.buildFConstant(
3675 Res: F32, Val: APFloat::getSmallestNormalized(Sem: APFloat::IEEEsingle()));
3676 auto IsLtSmallestNormal =
3677 B.buildFCmp(Pred: CmpInst::FCMP_OLT, Res: LLT::scalar(SizeInBits: 1), Op0: Src, Op1: SmallestNormal);
3678
3679 auto Scale32 = B.buildFConstant(Res: F32, Val: 0x1.0p+32);
3680 auto One = B.buildFConstant(Res: F32, Val: 1.0);
3681 auto ScaleFactor =
3682 B.buildSelect(Res: F32, Tst: IsLtSmallestNormal, Op0: Scale32, Op1: One, Flags);
3683 auto ScaledInput = B.buildFMul(Dst: F32, Src0: Src, Src1: ScaleFactor, Flags);
3684
3685 return {ScaledInput.getReg(Idx: 0), IsLtSmallestNormal.getReg(Idx: 0)};
3686}
3687
3688bool AMDGPULegalizerInfo::legalizeFlog2(MachineInstr &MI,
3689 MachineIRBuilder &B) const {
3690 // v_log_f32 is good enough for OpenCL, except it doesn't handle denormals.
3691 // If we have to handle denormals, scale up the input and adjust the result.
3692
3693 // scaled = x * (is_denormal ? 0x1.0p+32 : 1.0)
3694 // log2 = amdgpu_log2 - (is_denormal ? 32.0 : 0.0)
3695
3696 Register Dst = MI.getOperand(i: 0).getReg();
3697 Register Src = MI.getOperand(i: 1).getReg();
3698 LLT Ty = B.getMRI()->getType(Reg: Dst);
3699 unsigned Flags = MI.getFlags();
3700
3701 if (Ty == LLT::float16()) {
3702 const LLT F32 = LLT::float32();
3703 // Nothing in half is a denormal when promoted to f32.
3704 auto Ext = B.buildFPExt(Res: F32, Op: Src, Flags);
3705 auto Log2 = B.buildIntrinsic(ID: Intrinsic::amdgcn_log, Res: {F32})
3706 .addUse(RegNo: Ext.getReg(Idx: 0))
3707 .setMIFlags(Flags);
3708 B.buildFPTrunc(Res: Dst, Op: Log2, Flags);
3709 MI.eraseFromParent();
3710 return true;
3711 }
3712
3713 assert(Ty == LLT::float32());
3714
3715 auto [ScaledInput, IsLtSmallestNormal] = getScaledLogInput(B, Src, Flags);
3716 if (!ScaledInput) {
3717 B.buildIntrinsic(ID: Intrinsic::amdgcn_log, Res: {MI.getOperand(i: 0)})
3718 .addUse(RegNo: Src)
3719 .setMIFlags(Flags);
3720 MI.eraseFromParent();
3721 return true;
3722 }
3723
3724 auto Log2 = B.buildIntrinsic(ID: Intrinsic::amdgcn_log, Res: {Ty})
3725 .addUse(RegNo: ScaledInput)
3726 .setMIFlags(Flags);
3727
3728 auto ThirtyTwo = B.buildFConstant(Res: Ty, Val: 32.0);
3729 auto Zero = B.buildFConstant(Res: Ty, Val: 0.0);
3730 auto ResultOffset =
3731 B.buildSelect(Res: Ty, Tst: IsLtSmallestNormal, Op0: ThirtyTwo, Op1: Zero, Flags);
3732 B.buildFSub(Dst, Src0: Log2, Src1: ResultOffset, Flags);
3733
3734 MI.eraseFromParent();
3735 return true;
3736}
3737
3738static Register getMad(MachineIRBuilder &B, LLT Ty, Register X, Register Y,
3739 Register Z, unsigned Flags) {
3740 auto FMul = B.buildFMul(Dst: Ty, Src0: X, Src1: Y, Flags);
3741 return B.buildFAdd(Dst: Ty, Src0: FMul, Src1: Z, Flags).getReg(Idx: 0);
3742}
3743
3744bool AMDGPULegalizerInfo::legalizeFlogCommon(MachineInstr &MI,
3745 MachineIRBuilder &B) const {
3746 const bool IsLog10 = MI.getOpcode() == TargetOpcode::G_FLOG10;
3747 assert(IsLog10 || MI.getOpcode() == TargetOpcode::G_FLOG);
3748
3749 MachineRegisterInfo &MRI = *B.getMRI();
3750 Register Dst = MI.getOperand(i: 0).getReg();
3751 Register X = MI.getOperand(i: 1).getReg();
3752 unsigned Flags = MI.getFlags();
3753 const LLT Ty = MRI.getType(Reg: X);
3754
3755 const LLT F32 = LLT::float32();
3756 const LLT F16 = LLT::float16();
3757
3758 if (Ty == F16 || MI.getFlag(Flag: MachineInstr::FmAfn)) {
3759 // TODO: The direct f16 path is 1.79 ulp for f16. This should be used
3760 // depending on !fpmath metadata.
3761 bool PromoteToF32 =
3762 Ty == F16 && (!MI.getFlag(Flag: MachineInstr::FmAfn) || !ST.has16BitInsts());
3763 if (PromoteToF32) {
3764 Register LogVal = MRI.createGenericVirtualRegister(Ty: F32);
3765 auto PromoteSrc = B.buildFPExt(Res: F32, Op: X);
3766 legalizeFlogUnsafe(B, Dst: LogVal, Src: PromoteSrc.getReg(Idx: 0), IsLog10, Flags);
3767 B.buildFPTrunc(Res: Dst, Op: LogVal);
3768 } else {
3769 legalizeFlogUnsafe(B, Dst, Src: X, IsLog10, Flags);
3770 }
3771
3772 MI.eraseFromParent();
3773 return true;
3774 }
3775
3776 auto [ScaledInput, IsScaled] = getScaledLogInput(B, Src: X, Flags);
3777 if (ScaledInput)
3778 X = ScaledInput;
3779
3780 auto Y =
3781 B.buildIntrinsic(ID: Intrinsic::amdgcn_log, Res: {Ty}).addUse(RegNo: X).setMIFlags(Flags);
3782
3783 Register R;
3784 if (ST.hasFastFMAF32()) {
3785 // c+cc are ln(2)/ln(10) to more than 49 bits
3786 const float c_log10 = 0x1.344134p-2f;
3787 const float cc_log10 = 0x1.09f79ep-26f;
3788
3789 // c + cc is ln(2) to more than 49 bits
3790 const float c_log = 0x1.62e42ep-1f;
3791 const float cc_log = 0x1.efa39ep-25f;
3792
3793 auto C = B.buildFConstant(Res: Ty, Val: IsLog10 ? c_log10 : c_log);
3794 auto CC = B.buildFConstant(Res: Ty, Val: IsLog10 ? cc_log10 : cc_log);
3795 // This adds correction terms for which contraction may lead to an increase
3796 // in the error of the approximation, so disable it.
3797 auto NewFlags = Flags & ~(MachineInstr::FmContract);
3798 R = B.buildFMul(Dst: Ty, Src0: Y, Src1: C, Flags: NewFlags).getReg(Idx: 0);
3799 auto NegR = B.buildFNeg(Dst: Ty, Src0: R, Flags: NewFlags);
3800 auto FMA0 = B.buildFMA(Dst: Ty, Src0: Y, Src1: C, Src2: NegR, Flags: NewFlags);
3801 auto FMA1 = B.buildFMA(Dst: Ty, Src0: Y, Src1: CC, Src2: FMA0, Flags: NewFlags);
3802 R = B.buildFAdd(Dst: Ty, Src0: R, Src1: FMA1, Flags: NewFlags).getReg(Idx: 0);
3803 } else {
3804 // ch+ct is ln(2)/ln(10) to more than 36 bits
3805 const float ch_log10 = 0x1.344000p-2f;
3806 const float ct_log10 = 0x1.3509f6p-18f;
3807
3808 // ch + ct is ln(2) to more than 36 bits
3809 const float ch_log = 0x1.62e000p-1f;
3810 const float ct_log = 0x1.0bfbe8p-15f;
3811
3812 auto CH = B.buildFConstant(Res: Ty, Val: IsLog10 ? ch_log10 : ch_log);
3813 auto CT = B.buildFConstant(Res: Ty, Val: IsLog10 ? ct_log10 : ct_log);
3814
3815 auto MaskConst = B.buildConstant(Res: Ty, Val: 0xfffff000);
3816 auto YH = B.buildAnd(Dst: Ty, Src0: Y, Src1: MaskConst);
3817 auto YT = B.buildFSub(Dst: Ty, Src0: Y, Src1: YH, Flags);
3818 // This adds correction terms for which contraction may lead to an increase
3819 // in the error of the approximation, so disable it.
3820 auto NewFlags = Flags & ~(MachineInstr::FmContract);
3821 auto YTCT = B.buildFMul(Dst: Ty, Src0: YT, Src1: CT, Flags: NewFlags);
3822
3823 Register Mad0 =
3824 getMad(B, Ty, X: YH.getReg(Idx: 0), Y: CT.getReg(Idx: 0), Z: YTCT.getReg(Idx: 0), Flags: NewFlags);
3825 Register Mad1 = getMad(B, Ty, X: YT.getReg(Idx: 0), Y: CH.getReg(Idx: 0), Z: Mad0, Flags: NewFlags);
3826 R = getMad(B, Ty, X: YH.getReg(Idx: 0), Y: CH.getReg(Idx: 0), Z: Mad1, Flags: NewFlags);
3827 }
3828
3829 const bool IsFiniteOnly =
3830 MI.getFlag(Flag: MachineInstr::FmNoNans) && MI.getFlag(Flag: MachineInstr::FmNoInfs);
3831
3832 if (!IsFiniteOnly) {
3833 // Expand isfinite(x) => fabs(x) < inf
3834 auto Inf = B.buildFConstant(Res: Ty, Val: APFloat::getInf(Sem: APFloat::IEEEsingle()));
3835 auto Fabs = B.buildFAbs(Dst: Ty, Src0: Y);
3836 auto IsFinite =
3837 B.buildFCmp(Pred: CmpInst::FCMP_OLT, Res: LLT::scalar(SizeInBits: 1), Op0: Fabs, Op1: Inf, Flags);
3838 R = B.buildSelect(Res: Ty, Tst: IsFinite, Op0: R, Op1: Y, Flags).getReg(Idx: 0);
3839 }
3840
3841 if (ScaledInput) {
3842 auto Zero = B.buildFConstant(Res: Ty, Val: 0.0);
3843 auto ShiftK =
3844 B.buildFConstant(Res: Ty, Val: IsLog10 ? 0x1.344136p+3f : 0x1.62e430p+4f);
3845 auto Shift = B.buildSelect(Res: Ty, Tst: IsScaled, Op0: ShiftK, Op1: Zero, Flags);
3846 B.buildFSub(Dst, Src0: R, Src1: Shift, Flags);
3847 } else {
3848 B.buildCopy(Res: Dst, Op: R);
3849 }
3850
3851 MI.eraseFromParent();
3852 return true;
3853}
3854
3855bool AMDGPULegalizerInfo::legalizeFlogUnsafe(MachineIRBuilder &B, Register Dst,
3856 Register Src, bool IsLog10,
3857 unsigned Flags) const {
3858 const double Log2BaseInverted =
3859 IsLog10 ? numbers::ln2 / numbers::ln10 : numbers::ln2;
3860
3861 LLT Ty = B.getMRI()->getType(Reg: Dst);
3862
3863 if (Ty == LLT::float32()) {
3864 auto [ScaledInput, IsScaled] = getScaledLogInput(B, Src, Flags);
3865 if (ScaledInput) {
3866 auto LogSrc = B.buildIntrinsic(ID: Intrinsic::amdgcn_log, Res: {Ty})
3867 .addUse(RegNo: Src)
3868 .setMIFlags(Flags);
3869 auto ScaledResultOffset = B.buildFConstant(Res: Ty, Val: -32.0 * Log2BaseInverted);
3870 auto Zero = B.buildFConstant(Res: Ty, Val: 0.0);
3871 auto ResultOffset =
3872 B.buildSelect(Res: Ty, Tst: IsScaled, Op0: ScaledResultOffset, Op1: Zero, Flags);
3873 auto Log2Inv = B.buildFConstant(Res: Ty, Val: Log2BaseInverted);
3874
3875 if (ST.hasFastFMAF32())
3876 B.buildFMA(Dst, Src0: LogSrc, Src1: Log2Inv, Src2: ResultOffset, Flags);
3877 else {
3878 auto Mul = B.buildFMul(Dst: Ty, Src0: LogSrc, Src1: Log2Inv, Flags);
3879 B.buildFAdd(Dst, Src0: Mul, Src1: ResultOffset, Flags);
3880 }
3881
3882 return true;
3883 }
3884 }
3885
3886 auto Log2Operand = Ty == LLT::float16()
3887 ? B.buildFLog2(Dst: Ty, Src, Flags)
3888 : B.buildIntrinsic(ID: Intrinsic::amdgcn_log, Res: {Ty})
3889 .addUse(RegNo: Src)
3890 .setMIFlags(Flags);
3891 auto Log2BaseInvertedOperand = B.buildFConstant(Res: Ty, Val: Log2BaseInverted);
3892 B.buildFMul(Dst, Src0: Log2Operand, Src1: Log2BaseInvertedOperand, Flags);
3893 return true;
3894}
3895
3896bool AMDGPULegalizerInfo::legalizeFExp2(MachineInstr &MI,
3897 MachineIRBuilder &B) const {
3898 // v_exp_f32 is good enough for OpenCL, except it doesn't handle denormals.
3899 // If we have to handle denormals, scale up the input and adjust the result.
3900
3901 Register Dst = MI.getOperand(i: 0).getReg();
3902 Register Src = MI.getOperand(i: 1).getReg();
3903 unsigned Flags = MI.getFlags();
3904 LLT Ty = B.getMRI()->getType(Reg: Dst);
3905 const LLT F16 = LLT::float16();
3906 const LLT F32 = LLT::float32();
3907 const LLT F64 = LLT::float64();
3908
3909 if (Ty == F64)
3910 return legalizeFEXPF64(MI, B);
3911
3912 if (Ty == F16) {
3913 // Nothing in half is a denormal when promoted to f32.
3914 auto Ext = B.buildFPExt(Res: F32, Op: Src, Flags);
3915 auto Log2 = B.buildIntrinsic(ID: Intrinsic::amdgcn_exp2, Res: {F32})
3916 .addUse(RegNo: Ext.getReg(Idx: 0))
3917 .setMIFlags(Flags);
3918 B.buildFPTrunc(Res: Dst, Op: Log2, Flags);
3919 MI.eraseFromParent();
3920 return true;
3921 }
3922
3923 assert(Ty == F32);
3924
3925 if (!needsDenormHandlingF32(MF: B.getMF(), Src, Flags)) {
3926 B.buildIntrinsic(ID: Intrinsic::amdgcn_exp2, Res: ArrayRef<Register>{Dst})
3927 .addUse(RegNo: Src)
3928 .setMIFlags(Flags);
3929 MI.eraseFromParent();
3930 return true;
3931 }
3932
3933 // bool needs_scaling = x < -0x1.f80000p+6f;
3934 // v_exp_f32(x + (s ? 0x1.0p+6f : 0.0f)) * (s ? 0x1.0p-64f : 1.0f);
3935
3936 // -nextafter(128.0, -1)
3937 auto RangeCheckConst = B.buildFConstant(Res: Ty, Val: -0x1.f80000p+6f);
3938 auto NeedsScaling = B.buildFCmp(Pred: CmpInst::FCMP_OLT, Res: LLT::scalar(SizeInBits: 1), Op0: Src,
3939 Op1: RangeCheckConst, Flags);
3940
3941 auto SixtyFour = B.buildFConstant(Res: Ty, Val: 0x1.0p+6f);
3942 auto Zero = B.buildFConstant(Res: Ty, Val: 0.0);
3943 auto AddOffset = B.buildSelect(Res: F32, Tst: NeedsScaling, Op0: SixtyFour, Op1: Zero, Flags);
3944 auto AddInput = B.buildFAdd(Dst: F32, Src0: Src, Src1: AddOffset, Flags);
3945
3946 auto Exp2 = B.buildIntrinsic(ID: Intrinsic::amdgcn_exp2, Res: {Ty})
3947 .addUse(RegNo: AddInput.getReg(Idx: 0))
3948 .setMIFlags(Flags);
3949
3950 auto TwoExpNeg64 = B.buildFConstant(Res: Ty, Val: 0x1.0p-64f);
3951 auto One = B.buildFConstant(Res: Ty, Val: 1.0);
3952 auto ResultScale = B.buildSelect(Res: F32, Tst: NeedsScaling, Op0: TwoExpNeg64, Op1: One, Flags);
3953 B.buildFMul(Dst, Src0: Exp2, Src1: ResultScale, Flags);
3954 MI.eraseFromParent();
3955 return true;
3956}
3957
3958static MachineInstrBuilder buildExp(MachineIRBuilder &B, const DstOp &Dst,
3959 const SrcOp &Src, unsigned Flags) {
3960 LLT Ty = Dst.getLLTTy(MRI: *B.getMRI());
3961
3962 if (Ty == LLT::float32()) {
3963 return B.buildIntrinsic(ID: Intrinsic::amdgcn_exp2, Res: {Dst})
3964 .addUse(RegNo: Src.getReg())
3965 .setMIFlags(Flags);
3966 }
3967 return B.buildFExp2(Dst, Src, Flags);
3968}
3969
3970bool AMDGPULegalizerInfo::legalizeFExpUnsafeImpl(MachineIRBuilder &B,
3971 Register Dst, Register X,
3972 unsigned Flags,
3973 bool IsExp10) const {
3974 LLT Ty = B.getMRI()->getType(Reg: X);
3975
3976 // exp(x) -> exp2(M_LOG2E_F * x);
3977 // exp10(x) -> exp2(log2(10) * x);
3978 auto Const = B.buildFConstant(Res: Ty, Val: IsExp10 ? 0x1.a934f0p+1f : numbers::log2e);
3979 auto Mul = B.buildFMul(Dst: Ty, Src0: X, Src1: Const, Flags);
3980 buildExp(B, Dst, Src: Mul, Flags);
3981 return true;
3982}
3983
3984bool AMDGPULegalizerInfo::legalizeFExpUnsafe(MachineIRBuilder &B, Register Dst,
3985 Register X, unsigned Flags) const {
3986 LLT Ty = B.getMRI()->getType(Reg: Dst);
3987 LLT F32 = LLT::float32();
3988
3989 if (Ty != F32 || !needsDenormHandlingF32(MF: B.getMF(), Src: X, Flags)) {
3990 return legalizeFExpUnsafeImpl(B, Dst, X, Flags, /*IsExp10=*/false);
3991 }
3992
3993 auto Threshold = B.buildFConstant(Res: Ty, Val: -0x1.5d58a0p+6f);
3994 auto NeedsScaling =
3995 B.buildFCmp(Pred: CmpInst::FCMP_OLT, Res: LLT::scalar(SizeInBits: 1), Op0: X, Op1: Threshold, Flags);
3996 auto ScaleOffset = B.buildFConstant(Res: Ty, Val: 0x1.0p+6f);
3997 auto ScaledX = B.buildFAdd(Dst: Ty, Src0: X, Src1: ScaleOffset, Flags);
3998 auto AdjustedX = B.buildSelect(Res: Ty, Tst: NeedsScaling, Op0: ScaledX, Op1: X, Flags);
3999
4000 auto Log2E = B.buildFConstant(Res: Ty, Val: numbers::log2e);
4001 auto ExpInput = B.buildFMul(Dst: Ty, Src0: AdjustedX, Src1: Log2E, Flags);
4002
4003 auto Exp2 = B.buildIntrinsic(ID: Intrinsic::amdgcn_exp2, Res: {Ty})
4004 .addUse(RegNo: ExpInput.getReg(Idx: 0))
4005 .setMIFlags(Flags);
4006
4007 auto ResultScaleFactor = B.buildFConstant(Res: Ty, Val: 0x1.969d48p-93f);
4008 auto AdjustedResult = B.buildFMul(Dst: Ty, Src0: Exp2, Src1: ResultScaleFactor, Flags);
4009 B.buildSelect(Res: Dst, Tst: NeedsScaling, Op0: AdjustedResult, Op1: Exp2, Flags);
4010 return true;
4011}
4012
4013bool AMDGPULegalizerInfo::legalizeFExp10Unsafe(MachineIRBuilder &B,
4014 Register Dst, Register X,
4015 unsigned Flags) const {
4016 LLT Ty = B.getMRI()->getType(Reg: Dst);
4017 LLT F32 = LLT::float32();
4018
4019 if (Ty != F32 || !needsDenormHandlingF32(MF: B.getMF(), Src: X, Flags)) {
4020 // exp2(x * 0x1.a92000p+1f) * exp2(x * 0x1.4f0978p-11f);
4021 auto K0 = B.buildFConstant(Res: Ty, Val: 0x1.a92000p+1f);
4022 auto K1 = B.buildFConstant(Res: Ty, Val: 0x1.4f0978p-11f);
4023
4024 auto Mul1 = B.buildFMul(Dst: Ty, Src0: X, Src1: K1, Flags);
4025 auto Exp2_1 = buildExp(B, Dst: Ty, Src: Mul1, Flags);
4026 auto Mul0 = B.buildFMul(Dst: Ty, Src0: X, Src1: K0, Flags);
4027 auto Exp2_0 = buildExp(B, Dst: Ty, Src: Mul0, Flags);
4028 B.buildFMul(Dst, Src0: Exp2_0, Src1: Exp2_1, Flags);
4029 return true;
4030 }
4031
4032 // bool s = x < -0x1.2f7030p+5f;
4033 // x += s ? 0x1.0p+5f : 0.0f;
4034 // exp10 = exp2(x * 0x1.a92000p+1f) *
4035 // exp2(x * 0x1.4f0978p-11f) *
4036 // (s ? 0x1.9f623ep-107f : 1.0f);
4037
4038 auto Threshold = B.buildFConstant(Res: Ty, Val: -0x1.2f7030p+5f);
4039 auto NeedsScaling =
4040 B.buildFCmp(Pred: CmpInst::FCMP_OLT, Res: LLT::scalar(SizeInBits: 1), Op0: X, Op1: Threshold);
4041
4042 auto ScaleOffset = B.buildFConstant(Res: Ty, Val: 0x1.0p+5f);
4043 auto ScaledX = B.buildFAdd(Dst: Ty, Src0: X, Src1: ScaleOffset, Flags);
4044 auto AdjustedX = B.buildSelect(Res: Ty, Tst: NeedsScaling, Op0: ScaledX, Op1: X);
4045
4046 auto K0 = B.buildFConstant(Res: Ty, Val: 0x1.a92000p+1f);
4047 auto K1 = B.buildFConstant(Res: Ty, Val: 0x1.4f0978p-11f);
4048
4049 auto Mul1 = B.buildFMul(Dst: Ty, Src0: AdjustedX, Src1: K1, Flags);
4050 auto Exp2_1 = buildExp(B, Dst: Ty, Src: Mul1, Flags);
4051 auto Mul0 = B.buildFMul(Dst: Ty, Src0: AdjustedX, Src1: K0, Flags);
4052 auto Exp2_0 = buildExp(B, Dst: Ty, Src: Mul0, Flags);
4053
4054 auto MulExps = B.buildFMul(Dst: Ty, Src0: Exp2_0, Src1: Exp2_1, Flags);
4055 auto ResultScaleFactor = B.buildFConstant(Res: Ty, Val: 0x1.9f623ep-107f);
4056 auto AdjustedResult = B.buildFMul(Dst: Ty, Src0: MulExps, Src1: ResultScaleFactor, Flags);
4057
4058 B.buildSelect(Res: Dst, Tst: NeedsScaling, Op0: AdjustedResult, Op1: MulExps);
4059 return true;
4060}
4061
4062// This expansion gives a result slightly better than 1ulp.
4063bool AMDGPULegalizerInfo::legalizeFEXPF64(MachineInstr &MI,
4064 MachineIRBuilder &B) const {
4065
4066 Register X = MI.getOperand(i: 1).getReg();
4067 LLT F64 = LLT::float64();
4068 LLT I32 = LLT::integer(SizeInBits: 32);
4069 LLT S1 = LLT::scalar(SizeInBits: 1);
4070
4071 // TODO: Check if reassoc is safe. There is an output change in exp2 and
4072 // exp10, which slightly increases ulp.
4073 unsigned Flags = MI.getFlags() & ~MachineInstr::FmReassoc;
4074
4075 Register Dn, F, T;
4076
4077 if (MI.getOpcode() == TargetOpcode::G_FEXP2) {
4078 // Dn = rint(X)
4079 Dn = B.buildFRint(Dst: F64, Src0: X, Flags).getReg(Idx: 0);
4080 // F = X - Dn
4081 F = B.buildFSub(Dst: F64, Src0: X, Src1: Dn, Flags).getReg(Idx: 0);
4082 // T = F*C1 + F*C2
4083 auto C1 = B.buildFConstant(Res: F64, Val: APFloat(0x1.62e42fefa39efp-1));
4084 auto C2 = B.buildFConstant(Res: F64, Val: APFloat(0x1.abc9e3b39803fp-56));
4085 auto Mul2 = B.buildFMul(Dst: F64, Src0: F, Src1: C2, Flags).getReg(Idx: 0);
4086 T = B.buildFMA(Dst: F64, Src0: F, Src1: C1, Src2: Mul2, Flags).getReg(Idx: 0);
4087
4088 } else if (MI.getOpcode() == TargetOpcode::G_FEXP10) {
4089 auto C1 = B.buildFConstant(Res: F64, Val: APFloat(0x1.a934f0979a371p+1));
4090 auto Mul = B.buildFMul(Dst: F64, Src0: X, Src1: C1, Flags).getReg(Idx: 0);
4091 Dn = B.buildFRint(Dst: F64, Src0: Mul, Flags).getReg(Idx: 0);
4092
4093 auto NegDn = B.buildFNeg(Dst: F64, Src0: Dn, Flags).getReg(Idx: 0);
4094 auto C2 = B.buildFConstant(Res: F64, Val: APFloat(-0x1.9dc1da994fd21p-59));
4095 auto C3 = B.buildFConstant(Res: F64, Val: APFloat(0x1.34413509f79ffp-2));
4096 auto Inner = B.buildFMA(Dst: F64, Src0: NegDn, Src1: C3, Src2: X, Flags).getReg(Idx: 0);
4097 F = B.buildFMA(Dst: F64, Src0: NegDn, Src1: C2, Src2: Inner, Flags).getReg(Idx: 0);
4098
4099 auto C4 = B.buildFConstant(Res: F64, Val: APFloat(0x1.26bb1bbb55516p+1));
4100 auto C5 = B.buildFConstant(Res: F64, Val: APFloat(-0x1.f48ad494ea3e9p-53));
4101 auto MulF = B.buildFMul(Dst: F64, Src0: F, Src1: C5, Flags).getReg(Idx: 0);
4102 T = B.buildFMA(Dst: F64, Src0: F, Src1: C4, Src2: MulF, Flags).getReg(Idx: 0);
4103
4104 } else { // G_FEXP
4105 auto C1 = B.buildFConstant(Res: F64, Val: APFloat(0x1.71547652b82fep+0));
4106 auto Mul = B.buildFMul(Dst: F64, Src0: X, Src1: C1, Flags).getReg(Idx: 0);
4107 Dn = B.buildFRint(Dst: F64, Src0: Mul, Flags).getReg(Idx: 0);
4108
4109 auto NegDn = B.buildFNeg(Dst: F64, Src0: Dn, Flags).getReg(Idx: 0);
4110 auto C2 = B.buildFConstant(Res: F64, Val: APFloat(0x1.abc9e3b39803fp-56));
4111 auto C3 = B.buildFConstant(Res: F64, Val: APFloat(0x1.62e42fefa39efp-1));
4112 auto Inner = B.buildFMA(Dst: F64, Src0: NegDn, Src1: C3, Src2: X, Flags).getReg(Idx: 0);
4113 T = B.buildFMA(Dst: F64, Src0: NegDn, Src1: C2, Src2: Inner, Flags).getReg(Idx: 0);
4114 }
4115
4116 // Polynomial chain for P
4117 auto P = B.buildFConstant(Res: F64, Val: 0x1.ade156a5dcb37p-26);
4118 P = B.buildFMA(Dst: F64, Src0: T, Src1: P, Src2: B.buildFConstant(Res: F64, Val: 0x1.28af3fca7ab0cp-22),
4119 Flags);
4120 P = B.buildFMA(Dst: F64, Src0: T, Src1: P, Src2: B.buildFConstant(Res: F64, Val: 0x1.71dee623fde64p-19),
4121 Flags);
4122 P = B.buildFMA(Dst: F64, Src0: T, Src1: P, Src2: B.buildFConstant(Res: F64, Val: 0x1.a01997c89e6b0p-16),
4123 Flags);
4124 P = B.buildFMA(Dst: F64, Src0: T, Src1: P, Src2: B.buildFConstant(Res: F64, Val: 0x1.a01a014761f6ep-13),
4125 Flags);
4126 P = B.buildFMA(Dst: F64, Src0: T, Src1: P, Src2: B.buildFConstant(Res: F64, Val: 0x1.6c16c1852b7b0p-10),
4127 Flags);
4128 P = B.buildFMA(Dst: F64, Src0: T, Src1: P, Src2: B.buildFConstant(Res: F64, Val: 0x1.1111111122322p-7), Flags);
4129 P = B.buildFMA(Dst: F64, Src0: T, Src1: P, Src2: B.buildFConstant(Res: F64, Val: 0x1.55555555502a1p-5), Flags);
4130 P = B.buildFMA(Dst: F64, Src0: T, Src1: P, Src2: B.buildFConstant(Res: F64, Val: 0x1.5555555555511p-3), Flags);
4131 P = B.buildFMA(Dst: F64, Src0: T, Src1: P, Src2: B.buildFConstant(Res: F64, Val: 0x1.000000000000bp-1), Flags);
4132
4133 auto One = B.buildFConstant(Res: F64, Val: 1.0);
4134 P = B.buildFMA(Dst: F64, Src0: T, Src1: P, Src2: One, Flags);
4135 P = B.buildFMA(Dst: F64, Src0: T, Src1: P, Src2: One, Flags);
4136
4137 // Z = FLDEXP(P, (int)Dn)
4138 auto DnInt = B.buildFPTOSI(Dst: I32, Src0: Dn);
4139 auto Z = B.buildFLdexp(Dst: F64, Src0: P, Src1: DnInt, Flags);
4140
4141 if (!(Flags & MachineInstr::FmNoInfs)) {
4142 // Overflow guard: if X <= 1024.0 then Z else +inf
4143 auto CondHi = B.buildFCmp(Pred: CmpInst::FCMP_ULE, Res: S1, Op0: X,
4144 Op1: B.buildFConstant(Res: F64, Val: APFloat(1024.0)));
4145 auto PInf = B.buildFConstant(Res: F64, Val: APFloat::getInf(Sem: APFloat::IEEEdouble()));
4146 Z = B.buildSelect(Res: F64, Tst: CondHi, Op0: Z, Op1: PInf, Flags);
4147 }
4148
4149 // Underflow guard: if X >= -1075.0 then Z else 0.0
4150 auto CondLo = B.buildFCmp(Pred: CmpInst::FCMP_UGE, Res: S1, Op0: X,
4151 Op1: B.buildFConstant(Res: F64, Val: APFloat(-1075.0)));
4152 auto Zero = B.buildFConstant(Res: F64, Val: APFloat(0.0));
4153 B.buildSelect(Res: MI.getOperand(i: 0).getReg(), Tst: CondLo, Op0: Z, Op1: Zero, Flags);
4154
4155 MI.eraseFromParent();
4156 return true;
4157}
4158
4159bool AMDGPULegalizerInfo::legalizeFExp(MachineInstr &MI,
4160 MachineIRBuilder &B) const {
4161 Register Dst = MI.getOperand(i: 0).getReg();
4162 Register X = MI.getOperand(i: 1).getReg();
4163 const unsigned Flags = MI.getFlags();
4164 MachineFunction &MF = B.getMF();
4165 MachineRegisterInfo &MRI = *B.getMRI();
4166 LLT Ty = MRI.getType(Reg: Dst);
4167
4168 const LLT F64 = LLT::float64();
4169
4170 if (Ty == F64)
4171 return legalizeFEXPF64(MI, B);
4172
4173 const LLT F16 = LLT::float16();
4174 const LLT F32 = LLT::float32();
4175 const bool IsExp10 = MI.getOpcode() == TargetOpcode::G_FEXP10;
4176
4177 if (Ty == F16) {
4178 // v_exp_f16 (fmul x, log2e)
4179 if (allowApproxFunc(MF, Flags)) {
4180 // TODO: Does this really require fast?
4181 IsExp10 ? legalizeFExp10Unsafe(B, Dst, X, Flags)
4182 : legalizeFExpUnsafe(B, Dst, X, Flags);
4183 MI.eraseFromParent();
4184 return true;
4185 }
4186
4187 // Nothing in half is a denormal when promoted to f32.
4188 //
4189 // exp(f16 x) ->
4190 // fptrunc (v_exp_f32 (fmul (fpext x), log2e))
4191 //
4192 // exp10(f16 x) ->
4193 // fptrunc (v_exp_f32 (fmul (fpext x), log2(10)))
4194 auto Ext = B.buildFPExt(Res: F32, Op: X, Flags);
4195 Register Lowered = MRI.createGenericVirtualRegister(Ty: F32);
4196 legalizeFExpUnsafeImpl(B, Dst: Lowered, X: Ext.getReg(Idx: 0), Flags, IsExp10);
4197 B.buildFPTrunc(Res: Dst, Op: Lowered, Flags);
4198 MI.eraseFromParent();
4199 return true;
4200 }
4201
4202 assert(Ty == F32);
4203
4204 // TODO: Interpret allowApproxFunc as ignoring DAZ. This is currently copying
4205 // library behavior. Also, is known-not-daz source sufficient?
4206 if (allowApproxFunc(MF, Flags)) {
4207 IsExp10 ? legalizeFExp10Unsafe(B, Dst, X, Flags)
4208 : legalizeFExpUnsafe(B, Dst, X, Flags);
4209 MI.eraseFromParent();
4210 return true;
4211 }
4212
4213 // Algorithm:
4214 //
4215 // e^x = 2^(x/ln(2)) = 2^(x*(64/ln(2))/64)
4216 //
4217 // x*(64/ln(2)) = n + f, |f| <= 0.5, n is integer
4218 // n = 64*m + j, 0 <= j < 64
4219 //
4220 // e^x = 2^((64*m + j + f)/64)
4221 // = (2^m) * (2^(j/64)) * 2^(f/64)
4222 // = (2^m) * (2^(j/64)) * e^(f*(ln(2)/64))
4223 //
4224 // f = x*(64/ln(2)) - n
4225 // r = f*(ln(2)/64) = x - n*(ln(2)/64)
4226 //
4227 // e^x = (2^m) * (2^(j/64)) * e^r
4228 //
4229 // (2^(j/64)) is precomputed
4230 //
4231 // e^r = 1 + r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5!
4232 // e^r = 1 + q
4233 //
4234 // q = r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5!
4235 //
4236 // e^x = (2^m) * ( (2^(j/64)) + q*(2^(j/64)) )
4237 const unsigned FlagsNoContract = Flags & ~MachineInstr::FmContract;
4238 Register PH, PL;
4239
4240 if (ST.hasFastFMAF32()) {
4241 const float c_exp = numbers::log2ef;
4242 const float cc_exp = 0x1.4ae0bep-26f; // c+cc are 49 bits
4243 const float c_exp10 = 0x1.a934f0p+1f;
4244 const float cc_exp10 = 0x1.2f346ep-24f;
4245
4246 auto C = B.buildFConstant(Res: Ty, Val: IsExp10 ? c_exp10 : c_exp);
4247 PH = B.buildFMul(Dst: Ty, Src0: X, Src1: C, Flags).getReg(Idx: 0);
4248 auto NegPH = B.buildFNeg(Dst: Ty, Src0: PH, Flags);
4249 auto FMA0 = B.buildFMA(Dst: Ty, Src0: X, Src1: C, Src2: NegPH, Flags);
4250
4251 auto CC = B.buildFConstant(Res: Ty, Val: IsExp10 ? cc_exp10 : cc_exp);
4252 PL = B.buildFMA(Dst: Ty, Src0: X, Src1: CC, Src2: FMA0, Flags).getReg(Idx: 0);
4253 } else {
4254 const float ch_exp = 0x1.714000p+0f;
4255 const float cl_exp = 0x1.47652ap-12f; // ch + cl are 36 bits
4256
4257 const float ch_exp10 = 0x1.a92000p+1f;
4258 const float cl_exp10 = 0x1.4f0978p-11f;
4259
4260 auto MaskConst = B.buildConstant(Res: Ty, Val: 0xfffff000);
4261 auto XH = B.buildAnd(Dst: Ty, Src0: X, Src1: MaskConst);
4262 auto XL = B.buildFSub(Dst: Ty, Src0: X, Src1: XH, Flags);
4263
4264 auto CH = B.buildFConstant(Res: Ty, Val: IsExp10 ? ch_exp10 : ch_exp);
4265 PH = B.buildFMul(Dst: Ty, Src0: XH, Src1: CH, Flags).getReg(Idx: 0);
4266
4267 auto CL = B.buildFConstant(Res: Ty, Val: IsExp10 ? cl_exp10 : cl_exp);
4268 auto XLCL = B.buildFMul(Dst: Ty, Src0: XL, Src1: CL, Flags);
4269
4270 Register Mad0 =
4271 getMad(B, Ty, X: XL.getReg(Idx: 0), Y: CH.getReg(Idx: 0), Z: XLCL.getReg(Idx: 0), Flags);
4272 PL = getMad(B, Ty, X: XH.getReg(Idx: 0), Y: CL.getReg(Idx: 0), Z: Mad0, Flags);
4273 }
4274
4275 auto E = B.buildIntrinsicRoundeven(Dst: Ty, Src0: PH, Flags);
4276
4277 // It is unsafe to contract this fsub into the PH multiply.
4278 auto PHSubE = B.buildFSub(Dst: Ty, Src0: PH, Src1: E, Flags: FlagsNoContract);
4279 auto A = B.buildFAdd(Dst: Ty, Src0: PHSubE, Src1: PL, Flags);
4280 const LLT I32 = LLT::integer(SizeInBits: 32);
4281 auto IntE = B.buildFPTOSI(Dst: I32, Src0: E);
4282
4283 auto Exp2 = B.buildIntrinsic(ID: Intrinsic::amdgcn_exp2, Res: {Ty})
4284 .addUse(RegNo: A.getReg(Idx: 0))
4285 .setMIFlags(Flags);
4286 auto R = B.buildFLdexp(Dst: Ty, Src0: Exp2, Src1: IntE, Flags);
4287
4288 auto UnderflowCheckConst =
4289 B.buildFConstant(Res: Ty, Val: IsExp10 ? -0x1.66d3e8p+5f : -0x1.9d1da0p+6f);
4290 auto Zero = B.buildFConstant(Res: Ty, Val: 0.0);
4291 auto Underflow =
4292 B.buildFCmp(Pred: CmpInst::FCMP_OLT, Res: LLT::scalar(SizeInBits: 1), Op0: X, Op1: UnderflowCheckConst);
4293
4294 R = B.buildSelect(Res: Ty, Tst: Underflow, Op0: Zero, Op1: R);
4295
4296 if (!(Flags & MachineInstr::FmNoInfs)) {
4297 auto OverflowCheckConst =
4298 B.buildFConstant(Res: Ty, Val: IsExp10 ? 0x1.344136p+5f : 0x1.62e430p+6f);
4299
4300 auto Overflow =
4301 B.buildFCmp(Pred: CmpInst::FCMP_OGT, Res: LLT::scalar(SizeInBits: 1), Op0: X, Op1: OverflowCheckConst);
4302 auto Inf = B.buildFConstant(Res: Ty, Val: APFloat::getInf(Sem: APFloat::IEEEsingle()));
4303 R = B.buildSelect(Res: Ty, Tst: Overflow, Op0: Inf, Op1: R, Flags);
4304 }
4305
4306 B.buildCopy(Res: Dst, Op: R);
4307 MI.eraseFromParent();
4308 return true;
4309}
4310
4311bool AMDGPULegalizerInfo::legalizeFPow(MachineInstr &MI,
4312 MachineIRBuilder &B) const {
4313 Register Dst = MI.getOperand(i: 0).getReg();
4314 Register Src0 = MI.getOperand(i: 1).getReg();
4315 Register Src1 = MI.getOperand(i: 2).getReg();
4316 unsigned Flags = MI.getFlags();
4317 LLT Ty = B.getMRI()->getType(Reg: Dst);
4318 const LLT F16 = LLT::float16();
4319 const LLT F32 = LLT::float32();
4320
4321 if (Ty == F32) {
4322 auto Log = B.buildFLog2(Dst: F32, Src: Src0, Flags);
4323 auto Mul = B.buildIntrinsic(ID: Intrinsic::amdgcn_fmul_legacy, Res: {F32})
4324 .addUse(RegNo: Log.getReg(Idx: 0))
4325 .addUse(RegNo: Src1)
4326 .setMIFlags(Flags);
4327 B.buildFExp2(Dst, Src: Mul, Flags);
4328 } else if (Ty == F16) {
4329 // There's no f16 fmul_legacy, so we need to convert for it.
4330 auto Log = B.buildFLog2(Dst: F16, Src: Src0, Flags);
4331 auto Ext0 = B.buildFPExt(Res: F32, Op: Log, Flags);
4332 auto Ext1 = B.buildFPExt(Res: F32, Op: Src1, Flags);
4333 auto Mul = B.buildIntrinsic(ID: Intrinsic::amdgcn_fmul_legacy, Res: {F32})
4334 .addUse(RegNo: Ext0.getReg(Idx: 0))
4335 .addUse(RegNo: Ext1.getReg(Idx: 0))
4336 .setMIFlags(Flags);
4337 B.buildFExp2(Dst, Src: B.buildFPTrunc(Res: F16, Op: Mul), Flags);
4338 } else
4339 return false;
4340
4341 MI.eraseFromParent();
4342 return true;
4343}
4344
4345// Find a source register, ignoring any possible source modifiers.
4346static Register stripAnySourceMods(Register OrigSrc, MachineRegisterInfo &MRI) {
4347 Register ModSrc = OrigSrc;
4348 if (MachineInstr *SrcFNeg = getOpcodeDef(Opcode: AMDGPU::G_FNEG, Reg: ModSrc, MRI)) {
4349 ModSrc = SrcFNeg->getOperand(i: 1).getReg();
4350 if (MachineInstr *SrcFAbs = getOpcodeDef(Opcode: AMDGPU::G_FABS, Reg: ModSrc, MRI))
4351 ModSrc = SrcFAbs->getOperand(i: 1).getReg();
4352 } else if (MachineInstr *SrcFAbs = getOpcodeDef(Opcode: AMDGPU::G_FABS, Reg: ModSrc, MRI))
4353 ModSrc = SrcFAbs->getOperand(i: 1).getReg();
4354 return ModSrc;
4355}
4356
4357bool AMDGPULegalizerInfo::legalizeFFloor(MachineInstr &MI,
4358 MachineRegisterInfo &MRI,
4359 MachineIRBuilder &B) const {
4360
4361 const LLT S1 = LLT::scalar(SizeInBits: 1);
4362 const LLT F64 = LLT::float64();
4363 Register Dst = MI.getOperand(i: 0).getReg();
4364 Register OrigSrc = MI.getOperand(i: 1).getReg();
4365 unsigned Flags = MI.getFlags();
4366 assert(ST.hasFractBug() && MRI.getType(Dst) == F64 &&
4367 "this should not have been custom lowered");
4368
4369 // V_FRACT is buggy on SI, so the F32 version is never used and (x-floor(x))
4370 // is used instead. However, SI doesn't have V_FLOOR_F64, so the most
4371 // efficient way to implement it is using V_FRACT_F64. The workaround for the
4372 // V_FRACT bug is:
4373 // fract(x) = isnan(x) ? x : min(V_FRACT(x), 0.99999999999999999)
4374 //
4375 // Convert floor(x) to (x - fract(x))
4376
4377 auto Fract = B.buildIntrinsic(ID: Intrinsic::amdgcn_fract, Res: {F64})
4378 .addUse(RegNo: OrigSrc)
4379 .setMIFlags(Flags);
4380
4381 // Give source modifier matching some assistance before obscuring a foldable
4382 // pattern.
4383
4384 // TODO: We can avoid the neg on the fract? The input sign to fract
4385 // shouldn't matter?
4386 Register ModSrc = stripAnySourceMods(OrigSrc, MRI);
4387
4388 auto Const =
4389 B.buildFConstant(Res: F64, Val: llvm::bit_cast<double>(from: 0x3fefffffffffffff));
4390
4391 Register Min = MRI.createGenericVirtualRegister(Ty: F64);
4392
4393 // We don't need to concern ourselves with the snan handling difference, so
4394 // use the one which will directly select.
4395 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
4396 if (MFI->getMode().IEEE)
4397 B.buildFMinNumIEEE(Dst: Min, Src0: Fract, Src1: Const, Flags);
4398 else
4399 B.buildFMinNum(Dst: Min, Src0: Fract, Src1: Const, Flags);
4400
4401 Register CorrectedFract = Min;
4402 if (!MI.getFlag(Flag: MachineInstr::FmNoNans)) {
4403 auto IsNan = B.buildFCmp(Pred: CmpInst::FCMP_ORD, Res: S1, Op0: ModSrc, Op1: ModSrc, Flags);
4404 CorrectedFract = B.buildSelect(Res: F64, Tst: IsNan, Op0: ModSrc, Op1: Min, Flags).getReg(Idx: 0);
4405 }
4406
4407 auto NegFract = B.buildFNeg(Dst: F64, Src0: CorrectedFract, Flags);
4408 B.buildFAdd(Dst, Src0: OrigSrc, Src1: NegFract, Flags);
4409
4410 MI.eraseFromParent();
4411 return true;
4412}
4413
4414// Turn an illegal packed v2i16/v2f16 build vector into bit operations.
4415// TODO: This should probably be a bitcast action in LegalizerHelper.
4416bool AMDGPULegalizerInfo::legalizeBuildVector(
4417 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
4418 Register Dst = MI.getOperand(i: 0).getReg();
4419 const LLT I32 = LLT::integer(SizeInBits: 32);
4420 const LLT I16 = LLT::integer(SizeInBits: 16);
4421 assert(MRI.getType(Dst).isVector() &&
4422 MRI.getType(Dst).getNumElements() == 2 &&
4423 MRI.getType(Dst).getScalarSizeInBits() == 16);
4424
4425 Register Src0 = MI.getOperand(i: 1).getReg();
4426 Register Src1 = MI.getOperand(i: 2).getReg();
4427
4428 if (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC) {
4429 assert(MRI.getType(Src0) == I32);
4430 Src0 = B.buildTrunc(Res: I16, Op: MI.getOperand(i: 1).getReg()).getReg(Idx: 0);
4431 Src1 = B.buildTrunc(Res: I16, Op: MI.getOperand(i: 2).getReg()).getReg(Idx: 0);
4432 }
4433
4434 auto Merge = B.buildMergeLikeInstr(Res: I32, Ops: {Src0, Src1});
4435 B.buildBitcast(Dst, Src: Merge);
4436
4437 MI.eraseFromParent();
4438 return true;
4439}
4440
4441// Build a big integer multiply or multiply-add using MAD_64_32 instructions.
4442//
4443// Source and accumulation registers must all be 32-bits.
4444//
4445// TODO: When the multiply is uniform, we should produce a code sequence
4446// that is better suited to instruction selection on the SALU. Instead of
4447// the outer loop going over parts of the result, the outer loop should go
4448// over parts of one of the factors. This should result in instruction
4449// selection that makes full use of S_ADDC_U32 instructions.
4450void AMDGPULegalizerInfo::buildMultiply(LegalizerHelper &Helper,
4451 MutableArrayRef<Register> Accum,
4452 ArrayRef<Register> Src0,
4453 ArrayRef<Register> Src1,
4454 bool UsePartialMad64_32,
4455 bool SeparateOddAlignedProducts) const {
4456 // Use (possibly empty) vectors of S1 registers to represent the set of
4457 // carries from one pair of positions to the next.
4458 using Carry = SmallVector<Register, 2>;
4459
4460 MachineIRBuilder &B = Helper.MIRBuilder;
4461 GISelValueTracking &VT = *Helper.getValueTracking();
4462
4463 const LLT S1 = LLT::scalar(SizeInBits: 1);
4464 const LLT I32 = LLT::integer(SizeInBits: 32);
4465 const LLT I64 = LLT::integer(SizeInBits: 64);
4466
4467 Register Zero32;
4468 Register Zero64;
4469
4470 auto getZero32 = [&]() -> Register {
4471 if (!Zero32)
4472 Zero32 = B.buildConstant(Res: I32, Val: 0).getReg(Idx: 0);
4473 return Zero32;
4474 };
4475 auto getZero64 = [&]() -> Register {
4476 if (!Zero64)
4477 Zero64 = B.buildConstant(Res: I64, Val: 0).getReg(Idx: 0);
4478 return Zero64;
4479 };
4480
4481 SmallVector<bool, 2> Src0KnownZeros, Src1KnownZeros;
4482 for (unsigned i = 0; i < Src0.size(); ++i) {
4483 Src0KnownZeros.push_back(Elt: VT.getKnownBits(R: Src0[i]).isZero());
4484 Src1KnownZeros.push_back(Elt: VT.getKnownBits(R: Src1[i]).isZero());
4485 }
4486
4487 // Merge the given carries into the 32-bit LocalAccum, which is modified
4488 // in-place.
4489 //
4490 // Returns the carry-out, which is a single S1 register or null.
4491 auto mergeCarry =
4492 [&](Register &LocalAccum, const Carry &CarryIn) -> Register {
4493 if (CarryIn.empty())
4494 return Register();
4495
4496 bool HaveCarryOut = true;
4497 Register CarryAccum;
4498 if (CarryIn.size() == 1) {
4499 if (!LocalAccum) {
4500 LocalAccum = B.buildZExt(Res: I32, Op: CarryIn[0]).getReg(Idx: 0);
4501 return Register();
4502 }
4503
4504 CarryAccum = getZero32();
4505 } else {
4506 CarryAccum = B.buildZExt(Res: I32, Op: CarryIn[0]).getReg(Idx: 0);
4507 for (unsigned i = 1; i + 1 < CarryIn.size(); ++i) {
4508 CarryAccum =
4509 B.buildUAdde(Res: I32, CarryOut: S1, Op0: CarryAccum, Op1: getZero32(), CarryIn: CarryIn[i])
4510 .getReg(Idx: 0);
4511 }
4512
4513 if (!LocalAccum) {
4514 LocalAccum = getZero32();
4515 HaveCarryOut = false;
4516 }
4517 }
4518
4519 auto Add =
4520 B.buildUAdde(Res: I32, CarryOut: S1, Op0: CarryAccum, Op1: LocalAccum, CarryIn: CarryIn.back());
4521 LocalAccum = Add.getReg(Idx: 0);
4522 return HaveCarryOut ? Add.getReg(Idx: 1) : Register();
4523 };
4524
4525 // Build a multiply-add chain to compute
4526 //
4527 // LocalAccum + (partial products at DstIndex)
4528 // + (opportunistic subset of CarryIn)
4529 //
4530 // LocalAccum is an array of one or two 32-bit registers that are updated
4531 // in-place. The incoming registers may be null.
4532 //
4533 // In some edge cases, carry-ins can be consumed "for free". In that case,
4534 // the consumed carry bits are removed from CarryIn in-place.
4535 auto buildMadChain =
4536 [&](MutableArrayRef<Register> LocalAccum, unsigned DstIndex, Carry &CarryIn)
4537 -> Carry {
4538 assert((DstIndex + 1 < Accum.size() && LocalAccum.size() == 2) ||
4539 (DstIndex + 1 >= Accum.size() && LocalAccum.size() == 1));
4540
4541 Carry CarryOut;
4542 unsigned j0 = 0;
4543
4544 // Use plain 32-bit multiplication for the most significant part of the
4545 // result by default.
4546 if (LocalAccum.size() == 1 &&
4547 (!UsePartialMad64_32 || !CarryIn.empty())) {
4548 do {
4549 // Skip multiplication if one of the operands is 0
4550 unsigned j1 = DstIndex - j0;
4551 if (Src0KnownZeros[j0] || Src1KnownZeros[j1]) {
4552 ++j0;
4553 continue;
4554 }
4555 auto Mul = B.buildMul(Dst: I32, Src0: Src0[j0], Src1: Src1[j1]);
4556 if (!LocalAccum[0] || VT.getKnownBits(R: LocalAccum[0]).isZero()) {
4557 LocalAccum[0] = Mul.getReg(Idx: 0);
4558 } else {
4559 if (CarryIn.empty()) {
4560 LocalAccum[0] = B.buildAdd(Dst: I32, Src0: LocalAccum[0], Src1: Mul).getReg(Idx: 0);
4561 } else {
4562 LocalAccum[0] =
4563 B.buildUAdde(Res: I32, CarryOut: S1, Op0: LocalAccum[0], Op1: Mul, CarryIn: CarryIn.back())
4564 .getReg(Idx: 0);
4565 CarryIn.pop_back();
4566 }
4567 }
4568 ++j0;
4569 } while (j0 <= DstIndex && (!UsePartialMad64_32 || !CarryIn.empty()));
4570 }
4571
4572 // Build full 64-bit multiplies.
4573 if (j0 <= DstIndex) {
4574 bool HaveSmallAccum = false;
4575 Register Tmp;
4576
4577 if (LocalAccum[0]) {
4578 if (LocalAccum.size() == 1) {
4579 Tmp = B.buildAnyExt(Res: I64, Op: LocalAccum[0]).getReg(Idx: 0);
4580 HaveSmallAccum = true;
4581 } else if (LocalAccum[1]) {
4582 Tmp = B.buildMergeLikeInstr(Res: I64, Ops: LocalAccum).getReg(Idx: 0);
4583 HaveSmallAccum = false;
4584 } else {
4585 Tmp = B.buildZExt(Res: I64, Op: LocalAccum[0]).getReg(Idx: 0);
4586 HaveSmallAccum = true;
4587 }
4588 } else {
4589 assert(LocalAccum.size() == 1 || !LocalAccum[1]);
4590 Tmp = getZero64();
4591 HaveSmallAccum = true;
4592 }
4593
4594 do {
4595 unsigned j1 = DstIndex - j0;
4596 if (Src0KnownZeros[j0] || Src1KnownZeros[j1]) {
4597 ++j0;
4598 continue;
4599 }
4600 auto Mad = B.buildInstr(Opc: AMDGPU::G_AMDGPU_MAD_U64_U32, DstOps: {I64, S1},
4601 SrcOps: {Src0[j0], Src1[j1], Tmp});
4602 Tmp = Mad.getReg(Idx: 0);
4603 if (!HaveSmallAccum)
4604 CarryOut.push_back(Elt: Mad.getReg(Idx: 1));
4605 HaveSmallAccum = false;
4606
4607 ++j0;
4608 } while (j0 <= DstIndex);
4609
4610 auto Unmerge = B.buildUnmerge(Res: I32, Op: Tmp);
4611 LocalAccum[0] = Unmerge.getReg(Idx: 0);
4612 if (LocalAccum.size() > 1)
4613 LocalAccum[1] = Unmerge.getReg(Idx: 1);
4614 }
4615
4616 return CarryOut;
4617 };
4618
4619 // Outer multiply loop, iterating over destination parts from least
4620 // significant to most significant parts.
4621 //
4622 // The columns of the following diagram correspond to the destination parts
4623 // affected by one iteration of the outer loop (ignoring boundary
4624 // conditions).
4625 //
4626 // Dest index relative to 2 * i: 1 0 -1
4627 // ------
4628 // Carries from previous iteration: e o
4629 // Even-aligned partial product sum: E E .
4630 // Odd-aligned partial product sum: O O
4631 //
4632 // 'o' is OddCarry, 'e' is EvenCarry.
4633 // EE and OO are computed from partial products via buildMadChain and use
4634 // accumulation where possible and appropriate.
4635 //
4636 Register SeparateOddCarry;
4637 Carry EvenCarry;
4638 Carry OddCarry;
4639
4640 for (unsigned i = 0; i <= Accum.size() / 2; ++i) {
4641 Carry OddCarryIn = std::move(OddCarry);
4642 Carry EvenCarryIn = std::move(EvenCarry);
4643 OddCarry.clear();
4644 EvenCarry.clear();
4645
4646 // Partial products at offset 2 * i.
4647 if (2 * i < Accum.size()) {
4648 auto LocalAccum = Accum.drop_front(N: 2 * i).take_front(N: 2);
4649 EvenCarry = buildMadChain(LocalAccum, 2 * i, EvenCarryIn);
4650 }
4651
4652 // Partial products at offset 2 * i - 1.
4653 if (i > 0) {
4654 if (!SeparateOddAlignedProducts) {
4655 auto LocalAccum = Accum.drop_front(N: 2 * i - 1).take_front(N: 2);
4656 OddCarry = buildMadChain(LocalAccum, 2 * i - 1, OddCarryIn);
4657 } else {
4658 bool IsHighest = 2 * i >= Accum.size();
4659 Register SeparateOddOut[2];
4660 auto LocalAccum = MutableArrayRef(SeparateOddOut)
4661 .take_front(N: IsHighest ? 1 : 2);
4662 OddCarry = buildMadChain(LocalAccum, 2 * i - 1, OddCarryIn);
4663
4664 MachineInstr *Lo;
4665
4666 if (i == 1) {
4667 if (!IsHighest)
4668 Lo = B.buildUAddo(Res: I32, CarryOut: S1, Op0: Accum[2 * i - 1], Op1: SeparateOddOut[0]);
4669 else
4670 Lo = B.buildAdd(Dst: I32, Src0: Accum[2 * i - 1], Src1: SeparateOddOut[0]);
4671 } else {
4672 Lo = B.buildUAdde(Res: I32, CarryOut: S1, Op0: Accum[2 * i - 1], Op1: SeparateOddOut[0],
4673 CarryIn: SeparateOddCarry);
4674 }
4675 Accum[2 * i - 1] = Lo->getOperand(i: 0).getReg();
4676
4677 if (!IsHighest) {
4678 auto Hi = B.buildUAdde(Res: I32, CarryOut: S1, Op0: Accum[2 * i], Op1: SeparateOddOut[1],
4679 CarryIn: Lo->getOperand(i: 1).getReg());
4680 Accum[2 * i] = Hi.getReg(Idx: 0);
4681 SeparateOddCarry = Hi.getReg(Idx: 1);
4682 }
4683 }
4684 }
4685
4686 // Add in the carries from the previous iteration
4687 if (i > 0) {
4688 if (Register CarryOut = mergeCarry(Accum[2 * i - 1], OddCarryIn))
4689 EvenCarryIn.push_back(Elt: CarryOut);
4690
4691 if (2 * i < Accum.size()) {
4692 if (Register CarryOut = mergeCarry(Accum[2 * i], EvenCarryIn))
4693 OddCarry.push_back(Elt: CarryOut);
4694 }
4695 }
4696 }
4697}
4698
4699// Custom narrowing of wide multiplies using wide multiply-add instructions.
4700//
4701// TODO: If the multiply is followed by an addition, we should attempt to
4702// integrate it to make better use of V_MAD_U64_U32's multiply-add capabilities.
4703bool AMDGPULegalizerInfo::legalizeMul(LegalizerHelper &Helper,
4704 MachineInstr &MI) const {
4705 assert(ST.hasMad64_32());
4706 assert(MI.getOpcode() == TargetOpcode::G_MUL);
4707
4708 MachineIRBuilder &B = Helper.MIRBuilder;
4709 MachineRegisterInfo &MRI = *B.getMRI();
4710
4711 Register DstReg = MI.getOperand(i: 0).getReg();
4712 Register Src0 = MI.getOperand(i: 1).getReg();
4713 Register Src1 = MI.getOperand(i: 2).getReg();
4714
4715 LLT Ty = MRI.getType(Reg: DstReg);
4716 assert(Ty.isScalar());
4717
4718 unsigned Size = Ty.getSizeInBits();
4719 if (ST.hasVMulU64Inst() && Size == 64)
4720 return true;
4721
4722 unsigned NumParts = Size / 32;
4723 assert((Size % 32) == 0);
4724 assert(NumParts >= 2);
4725
4726 // Whether to use MAD_64_32 for partial products whose high half is
4727 // discarded. This avoids some ADD instructions but risks false dependency
4728 // stalls on some subtargets in some cases.
4729 const bool UsePartialMad64_32 = ST.getGeneration() < AMDGPUSubtarget::GFX10;
4730
4731 // Whether to compute odd-aligned partial products separately. This is
4732 // advisable on subtargets where the accumulator of MAD_64_32 must be placed
4733 // in an even-aligned VGPR.
4734 const bool SeparateOddAlignedProducts = ST.hasFullRate64Ops();
4735
4736 LLT I32 = LLT::integer(SizeInBits: 32);
4737 SmallVector<Register, 2> Src0Parts, Src1Parts;
4738 for (unsigned i = 0; i < NumParts; ++i) {
4739 Src0Parts.push_back(Elt: MRI.createGenericVirtualRegister(Ty: I32));
4740 Src1Parts.push_back(Elt: MRI.createGenericVirtualRegister(Ty: I32));
4741 }
4742 B.buildUnmerge(Res: Src0Parts, Op: Src0);
4743 B.buildUnmerge(Res: Src1Parts, Op: Src1);
4744
4745 SmallVector<Register, 2> AccumRegs(NumParts);
4746 buildMultiply(Helper, Accum: AccumRegs, Src0: Src0Parts, Src1: Src1Parts, UsePartialMad64_32,
4747 SeparateOddAlignedProducts);
4748
4749 B.buildMergeLikeInstr(Res: DstReg, Ops: AccumRegs);
4750 MI.eraseFromParent();
4751 return true;
4752}
4753
4754// Legalize ctlz/cttz to ffbh/ffbl instead of the default legalization to
4755// ctlz/cttz_zero_poison. This allows us to fix up the result for the zero input
4756// case with a single min instruction instead of a compare+select.
4757bool AMDGPULegalizerInfo::legalizeCTLZ_CTTZ(MachineInstr &MI,
4758 MachineRegisterInfo &MRI,
4759 MachineIRBuilder &B) const {
4760 Register Dst = MI.getOperand(i: 0).getReg();
4761 Register Src = MI.getOperand(i: 1).getReg();
4762 LLT DstTy = MRI.getType(Reg: Dst);
4763 LLT SrcTy = MRI.getType(Reg: Src);
4764
4765 unsigned NewOpc = MI.getOpcode() == AMDGPU::G_CTLZ
4766 ? AMDGPU::G_AMDGPU_FFBH_U32
4767 : AMDGPU::G_AMDGPU_FFBL_B32;
4768 auto Tmp = B.buildInstr(Opc: NewOpc, DstOps: {DstTy}, SrcOps: {Src});
4769 B.buildUMin(Dst, Src0: Tmp, Src1: B.buildConstant(Res: DstTy, Val: SrcTy.getSizeInBits()));
4770
4771 MI.eraseFromParent();
4772 return true;
4773}
4774
4775bool AMDGPULegalizerInfo::legalizeCTLZ_ZERO_POISON(MachineInstr &MI,
4776 MachineRegisterInfo &MRI,
4777 MachineIRBuilder &B) const {
4778 Register Dst = MI.getOperand(i: 0).getReg();
4779 Register Src = MI.getOperand(i: 1).getReg();
4780 LLT SrcTy = MRI.getType(Reg: Src);
4781 TypeSize NumBits = SrcTy.getSizeInBits();
4782
4783 assert(NumBits < 32u);
4784
4785 const LLT I32 = LLT::integer(SizeInBits: 32);
4786 auto ShiftAmt = B.buildConstant(Res: I32, Val: 32u - NumBits);
4787 auto Extend = B.buildAnyExt(Res: I32, Op: {Src}).getReg(Idx: 0u);
4788 auto Shift = B.buildShl(Dst: I32, Src0: Extend, Src1: ShiftAmt);
4789 auto Ctlz = B.buildInstr(Opc: AMDGPU::G_AMDGPU_FFBH_U32, DstOps: {I32}, SrcOps: {Shift});
4790 B.buildTrunc(Res: Dst, Op: Ctlz);
4791 MI.eraseFromParent();
4792 return true;
4793}
4794
4795bool AMDGPULegalizerInfo::legalizeCTLS(MachineInstr &MI,
4796 MachineRegisterInfo &MRI,
4797 MachineIRBuilder &B) const {
4798 Register Dst = MI.getOperand(i: 0).getReg();
4799 Register Src = MI.getOperand(i: 1).getReg();
4800 LLT SrcTy = MRI.getType(Reg: Src);
4801 const LLT I32 = LLT::integer(SizeInBits: 32);
4802 assert(SrcTy == I32 && "legalizeCTLS only supports i32");
4803 unsigned BitWidth = SrcTy.getSizeInBits();
4804
4805 auto Sffbh = B.buildIntrinsic(ID: Intrinsic::amdgcn_sffbh, Res: {I32}).addUse(RegNo: Src);
4806 auto Clamped = B.buildUMin(Dst: I32, Src0: Sffbh, Src1: B.buildConstant(Res: I32, Val: BitWidth));
4807 B.buildSub(Dst, Src0: Clamped, Src1: B.buildConstant(Res: I32, Val: 1));
4808 MI.eraseFromParent();
4809 return true;
4810}
4811
4812// Check that this is a G_XOR x, -1
4813static bool isNot(const MachineRegisterInfo &MRI, const MachineInstr &MI) {
4814 if (MI.getOpcode() != TargetOpcode::G_XOR)
4815 return false;
4816 auto ConstVal = getIConstantVRegSExtVal(VReg: MI.getOperand(i: 2).getReg(), MRI);
4817 return ConstVal == -1;
4818}
4819
4820// Return the use branch instruction, otherwise null if the usage is invalid.
4821static MachineInstr *
4822verifyCFIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, MachineInstr *&Br,
4823 MachineBasicBlock *&UncondBrTarget, bool &Negated) {
4824 Register CondDef = MI.getOperand(i: 0).getReg();
4825 if (!MRI.hasOneNonDBGUse(RegNo: CondDef))
4826 return nullptr;
4827
4828 MachineBasicBlock *Parent = MI.getParent();
4829 MachineInstr *UseMI = &*MRI.use_instr_nodbg_begin(RegNo: CondDef);
4830
4831 if (isNot(MRI, MI: *UseMI)) {
4832 Register NegatedCond = UseMI->getOperand(i: 0).getReg();
4833 if (!MRI.hasOneNonDBGUse(RegNo: NegatedCond))
4834 return nullptr;
4835
4836 // We're deleting the def of this value, so we need to remove it.
4837 eraseInstr(MI&: *UseMI, MRI);
4838
4839 UseMI = &*MRI.use_instr_nodbg_begin(RegNo: NegatedCond);
4840 Negated = true;
4841 }
4842
4843 if (UseMI->getParent() != Parent || UseMI->getOpcode() != AMDGPU::G_BRCOND)
4844 return nullptr;
4845
4846 // Make sure the cond br is followed by a G_BR, or is the last instruction.
4847 MachineBasicBlock::iterator Next = std::next(x: UseMI->getIterator());
4848 if (Next == Parent->end()) {
4849 MachineFunction::iterator NextMBB = std::next(x: Parent->getIterator());
4850 if (NextMBB == Parent->getParent()->end()) // Illegal intrinsic use.
4851 return nullptr;
4852 UncondBrTarget = &*NextMBB;
4853 } else {
4854 if (Next->getOpcode() != AMDGPU::G_BR)
4855 return nullptr;
4856 Br = &*Next;
4857 UncondBrTarget = Br->getOperand(i: 0).getMBB();
4858 }
4859
4860 return UseMI;
4861}
4862
4863void AMDGPULegalizerInfo::buildLoadInputValue(Register DstReg,
4864 MachineIRBuilder &B,
4865 const ArgDescriptor *Arg,
4866 const TargetRegisterClass *ArgRC,
4867 LLT ArgTy) const {
4868 MCRegister SrcReg = Arg->getRegister();
4869 assert(SrcReg.isPhysical() && "Physical register expected");
4870 assert(DstReg.isVirtual() && "Virtual register expected");
4871
4872 Register LiveIn = getFunctionLiveInPhysReg(MF&: B.getMF(), TII: B.getTII(), PhysReg: SrcReg,
4873 RC: *ArgRC, DL: B.getDebugLoc(), RegTy: ArgTy);
4874 if (Arg->isMasked()) {
4875 // TODO: Should we try to emit this once in the entry block?
4876 const LLT I32 = LLT::integer(SizeInBits: 32);
4877 const unsigned Mask = Arg->getMask();
4878 const unsigned Shift = llvm::countr_zero<unsigned>(Val: Mask);
4879
4880 Register AndMaskSrc = LiveIn;
4881
4882 // TODO: Avoid clearing the high bits if we know workitem id y/z are always
4883 // 0.
4884 if (Shift != 0) {
4885 auto ShiftAmt = B.buildConstant(Res: I32, Val: Shift);
4886 AndMaskSrc = B.buildLShr(Dst: I32, Src0: LiveIn, Src1: ShiftAmt).getReg(Idx: 0);
4887 }
4888
4889 B.buildAnd(Dst: DstReg, Src0: AndMaskSrc, Src1: B.buildConstant(Res: I32, Val: Mask >> Shift));
4890 } else {
4891 B.buildCopy(Res: DstReg, Op: LiveIn);
4892 }
4893}
4894
4895bool AMDGPULegalizerInfo::legalizeWorkGroupId(
4896 MachineInstr &MI, MachineIRBuilder &B,
4897 AMDGPUFunctionArgInfo::PreloadedValue WorkGroupIdPV,
4898 AMDGPUFunctionArgInfo::PreloadedValue ClusterMaxIdPV,
4899 AMDGPUFunctionArgInfo::PreloadedValue ClusterWorkGroupIdPV) const {
4900 Register DstReg = MI.getOperand(i: 0).getReg();
4901 if (!ST.hasClusters()) {
4902 if (!loadInputValue(DstReg, B, ArgType: WorkGroupIdPV))
4903 return false;
4904 MI.eraseFromParent();
4905 return true;
4906 }
4907
4908 // Clusters are supported. Return the global position in the grid. If clusters
4909 // are enabled, WorkGroupIdPV returns the cluster ID not the workgroup ID.
4910
4911 // WorkGroupIdXYZ = ClusterId == 0 ?
4912 // ClusterIdXYZ :
4913 // ClusterIdXYZ * (ClusterMaxIdXYZ + 1) + ClusterWorkGroupIdXYZ
4914 MachineRegisterInfo &MRI = *B.getMRI();
4915 const LLT I32 = LLT::integer(SizeInBits: 32);
4916 Register ClusterIdXYZ = MRI.createGenericVirtualRegister(Ty: I32);
4917 Register ClusterMaxIdXYZ = MRI.createGenericVirtualRegister(Ty: I32);
4918 Register ClusterWorkGroupIdXYZ = MRI.createGenericVirtualRegister(Ty: I32);
4919 if (!loadInputValue(DstReg: ClusterIdXYZ, B, ArgType: WorkGroupIdPV) ||
4920 !loadInputValue(DstReg: ClusterWorkGroupIdXYZ, B, ArgType: ClusterWorkGroupIdPV) ||
4921 !loadInputValue(DstReg: ClusterMaxIdXYZ, B, ArgType: ClusterMaxIdPV))
4922 return false;
4923
4924 auto One = B.buildConstant(Res: I32, Val: 1);
4925 auto ClusterSizeXYZ = B.buildAdd(Dst: I32, Src0: ClusterMaxIdXYZ, Src1: One);
4926 auto GlobalIdXYZ = B.buildAdd(Dst: I32, Src0: ClusterWorkGroupIdXYZ,
4927 Src1: B.buildMul(Dst: I32, Src0: ClusterIdXYZ, Src1: ClusterSizeXYZ));
4928
4929 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
4930
4931 switch (MFI->getClusterDims().getKind()) {
4932 case AMDGPU::ClusterDimsAttr::Kind::FixedDims:
4933 case AMDGPU::ClusterDimsAttr::Kind::VariableDims: {
4934 B.buildCopy(Res: DstReg, Op: GlobalIdXYZ);
4935 MI.eraseFromParent();
4936 return true;
4937 }
4938 case AMDGPU::ClusterDimsAttr::Kind::NoCluster: {
4939 B.buildCopy(Res: DstReg, Op: ClusterIdXYZ);
4940 MI.eraseFromParent();
4941 return true;
4942 }
4943 case AMDGPU::ClusterDimsAttr::Kind::Unknown: {
4944 using namespace AMDGPU::Hwreg;
4945 unsigned ClusterIdField = HwregEncoding::encode(Values: ID_IB_STS2, Values: 6, Values: 4);
4946 Register ClusterId = MRI.createGenericVirtualRegister(Ty: I32);
4947 MRI.setRegClass(Reg: ClusterId, RC: &AMDGPU::SReg_32RegClass);
4948 B.buildInstr(Opcode: AMDGPU::S_GETREG_B32_const)
4949 .addDef(RegNo: ClusterId)
4950 .addImm(Val: ClusterIdField);
4951 auto Zero = B.buildConstant(Res: I32, Val: 0);
4952 auto NoClusters =
4953 B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: LLT::scalar(SizeInBits: 1), Op0: ClusterId, Op1: Zero);
4954 B.buildSelect(Res: DstReg, Tst: NoClusters, Op0: ClusterIdXYZ, Op1: GlobalIdXYZ);
4955 MI.eraseFromParent();
4956 return true;
4957 }
4958 }
4959
4960 llvm_unreachable("nothing should reach here");
4961}
4962
4963bool AMDGPULegalizerInfo::loadInputValue(
4964 Register DstReg, MachineIRBuilder &B,
4965 AMDGPUFunctionArgInfo::PreloadedValue ArgType) const {
4966 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
4967 const ArgDescriptor *Arg = nullptr;
4968 const TargetRegisterClass *ArgRC = nullptr;
4969 LLT ArgTy;
4970
4971 CallingConv::ID CC = B.getMF().getFunction().getCallingConv();
4972 const ArgDescriptor WorkGroupIDX =
4973 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP9);
4974 // If GridZ is not programmed in an entry function then the hardware will set
4975 // it to all zeros, so there is no need to mask the GridY value in the low
4976 // order bits.
4977 const ArgDescriptor WorkGroupIDY = ArgDescriptor::createRegister(
4978 Reg: AMDGPU::TTMP7,
4979 Mask: AMDGPU::isEntryFunctionCC(CC) && !MFI->hasWorkGroupIDZ() ? ~0u : 0xFFFFu);
4980 const ArgDescriptor WorkGroupIDZ =
4981 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP7, Mask: 0xFFFF0000u);
4982 const ArgDescriptor ClusterWorkGroupIDX =
4983 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x0000000Fu);
4984 const ArgDescriptor ClusterWorkGroupIDY =
4985 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x000000F0u);
4986 const ArgDescriptor ClusterWorkGroupIDZ =
4987 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x00000F00u);
4988 const ArgDescriptor ClusterWorkGroupMaxIDX =
4989 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x0000F000u);
4990 const ArgDescriptor ClusterWorkGroupMaxIDY =
4991 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x000F0000u);
4992 const ArgDescriptor ClusterWorkGroupMaxIDZ =
4993 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x00F00000u);
4994 const ArgDescriptor ClusterWorkGroupMaxFlatID =
4995 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x0F000000u);
4996
4997 auto LoadConstant = [&](unsigned N) {
4998 B.buildConstant(Res: DstReg, Val: N);
4999 return true;
5000 };
5001
5002 if (ST.hasArchitectedSGPRs() &&
5003 (AMDGPU::isCompute(CC) || CC == CallingConv::AMDGPU_Gfx)) {
5004 AMDGPU::ClusterDimsAttr ClusterDims = MFI->getClusterDims();
5005 bool HasFixedDims = ClusterDims.isFixedDims();
5006
5007 switch (ArgType) {
5008 case AMDGPUFunctionArgInfo::WORKGROUP_ID_X:
5009 Arg = &WorkGroupIDX;
5010 ArgRC = &AMDGPU::SReg_32RegClass;
5011 ArgTy = LLT::integer(SizeInBits: 32);
5012 break;
5013 case AMDGPUFunctionArgInfo::WORKGROUP_ID_Y:
5014 Arg = &WorkGroupIDY;
5015 ArgRC = &AMDGPU::SReg_32RegClass;
5016 ArgTy = LLT::integer(SizeInBits: 32);
5017 break;
5018 case AMDGPUFunctionArgInfo::WORKGROUP_ID_Z:
5019 Arg = &WorkGroupIDZ;
5020 ArgRC = &AMDGPU::SReg_32RegClass;
5021 ArgTy = LLT::integer(SizeInBits: 32);
5022 break;
5023 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X:
5024 if (HasFixedDims && ClusterDims.getDims()[0] == 1)
5025 return LoadConstant(0);
5026 Arg = &ClusterWorkGroupIDX;
5027 ArgRC = &AMDGPU::SReg_32RegClass;
5028 ArgTy = LLT::integer(SizeInBits: 32);
5029 break;
5030 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y:
5031 if (HasFixedDims && ClusterDims.getDims()[1] == 1)
5032 return LoadConstant(0);
5033 Arg = &ClusterWorkGroupIDY;
5034 ArgRC = &AMDGPU::SReg_32RegClass;
5035 ArgTy = LLT::integer(SizeInBits: 32);
5036 break;
5037 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z:
5038 if (HasFixedDims && ClusterDims.getDims()[2] == 1)
5039 return LoadConstant(0);
5040 Arg = &ClusterWorkGroupIDZ;
5041 ArgRC = &AMDGPU::SReg_32RegClass;
5042 ArgTy = LLT::integer(SizeInBits: 32);
5043 break;
5044 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X:
5045 if (HasFixedDims)
5046 return LoadConstant(ClusterDims.getDims()[0] - 1);
5047 Arg = &ClusterWorkGroupMaxIDX;
5048 ArgRC = &AMDGPU::SReg_32RegClass;
5049 ArgTy = LLT::integer(SizeInBits: 32);
5050 break;
5051 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y:
5052 if (HasFixedDims)
5053 return LoadConstant(ClusterDims.getDims()[1] - 1);
5054 Arg = &ClusterWorkGroupMaxIDY;
5055 ArgRC = &AMDGPU::SReg_32RegClass;
5056 ArgTy = LLT::integer(SizeInBits: 32);
5057 break;
5058 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z:
5059 if (HasFixedDims)
5060 return LoadConstant(ClusterDims.getDims()[2] - 1);
5061 Arg = &ClusterWorkGroupMaxIDZ;
5062 ArgRC = &AMDGPU::SReg_32RegClass;
5063 ArgTy = LLT::integer(SizeInBits: 32);
5064 break;
5065 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_FLAT_ID:
5066 Arg = &ClusterWorkGroupMaxFlatID;
5067 ArgRC = &AMDGPU::SReg_32RegClass;
5068 ArgTy = LLT::integer(SizeInBits: 32);
5069 break;
5070 default:
5071 break;
5072 }
5073 }
5074
5075 if (!Arg)
5076 std::tie(args&: Arg, args&: ArgRC, args&: ArgTy) = MFI->getPreloadedValue(Value: ArgType);
5077
5078 if (!Arg) {
5079 if (ArgType == AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR) {
5080 // The intrinsic may appear when we have a 0 sized kernarg segment, in
5081 // which case the pointer argument may be missing and we use null.
5082 return LoadConstant(0);
5083 }
5084
5085 // It's undefined behavior if a function marked with the amdgpu-no-*
5086 // attributes uses the corresponding intrinsic.
5087 B.buildUndef(Res: DstReg);
5088 return true;
5089 }
5090
5091 if (!Arg->isRegister() || !Arg->getRegister().isValid())
5092 return false; // TODO: Handle these
5093 buildLoadInputValue(DstReg, B, Arg, ArgRC, ArgTy);
5094 return true;
5095}
5096
5097bool AMDGPULegalizerInfo::legalizePreloadedArgIntrin(
5098 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B,
5099 AMDGPUFunctionArgInfo::PreloadedValue ArgType) const {
5100 if (!loadInputValue(DstReg: MI.getOperand(i: 0).getReg(), B, ArgType))
5101 return false;
5102
5103 MI.eraseFromParent();
5104 return true;
5105}
5106
5107static bool replaceWithConstant(MachineIRBuilder &B, MachineInstr &MI,
5108 int64_t C) {
5109 B.buildConstant(Res: MI.getOperand(i: 0).getReg(), Val: C);
5110 MI.eraseFromParent();
5111 return true;
5112}
5113
5114bool AMDGPULegalizerInfo::legalizeWorkitemIDIntrinsic(
5115 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B,
5116 unsigned Dim, AMDGPUFunctionArgInfo::PreloadedValue ArgType) const {
5117 unsigned MaxID = ST.getMaxWorkitemID(Kernel: B.getMF().getFunction(), Dimension: Dim);
5118 if (MaxID == 0)
5119 return replaceWithConstant(B, MI, C: 0);
5120
5121 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
5122 const ArgDescriptor *Arg;
5123 const TargetRegisterClass *ArgRC;
5124 LLT ArgTy;
5125 std::tie(args&: Arg, args&: ArgRC, args&: ArgTy) = MFI->getPreloadedValue(Value: ArgType);
5126
5127 Register DstReg = MI.getOperand(i: 0).getReg();
5128 if (!Arg) {
5129 // It's undefined behavior if a function marked with the amdgpu-no-*
5130 // attributes uses the corresponding intrinsic.
5131 B.buildUndef(Res: DstReg);
5132 MI.eraseFromParent();
5133 return true;
5134 }
5135
5136 if (Arg->isMasked()) {
5137 // Don't bother inserting AssertZext for packed IDs since we're emitting the
5138 // masking operations anyway.
5139 //
5140 // TODO: We could assert the top bit is 0 for the source copy.
5141 if (!loadInputValue(DstReg, B, ArgType))
5142 return false;
5143 } else {
5144 Register TmpReg = MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: 32));
5145 if (!loadInputValue(DstReg: TmpReg, B, ArgType))
5146 return false;
5147 B.buildAssertZExt(Res: DstReg, Op: TmpReg, Size: llvm::bit_width(Value: MaxID));
5148 }
5149
5150 MI.eraseFromParent();
5151 return true;
5152}
5153
5154MachinePointerInfo
5155AMDGPULegalizerInfo::getKernargSegmentPtrInfo(MachineFunction &MF) const {
5156 // This isn't really a constant pool but close enough.
5157 MachinePointerInfo PtrInfo(MF.getPSVManager().getConstantPool());
5158 PtrInfo.AddrSpace = AMDGPUAS::CONSTANT_ADDRESS;
5159 return PtrInfo;
5160}
5161
5162Register AMDGPULegalizerInfo::getKernargParameterPtr(MachineIRBuilder &B,
5163 int64_t Offset) const {
5164 LLT PtrTy = LLT::pointer(AddressSpace: AMDGPUAS::CONSTANT_ADDRESS, SizeInBits: 64);
5165 Register KernArgReg = B.getMRI()->createGenericVirtualRegister(Ty: PtrTy);
5166
5167 // TODO: If we passed in the base kernel offset we could have a better
5168 // alignment than 4, but we don't really need it.
5169 if (!loadInputValue(DstReg: KernArgReg, B,
5170 ArgType: AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR))
5171 llvm_unreachable("failed to find kernarg segment ptr");
5172
5173 auto COffset = B.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: Offset);
5174 return B.buildObjectPtrOffset(Res: PtrTy, Op0: KernArgReg, Op1: COffset).getReg(Idx: 0);
5175}
5176
5177/// Legalize a value that's loaded from kernel arguments. This is only used by
5178/// legacy intrinsics.
5179bool AMDGPULegalizerInfo::legalizeKernargMemParameter(MachineInstr &MI,
5180 MachineIRBuilder &B,
5181 uint64_t Offset,
5182 Align Alignment) const {
5183 Register DstReg = MI.getOperand(i: 0).getReg();
5184
5185 assert(B.getMRI()->getType(DstReg) == LLT::integer(32) &&
5186 "unexpected kernarg parameter type");
5187
5188 Register Ptr = getKernargParameterPtr(B, Offset);
5189 MachinePointerInfo PtrInfo = getKernargSegmentPtrInfo(MF&: B.getMF());
5190 B.buildLoad(Res: DstReg, Addr: Ptr, PtrInfo: PtrInfo.getWithOffset(O: Offset), Alignment: Align(4),
5191 MMOFlags: MachineMemOperand::MODereferenceable |
5192 MachineMemOperand::MOInvariant);
5193 MI.eraseFromParent();
5194 return true;
5195}
5196
5197bool AMDGPULegalizerInfo::legalizeFDIV(MachineInstr &MI,
5198 MachineRegisterInfo &MRI,
5199 MachineIRBuilder &B) const {
5200 Register Dst = MI.getOperand(i: 0).getReg();
5201 LLT DstTy = MRI.getType(Reg: Dst);
5202 LLT F16 = LLT::float16();
5203 LLT F32 = LLT::float32();
5204 LLT F64 = LLT::float64();
5205
5206 if (DstTy == F16)
5207 return legalizeFDIV16(MI, MRI, B);
5208 if (DstTy == F32)
5209 return legalizeFDIV32(MI, MRI, B);
5210 if (DstTy == F64)
5211 return legalizeFDIV64(MI, MRI, B);
5212
5213 return false;
5214}
5215
5216void AMDGPULegalizerInfo::legalizeUnsignedDIV_REM32Impl(MachineIRBuilder &B,
5217 Register DstDivReg,
5218 Register DstRemReg,
5219 Register X,
5220 Register Y) const {
5221 const LLT S1 = LLT::scalar(SizeInBits: 1);
5222 const LLT I32 = LLT::integer(SizeInBits: 32);
5223 const LLT F32 = LLT::float32();
5224
5225 // See AMDGPUCodeGenPrepare::expandDivRem32 for a description of the
5226 // algorithm used here.
5227
5228 // Initial estimate of inv(y).
5229 auto FloatY = B.buildUITOFP(Dst: F32, Src0: Y);
5230 auto RcpIFlag = B.buildInstr(Opc: AMDGPU::G_AMDGPU_RCP_IFLAG, DstOps: {F32}, SrcOps: {FloatY});
5231 auto Scale = B.buildFConstant(Res: F32, Val: llvm::bit_cast<float>(from: 0x4f7ffffe));
5232 auto ScaledY = B.buildFMul(Dst: F32, Src0: RcpIFlag, Src1: Scale);
5233 auto Z = B.buildFPTOUI(Dst: I32, Src0: ScaledY);
5234
5235 // One round of UNR.
5236 auto NegY = B.buildSub(Dst: I32, Src0: B.buildConstant(Res: I32, Val: 0), Src1: Y);
5237 auto NegYZ = B.buildMul(Dst: I32, Src0: NegY, Src1: Z);
5238 Z = B.buildAdd(Dst: I32, Src0: Z, Src1: B.buildUMulH(Dst: I32, Src0: Z, Src1: NegYZ));
5239
5240 // Quotient/remainder estimate.
5241 auto Q = B.buildUMulH(Dst: I32, Src0: X, Src1: Z);
5242 auto R = B.buildSub(Dst: I32, Src0: X, Src1: B.buildMul(Dst: I32, Src0: Q, Src1: Y));
5243
5244 // First quotient/remainder refinement.
5245 auto One = B.buildConstant(Res: I32, Val: 1);
5246 auto Cond = B.buildICmp(Pred: CmpInst::ICMP_UGE, Res: S1, Op0: R, Op1: Y);
5247 if (DstDivReg)
5248 Q = B.buildSelect(Res: I32, Tst: Cond, Op0: B.buildAdd(Dst: I32, Src0: Q, Src1: One), Op1: Q);
5249 R = B.buildSelect(Res: I32, Tst: Cond, Op0: B.buildSub(Dst: I32, Src0: R, Src1: Y), Op1: R);
5250
5251 // Second quotient/remainder refinement.
5252 Cond = B.buildICmp(Pred: CmpInst::ICMP_UGE, Res: S1, Op0: R, Op1: Y);
5253 if (DstDivReg)
5254 B.buildSelect(Res: DstDivReg, Tst: Cond, Op0: B.buildAdd(Dst: I32, Src0: Q, Src1: One), Op1: Q);
5255
5256 if (DstRemReg)
5257 B.buildSelect(Res: DstRemReg, Tst: Cond, Op0: B.buildSub(Dst: I32, Src0: R, Src1: Y), Op1: R);
5258}
5259
5260// Build integer reciprocal sequence around V_RCP_IFLAG_F32
5261//
5262// Return lo, hi of result
5263//
5264// %cvt.lo = G_UITOFP Val.lo
5265// %cvt.hi = G_UITOFP Val.hi
5266// %mad = G_FMAD %cvt.hi, 2**32, %cvt.lo
5267// %rcp = G_AMDGPU_RCP_IFLAG %mad
5268// %mul1 = G_FMUL %rcp, 0x5f7ffffc
5269// %mul2 = G_FMUL %mul1, 2**(-32)
5270// %trunc = G_INTRINSIC_TRUNC %mul2
5271// %mad2 = G_FMAD %trunc, -(2**32), %mul1
5272// return {G_FPTOUI %mad2, G_FPTOUI %trunc}
5273static std::pair<Register, Register> emitReciprocalU64(MachineIRBuilder &B,
5274 Register Val) {
5275 const LLT I32 = LLT::integer(SizeInBits: 32);
5276 const LLT F32 = LLT::float32();
5277 auto Unmerge = B.buildUnmerge(Res: I32, Op: Val);
5278
5279 auto CvtLo = B.buildUITOFP(Dst: F32, Src0: Unmerge.getReg(Idx: 0));
5280 auto CvtHi = B.buildUITOFP(Dst: F32, Src0: Unmerge.getReg(Idx: 1));
5281
5282 auto Mad = B.buildFMAD(
5283 Dst: F32, Src0: CvtHi, // 2**32
5284 Src1: B.buildFConstant(Res: F32, Val: llvm::bit_cast<float>(from: 0x4f800000)), Src2: CvtLo);
5285
5286 auto Rcp = B.buildInstr(Opc: AMDGPU::G_AMDGPU_RCP_IFLAG, DstOps: {F32}, SrcOps: {Mad});
5287 auto Mul1 = B.buildFMul(
5288 Dst: F32, Src0: Rcp, Src1: B.buildFConstant(Res: F32, Val: llvm::bit_cast<float>(from: 0x5f7ffffc)));
5289
5290 // 2**(-32)
5291 auto Mul2 = B.buildFMul(
5292 Dst: F32, Src0: Mul1, Src1: B.buildFConstant(Res: F32, Val: llvm::bit_cast<float>(from: 0x2f800000)));
5293 auto Trunc = B.buildIntrinsicTrunc(Dst: F32, Src0: Mul2);
5294
5295 // -(2**32)
5296 auto Mad2 = B.buildFMAD(
5297 Dst: F32, Src0: Trunc, Src1: B.buildFConstant(Res: F32, Val: llvm::bit_cast<float>(from: 0xcf800000)),
5298 Src2: Mul1);
5299
5300 auto ResultLo = B.buildFPTOUI(Dst: I32, Src0: Mad2);
5301 auto ResultHi = B.buildFPTOUI(Dst: I32, Src0: Trunc);
5302
5303 return {ResultLo.getReg(Idx: 0), ResultHi.getReg(Idx: 0)};
5304}
5305
5306void AMDGPULegalizerInfo::legalizeUnsignedDIV_REM64Impl(MachineIRBuilder &B,
5307 Register DstDivReg,
5308 Register DstRemReg,
5309 Register Numer,
5310 Register Denom) const {
5311 const LLT I32 = LLT::integer(SizeInBits: 32);
5312 const LLT I64 = LLT::integer(SizeInBits: 64);
5313 const LLT S1 = LLT::scalar(SizeInBits: 1);
5314 Register RcpLo, RcpHi;
5315
5316 std::tie(args&: RcpLo, args&: RcpHi) = emitReciprocalU64(B, Val: Denom);
5317
5318 auto Rcp = B.buildMergeLikeInstr(Res: I64, Ops: {RcpLo, RcpHi});
5319
5320 auto Zero64 = B.buildConstant(Res: I64, Val: 0);
5321 auto NegDenom = B.buildSub(Dst: I64, Src0: Zero64, Src1: Denom);
5322
5323 auto MulLo1 = B.buildMul(Dst: I64, Src0: NegDenom, Src1: Rcp);
5324 auto MulHi1 = B.buildUMulH(Dst: I64, Src0: Rcp, Src1: MulLo1);
5325
5326 auto UnmergeMulHi1 = B.buildUnmerge(Res: I32, Op: MulHi1);
5327 Register MulHi1_Lo = UnmergeMulHi1.getReg(Idx: 0);
5328 Register MulHi1_Hi = UnmergeMulHi1.getReg(Idx: 1);
5329
5330 auto Add1_Lo = B.buildUAddo(Res: I32, CarryOut: S1, Op0: RcpLo, Op1: MulHi1_Lo);
5331 auto Add1_Hi = B.buildUAdde(Res: I32, CarryOut: S1, Op0: RcpHi, Op1: MulHi1_Hi, CarryIn: Add1_Lo.getReg(Idx: 1));
5332 auto Add1 = B.buildMergeLikeInstr(Res: I64, Ops: {Add1_Lo, Add1_Hi});
5333
5334 auto MulLo2 = B.buildMul(Dst: I64, Src0: NegDenom, Src1: Add1);
5335 auto MulHi2 = B.buildUMulH(Dst: I64, Src0: Add1, Src1: MulLo2);
5336 auto UnmergeMulHi2 = B.buildUnmerge(Res: I32, Op: MulHi2);
5337 Register MulHi2_Lo = UnmergeMulHi2.getReg(Idx: 0);
5338 Register MulHi2_Hi = UnmergeMulHi2.getReg(Idx: 1);
5339
5340 auto Zero32 = B.buildConstant(Res: I32, Val: 0);
5341 auto Add2_Lo = B.buildUAddo(Res: I32, CarryOut: S1, Op0: Add1_Lo, Op1: MulHi2_Lo);
5342 auto Add2_Hi = B.buildUAdde(Res: I32, CarryOut: S1, Op0: Add1_Hi, Op1: MulHi2_Hi, CarryIn: Add2_Lo.getReg(Idx: 1));
5343 auto Add2 = B.buildMergeLikeInstr(Res: I64, Ops: {Add2_Lo, Add2_Hi});
5344
5345 auto UnmergeNumer = B.buildUnmerge(Res: I32, Op: Numer);
5346 Register NumerLo = UnmergeNumer.getReg(Idx: 0);
5347 Register NumerHi = UnmergeNumer.getReg(Idx: 1);
5348
5349 auto MulHi3 = B.buildUMulH(Dst: I64, Src0: Numer, Src1: Add2);
5350 auto Mul3 = B.buildMul(Dst: I64, Src0: Denom, Src1: MulHi3);
5351 auto UnmergeMul3 = B.buildUnmerge(Res: I32, Op: Mul3);
5352 Register Mul3_Lo = UnmergeMul3.getReg(Idx: 0);
5353 Register Mul3_Hi = UnmergeMul3.getReg(Idx: 1);
5354 auto Sub1_Lo = B.buildUSubo(Res: I32, CarryOut: S1, Op0: NumerLo, Op1: Mul3_Lo);
5355 auto Sub1_Hi = B.buildUSube(Res: I32, CarryOut: S1, Op0: NumerHi, Op1: Mul3_Hi, CarryIn: Sub1_Lo.getReg(Idx: 1));
5356 auto Sub1_Mi = B.buildSub(Dst: I32, Src0: NumerHi, Src1: Mul3_Hi);
5357 auto Sub1 = B.buildMergeLikeInstr(Res: I64, Ops: {Sub1_Lo, Sub1_Hi});
5358
5359 auto UnmergeDenom = B.buildUnmerge(Res: I32, Op: Denom);
5360 Register DenomLo = UnmergeDenom.getReg(Idx: 0);
5361 Register DenomHi = UnmergeDenom.getReg(Idx: 1);
5362
5363 auto CmpHi = B.buildICmp(Pred: CmpInst::ICMP_UGE, Res: S1, Op0: Sub1_Hi, Op1: DenomHi);
5364 auto C1 = B.buildSExt(Res: I32, Op: CmpHi);
5365
5366 auto CmpLo = B.buildICmp(Pred: CmpInst::ICMP_UGE, Res: S1, Op0: Sub1_Lo, Op1: DenomLo);
5367 auto C2 = B.buildSExt(Res: I32, Op: CmpLo);
5368
5369 auto CmpEq = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: S1, Op0: Sub1_Hi, Op1: DenomHi);
5370 auto C3 = B.buildSelect(Res: I32, Tst: CmpEq, Op0: C2, Op1: C1);
5371
5372 // TODO: Here and below portions of the code can be enclosed into if/endif.
5373 // Currently control flow is unconditional and we have 4 selects after
5374 // potential endif to substitute PHIs.
5375
5376 // if C3 != 0 ...
5377 auto Sub2_Lo = B.buildUSubo(Res: I32, CarryOut: S1, Op0: Sub1_Lo, Op1: DenomLo);
5378 auto Sub2_Mi = B.buildUSube(Res: I32, CarryOut: S1, Op0: Sub1_Mi, Op1: DenomHi, CarryIn: Sub1_Lo.getReg(Idx: 1));
5379 auto Sub2_Hi = B.buildUSube(Res: I32, CarryOut: S1, Op0: Sub2_Mi, Op1: Zero32, CarryIn: Sub2_Lo.getReg(Idx: 1));
5380 auto Sub2 = B.buildMergeLikeInstr(Res: I64, Ops: {Sub2_Lo, Sub2_Hi});
5381
5382 auto One64 = B.buildConstant(Res: I64, Val: 1);
5383 auto Add3 = B.buildAdd(Dst: I64, Src0: MulHi3, Src1: One64);
5384
5385 auto C4 =
5386 B.buildSExt(Res: I32, Op: B.buildICmp(Pred: CmpInst::ICMP_UGE, Res: S1, Op0: Sub2_Hi, Op1: DenomHi));
5387 auto C5 =
5388 B.buildSExt(Res: I32, Op: B.buildICmp(Pred: CmpInst::ICMP_UGE, Res: S1, Op0: Sub2_Lo, Op1: DenomLo));
5389 auto C6 = B.buildSelect(
5390 Res: I32, Tst: B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: S1, Op0: Sub2_Hi, Op1: DenomHi), Op0: C5, Op1: C4);
5391
5392 // if (C6 != 0)
5393 auto Add4 = B.buildAdd(Dst: I64, Src0: Add3, Src1: One64);
5394 auto Sub3_Lo = B.buildUSubo(Res: I32, CarryOut: S1, Op0: Sub2_Lo, Op1: DenomLo);
5395
5396 auto Sub3_Mi = B.buildUSube(Res: I32, CarryOut: S1, Op0: Sub2_Mi, Op1: DenomHi, CarryIn: Sub2_Lo.getReg(Idx: 1));
5397 auto Sub3_Hi = B.buildUSube(Res: I32, CarryOut: S1, Op0: Sub3_Mi, Op1: Zero32, CarryIn: Sub3_Lo.getReg(Idx: 1));
5398 auto Sub3 = B.buildMergeLikeInstr(Res: I64, Ops: {Sub3_Lo, Sub3_Hi});
5399
5400 // endif C6
5401 // endif C3
5402
5403 if (DstDivReg) {
5404 auto Sel1 = B.buildSelect(
5405 Res: I64, Tst: B.buildICmp(Pred: CmpInst::ICMP_NE, Res: S1, Op0: C6, Op1: Zero32), Op0: Add4, Op1: Add3);
5406 B.buildSelect(Res: DstDivReg, Tst: B.buildICmp(Pred: CmpInst::ICMP_NE, Res: S1, Op0: C3, Op1: Zero32),
5407 Op0: Sel1, Op1: MulHi3);
5408 }
5409
5410 if (DstRemReg) {
5411 auto Sel2 = B.buildSelect(
5412 Res: I64, Tst: B.buildICmp(Pred: CmpInst::ICMP_NE, Res: S1, Op0: C6, Op1: Zero32), Op0: Sub3, Op1: Sub2);
5413 B.buildSelect(Res: DstRemReg, Tst: B.buildICmp(Pred: CmpInst::ICMP_NE, Res: S1, Op0: C3, Op1: Zero32),
5414 Op0: Sel2, Op1: Sub1);
5415 }
5416}
5417
5418bool AMDGPULegalizerInfo::legalizeUnsignedDIV_REM(MachineInstr &MI,
5419 MachineRegisterInfo &MRI,
5420 MachineIRBuilder &B) const {
5421 Register DstDivReg, DstRemReg;
5422 switch (MI.getOpcode()) {
5423 default:
5424 llvm_unreachable("Unexpected opcode!");
5425 case AMDGPU::G_UDIV: {
5426 DstDivReg = MI.getOperand(i: 0).getReg();
5427 break;
5428 }
5429 case AMDGPU::G_UREM: {
5430 DstRemReg = MI.getOperand(i: 0).getReg();
5431 break;
5432 }
5433 case AMDGPU::G_UDIVREM: {
5434 DstDivReg = MI.getOperand(i: 0).getReg();
5435 DstRemReg = MI.getOperand(i: 1).getReg();
5436 break;
5437 }
5438 }
5439
5440 const LLT I64 = LLT::integer(SizeInBits: 64);
5441 const LLT I32 = LLT::integer(SizeInBits: 32);
5442 const unsigned FirstSrcOpIdx = MI.getNumExplicitDefs();
5443 Register Num = MI.getOperand(i: FirstSrcOpIdx).getReg();
5444 Register Den = MI.getOperand(i: FirstSrcOpIdx + 1).getReg();
5445 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5446
5447 if (Ty == I32)
5448 legalizeUnsignedDIV_REM32Impl(B, DstDivReg, DstRemReg, X: Num, Y: Den);
5449 else if (Ty == I64)
5450 legalizeUnsignedDIV_REM64Impl(B, DstDivReg, DstRemReg, Numer: Num, Denom: Den);
5451 else
5452 return false;
5453
5454 MI.eraseFromParent();
5455 return true;
5456}
5457
5458bool AMDGPULegalizerInfo::legalizeSignedDIV_REM(MachineInstr &MI,
5459 MachineRegisterInfo &MRI,
5460 MachineIRBuilder &B) const {
5461 const LLT I64 = LLT::integer(SizeInBits: 64);
5462 const LLT I32 = LLT::integer(SizeInBits: 32);
5463
5464 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
5465 if (Ty != I32 && Ty != I64)
5466 return false;
5467
5468 const unsigned FirstSrcOpIdx = MI.getNumExplicitDefs();
5469 Register LHS = MI.getOperand(i: FirstSrcOpIdx).getReg();
5470 Register RHS = MI.getOperand(i: FirstSrcOpIdx + 1).getReg();
5471
5472 auto SignBitOffset = B.buildConstant(Res: I32, Val: Ty.getSizeInBits() - 1);
5473 auto LHSign = B.buildAShr(Dst: Ty, Src0: LHS, Src1: SignBitOffset);
5474 auto RHSign = B.buildAShr(Dst: Ty, Src0: RHS, Src1: SignBitOffset);
5475
5476 LHS = B.buildAdd(Dst: Ty, Src0: LHS, Src1: LHSign).getReg(Idx: 0);
5477 RHS = B.buildAdd(Dst: Ty, Src0: RHS, Src1: RHSign).getReg(Idx: 0);
5478
5479 LHS = B.buildXor(Dst: Ty, Src0: LHS, Src1: LHSign).getReg(Idx: 0);
5480 RHS = B.buildXor(Dst: Ty, Src0: RHS, Src1: RHSign).getReg(Idx: 0);
5481
5482 Register DstDivReg, DstRemReg, TmpDivReg, TmpRemReg;
5483 switch (MI.getOpcode()) {
5484 default:
5485 llvm_unreachable("Unexpected opcode!");
5486 case AMDGPU::G_SDIV: {
5487 DstDivReg = MI.getOperand(i: 0).getReg();
5488 TmpDivReg = MRI.createGenericVirtualRegister(Ty);
5489 break;
5490 }
5491 case AMDGPU::G_SREM: {
5492 DstRemReg = MI.getOperand(i: 0).getReg();
5493 TmpRemReg = MRI.createGenericVirtualRegister(Ty);
5494 break;
5495 }
5496 case AMDGPU::G_SDIVREM: {
5497 DstDivReg = MI.getOperand(i: 0).getReg();
5498 DstRemReg = MI.getOperand(i: 1).getReg();
5499 TmpDivReg = MRI.createGenericVirtualRegister(Ty);
5500 TmpRemReg = MRI.createGenericVirtualRegister(Ty);
5501 break;
5502 }
5503 }
5504
5505 if (Ty == I32)
5506 legalizeUnsignedDIV_REM32Impl(B, DstDivReg: TmpDivReg, DstRemReg: TmpRemReg, X: LHS, Y: RHS);
5507 else
5508 legalizeUnsignedDIV_REM64Impl(B, DstDivReg: TmpDivReg, DstRemReg: TmpRemReg, Numer: LHS, Denom: RHS);
5509
5510 if (DstDivReg) {
5511 auto Sign = B.buildXor(Dst: Ty, Src0: LHSign, Src1: RHSign).getReg(Idx: 0);
5512 auto SignXor = B.buildXor(Dst: Ty, Src0: TmpDivReg, Src1: Sign).getReg(Idx: 0);
5513 B.buildSub(Dst: DstDivReg, Src0: SignXor, Src1: Sign);
5514 }
5515
5516 if (DstRemReg) {
5517 auto Sign = LHSign.getReg(Idx: 0); // Remainder sign is the same as LHS
5518 auto SignXor = B.buildXor(Dst: Ty, Src0: TmpRemReg, Src1: Sign).getReg(Idx: 0);
5519 B.buildSub(Dst: DstRemReg, Src0: SignXor, Src1: Sign);
5520 }
5521
5522 MI.eraseFromParent();
5523 return true;
5524}
5525
5526bool AMDGPULegalizerInfo::legalizeFastUnsafeFDIV(MachineInstr &MI,
5527 MachineRegisterInfo &MRI,
5528 MachineIRBuilder &B) const {
5529 Register Res = MI.getOperand(i: 0).getReg();
5530 Register LHS = MI.getOperand(i: 1).getReg();
5531 Register RHS = MI.getOperand(i: 2).getReg();
5532 uint16_t Flags = MI.getFlags();
5533 LLT ResTy = MRI.getType(Reg: Res);
5534
5535 bool AllowInaccurateRcp = MI.getFlag(Flag: MachineInstr::FmAfn);
5536
5537 if (const auto *CLHS = getConstantFPVRegVal(VReg: LHS, MRI)) {
5538 if (!AllowInaccurateRcp && ResTy != LLT::float16())
5539 return false;
5540
5541 // v_rcp_f32 and v_rsq_f32 do not support denormals, and according to
5542 // the CI documentation has a worst case error of 1 ulp.
5543 // OpenCL requires <= 2.5 ulp for 1.0 / x, so it should always be OK to
5544 // use it as long as we aren't trying to use denormals.
5545 //
5546 // v_rcp_f16 and v_rsq_f16 DO support denormals and 0.51ulp.
5547
5548 // 1 / x -> RCP(x)
5549 if (CLHS->isOne()) {
5550 B.buildIntrinsic(ID: Intrinsic::amdgcn_rcp, Res)
5551 .addUse(RegNo: RHS)
5552 .setMIFlags(Flags);
5553
5554 MI.eraseFromParent();
5555 return true;
5556 }
5557
5558 // -1 / x -> RCP( FNEG(x) )
5559 if (CLHS->isMinusOne()) {
5560 auto FNeg = B.buildFNeg(Dst: ResTy, Src0: RHS, Flags);
5561 B.buildIntrinsic(ID: Intrinsic::amdgcn_rcp, Res)
5562 .addUse(RegNo: FNeg.getReg(Idx: 0))
5563 .setMIFlags(Flags);
5564
5565 MI.eraseFromParent();
5566 return true;
5567 }
5568 }
5569
5570 // For f16 require afn or arcp.
5571 // For f32 require afn.
5572 if (!AllowInaccurateRcp &&
5573 (ResTy != LLT::float16() || !MI.getFlag(Flag: MachineInstr::FmArcp)))
5574 return false;
5575
5576 // x / y -> x * (1.0 / y)
5577 auto RCP = B.buildIntrinsic(ID: Intrinsic::amdgcn_rcp, Res: {ResTy})
5578 .addUse(RegNo: RHS)
5579 .setMIFlags(Flags);
5580 B.buildFMul(Dst: Res, Src0: LHS, Src1: RCP, Flags);
5581
5582 MI.eraseFromParent();
5583 return true;
5584}
5585
5586bool AMDGPULegalizerInfo::legalizeFastUnsafeFDIV64(MachineInstr &MI,
5587 MachineRegisterInfo &MRI,
5588 MachineIRBuilder &B) const {
5589 Register Res = MI.getOperand(i: 0).getReg();
5590 Register X = MI.getOperand(i: 1).getReg();
5591 Register Y = MI.getOperand(i: 2).getReg();
5592 uint16_t Flags = MI.getFlags();
5593 LLT ResTy = MRI.getType(Reg: Res);
5594
5595 bool AllowInaccurateRcp = MI.getFlag(Flag: MachineInstr::FmAfn);
5596
5597 if (!AllowInaccurateRcp)
5598 return false;
5599
5600 const ConstantFP *CLHS = getConstantFPVRegVal(VReg: X, MRI);
5601 bool IsNegRcp = CLHS && CLHS->isMinusOne();
5602
5603 // Pull out the negation so it folds for free into the source modifiers.
5604 if (IsNegRcp)
5605 X = B.buildFConstant(Res: ResTy, Val: 1.0).getReg(Idx: 0);
5606
5607 Register NegY = IsNegRcp ? Y : B.buildFNeg(Dst: ResTy, Src0: Y).getReg(Idx: 0);
5608 auto One = B.buildFConstant(Res: ResTy, Val: 1.0);
5609
5610 auto R = B.buildIntrinsic(ID: Intrinsic::amdgcn_rcp, Res: {ResTy})
5611 .addUse(RegNo: Y)
5612 .setMIFlags(Flags);
5613 if (IsNegRcp)
5614 R = B.buildFNeg(Dst: ResTy, Src0: R);
5615
5616 auto Tmp0 = B.buildFMA(Dst: ResTy, Src0: NegY, Src1: R, Src2: One);
5617 R = B.buildFMA(Dst: ResTy, Src0: Tmp0, Src1: R, Src2: R);
5618
5619 auto Tmp1 = B.buildFMA(Dst: ResTy, Src0: NegY, Src1: R, Src2: One);
5620 R = B.buildFMA(Dst: ResTy, Src0: Tmp1, Src1: R, Src2: R);
5621
5622 // Skip the last 2 correction terms for reciprocal.
5623 if (IsNegRcp || (CLHS && CLHS->isOne())) {
5624 B.buildCopy(Res, Op: R);
5625 MI.eraseFromParent();
5626 return true;
5627 }
5628
5629 auto Ret = B.buildFMul(Dst: ResTy, Src0: X, Src1: R);
5630 auto Tmp2 = B.buildFMA(Dst: ResTy, Src0: NegY, Src1: Ret, Src2: X);
5631
5632 B.buildFMA(Dst: Res, Src0: Tmp2, Src1: R, Src2: Ret);
5633 MI.eraseFromParent();
5634 return true;
5635}
5636
5637bool AMDGPULegalizerInfo::legalizeFDIV16(MachineInstr &MI,
5638 MachineRegisterInfo &MRI,
5639 MachineIRBuilder &B) const {
5640 if (legalizeFastUnsafeFDIV(MI, MRI, B))
5641 return true;
5642
5643 Register Res = MI.getOperand(i: 0).getReg();
5644 Register LHS = MI.getOperand(i: 1).getReg();
5645 Register RHS = MI.getOperand(i: 2).getReg();
5646
5647 uint16_t Flags = MI.getFlags();
5648
5649 LLT F16 = LLT::float16();
5650 LLT F32 = LLT::float32();
5651 LLT I32 = LLT::integer(SizeInBits: 32);
5652
5653 // a32.u = opx(V_CVT_F32_F16, a.u); // CVT to F32
5654 // b32.u = opx(V_CVT_F32_F16, b.u); // CVT to F32
5655 // r32.u = opx(V_RCP_F32, b32.u); // rcp = 1 / d
5656 // q32.u = opx(V_MUL_F32, a32.u, r32.u); // q = n * rcp
5657 // e32.u = opx(V_MAD_F32, (b32.u^_neg32), q32.u, a32.u); // err = -d * q + n
5658 // q32.u = opx(V_MAD_F32, e32.u, r32.u, q32.u); // q = n * rcp
5659 // e32.u = opx(V_MAD_F32, (b32.u^_neg32), q32.u, a32.u); // err = -d * q + n
5660 // tmp.u = opx(V_MUL_F32, e32.u, r32.u);
5661 // tmp.u = opx(V_AND_B32, tmp.u, 0xff800000)
5662 // q32.u = opx(V_ADD_F32, tmp.u, q32.u);
5663 // q16.u = opx(V_CVT_F16_F32, q32.u);
5664 // q16.u = opx(V_DIV_FIXUP_F16, q16.u, b.u, a.u); // q = touchup(q, d, n)
5665
5666 auto LHSExt = B.buildFPExt(Res: F32, Op: LHS, Flags);
5667 auto RHSExt = B.buildFPExt(Res: F32, Op: RHS, Flags);
5668 auto NegRHSExt = B.buildFNeg(Dst: F32, Src0: RHSExt);
5669 auto Rcp = B.buildIntrinsic(ID: Intrinsic::amdgcn_rcp, Res: {F32})
5670 .addUse(RegNo: RHSExt.getReg(Idx: 0))
5671 .setMIFlags(Flags);
5672 auto Quot = B.buildFMul(Dst: F32, Src0: LHSExt, Src1: Rcp, Flags);
5673 MachineInstrBuilder Err;
5674 if (ST.hasMadMacF32Insts()) {
5675 Err = B.buildFMAD(Dst: F32, Src0: NegRHSExt, Src1: Quot, Src2: LHSExt, Flags);
5676 Quot = B.buildFMAD(Dst: F32, Src0: Err, Src1: Rcp, Src2: Quot, Flags);
5677 Err = B.buildFMAD(Dst: F32, Src0: NegRHSExt, Src1: Quot, Src2: LHSExt, Flags);
5678 } else {
5679 Err = B.buildFMA(Dst: F32, Src0: NegRHSExt, Src1: Quot, Src2: LHSExt, Flags);
5680 Quot = B.buildFMA(Dst: F32, Src0: Err, Src1: Rcp, Src2: Quot, Flags);
5681 Err = B.buildFMA(Dst: F32, Src0: NegRHSExt, Src1: Quot, Src2: LHSExt, Flags);
5682 }
5683 auto Tmp = B.buildFMul(Dst: F32, Src0: Err, Src1: Rcp, Flags);
5684 auto TmpInt = B.buildBitcast(Dst: I32, Src: Tmp);
5685 auto MaskedInt = B.buildAnd(Dst: I32, Src0: TmpInt, Src1: B.buildConstant(Res: I32, Val: 0xff800000));
5686 auto Masked = B.buildBitcast(Dst: F32, Src: MaskedInt);
5687 Quot = B.buildFAdd(Dst: F32, Src0: Masked, Src1: Quot, Flags);
5688 auto RDst = B.buildFPTrunc(Res: F16, Op: Quot, Flags);
5689 B.buildIntrinsic(ID: Intrinsic::amdgcn_div_fixup, Res)
5690 .addUse(RegNo: RDst.getReg(Idx: 0))
5691 .addUse(RegNo: RHS)
5692 .addUse(RegNo: LHS)
5693 .setMIFlags(Flags);
5694
5695 MI.eraseFromParent();
5696 return true;
5697}
5698
5699static constexpr unsigned SPDenormModeBitField =
5700 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 4, Values: 2);
5701
5702// Enable or disable FP32 denorm mode. When 'Enable' is true, emit instructions
5703// to enable denorm mode. When 'Enable' is false, disable denorm mode.
5704static void toggleSPDenormMode(bool Enable, MachineIRBuilder &B,
5705 const GCNSubtarget &ST,
5706 SIModeRegisterDefaults Mode) {
5707 // Set SP denorm mode to this value.
5708 unsigned SPDenormMode =
5709 Enable ? FP_DENORM_FLUSH_NONE : Mode.fpDenormModeSPValue();
5710
5711 if (ST.hasDenormModeInst()) {
5712 // Preserve default FP64FP16 denorm mode while updating FP32 mode.
5713 uint32_t DPDenormModeDefault = Mode.fpDenormModeDPValue();
5714
5715 uint32_t NewDenormModeValue = SPDenormMode | (DPDenormModeDefault << 2);
5716 B.buildInstr(Opcode: AMDGPU::S_DENORM_MODE)
5717 .addImm(Val: NewDenormModeValue);
5718
5719 } else {
5720 B.buildInstr(Opcode: AMDGPU::S_SETREG_IMM32_B32)
5721 .addImm(Val: SPDenormMode)
5722 .addImm(Val: SPDenormModeBitField);
5723 }
5724}
5725
5726bool AMDGPULegalizerInfo::legalizeFDIV32(MachineInstr &MI,
5727 MachineRegisterInfo &MRI,
5728 MachineIRBuilder &B) const {
5729 if (legalizeFastUnsafeFDIV(MI, MRI, B))
5730 return true;
5731
5732 Register Res = MI.getOperand(i: 0).getReg();
5733 Register LHS = MI.getOperand(i: 1).getReg();
5734 Register RHS = MI.getOperand(i: 2).getReg();
5735 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
5736 SIModeRegisterDefaults Mode = MFI->getMode();
5737
5738 uint16_t Flags = MI.getFlags();
5739
5740 LLT F32 = LLT::float32();
5741 LLT S1 = LLT::scalar(SizeInBits: 1);
5742
5743 auto One = B.buildFConstant(Res: F32, Val: 1.0f);
5744
5745 auto DenominatorScaled =
5746 B.buildIntrinsic(ID: Intrinsic::amdgcn_div_scale, Res: {F32, S1})
5747 .addUse(RegNo: LHS)
5748 .addUse(RegNo: RHS)
5749 .addImm(Val: 0)
5750 .setMIFlags(Flags);
5751 auto NumeratorScaled =
5752 B.buildIntrinsic(ID: Intrinsic::amdgcn_div_scale, Res: {F32, S1})
5753 .addUse(RegNo: LHS)
5754 .addUse(RegNo: RHS)
5755 .addImm(Val: 1)
5756 .setMIFlags(Flags);
5757
5758 auto ApproxRcp = B.buildIntrinsic(ID: Intrinsic::amdgcn_rcp, Res: {F32})
5759 .addUse(RegNo: DenominatorScaled.getReg(Idx: 0))
5760 .setMIFlags(Flags);
5761 auto NegDivScale0 = B.buildFNeg(Dst: F32, Src0: DenominatorScaled, Flags);
5762
5763 const bool PreservesDenormals = Mode.FP32Denormals == DenormalMode::getIEEE();
5764 const bool HasDynamicDenormals =
5765 (Mode.FP32Denormals.Input == DenormalMode::Dynamic) ||
5766 (Mode.FP32Denormals.Output == DenormalMode::Dynamic);
5767
5768 Register SavedSPDenormMode;
5769 if (!PreservesDenormals) {
5770 if (HasDynamicDenormals) {
5771 SavedSPDenormMode = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5772 B.buildInstr(Opcode: AMDGPU::S_GETREG_B32)
5773 .addDef(RegNo: SavedSPDenormMode)
5774 .addImm(Val: SPDenormModeBitField);
5775 }
5776 toggleSPDenormMode(Enable: true, B, ST, Mode);
5777 }
5778
5779 auto Fma0 = B.buildFMA(Dst: F32, Src0: NegDivScale0, Src1: ApproxRcp, Src2: One, Flags);
5780 auto Fma1 = B.buildFMA(Dst: F32, Src0: Fma0, Src1: ApproxRcp, Src2: ApproxRcp, Flags);
5781 auto Mul = B.buildFMul(Dst: F32, Src0: NumeratorScaled, Src1: Fma1, Flags);
5782 auto Fma2 = B.buildFMA(Dst: F32, Src0: NegDivScale0, Src1: Mul, Src2: NumeratorScaled, Flags);
5783 auto Fma3 = B.buildFMA(Dst: F32, Src0: Fma2, Src1: Fma1, Src2: Mul, Flags);
5784 auto Fma4 = B.buildFMA(Dst: F32, Src0: NegDivScale0, Src1: Fma3, Src2: NumeratorScaled, Flags);
5785
5786 if (!PreservesDenormals) {
5787 if (HasDynamicDenormals) {
5788 assert(SavedSPDenormMode);
5789 B.buildInstr(Opcode: AMDGPU::S_SETREG_B32)
5790 .addReg(RegNo: SavedSPDenormMode)
5791 .addImm(Val: SPDenormModeBitField);
5792 } else
5793 toggleSPDenormMode(Enable: false, B, ST, Mode);
5794 }
5795
5796 auto Fmas = B.buildIntrinsic(ID: Intrinsic::amdgcn_div_fmas, Res: {F32})
5797 .addUse(RegNo: Fma4.getReg(Idx: 0))
5798 .addUse(RegNo: Fma1.getReg(Idx: 0))
5799 .addUse(RegNo: Fma3.getReg(Idx: 0))
5800 .addUse(RegNo: NumeratorScaled.getReg(Idx: 1))
5801 .setMIFlags(Flags);
5802
5803 B.buildIntrinsic(ID: Intrinsic::amdgcn_div_fixup, Res)
5804 .addUse(RegNo: Fmas.getReg(Idx: 0))
5805 .addUse(RegNo: RHS)
5806 .addUse(RegNo: LHS)
5807 .setMIFlags(Flags);
5808
5809 MI.eraseFromParent();
5810 return true;
5811}
5812
5813bool AMDGPULegalizerInfo::legalizeFDIV64(MachineInstr &MI,
5814 MachineRegisterInfo &MRI,
5815 MachineIRBuilder &B) const {
5816 if (legalizeFastUnsafeFDIV64(MI, MRI, B))
5817 return true;
5818
5819 Register Res = MI.getOperand(i: 0).getReg();
5820 Register LHS = MI.getOperand(i: 1).getReg();
5821 Register RHS = MI.getOperand(i: 2).getReg();
5822
5823 uint16_t Flags = MI.getFlags();
5824
5825 LLT F64 = LLT::float64();
5826 LLT S1 = LLT::scalar(SizeInBits: 1);
5827
5828 auto One = B.buildFConstant(Res: F64, Val: 1.0);
5829
5830 auto DivScale0 = B.buildIntrinsic(ID: Intrinsic::amdgcn_div_scale, Res: {F64, S1})
5831 .addUse(RegNo: LHS)
5832 .addUse(RegNo: RHS)
5833 .addImm(Val: 0)
5834 .setMIFlags(Flags);
5835
5836 auto NegDivScale0 = B.buildFNeg(Dst: F64, Src0: DivScale0.getReg(Idx: 0), Flags);
5837
5838 auto Rcp = B.buildIntrinsic(ID: Intrinsic::amdgcn_rcp, Res: {F64})
5839 .addUse(RegNo: DivScale0.getReg(Idx: 0))
5840 .setMIFlags(Flags);
5841
5842 auto Fma0 = B.buildFMA(Dst: F64, Src0: NegDivScale0, Src1: Rcp, Src2: One, Flags);
5843 auto Fma1 = B.buildFMA(Dst: F64, Src0: Rcp, Src1: Fma0, Src2: Rcp, Flags);
5844 auto Fma2 = B.buildFMA(Dst: F64, Src0: NegDivScale0, Src1: Fma1, Src2: One, Flags);
5845
5846 auto DivScale1 = B.buildIntrinsic(ID: Intrinsic::amdgcn_div_scale, Res: {F64, S1})
5847 .addUse(RegNo: LHS)
5848 .addUse(RegNo: RHS)
5849 .addImm(Val: 1)
5850 .setMIFlags(Flags);
5851
5852 auto Fma3 = B.buildFMA(Dst: F64, Src0: Fma1, Src1: Fma2, Src2: Fma1, Flags);
5853 auto Mul = B.buildFMul(Dst: F64, Src0: DivScale1.getReg(Idx: 0), Src1: Fma3, Flags);
5854 auto Fma4 = B.buildFMA(Dst: F64, Src0: NegDivScale0, Src1: Mul, Src2: DivScale1.getReg(Idx: 0), Flags);
5855
5856 Register Scale;
5857 if (!ST.hasUsableDivScaleConditionOutput()) {
5858 // Workaround a hardware bug on SI where the condition output from div_scale
5859 // is not usable.
5860
5861 LLT I32 = LLT::integer(SizeInBits: 32);
5862 LLT I64 = LLT::integer(SizeInBits: 64);
5863
5864 auto NumUnmerge = B.buildUnmerge(Res: I32, Op: B.buildBitcast(Dst: I64, Src: LHS));
5865 auto DenUnmerge = B.buildUnmerge(Res: I32, Op: B.buildBitcast(Dst: I64, Src: RHS));
5866 auto Scale0Unmerge = B.buildUnmerge(Res: I32, Op: B.buildBitcast(Dst: I64, Src: DivScale0));
5867 auto Scale1Unmerge = B.buildUnmerge(Res: I32, Op: B.buildBitcast(Dst: I64, Src: DivScale1));
5868
5869 auto CmpNum = B.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: S1, Op0: NumUnmerge.getReg(Idx: 1),
5870 Op1: Scale1Unmerge.getReg(Idx: 1));
5871 auto CmpDen = B.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: S1, Op0: DenUnmerge.getReg(Idx: 1),
5872 Op1: Scale0Unmerge.getReg(Idx: 1));
5873 Scale = B.buildXor(Dst: S1, Src0: CmpNum, Src1: CmpDen).getReg(Idx: 0);
5874 } else {
5875 Scale = DivScale1.getReg(Idx: 1);
5876 }
5877
5878 auto Fmas = B.buildIntrinsic(ID: Intrinsic::amdgcn_div_fmas, Res: {F64})
5879 .addUse(RegNo: Fma4.getReg(Idx: 0))
5880 .addUse(RegNo: Fma3.getReg(Idx: 0))
5881 .addUse(RegNo: Mul.getReg(Idx: 0))
5882 .addUse(RegNo: Scale)
5883 .setMIFlags(Flags);
5884
5885 B.buildIntrinsic(ID: Intrinsic::amdgcn_div_fixup, Res: ArrayRef(Res))
5886 .addUse(RegNo: Fmas.getReg(Idx: 0))
5887 .addUse(RegNo: RHS)
5888 .addUse(RegNo: LHS)
5889 .setMIFlags(Flags);
5890
5891 MI.eraseFromParent();
5892 return true;
5893}
5894
5895bool AMDGPULegalizerInfo::legalizeFFREXP(MachineInstr &MI,
5896 MachineRegisterInfo &MRI,
5897 MachineIRBuilder &B) const {
5898 Register Res0 = MI.getOperand(i: 0).getReg();
5899 Register Res1 = MI.getOperand(i: 1).getReg();
5900 Register Val = MI.getOperand(i: 2).getReg();
5901 uint16_t Flags = MI.getFlags();
5902
5903 LLT Ty = MRI.getType(Reg: Res0);
5904 LLT InstrExpTy = Ty == LLT::float16() ? LLT::integer(SizeInBits: 16) : LLT::integer(SizeInBits: 32);
5905
5906 auto Mant = B.buildIntrinsic(ID: Intrinsic::amdgcn_frexp_mant, Res: {Ty})
5907 .addUse(RegNo: Val)
5908 .setMIFlags(Flags);
5909 auto Exp = B.buildIntrinsic(ID: Intrinsic::amdgcn_frexp_exp, Res: {InstrExpTy})
5910 .addUse(RegNo: Val)
5911 .setMIFlags(Flags);
5912
5913 if (ST.hasFractBug()) {
5914 auto Fabs = B.buildFAbs(Dst: Ty, Src0: Val);
5915 auto Inf = B.buildFConstant(Res: Ty, Val: APFloat::getInf(Sem: getFltSemanticForLLT(Ty)));
5916 auto IsFinite =
5917 B.buildFCmp(Pred: CmpInst::FCMP_OLT, Res: LLT::scalar(SizeInBits: 1), Op0: Fabs, Op1: Inf, Flags);
5918 auto Zero = B.buildConstant(Res: InstrExpTy, Val: 0);
5919 Exp = B.buildSelect(Res: InstrExpTy, Tst: IsFinite, Op0: Exp, Op1: Zero);
5920 Mant = B.buildSelect(Res: Ty, Tst: IsFinite, Op0: Mant, Op1: Val);
5921 }
5922
5923 B.buildCopy(Res: Res0, Op: Mant);
5924 B.buildSExtOrTrunc(Res: Res1, Op: Exp);
5925
5926 MI.eraseFromParent();
5927 return true;
5928}
5929
5930bool AMDGPULegalizerInfo::legalizeFDIVFastIntrin(MachineInstr &MI,
5931 MachineRegisterInfo &MRI,
5932 MachineIRBuilder &B) const {
5933 Register Res = MI.getOperand(i: 0).getReg();
5934 Register LHS = MI.getOperand(i: 2).getReg();
5935 Register RHS = MI.getOperand(i: 3).getReg();
5936 uint16_t Flags = MI.getFlags();
5937
5938 LLT F32 = LLT::float32();
5939 LLT S1 = LLT::scalar(SizeInBits: 1);
5940
5941 auto Abs = B.buildFAbs(Dst: F32, Src0: RHS, Flags);
5942 const APFloat C0Val(1.0f);
5943
5944 auto C0 = B.buildFConstant(Res: F32, Val: 0x1p+96f);
5945 auto C1 = B.buildFConstant(Res: F32, Val: 0x1p-32f);
5946 auto C2 = B.buildFConstant(Res: F32, Val: 1.0f);
5947
5948 auto CmpRes = B.buildFCmp(Pred: CmpInst::FCMP_OGT, Res: S1, Op0: Abs, Op1: C0, Flags);
5949 auto Sel = B.buildSelect(Res: F32, Tst: CmpRes, Op0: C1, Op1: C2, Flags);
5950
5951 auto Mul0 = B.buildFMul(Dst: F32, Src0: RHS, Src1: Sel, Flags);
5952
5953 auto RCP = B.buildIntrinsic(ID: Intrinsic::amdgcn_rcp, Res: {F32})
5954 .addUse(RegNo: Mul0.getReg(Idx: 0))
5955 .setMIFlags(Flags);
5956
5957 auto Mul1 = B.buildFMul(Dst: F32, Src0: LHS, Src1: RCP, Flags);
5958
5959 B.buildFMul(Dst: Res, Src0: Sel, Src1: Mul1, Flags);
5960
5961 MI.eraseFromParent();
5962 return true;
5963}
5964
5965bool AMDGPULegalizerInfo::legalizeFSQRTF16(MachineInstr &MI,
5966 MachineRegisterInfo &MRI,
5967 MachineIRBuilder &B) const {
5968 // Bypass the correct expansion a standard promotion through G_FSQRT would
5969 // get. The f32 op is accurate enough for the f16 cas.
5970 unsigned Flags = MI.getFlags();
5971 assert(!ST.has16BitInsts());
5972 const LLT F32 = LLT::float32();
5973 auto Ext = B.buildFPExt(Res: F32, Op: MI.getOperand(i: 1), Flags);
5974 auto Log2 = B.buildIntrinsic(ID: Intrinsic::amdgcn_sqrt, Res: {F32})
5975 .addUse(RegNo: Ext.getReg(Idx: 0))
5976 .setMIFlags(Flags);
5977 B.buildFPTrunc(Res: MI.getOperand(i: 0), Op: Log2, Flags);
5978 MI.eraseFromParent();
5979 return true;
5980}
5981
5982bool AMDGPULegalizerInfo::legalizeFSQRTF32(MachineInstr &MI,
5983 MachineRegisterInfo &MRI,
5984 MachineIRBuilder &B) const {
5985 MachineFunction &MF = B.getMF();
5986 Register Dst = MI.getOperand(i: 0).getReg();
5987 Register X = MI.getOperand(i: 1).getReg();
5988 const unsigned Flags = MI.getFlags();
5989 const LLT S1 = LLT::scalar(SizeInBits: 1);
5990 const LLT F32 = LLT::float32();
5991 const LLT I32 = LLT::integer(SizeInBits: 32);
5992
5993 if (allowApproxFunc(MF, Flags)) {
5994 B.buildIntrinsic(ID: Intrinsic::amdgcn_sqrt, Res: ArrayRef<Register>({Dst}))
5995 .addUse(RegNo: X)
5996 .setMIFlags(Flags);
5997 MI.eraseFromParent();
5998 return true;
5999 }
6000
6001 auto ScaleThreshold = B.buildFConstant(Res: F32, Val: 0x1.0p-96f);
6002 auto NeedScale = B.buildFCmp(Pred: CmpInst::FCMP_OGT, Res: S1, Op0: ScaleThreshold, Op1: X, Flags);
6003 auto ScaleUpFactor = B.buildFConstant(Res: F32, Val: 0x1.0p+32f);
6004 auto ScaledX = B.buildFMul(Dst: F32, Src0: X, Src1: ScaleUpFactor, Flags);
6005 auto SqrtX = B.buildSelect(Res: F32, Tst: NeedScale, Op0: ScaledX, Op1: X, Flags);
6006
6007 Register SqrtS = MRI.createGenericVirtualRegister(Ty: F32);
6008 if (needsDenormHandlingF32(MF, Src: X, Flags)) {
6009 B.buildIntrinsic(ID: Intrinsic::amdgcn_sqrt, Res: ArrayRef<Register>({SqrtS}))
6010 .addUse(RegNo: SqrtX.getReg(Idx: 0))
6011 .setMIFlags(Flags);
6012
6013 auto SqrtSInt = B.buildBitcast(Dst: I32, Src: SqrtS);
6014 auto NegOne = B.buildConstant(Res: I32, Val: -1);
6015 auto SqrtSNextDown = B.buildBitcast(Dst: F32, Src: B.buildAdd(Dst: I32, Src0: SqrtSInt, Src1: NegOne));
6016
6017 auto NegSqrtSNextDown = B.buildFNeg(Dst: F32, Src0: SqrtSNextDown, Flags);
6018 auto SqrtVP = B.buildFMA(Dst: F32, Src0: NegSqrtSNextDown, Src1: SqrtS, Src2: SqrtX, Flags);
6019
6020 auto PosOne = B.buildConstant(Res: I32, Val: 1);
6021 auto SqrtSNextUp = B.buildBitcast(Dst: F32, Src: B.buildAdd(Dst: I32, Src0: SqrtSInt, Src1: PosOne));
6022
6023 auto NegSqrtSNextUp = B.buildFNeg(Dst: F32, Src0: SqrtSNextUp, Flags);
6024 auto SqrtVS = B.buildFMA(Dst: F32, Src0: NegSqrtSNextUp, Src1: SqrtS, Src2: SqrtX, Flags);
6025
6026 auto Zero = B.buildFConstant(Res: F32, Val: 0.0f);
6027 auto SqrtVPLE0 = B.buildFCmp(Pred: CmpInst::FCMP_OLE, Res: S1, Op0: SqrtVP, Op1: Zero, Flags);
6028
6029 SqrtS =
6030 B.buildSelect(Res: F32, Tst: SqrtVPLE0, Op0: SqrtSNextDown, Op1: SqrtS, Flags).getReg(Idx: 0);
6031
6032 auto SqrtVPVSGT0 = B.buildFCmp(Pred: CmpInst::FCMP_OGT, Res: S1, Op0: SqrtVS, Op1: Zero, Flags);
6033 SqrtS =
6034 B.buildSelect(Res: F32, Tst: SqrtVPVSGT0, Op0: SqrtSNextUp, Op1: SqrtS, Flags).getReg(Idx: 0);
6035 } else {
6036 auto SqrtR =
6037 B.buildIntrinsic(ID: Intrinsic::amdgcn_rsq, Res: {F32}).addReg(RegNo: SqrtX.getReg(Idx: 0));
6038 B.buildFMul(Dst: SqrtS, Src0: SqrtX, Src1: SqrtR, Flags);
6039
6040 auto Half = B.buildFConstant(Res: F32, Val: 0.5f);
6041 auto SqrtH = B.buildFMul(Dst: F32, Src0: SqrtR, Src1: Half, Flags);
6042 auto NegSqrtH = B.buildFNeg(Dst: F32, Src0: SqrtH, Flags);
6043 auto SqrtE = B.buildFMA(Dst: F32, Src0: NegSqrtH, Src1: SqrtS, Src2: Half, Flags);
6044 SqrtH = B.buildFMA(Dst: F32, Src0: SqrtH, Src1: SqrtE, Src2: SqrtH, Flags);
6045 SqrtS = B.buildFMA(Dst: F32, Src0: SqrtS, Src1: SqrtE, Src2: SqrtS, Flags).getReg(Idx: 0);
6046 auto NegSqrtS = B.buildFNeg(Dst: F32, Src0: SqrtS, Flags);
6047 auto SqrtD = B.buildFMA(Dst: F32, Src0: NegSqrtS, Src1: SqrtS, Src2: SqrtX, Flags);
6048 SqrtS = B.buildFMA(Dst: F32, Src0: SqrtD, Src1: SqrtH, Src2: SqrtS, Flags).getReg(Idx: 0);
6049 }
6050
6051 auto ScaleDownFactor = B.buildFConstant(Res: F32, Val: 0x1.0p-16f);
6052
6053 auto ScaledDown = B.buildFMul(Dst: F32, Src0: SqrtS, Src1: ScaleDownFactor, Flags);
6054
6055 SqrtS = B.buildSelect(Res: F32, Tst: NeedScale, Op0: ScaledDown, Op1: SqrtS, Flags).getReg(Idx: 0);
6056
6057 auto IsZeroOrInf = B.buildIsFPClass(Res: LLT::scalar(SizeInBits: 1), Src: SqrtX, Mask: fcZero | fcPosInf);
6058 B.buildSelect(Res: Dst, Tst: IsZeroOrInf, Op0: SqrtX, Op1: SqrtS, Flags);
6059
6060 MI.eraseFromParent();
6061 return true;
6062}
6063
6064bool AMDGPULegalizerInfo::legalizeFSQRTF64(MachineInstr &MI,
6065 MachineRegisterInfo &MRI,
6066 MachineIRBuilder &B) const {
6067 // For double type, the SQRT and RSQ instructions don't have required
6068 // precision, we apply Goldschmidt's algorithm to improve the result:
6069 //
6070 // y0 = rsq(x)
6071 // g0 = x * y0
6072 // h0 = 0.5 * y0
6073 //
6074 // r0 = 0.5 - h0 * g0
6075 // g1 = g0 * r0 + g0
6076 // h1 = h0 * r0 + h0
6077 //
6078 // r1 = 0.5 - h1 * g1 => d0 = x - g1 * g1
6079 // g2 = g1 * r1 + g1 g2 = d0 * h1 + g1
6080 // h2 = h1 * r1 + h1
6081 //
6082 // r2 = 0.5 - h2 * g2 => d1 = x - g2 * g2
6083 // g3 = g2 * r2 + g2 g3 = d1 * h1 + g2
6084 //
6085 // sqrt(x) = g3
6086
6087 const LLT S1 = LLT::scalar(SizeInBits: 1);
6088 const LLT I32 = LLT::integer(SizeInBits: 32);
6089 const LLT F64 = LLT::float64();
6090
6091 Register Dst = MI.getOperand(i: 0).getReg();
6092 assert(MRI.getType(Dst) == F64 && "only expect to lower f64 sqrt");
6093
6094 Register X = MI.getOperand(i: 1).getReg();
6095 unsigned Flags = MI.getFlags();
6096
6097 Register SqrtX = X;
6098 Register Scaling, ZeroInt;
6099 if (!MI.getFlag(Flag: MachineInstr::FmAfn)) {
6100 auto ScaleConstant = B.buildFConstant(Res: F64, Val: 0x1.0p-767);
6101
6102 ZeroInt = B.buildConstant(Res: I32, Val: 0).getReg(Idx: 0);
6103 Scaling = B.buildFCmp(Pred: FCmpInst::FCMP_OLT, Res: S1, Op0: X, Op1: ScaleConstant).getReg(Idx: 0);
6104
6105 // Scale up input if it is too small.
6106 auto ScaleUpFactor = B.buildConstant(Res: I32, Val: 256);
6107 auto ScaleUp = B.buildSelect(Res: I32, Tst: Scaling, Op0: ScaleUpFactor, Op1: ZeroInt);
6108 SqrtX = B.buildFLdexp(Dst: F64, Src0: X, Src1: ScaleUp, Flags).getReg(Idx: 0);
6109 }
6110
6111 auto SqrtY = B.buildIntrinsic(ID: Intrinsic::amdgcn_rsq, Res: {F64}).addReg(RegNo: SqrtX);
6112
6113 auto Half = B.buildFConstant(Res: F64, Val: 0.5);
6114 auto SqrtH0 = B.buildFMul(Dst: F64, Src0: SqrtY, Src1: Half);
6115 auto SqrtS0 = B.buildFMul(Dst: F64, Src0: SqrtX, Src1: SqrtY);
6116
6117 auto NegSqrtH0 = B.buildFNeg(Dst: F64, Src0: SqrtH0);
6118 auto SqrtR0 = B.buildFMA(Dst: F64, Src0: NegSqrtH0, Src1: SqrtS0, Src2: Half);
6119
6120 auto SqrtS1 = B.buildFMA(Dst: F64, Src0: SqrtS0, Src1: SqrtR0, Src2: SqrtS0);
6121 auto SqrtH1 = B.buildFMA(Dst: F64, Src0: SqrtH0, Src1: SqrtR0, Src2: SqrtH0);
6122
6123 auto NegSqrtS1 = B.buildFNeg(Dst: F64, Src0: SqrtS1);
6124 auto SqrtD0 = B.buildFMA(Dst: F64, Src0: NegSqrtS1, Src1: SqrtS1, Src2: SqrtX);
6125
6126 auto SqrtS2 = B.buildFMA(Dst: F64, Src0: SqrtD0, Src1: SqrtH1, Src2: SqrtS1);
6127
6128 Register SqrtRet = SqrtS2.getReg(Idx: 0);
6129 if (!MI.getFlag(Flag: MachineInstr::FmAfn)) {
6130 auto NegSqrtS2 = B.buildFNeg(Dst: F64, Src0: SqrtS2);
6131 auto SqrtD1 = B.buildFMA(Dst: F64, Src0: NegSqrtS2, Src1: SqrtS2, Src2: SqrtX);
6132 auto SqrtD2 = B.buildFMA(Dst: F64, Src0: SqrtD1, Src1: SqrtH1, Src2: SqrtS2);
6133
6134 // Scale down the result.
6135 auto ScaleDownFactor = B.buildConstant(Res: I32, Val: -128);
6136 auto ScaleDown = B.buildSelect(Res: I32, Tst: Scaling, Op0: ScaleDownFactor, Op1: ZeroInt);
6137 SqrtRet = B.buildFLdexp(Dst: F64, Src0: SqrtD2, Src1: ScaleDown, Flags).getReg(Idx: 0);
6138 }
6139
6140 Register IsZeroOrInf;
6141 if (MI.getFlag(Flag: MachineInstr::FmNoInfs)) {
6142 auto ZeroFP = B.buildFConstant(Res: F64, Val: 0.0);
6143 IsZeroOrInf = B.buildFCmp(Pred: FCmpInst::FCMP_OEQ, Res: S1, Op0: SqrtX, Op1: ZeroFP).getReg(Idx: 0);
6144 } else {
6145 IsZeroOrInf = B.buildIsFPClass(Res: S1, Src: SqrtX, Mask: fcZero | fcPosInf).getReg(Idx: 0);
6146 }
6147
6148 // TODO: Check for DAZ and expand to subnormals
6149
6150 // If x is +INF, +0, or -0, use its original value
6151 B.buildSelect(Res: Dst, Tst: IsZeroOrInf, Op0: SqrtX, Op1: SqrtRet, Flags);
6152
6153 MI.eraseFromParent();
6154 return true;
6155}
6156
6157bool AMDGPULegalizerInfo::legalizeFSQRT(MachineInstr &MI,
6158 MachineRegisterInfo &MRI,
6159 MachineIRBuilder &B) const {
6160 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
6161 if (Ty == LLT::float32())
6162 return legalizeFSQRTF32(MI, MRI, B);
6163 if (Ty == LLT::float64())
6164 return legalizeFSQRTF64(MI, MRI, B);
6165 if (Ty == LLT::float16())
6166 return legalizeFSQRTF16(MI, MRI, B);
6167 return false;
6168}
6169
6170// Expand llvm.amdgcn.rsq.clamp on targets that don't support the instruction.
6171// FIXME: Why do we handle this one but not other removed instructions?
6172//
6173// Reciprocal square root. The clamp prevents infinite results, clamping
6174// infinities to max_float. D.f = 1.0 / sqrt(S0.f), result clamped to
6175// +-max_float.
6176bool AMDGPULegalizerInfo::legalizeRsqClampIntrinsic(MachineInstr &MI,
6177 MachineRegisterInfo &MRI,
6178 MachineIRBuilder &B) const {
6179 if (ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
6180 return true;
6181
6182 Register Dst = MI.getOperand(i: 0).getReg();
6183 Register Src = MI.getOperand(i: 2).getReg();
6184 auto Flags = MI.getFlags();
6185
6186 LLT Ty = MRI.getType(Reg: Dst);
6187
6188 const fltSemantics *FltSemantics;
6189 if (Ty == LLT::float32())
6190 FltSemantics = &APFloat::IEEEsingle();
6191 else if (Ty == LLT::float64())
6192 FltSemantics = &APFloat::IEEEdouble();
6193 else
6194 return false;
6195
6196 auto Rsq = B.buildIntrinsic(ID: Intrinsic::amdgcn_rsq, Res: {Ty})
6197 .addUse(RegNo: Src)
6198 .setMIFlags(Flags);
6199
6200 // We don't need to concern ourselves with the snan handling difference, since
6201 // the rsq quieted (or not) so use the one which will directly select.
6202 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
6203 const bool UseIEEE = MFI->getMode().IEEE;
6204
6205 auto MaxFlt = B.buildFConstant(Res: Ty, Val: APFloat::getLargest(Sem: *FltSemantics));
6206 auto ClampMax = UseIEEE ? B.buildFMinNumIEEE(Dst: Ty, Src0: Rsq, Src1: MaxFlt, Flags) :
6207 B.buildFMinNum(Dst: Ty, Src0: Rsq, Src1: MaxFlt, Flags);
6208
6209 auto MinFlt = B.buildFConstant(Res: Ty, Val: APFloat::getLargest(Sem: *FltSemantics, Negative: true));
6210
6211 if (UseIEEE)
6212 B.buildFMaxNumIEEE(Dst, Src0: ClampMax, Src1: MinFlt, Flags);
6213 else
6214 B.buildFMaxNum(Dst, Src0: ClampMax, Src1: MinFlt, Flags);
6215 MI.eraseFromParent();
6216 return true;
6217}
6218
6219// TODO: Fix pointer type handling
6220bool AMDGPULegalizerInfo::legalizeLaneOp(LegalizerHelper &Helper,
6221 MachineInstr &MI,
6222 Intrinsic::ID IID) const {
6223
6224 MachineIRBuilder &B = Helper.MIRBuilder;
6225 MachineRegisterInfo &MRI = *B.getMRI();
6226
6227 bool IsPermLane16 = IID == Intrinsic::amdgcn_permlane16 ||
6228 IID == Intrinsic::amdgcn_permlanex16;
6229 bool IsSetInactive = IID == Intrinsic::amdgcn_set_inactive ||
6230 IID == Intrinsic::amdgcn_set_inactive_chain_arg;
6231 bool IsPermlaneShuffle = IID == Intrinsic::amdgcn_permlane_bcast ||
6232 IID == Intrinsic::amdgcn_permlane_up ||
6233 IID == Intrinsic::amdgcn_permlane_down ||
6234 IID == Intrinsic::amdgcn_permlane_xor;
6235
6236 auto createLaneOp = [&IID, &B, &MI](Register Src0, Register Src1,
6237 Register Src2, LLT VT) -> Register {
6238 auto LaneOp = B.buildIntrinsic(ID: IID, Res: {VT}).addUse(RegNo: Src0);
6239 switch (IID) {
6240 case Intrinsic::amdgcn_readfirstlane:
6241 case Intrinsic::amdgcn_permlane64:
6242 return LaneOp.getReg(Idx: 0);
6243 case Intrinsic::amdgcn_readlane:
6244 case Intrinsic::amdgcn_set_inactive:
6245 case Intrinsic::amdgcn_set_inactive_chain_arg:
6246 return LaneOp.addUse(RegNo: Src1).getReg(Idx: 0);
6247 case Intrinsic::amdgcn_writelane:
6248 case Intrinsic::amdgcn_permlane_bcast:
6249 case Intrinsic::amdgcn_permlane_up:
6250 case Intrinsic::amdgcn_permlane_down:
6251 case Intrinsic::amdgcn_permlane_xor:
6252 return LaneOp.addUse(RegNo: Src1).addUse(RegNo: Src2).getReg(Idx: 0);
6253 case Intrinsic::amdgcn_permlane16:
6254 case Intrinsic::amdgcn_permlanex16: {
6255 Register Src3 = MI.getOperand(i: 5).getReg();
6256 int64_t Src4 = MI.getOperand(i: 6).getImm();
6257 int64_t Src5 = MI.getOperand(i: 7).getImm();
6258 return LaneOp.addUse(RegNo: Src1)
6259 .addUse(RegNo: Src2)
6260 .addUse(RegNo: Src3)
6261 .addImm(Val: Src4)
6262 .addImm(Val: Src5)
6263 .getReg(Idx: 0);
6264 }
6265 case Intrinsic::amdgcn_mov_dpp8:
6266 return LaneOp.addImm(Val: MI.getOperand(i: 3).getImm()).getReg(Idx: 0);
6267 case Intrinsic::amdgcn_update_dpp:
6268 return LaneOp.addUse(RegNo: Src1)
6269 .addImm(Val: MI.getOperand(i: 4).getImm())
6270 .addImm(Val: MI.getOperand(i: 5).getImm())
6271 .addImm(Val: MI.getOperand(i: 6).getImm())
6272 .addImm(Val: MI.getOperand(i: 7).getImm())
6273 .getReg(Idx: 0);
6274 default:
6275 llvm_unreachable("unhandled lane op");
6276 }
6277 };
6278
6279 Register DstReg = MI.getOperand(i: 0).getReg();
6280 Register Src0 = MI.getOperand(i: 2).getReg();
6281 Register Src1, Src2;
6282 if (IID == Intrinsic::amdgcn_readlane || IID == Intrinsic::amdgcn_writelane ||
6283 IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16 ||
6284 IsPermlaneShuffle) {
6285 Src1 = MI.getOperand(i: 3).getReg();
6286 if (IID == Intrinsic::amdgcn_writelane || IsPermLane16 ||
6287 IsPermlaneShuffle) {
6288 Src2 = MI.getOperand(i: 4).getReg();
6289 }
6290 }
6291
6292 LLT Ty = MRI.getType(Reg: DstReg);
6293 unsigned Size = Ty.getSizeInBits();
6294
6295 unsigned SplitSize = 32;
6296 if (IID == Intrinsic::amdgcn_update_dpp && (Size % 64 == 0) &&
6297 ST.hasDPALU_DPP() &&
6298 AMDGPU::isLegalDPALU_DPPControl(ST, DC: MI.getOperand(i: 4).getImm()))
6299 SplitSize = 64;
6300
6301 if (Size == SplitSize) {
6302 // Already legal
6303 return true;
6304 }
6305
6306 const LLT I32 = LLT::integer(SizeInBits: 32);
6307
6308 bool IsFloat = Ty.getScalarType().isFloat();
6309
6310 LLT IntTy = IsFloat ? LLT::integer(SizeInBits: Size) : Ty;
6311 if (IsFloat) {
6312 Src0 = B.buildBitcast(Dst: IntTy, Src: Src0).getReg(Idx: 0);
6313 if (Src1 && MRI.getType(Reg: Src1).getScalarType().isFloat())
6314 Src1 = B.buildBitcast(Dst: IntTy, Src: Src1).getReg(Idx: 0);
6315 if (Src2 && MRI.getType(Reg: Src2).getScalarType().isFloat())
6316 Src2 = B.buildBitcast(Dst: IntTy, Src: Src2).getReg(Idx: 0);
6317 }
6318
6319 if (Size < 32) {
6320 Src0 = B.buildAnyExt(Res: I32, Op: Src0).getReg(Idx: 0);
6321
6322 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
6323 Src1 = B.buildAnyExt(Res: I32, Op: Src1).getReg(Idx: 0);
6324
6325 if (IID == Intrinsic::amdgcn_writelane)
6326 Src2 = B.buildAnyExt(Res: I32, Op: Src2).getReg(Idx: 0);
6327
6328 Register LaneOpDst = createLaneOp(Src0, Src1, Src2, I32);
6329 if (IsFloat)
6330 B.buildBitcast(Dst: DstReg, Src: B.buildTrunc(Res: IntTy, Op: LaneOpDst));
6331 else
6332 B.buildTrunc(Res: DstReg, Op: LaneOpDst);
6333 MI.eraseFromParent();
6334 return true;
6335 }
6336
6337 if (Size % SplitSize != 0)
6338 return false;
6339
6340 LLT PartialResTy = LLT::integer(SizeInBits: SplitSize);
6341 bool NeedsBitcast = false;
6342 if (IntTy.isVector()) {
6343 LLT EltTy = IntTy.getElementType();
6344 unsigned EltSize = EltTy.getSizeInBits();
6345 if (EltSize == SplitSize) {
6346 PartialResTy = EltTy;
6347 } else if (EltSize == 16 || EltSize == 32) {
6348 unsigned NElem = SplitSize / EltSize;
6349 PartialResTy = IntTy.changeElementCount(EC: ElementCount::getFixed(MinVal: NElem));
6350 } else {
6351 NeedsBitcast = true;
6352 }
6353 }
6354
6355 SmallVector<Register, 4> PartialRes;
6356 unsigned NumParts = Size / SplitSize;
6357 MachineInstrBuilder Src0Parts = B.buildUnmerge(Res: PartialResTy, Op: Src0);
6358 MachineInstrBuilder Src1Parts, Src2Parts;
6359
6360 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
6361 Src1Parts = B.buildUnmerge(Res: PartialResTy, Op: Src1);
6362
6363 if (IID == Intrinsic::amdgcn_writelane)
6364 Src2Parts = B.buildUnmerge(Res: PartialResTy, Op: Src2);
6365
6366 for (unsigned i = 0; i < NumParts; ++i) {
6367 Src0 = Src0Parts.getReg(Idx: i);
6368
6369 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
6370 Src1 = Src1Parts.getReg(Idx: i);
6371
6372 if (IID == Intrinsic::amdgcn_writelane)
6373 Src2 = Src2Parts.getReg(Idx: i);
6374
6375 PartialRes.push_back(Elt: createLaneOp(Src0, Src1, Src2, PartialResTy));
6376 }
6377
6378 if (NeedsBitcast || IsFloat)
6379 B.buildBitcast(
6380 Dst: DstReg,
6381 Src: B.buildMergeLikeInstr(Res: LLT::integer(SizeInBits: IntTy.getSizeInBits()), Ops: PartialRes));
6382 else
6383 B.buildMergeLikeInstr(Res: DstReg, Ops: PartialRes);
6384
6385 MI.eraseFromParent();
6386 return true;
6387}
6388
6389bool AMDGPULegalizerInfo::getImplicitArgPtr(Register DstReg,
6390 MachineRegisterInfo &MRI,
6391 MachineIRBuilder &B) const {
6392 uint64_t Offset =
6393 ST.getTargetLowering()->getImplicitParameterOffset(
6394 MF: B.getMF(), Param: AMDGPUTargetLowering::FIRST_IMPLICIT);
6395 LLT DstTy = MRI.getType(Reg: DstReg);
6396 LLT IdxTy = LLT::integer(SizeInBits: DstTy.getSizeInBits());
6397
6398 Register KernargPtrReg = MRI.createGenericVirtualRegister(Ty: DstTy);
6399 if (!loadInputValue(DstReg: KernargPtrReg, B,
6400 ArgType: AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR))
6401 return false;
6402
6403 B.buildObjectPtrOffset(Res: DstReg, Op0: KernargPtrReg,
6404 Op1: B.buildConstant(Res: IdxTy, Val: Offset).getReg(Idx: 0));
6405 return true;
6406}
6407
6408/// To create a buffer resource from a 64-bit pointer, mask off the upper 32
6409/// bits of the pointer and replace them with the stride argument, then
6410/// merge_values everything together. In the common case of a raw buffer (the
6411/// stride component is 0), we can just AND off the upper half.
6412bool AMDGPULegalizerInfo::legalizePointerAsRsrcIntrin(
6413 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
6414 Register Result = MI.getOperand(i: 0).getReg();
6415 Register Pointer = MI.getOperand(i: 2).getReg();
6416 Register Stride = MI.getOperand(i: 3).getReg();
6417 Register NumRecords = MI.getOperand(i: 4).getReg();
6418 Register Flags = MI.getOperand(i: 5).getReg();
6419
6420 LLT I32 = LLT::integer(SizeInBits: 32);
6421 LLT I64 = LLT::integer(SizeInBits: 64);
6422
6423 B.setInsertPt(MBB&: B.getMBB(), II: ++B.getInsertPt());
6424
6425 auto ExtStride = B.buildAnyExt(Res: I32, Op: Stride);
6426
6427 if (ST.has45BitNumRecordsBufferResource()) {
6428 Register Zero = B.buildConstant(Res: I32, Val: 0).getReg(Idx: 0);
6429 // Build the lower 64-bit value, which has a 57-bit base and the lower 7-bit
6430 // num_records.
6431 LLT PtrIntTy = LLT::integer(SizeInBits: MRI.getType(Reg: Pointer).getSizeInBits());
6432 auto PointerInt = B.buildPtrToInt(Dst: PtrIntTy, Src: Pointer);
6433 auto ExtPointer = B.buildAnyExtOrTrunc(Res: I64, Op: PointerInt);
6434 auto NumRecordsLHS = B.buildShl(Dst: I64, Src0: NumRecords, Src1: B.buildConstant(Res: I32, Val: 57));
6435 Register LowHalf = B.buildOr(Dst: I64, Src0: ExtPointer, Src1: NumRecordsLHS).getReg(Idx: 0);
6436
6437 // Build the higher 64-bit value, which has the higher 38-bit num_records,
6438 // 6-bit zero (omit), 16-bit stride and scale and 4-bit flag.
6439 auto NumRecordsRHS = B.buildLShr(Dst: I64, Src0: NumRecords, Src1: B.buildConstant(Res: I32, Val: 7));
6440 auto ShiftedStride = B.buildShl(Dst: I32, Src0: ExtStride, Src1: B.buildConstant(Res: I32, Val: 12));
6441 auto ExtShiftedStride =
6442 B.buildMergeValues(Res: I64, Ops: {Zero, ShiftedStride.getReg(Idx: 0)});
6443 auto ShiftedFlags = B.buildShl(Dst: I32, Src0: Flags, Src1: B.buildConstant(Res: I32, Val: 28));
6444 auto ExtShiftedFlags =
6445 B.buildMergeValues(Res: I64, Ops: {Zero, ShiftedFlags.getReg(Idx: 0)});
6446 auto CombinedFields = B.buildOr(Dst: I64, Src0: NumRecordsRHS, Src1: ExtShiftedStride);
6447 Register HighHalf =
6448 B.buildOr(Dst: I64, Src0: CombinedFields, Src1: ExtShiftedFlags).getReg(Idx: 0);
6449 B.buildMergeValues(Res: Result, Ops: {LowHalf, HighHalf});
6450 } else {
6451 NumRecords = B.buildTrunc(Res: I32, Op: NumRecords).getReg(Idx: 0);
6452 auto Unmerge = B.buildUnmerge(Res: I32, Op: Pointer);
6453 auto LowHalf = Unmerge.getReg(Idx: 0);
6454 auto HighHalf = Unmerge.getReg(Idx: 1);
6455
6456 auto AndMask = B.buildConstant(Res: I32, Val: 0x0000ffff);
6457 auto Masked = B.buildAnd(Dst: I32, Src0: HighHalf, Src1: AndMask);
6458 auto ShiftConst = B.buildConstant(Res: I32, Val: 16);
6459 auto ShiftedStride = B.buildShl(Dst: I32, Src0: ExtStride, Src1: ShiftConst);
6460 auto NewHighHalf = B.buildOr(Dst: I32, Src0: Masked, Src1: ShiftedStride);
6461 Register NewHighHalfReg = NewHighHalf.getReg(Idx: 0);
6462 B.buildMergeValues(Res: Result, Ops: {LowHalf, NewHighHalfReg, NumRecords, Flags});
6463 }
6464
6465 MI.eraseFromParent();
6466 return true;
6467}
6468
6469bool AMDGPULegalizerInfo::legalizeImplicitArgPtr(MachineInstr &MI,
6470 MachineRegisterInfo &MRI,
6471 MachineIRBuilder &B) const {
6472 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
6473 if (!MFI->isEntryFunction()) {
6474 return legalizePreloadedArgIntrin(MI, MRI, B,
6475 ArgType: AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR);
6476 }
6477
6478 Register DstReg = MI.getOperand(i: 0).getReg();
6479 if (!getImplicitArgPtr(DstReg, MRI, B))
6480 return false;
6481
6482 MI.eraseFromParent();
6483 return true;
6484}
6485
6486bool AMDGPULegalizerInfo::getLDSKernelId(Register DstReg,
6487 MachineRegisterInfo &MRI,
6488 MachineIRBuilder &B) const {
6489 Function &F = B.getMF().getFunction();
6490 std::optional<uint32_t> KnownSize =
6491 AMDGPUMachineFunctionInfo::getLDSKernelIdMetadata(F);
6492 if (KnownSize.has_value())
6493 B.buildConstant(Res: DstReg, Val: *KnownSize);
6494 return false;
6495}
6496
6497bool AMDGPULegalizerInfo::legalizeLDSKernelId(MachineInstr &MI,
6498 MachineRegisterInfo &MRI,
6499 MachineIRBuilder &B) const {
6500
6501 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
6502 if (!MFI->isEntryFunction()) {
6503 return legalizePreloadedArgIntrin(MI, MRI, B,
6504 ArgType: AMDGPUFunctionArgInfo::LDS_KERNEL_ID);
6505 }
6506
6507 Register DstReg = MI.getOperand(i: 0).getReg();
6508 if (!getLDSKernelId(DstReg, MRI, B))
6509 return false;
6510
6511 MI.eraseFromParent();
6512 return true;
6513}
6514
6515bool AMDGPULegalizerInfo::legalizeIsAddrSpace(MachineInstr &MI,
6516 MachineRegisterInfo &MRI,
6517 MachineIRBuilder &B,
6518 unsigned AddrSpace) const {
6519 const LLT I32 = LLT::integer(SizeInBits: 32);
6520 auto Unmerge = B.buildUnmerge(Res: I32, Op: MI.getOperand(i: 2).getReg());
6521 Register Hi32 = Unmerge.getReg(Idx: 1);
6522
6523 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS &&
6524 ST.hasGloballyAddressableScratch()) {
6525 Register FlatScratchBaseHi =
6526 B.buildInstr(Opc: AMDGPU::S_MOV_B32, DstOps: {I32},
6527 SrcOps: {Register(AMDGPU::SRC_FLAT_SCRATCH_BASE_HI)})
6528 .getReg(Idx: 0);
6529 MRI.setRegClass(Reg: FlatScratchBaseHi, RC: &AMDGPU::SReg_32RegClass);
6530 // Test bits 63..58 against the aperture address.
6531 Register XOR = B.buildXor(Dst: I32, Src0: Hi32, Src1: FlatScratchBaseHi).getReg(Idx: 0);
6532 B.buildICmp(Pred: ICmpInst::ICMP_ULT, Res: MI.getOperand(i: 0), Op0: XOR,
6533 Op1: B.buildConstant(Res: I32, Val: 1u << 26));
6534 } else {
6535 Register ApertureReg = getSegmentAperture(AS: AddrSpace, MRI, B);
6536 B.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: MI.getOperand(i: 0), Op0: Hi32, Op1: ApertureReg);
6537 }
6538 MI.eraseFromParent();
6539 return true;
6540}
6541
6542// The raw.(t)buffer and struct.(t)buffer intrinsics have two offset args:
6543// offset (the offset that is included in bounds checking and swizzling, to be
6544// split between the instruction's voffset and immoffset fields) and soffset
6545// (the offset that is excluded from bounds checking and swizzling, to go in
6546// the instruction's soffset field). This function takes the first kind of
6547// offset and figures out how to split it between voffset and immoffset.
6548std::pair<Register, unsigned>
6549AMDGPULegalizerInfo::splitBufferOffsets(MachineIRBuilder &B,
6550 Register OrigOffset) const {
6551 const unsigned MaxImm = SIInstrInfo::getMaxMUBUFImmOffset(ST);
6552 Register BaseReg;
6553 unsigned ImmOffset;
6554 const LLT I32 = LLT::integer(SizeInBits: 32);
6555 MachineRegisterInfo &MRI = *B.getMRI();
6556
6557 // On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
6558 // being added, so we can only safely match a 32-bit addition with no unsigned
6559 // overflow.
6560 bool CheckNUW = ST.hasGFX1250Insts();
6561 std::tie(args&: BaseReg, args&: ImmOffset) = AMDGPU::getBaseWithConstantOffset(
6562 MRI, Reg: OrigOffset, /*KnownBits=*/ValueTracking: nullptr, CheckNUW);
6563
6564 // If BaseReg is a pointer, convert it to int.
6565 if (MRI.getType(Reg: BaseReg).isPointer())
6566 BaseReg = B.buildPtrToInt(Dst: MRI.getType(Reg: OrigOffset), Src: BaseReg).getReg(Idx: 0);
6567
6568 // If the immediate value is too big for the immoffset field, put only bits
6569 // that would normally fit in the immoffset field. The remaining value that
6570 // is copied/added for the voffset field is a large power of 2, and it
6571 // stands more chance of being CSEd with the copy/add for another similar
6572 // load/store.
6573 // However, do not do that rounding down if that is a negative
6574 // number, as it appears to be illegal to have a negative offset in the
6575 // vgpr, even if adding the immediate offset makes it positive.
6576 unsigned Overflow = ImmOffset & ~MaxImm;
6577 ImmOffset -= Overflow;
6578 if ((int32_t)Overflow < 0) {
6579 Overflow += ImmOffset;
6580 ImmOffset = 0;
6581 }
6582
6583 if (Overflow != 0) {
6584 if (!BaseReg) {
6585 BaseReg = B.buildConstant(Res: I32, Val: Overflow).getReg(Idx: 0);
6586 } else {
6587 auto OverflowVal = B.buildConstant(Res: I32, Val: Overflow);
6588 BaseReg = B.buildAdd(Dst: I32, Src0: BaseReg, Src1: OverflowVal).getReg(Idx: 0);
6589 }
6590 }
6591
6592 if (!BaseReg)
6593 BaseReg = B.buildConstant(Res: I32, Val: 0).getReg(Idx: 0);
6594
6595 return std::pair(BaseReg, ImmOffset);
6596}
6597
6598/// Handle register layout difference for f16 images for some subtargets.
6599Register AMDGPULegalizerInfo::handleD16VData(MachineIRBuilder &B,
6600 MachineRegisterInfo &MRI,
6601 Register Reg,
6602 bool ImageStore) const {
6603 const LLT I16 = LLT::integer(SizeInBits: 16);
6604 const LLT I32 = LLT::integer(SizeInBits: 32);
6605 LLT StoreVT = MRI.getType(Reg);
6606 assert(StoreVT.isVector() && StoreVT.getElementType().getSizeInBits() == 16);
6607
6608 LLT I16Vec = StoreVT.changeElementType(NewEltTy: I16);
6609 Register RegI16 =
6610 StoreVT == I16Vec ? Reg : B.buildBitcast(Dst: I16Vec, Src: Reg).getReg(Idx: 0);
6611
6612 if (ST.hasUnpackedD16VMem()) {
6613 auto Unmerge = B.buildUnmerge(Res: I16, Op: RegI16);
6614
6615 SmallVector<Register, 4> WideRegs;
6616 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
6617 WideRegs.push_back(Elt: B.buildAnyExt(Res: I32, Op: Unmerge.getReg(Idx: I)).getReg(Idx: 0));
6618
6619 int NumElts = StoreVT.getNumElements();
6620
6621 return B.buildBuildVector(Res: LLT::fixed_vector(NumElements: NumElts, ScalarTy: I32), Ops: WideRegs)
6622 .getReg(Idx: 0);
6623 }
6624
6625 if (ImageStore && ST.hasImageStoreD16Bug()) {
6626 if (StoreVT.getNumElements() == 2) {
6627 SmallVector<Register, 4> PackedRegs;
6628 Reg = B.buildBitcast(Dst: I32, Src: RegI16).getReg(Idx: 0);
6629 PackedRegs.push_back(Elt: Reg);
6630 PackedRegs.resize(N: 2, NV: B.buildUndef(Res: I32).getReg(Idx: 0));
6631 return B.buildBuildVector(Res: LLT::fixed_vector(NumElements: 2, ScalarTy: I32), Ops: PackedRegs)
6632 .getReg(Idx: 0);
6633 }
6634
6635 if (StoreVT.getNumElements() == 3) {
6636 SmallVector<Register, 4> PackedRegs;
6637 auto Unmerge = B.buildUnmerge(Res: I16, Op: RegI16);
6638 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
6639 PackedRegs.push_back(Elt: Unmerge.getReg(Idx: I));
6640 PackedRegs.resize(N: 6, NV: B.buildUndef(Res: I16).getReg(Idx: 0));
6641 Reg = B.buildBuildVector(Res: LLT::fixed_vector(NumElements: 6, ScalarTy: I16), Ops: PackedRegs).getReg(Idx: 0);
6642 return B.buildBitcast(Dst: LLT::fixed_vector(NumElements: 3, ScalarTy: I32), Src: Reg).getReg(Idx: 0);
6643 }
6644
6645 if (StoreVT.getNumElements() == 4) {
6646 SmallVector<Register, 4> PackedRegs;
6647 Reg = B.buildBitcast(Dst: LLT::fixed_vector(NumElements: 2, ScalarTy: I32), Src: RegI16).getReg(Idx: 0);
6648 auto Unmerge = B.buildUnmerge(Res: I32, Op: Reg);
6649 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
6650 PackedRegs.push_back(Elt: Unmerge.getReg(Idx: I));
6651 PackedRegs.resize(N: 4, NV: B.buildUndef(Res: I32).getReg(Idx: 0));
6652 return B.buildBuildVector(Res: LLT::fixed_vector(NumElements: 4, ScalarTy: I32), Ops: PackedRegs)
6653 .getReg(Idx: 0);
6654 }
6655
6656 llvm_unreachable("invalid data type");
6657 }
6658
6659 if (StoreVT.isVector() && StoreVT.getNumElements() == 3 &&
6660 StoreVT.getElementType().getSizeInBits() == 16) {
6661 Reg = B.buildPadVectorWithUndefElements(
6662 Res: LLT::fixed_vector(NumElements: 4, ScalarTy: StoreVT.getElementType()), Op0: Reg)
6663 .getReg(Idx: 0);
6664 }
6665 return Reg;
6666}
6667
6668Register AMDGPULegalizerInfo::fixStoreSourceType(MachineIRBuilder &B,
6669 Register VData, LLT MemTy,
6670 bool IsFormat) const {
6671 MachineRegisterInfo *MRI = B.getMRI();
6672 LLT Ty = MRI->getType(Reg: VData);
6673
6674 // Fixup buffer resources themselves needing to be v4i128.
6675 if (hasBufferRsrcWorkaround(Ty))
6676 return castBufferRsrcToV4I32(Pointer: VData, B);
6677
6678 if (shouldBitcastLoadStoreType(ST, Ty, MemTy)) {
6679 Ty = getBitcastRegisterType(Ty);
6680 VData = B.buildBitcast(Dst: Ty, Src: VData).getReg(Idx: 0);
6681 }
6682 // Fixup illegal register types for i8 stores.
6683 if (Ty == LLT::integer(SizeInBits: 8) || Ty == LLT::integer(SizeInBits: 16) || Ty == LLT::float16()) {
6684 Register AnyExt = B.buildAnyExt(Res: LLT::integer(SizeInBits: 32), Op: VData).getReg(Idx: 0);
6685 return AnyExt;
6686 }
6687
6688 if (Ty.isVector()) {
6689 if (Ty.getElementType().getSizeInBits() == 16 && Ty.getNumElements() <= 4) {
6690 if (IsFormat)
6691 return handleD16VData(B, MRI&: *MRI, Reg: VData);
6692 }
6693 }
6694
6695 return VData;
6696}
6697
6698bool AMDGPULegalizerInfo::legalizeBufferStore(MachineInstr &MI,
6699 LegalizerHelper &Helper,
6700 bool IsTyped,
6701 bool IsFormat) const {
6702 MachineIRBuilder &B = Helper.MIRBuilder;
6703 MachineRegisterInfo &MRI = *B.getMRI();
6704
6705 Register VData = MI.getOperand(i: 1).getReg();
6706 LLT Ty = MRI.getType(Reg: VData);
6707 LLT EltTy = Ty.getScalarType();
6708 const bool IsD16 = IsFormat && (EltTy.getSizeInBits() == 16);
6709 const LLT I32 = LLT::integer(SizeInBits: 32);
6710
6711 MachineMemOperand *MMO = *MI.memoperands_begin();
6712 const int MemSize = MMO->getSize().getValue();
6713 LLT MemTy = MMO->getMemoryType();
6714
6715 if (IsFormat && !IsTyped && !IsD16 && MemTy.getSizeInBits() < 32) {
6716 const Function &Fn = B.getMF().getFunction();
6717 Fn.getContext().diagnose(DI: DiagnosticInfoUnsupported(
6718 Fn, "unsupported sub-dword format buffer store", MI.getDebugLoc()));
6719 MI.eraseFromParent();
6720 return true;
6721 }
6722
6723 VData = fixStoreSourceType(B, VData, MemTy, IsFormat);
6724
6725 castBufferRsrcArgToV4I32(MI, B, Idx: 2);
6726 Register RSrc = MI.getOperand(i: 2).getReg();
6727
6728 unsigned ImmOffset;
6729
6730 // The typed intrinsics add an immediate after the registers.
6731 const unsigned NumVIndexOps = IsTyped ? 8 : 7;
6732
6733 // The struct intrinsic variants add one additional operand over raw.
6734 const bool HasVIndex = MI.getNumOperands() == NumVIndexOps;
6735 Register VIndex;
6736 int OpOffset = 0;
6737 if (HasVIndex) {
6738 VIndex = MI.getOperand(i: 3).getReg();
6739 OpOffset = 1;
6740 } else {
6741 VIndex = B.buildConstant(Res: I32, Val: 0).getReg(Idx: 0);
6742 }
6743
6744 Register VOffset = MI.getOperand(i: 3 + OpOffset).getReg();
6745 Register SOffset = MI.getOperand(i: 4 + OpOffset).getReg();
6746
6747 unsigned Format = 0;
6748 if (IsTyped) {
6749 Format = MI.getOperand(i: 5 + OpOffset).getImm();
6750 ++OpOffset;
6751 }
6752
6753 unsigned AuxiliaryData = MI.getOperand(i: 5 + OpOffset).getImm();
6754
6755 std::tie(args&: VOffset, args&: ImmOffset) = splitBufferOffsets(B, OrigOffset: VOffset);
6756
6757 unsigned Opc;
6758 if (IsTyped) {
6759 Opc = IsD16 ? AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16 :
6760 AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT;
6761 } else if (IsFormat) {
6762 Opc = IsD16 ? AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16 :
6763 AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT;
6764 } else {
6765 switch (MemSize) {
6766 case 1:
6767 Opc = AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE;
6768 break;
6769 case 2:
6770 Opc = AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT;
6771 break;
6772 default:
6773 Opc = AMDGPU::G_AMDGPU_BUFFER_STORE;
6774 break;
6775 }
6776 }
6777
6778 auto MIB = B.buildInstr(Opcode: Opc)
6779 .addUse(RegNo: VData) // vdata
6780 .addUse(RegNo: RSrc) // rsrc
6781 .addUse(RegNo: VIndex) // vindex
6782 .addUse(RegNo: VOffset) // voffset
6783 .addUse(RegNo: SOffset) // soffset
6784 .addImm(Val: ImmOffset); // offset(imm)
6785
6786 if (IsTyped)
6787 MIB.addImm(Val: Format);
6788
6789 MIB.addImm(Val: AuxiliaryData) // cachepolicy, swizzled buffer(imm)
6790 .addImm(Val: HasVIndex ? -1 : 0) // idxen(imm)
6791 .addMemOperand(MMO);
6792
6793 MI.eraseFromParent();
6794 return true;
6795}
6796
6797static void buildBufferLoad(unsigned Opc, Register LoadDstReg, Register RSrc,
6798 Register VIndex, Register VOffset, Register SOffset,
6799 unsigned ImmOffset, unsigned Format,
6800 unsigned AuxiliaryData, MachineMemOperand *MMO,
6801 bool IsTyped, bool HasVIndex, MachineIRBuilder &B) {
6802 auto MIB = B.buildInstr(Opcode: Opc)
6803 .addDef(RegNo: LoadDstReg) // vdata
6804 .addUse(RegNo: RSrc) // rsrc
6805 .addUse(RegNo: VIndex) // vindex
6806 .addUse(RegNo: VOffset) // voffset
6807 .addUse(RegNo: SOffset) // soffset
6808 .addImm(Val: ImmOffset); // offset(imm)
6809
6810 if (IsTyped)
6811 MIB.addImm(Val: Format);
6812
6813 MIB.addImm(Val: AuxiliaryData) // cachepolicy, swizzled buffer(imm)
6814 .addImm(Val: HasVIndex ? -1 : 0) // idxen(imm)
6815 .addMemOperand(MMO);
6816}
6817
6818bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
6819 LegalizerHelper &Helper,
6820 bool IsFormat,
6821 bool IsTyped) const {
6822 MachineIRBuilder &B = Helper.MIRBuilder;
6823 MachineRegisterInfo &MRI = *B.getMRI();
6824 GISelChangeObserver &Observer = Helper.Observer;
6825
6826 // FIXME: Verifier should enforce 1 MMO for these intrinsics.
6827 MachineMemOperand *MMO = *MI.memoperands_begin();
6828 const LLT MemTy = MMO->getMemoryType();
6829 const LLT I32 = LLT::integer(SizeInBits: 32);
6830
6831 Register Dst = MI.getOperand(i: 0).getReg();
6832
6833 Register StatusDst;
6834 int OpOffset = 0;
6835 assert(MI.getNumExplicitDefs() == 1 || MI.getNumExplicitDefs() == 2);
6836 bool IsTFE = MI.getNumExplicitDefs() == 2;
6837 if (IsTFE) {
6838 StatusDst = MI.getOperand(i: 1).getReg();
6839 ++OpOffset;
6840 }
6841
6842 castBufferRsrcArgToV4I32(MI, B, Idx: 2 + OpOffset);
6843 Register RSrc = MI.getOperand(i: 2 + OpOffset).getReg();
6844
6845 // The typed intrinsics add an immediate after the registers.
6846 const unsigned NumVIndexOps = IsTyped ? 8 : 7;
6847
6848 // The struct intrinsic variants add one additional operand over raw.
6849 const bool HasVIndex = MI.getNumOperands() == NumVIndexOps + OpOffset;
6850 Register VIndex;
6851 if (HasVIndex) {
6852 VIndex = MI.getOperand(i: 3 + OpOffset).getReg();
6853 ++OpOffset;
6854 } else {
6855 VIndex = B.buildConstant(Res: I32, Val: 0).getReg(Idx: 0);
6856 }
6857
6858 Register VOffset = MI.getOperand(i: 3 + OpOffset).getReg();
6859 Register SOffset = MI.getOperand(i: 4 + OpOffset).getReg();
6860
6861 unsigned Format = 0;
6862 if (IsTyped) {
6863 Format = MI.getOperand(i: 5 + OpOffset).getImm();
6864 ++OpOffset;
6865 }
6866
6867 unsigned AuxiliaryData = MI.getOperand(i: 5 + OpOffset).getImm();
6868 unsigned ImmOffset;
6869
6870 LLT Ty = MRI.getType(Reg: Dst);
6871 // Make addrspace 8 pointers loads into 4xi32 loads here, so the rest of the
6872 // logic doesn't have to handle that case.
6873 if (hasBufferRsrcWorkaround(Ty)) {
6874 Observer.changingInstr(MI);
6875 Ty = castBufferRsrcFromV4I32(MI, B, MRI, Idx: 0);
6876 Observer.changedInstr(MI);
6877 Dst = MI.getOperand(i: 0).getReg();
6878 B.setInsertPt(MBB&: B.getMBB(), II: MI);
6879 }
6880 if (shouldBitcastLoadStoreType(ST, Ty, MemTy)) {
6881 Ty = getBitcastRegisterType(Ty);
6882 Observer.changingInstr(MI);
6883 Helper.bitcastDst(MI, CastTy: Ty, OpIdx: 0);
6884 Observer.changedInstr(MI);
6885 Dst = MI.getOperand(i: 0).getReg();
6886 B.setInsertPt(MBB&: B.getMBB(), II: MI);
6887 }
6888
6889 LLT EltTy = Ty.getScalarType();
6890 const bool IsD16 = IsFormat && (EltTy.getSizeInBits() == 16);
6891 const bool Unpacked = ST.hasUnpackedD16VMem();
6892
6893 if (IsFormat && !IsTyped && !IsD16 && MemTy.getSizeInBits() < 32) {
6894 const Function &Fn = B.getMF().getFunction();
6895 Fn.getContext().diagnose(DI: DiagnosticInfoUnsupported(
6896 Fn, "unsupported sub-dword format buffer load", MI.getDebugLoc()));
6897 B.buildUndef(Res: Dst);
6898 if (IsTFE)
6899 B.buildUndef(Res: StatusDst);
6900 MI.eraseFromParent();
6901 return true;
6902 }
6903
6904 std::tie(args&: VOffset, args&: ImmOffset) = splitBufferOffsets(B, OrigOffset: VOffset);
6905
6906 unsigned Opc;
6907
6908 // TODO: Support TFE for typed and narrow loads.
6909 if (IsTyped) {
6910 if (IsTFE)
6911 return false;
6912 Opc = IsD16 ? AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 :
6913 AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT;
6914 } else if (IsFormat) {
6915 if (IsD16) {
6916 if (IsTFE)
6917 return false;
6918 Opc = AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16;
6919 } else {
6920 Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_TFE
6921 : AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT;
6922 }
6923 } else {
6924 switch (MemTy.getSizeInBits()) {
6925 case 8:
6926 Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE_TFE
6927 : AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE;
6928 break;
6929 case 16:
6930 Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT_TFE
6931 : AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT;
6932 break;
6933 default:
6934 Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_TFE
6935 : AMDGPU::G_AMDGPU_BUFFER_LOAD;
6936 break;
6937 }
6938 }
6939
6940 if (IsTFE) {
6941 unsigned NumValueDWords = divideCeil(Numerator: Ty.getSizeInBits(), Denominator: 32);
6942 unsigned NumLoadDWords = NumValueDWords + 1;
6943 LLT LoadTy = LLT::fixed_vector(NumElements: NumLoadDWords, ScalarTy: I32);
6944 Register LoadDstReg = B.getMRI()->createGenericVirtualRegister(Ty: LoadTy);
6945 buildBufferLoad(Opc, LoadDstReg, RSrc, VIndex, VOffset, SOffset, ImmOffset,
6946 Format, AuxiliaryData, MMO, IsTyped, HasVIndex, B);
6947 bool IsFloat = Ty.getScalarType().isFloat();
6948 LLT DstIntTy =
6949 IsFloat ? Ty.changeElementType(NewEltTy: LLT::integer(SizeInBits: EltTy.getSizeInBits()))
6950 : Ty;
6951 Register DstInt =
6952 IsFloat ? B.getMRI()->createGenericVirtualRegister(Ty: DstIntTy) : Dst;
6953 if (MemTy.getSizeInBits() < 32) {
6954 Register ExtDst = B.getMRI()->createGenericVirtualRegister(Ty: I32);
6955 B.buildUnmerge(Res: {ExtDst, StatusDst}, Op: LoadDstReg);
6956 B.buildTrunc(Res: DstInt, Op: ExtDst);
6957 } else if (NumValueDWords == 1) {
6958 B.buildUnmerge(Res: {DstInt, StatusDst}, Op: LoadDstReg);
6959 } else {
6960 SmallVector<Register, 5> LoadElts;
6961 for (unsigned I = 0; I != NumValueDWords; ++I)
6962 LoadElts.push_back(Elt: B.getMRI()->createGenericVirtualRegister(Ty: I32));
6963 LoadElts.push_back(Elt: StatusDst);
6964 B.buildUnmerge(Res: LoadElts, Op: LoadDstReg);
6965 LoadElts.truncate(N: NumValueDWords);
6966 B.buildMergeLikeInstr(Res: DstInt, Ops: LoadElts);
6967 }
6968 if (DstInt != Dst)
6969 B.buildBitcast(Dst, Src: DstInt);
6970 } else if ((!IsD16 && MemTy.getSizeInBits() < 32) ||
6971 (IsD16 && !Ty.isVector())) {
6972 Register LoadDstReg = B.getMRI()->createGenericVirtualRegister(Ty: I32);
6973 buildBufferLoad(Opc, LoadDstReg, RSrc, VIndex, VOffset, SOffset, ImmOffset,
6974 Format, AuxiliaryData, MMO, IsTyped, HasVIndex, B);
6975 B.setInsertPt(MBB&: B.getMBB(), II: ++B.getInsertPt());
6976 B.buildTrunc(Res: Dst, Op: LoadDstReg);
6977 } else if (Unpacked && IsD16 && Ty.isVector()) {
6978 LLT UnpackedTy = LLT::fixed_vector(NumElements: Ty.getNumElements(), ScalarTy: LLT::integer(SizeInBits: 32));
6979 Register LoadDstReg = B.getMRI()->createGenericVirtualRegister(Ty: UnpackedTy);
6980 buildBufferLoad(Opc, LoadDstReg, RSrc, VIndex, VOffset, SOffset, ImmOffset,
6981 Format, AuxiliaryData, MMO, IsTyped, HasVIndex, B);
6982 B.setInsertPt(MBB&: B.getMBB(), II: ++B.getInsertPt());
6983 // FIXME: G_TRUNC should work, but legalization currently fails
6984 auto Unmerge = B.buildUnmerge(Res: I32, Op: LoadDstReg);
6985 SmallVector<Register, 4> Repack;
6986 for (unsigned I = 0, N = Unmerge->getNumOperands() - 1; I != N; ++I)
6987 Repack.push_back(Elt: B.buildTrunc(Res: EltTy, Op: Unmerge.getReg(Idx: I)).getReg(Idx: 0));
6988 B.buildMergeLikeInstr(Res: Dst, Ops: Repack);
6989 } else {
6990 buildBufferLoad(Opc, LoadDstReg: Dst, RSrc, VIndex, VOffset, SOffset, ImmOffset, Format,
6991 AuxiliaryData, MMO, IsTyped, HasVIndex, B);
6992 }
6993
6994 MI.eraseFromParent();
6995 return true;
6996}
6997
6998static unsigned getBufferAtomicPseudo(Intrinsic::ID IntrID) {
6999 switch (IntrID) {
7000 case Intrinsic::amdgcn_raw_buffer_atomic_swap:
7001 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
7002 case Intrinsic::amdgcn_struct_buffer_atomic_swap:
7003 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
7004 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP;
7005 case Intrinsic::amdgcn_raw_buffer_atomic_add:
7006 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
7007 case Intrinsic::amdgcn_struct_buffer_atomic_add:
7008 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
7009 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD;
7010 case Intrinsic::amdgcn_raw_buffer_atomic_sub:
7011 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
7012 case Intrinsic::amdgcn_struct_buffer_atomic_sub:
7013 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
7014 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB;
7015 case Intrinsic::amdgcn_raw_buffer_atomic_smin:
7016 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
7017 case Intrinsic::amdgcn_struct_buffer_atomic_smin:
7018 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
7019 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN;
7020 case Intrinsic::amdgcn_raw_buffer_atomic_umin:
7021 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
7022 case Intrinsic::amdgcn_struct_buffer_atomic_umin:
7023 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
7024 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN;
7025 case Intrinsic::amdgcn_raw_buffer_atomic_smax:
7026 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
7027 case Intrinsic::amdgcn_struct_buffer_atomic_smax:
7028 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
7029 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX;
7030 case Intrinsic::amdgcn_raw_buffer_atomic_umax:
7031 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
7032 case Intrinsic::amdgcn_struct_buffer_atomic_umax:
7033 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
7034 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX;
7035 case Intrinsic::amdgcn_raw_buffer_atomic_and:
7036 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
7037 case Intrinsic::amdgcn_struct_buffer_atomic_and:
7038 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
7039 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND;
7040 case Intrinsic::amdgcn_raw_buffer_atomic_or:
7041 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
7042 case Intrinsic::amdgcn_struct_buffer_atomic_or:
7043 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
7044 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR;
7045 case Intrinsic::amdgcn_raw_buffer_atomic_xor:
7046 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
7047 case Intrinsic::amdgcn_struct_buffer_atomic_xor:
7048 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
7049 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR;
7050 case Intrinsic::amdgcn_raw_buffer_atomic_inc:
7051 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
7052 case Intrinsic::amdgcn_struct_buffer_atomic_inc:
7053 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
7054 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC;
7055 case Intrinsic::amdgcn_raw_buffer_atomic_dec:
7056 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
7057 case Intrinsic::amdgcn_struct_buffer_atomic_dec:
7058 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
7059 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC;
7060 case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
7061 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap:
7062 case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
7063 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap:
7064 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP;
7065 case Intrinsic::amdgcn_raw_buffer_atomic_fadd:
7066 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
7067 case Intrinsic::amdgcn_struct_buffer_atomic_fadd:
7068 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
7069 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD;
7070 case Intrinsic::amdgcn_raw_buffer_atomic_fmin:
7071 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
7072 case Intrinsic::amdgcn_struct_buffer_atomic_fmin:
7073 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
7074 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN;
7075 case Intrinsic::amdgcn_raw_buffer_atomic_fmax:
7076 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
7077 case Intrinsic::amdgcn_struct_buffer_atomic_fmax:
7078 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
7079 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX;
7080 case Intrinsic::amdgcn_raw_buffer_atomic_sub_clamp_u32:
7081 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub_clamp_u32:
7082 case Intrinsic::amdgcn_struct_buffer_atomic_sub_clamp_u32:
7083 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub_clamp_u32:
7084 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB_CLAMP_U32;
7085 case Intrinsic::amdgcn_raw_buffer_atomic_cond_sub_u32:
7086 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cond_sub_u32:
7087 case Intrinsic::amdgcn_struct_buffer_atomic_cond_sub_u32:
7088 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cond_sub_u32:
7089 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_COND_SUB_U32;
7090 default:
7091 llvm_unreachable("unhandled atomic opcode");
7092 }
7093}
7094
7095bool AMDGPULegalizerInfo::legalizeBufferAtomic(MachineInstr &MI,
7096 MachineIRBuilder &B,
7097 Intrinsic::ID IID) const {
7098 const bool IsCmpSwap =
7099 IID == Intrinsic::amdgcn_raw_buffer_atomic_cmpswap ||
7100 IID == Intrinsic::amdgcn_struct_buffer_atomic_cmpswap ||
7101 IID == Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap ||
7102 IID == Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap;
7103
7104 Register Dst = MI.getOperand(i: 0).getReg();
7105 // Since we don't have 128-bit atomics, we don't need to handle the case of
7106 // p8 argmunents to the atomic itself
7107 Register VData = MI.getOperand(i: 2).getReg();
7108
7109 Register CmpVal;
7110 int OpOffset = 0;
7111
7112 if (IsCmpSwap) {
7113 CmpVal = MI.getOperand(i: 3).getReg();
7114 ++OpOffset;
7115 }
7116
7117 castBufferRsrcArgToV4I32(MI, B, Idx: 3 + OpOffset);
7118 Register RSrc = MI.getOperand(i: 3 + OpOffset).getReg();
7119 const unsigned NumVIndexOps = IsCmpSwap ? 9 : 8;
7120
7121 // The struct intrinsic variants add one additional operand over raw.
7122 const bool HasVIndex = MI.getNumOperands() == NumVIndexOps;
7123 Register VIndex;
7124 if (HasVIndex) {
7125 VIndex = MI.getOperand(i: 4 + OpOffset).getReg();
7126 ++OpOffset;
7127 } else {
7128 VIndex = B.buildConstant(Res: LLT::integer(SizeInBits: 32), Val: 0).getReg(Idx: 0);
7129 }
7130
7131 Register VOffset = MI.getOperand(i: 4 + OpOffset).getReg();
7132 Register SOffset = MI.getOperand(i: 5 + OpOffset).getReg();
7133 unsigned AuxiliaryData = MI.getOperand(i: 6 + OpOffset).getImm();
7134
7135 MachineMemOperand *MMO = *MI.memoperands_begin();
7136
7137 unsigned ImmOffset;
7138 std::tie(args&: VOffset, args&: ImmOffset) = splitBufferOffsets(B, OrigOffset: VOffset);
7139
7140 auto MIB = B.buildInstr(Opcode: getBufferAtomicPseudo(IntrID: IID))
7141 .addDef(RegNo: Dst)
7142 .addUse(RegNo: VData); // vdata
7143
7144 if (IsCmpSwap)
7145 MIB.addReg(RegNo: CmpVal);
7146
7147 MIB.addUse(RegNo: RSrc) // rsrc
7148 .addUse(RegNo: VIndex) // vindex
7149 .addUse(RegNo: VOffset) // voffset
7150 .addUse(RegNo: SOffset) // soffset
7151 .addImm(Val: ImmOffset) // offset(imm)
7152 .addImm(Val: AuxiliaryData) // cachepolicy, swizzled buffer(imm)
7153 .addImm(Val: HasVIndex ? -1 : 0) // idxen(imm)
7154 .addMemOperand(MMO);
7155
7156 MI.eraseFromParent();
7157 return true;
7158}
7159
7160/// Turn a set of f16 typed registers in \p AddrRegs into a dword sized
7161/// vector with f16 typed elements.
7162static void packImage16bitOpsToDwords(MachineIRBuilder &B, MachineInstr &MI,
7163 SmallVectorImpl<Register> &PackedAddrs,
7164 unsigned ArgOffset,
7165 const AMDGPU::ImageDimIntrinsicInfo *Intr,
7166 bool IsA16, bool IsG16) {
7167 const LLT F16 = LLT::float16();
7168 const LLT V2F16 = LLT::fixed_vector(NumElements: 2, ScalarTy: F16);
7169 auto EndIdx = Intr->VAddrEnd;
7170
7171 for (unsigned I = Intr->VAddrStart; I < EndIdx; I++) {
7172 MachineOperand &SrcOp = MI.getOperand(i: ArgOffset + I);
7173 if (!SrcOp.isReg())
7174 continue; // _L to _LZ may have eliminated this.
7175
7176 Register AddrReg = SrcOp.getReg();
7177
7178 if ((I < Intr->GradientStart) ||
7179 (I >= Intr->GradientStart && I < Intr->CoordStart && !IsG16) ||
7180 (I >= Intr->CoordStart && !IsA16)) {
7181 if ((I < Intr->GradientStart) && IsA16 &&
7182 (B.getMRI()->getType(Reg: AddrReg) == F16)) {
7183 assert(I == Intr->BiasIndex && "Got unexpected 16-bit extra argument");
7184 // Special handling of bias when A16 is on. Bias is of type half but
7185 // occupies full 32-bit.
7186 PackedAddrs.push_back(
7187 Elt: B.buildBuildVector(Res: V2F16, Ops: {AddrReg, B.buildUndef(Res: F16).getReg(Idx: 0)})
7188 .getReg(Idx: 0));
7189 } else {
7190 assert((!IsA16 || Intr->NumBiasArgs == 0 || I != Intr->BiasIndex) &&
7191 "Bias needs to be converted to 16 bit in A16 mode");
7192 // Handle any gradient or coordinate operands that should not be packed
7193 AddrReg = B.buildBitcast(Dst: V2F16, Src: AddrReg).getReg(Idx: 0);
7194 PackedAddrs.push_back(Elt: AddrReg);
7195 }
7196 } else {
7197 const LLT EltTy = B.getMRI()->getType(Reg: AddrReg);
7198 const LLT V2EltTy = LLT::fixed_vector(NumElements: 2, ScalarTy: EltTy);
7199 // Dz/dh, dz/dv and the last odd coord are packed with undef. Also, in 1D,
7200 // derivatives dx/dh and dx/dv are packed with undef.
7201 if (((I + 1) >= EndIdx) ||
7202 ((Intr->NumGradients / 2) % 2 == 1 &&
7203 (I == static_cast<unsigned>(Intr->GradientStart +
7204 (Intr->NumGradients / 2) - 1) ||
7205 I == static_cast<unsigned>(Intr->GradientStart +
7206 Intr->NumGradients - 1))) ||
7207 // Check for _L to _LZ optimization
7208 !MI.getOperand(i: ArgOffset + I + 1).isReg()) {
7209 PackedAddrs.push_back(
7210 Elt: B.buildBuildVector(Res: V2EltTy,
7211 Ops: {AddrReg, B.buildUndef(Res: EltTy).getReg(Idx: 0)})
7212 .getReg(Idx: 0));
7213 } else {
7214 PackedAddrs.push_back(
7215 Elt: B.buildBuildVector(
7216 Res: V2EltTy, Ops: {AddrReg, MI.getOperand(i: ArgOffset + I + 1).getReg()})
7217 .getReg(Idx: 0));
7218 ++I;
7219 }
7220 }
7221 }
7222}
7223
7224/// Convert from separate vaddr components to a single vector address register,
7225/// and replace the remaining operands with $noreg.
7226static void convertImageAddrToPacked(MachineIRBuilder &B, MachineInstr &MI,
7227 int DimIdx, int NumVAddrs) {
7228 SmallVector<Register, 8> AddrRegs;
7229 for (int I = 0; I != NumVAddrs; ++I) {
7230 MachineOperand &SrcOp = MI.getOperand(i: DimIdx + I);
7231 if (SrcOp.isReg()) {
7232 Register Reg = SrcOp.getReg();
7233 LLT I32 = LLT::integer(SizeInBits: 32);
7234 assert(B.getMRI()->getType(Reg).getSizeInBits() == 32);
7235 if (B.getMRI()->getType(Reg) != I32)
7236 Reg = B.buildBitcast(Dst: I32, Src: Reg).getReg(Idx: 0);
7237 AddrRegs.push_back(Elt: Reg);
7238 }
7239 }
7240
7241 int NumAddrRegs = AddrRegs.size();
7242 if (NumAddrRegs != 1) {
7243 LLT EltTy = B.getMRI()->getType(Reg: AddrRegs[0]);
7244 auto VAddr =
7245 B.buildBuildVector(Res: LLT::fixed_vector(NumElements: NumAddrRegs, ScalarTy: EltTy), Ops: AddrRegs);
7246 MI.getOperand(i: DimIdx).setReg(VAddr.getReg(Idx: 0));
7247 }
7248
7249 for (int I = 1; I != NumVAddrs; ++I) {
7250 MachineOperand &SrcOp = MI.getOperand(i: DimIdx + I);
7251 if (SrcOp.isReg())
7252 MI.getOperand(i: DimIdx + I).setReg(AMDGPU::NoRegister);
7253 }
7254}
7255
7256/// Rewrite image intrinsics to use register layouts expected by the subtarget.
7257///
7258/// Depending on the subtarget, load/store with 16-bit element data need to be
7259/// rewritten to use the low half of 32-bit registers, or directly use a packed
7260/// layout. 16-bit addresses should also sometimes be packed into 32-bit
7261/// registers.
7262///
7263/// We don't want to directly select image instructions just yet, but also want
7264/// to exposes all register repacking to the legalizer/combiners. We also don't
7265/// want a selected instruction entering RegBankSelect. In order to avoid
7266/// defining a multitude of intermediate image instructions, directly hack on
7267/// the intrinsic's arguments. In cases like a16 addresses, this requires
7268/// padding now unnecessary arguments with $noreg.
7269bool AMDGPULegalizerInfo::legalizeImageIntrinsic(
7270 MachineInstr &MI, MachineIRBuilder &B, GISelChangeObserver &Observer,
7271 const AMDGPU::ImageDimIntrinsicInfo *Intr) const {
7272
7273 const MachineFunction &MF = *MI.getMF();
7274 const unsigned NumDefs = MI.getNumExplicitDefs();
7275 const unsigned ArgOffset = NumDefs + 1;
7276 bool IsTFE = NumDefs == 2;
7277 // We are only processing the operands of d16 image operations on subtargets
7278 // that use the unpacked register layout, or need to repack the TFE result.
7279
7280 // TODO: Do we need to guard against already legalized intrinsics?
7281 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
7282 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Intr->BaseOpcode);
7283
7284 MachineRegisterInfo *MRI = B.getMRI();
7285 const LLT I32 = LLT::integer(SizeInBits: 32);
7286 const LLT I16 = LLT::integer(SizeInBits: 16);
7287 const LLT F16 = LLT::float16();
7288 const LLT V2I16 = LLT::fixed_vector(NumElements: 2, ScalarTy: I16);
7289 const LLT V2F16 = LLT::fixed_vector(NumElements: 2, ScalarTy: F16);
7290
7291 unsigned DMask = 0;
7292 Register VData;
7293 LLT Ty;
7294
7295 if (!BaseOpcode->NoReturn || BaseOpcode->Store) {
7296 VData = MI.getOperand(i: NumDefs == 0 ? 1 : 0).getReg();
7297 Ty = MRI->getType(Reg: VData);
7298 }
7299
7300 const bool IsAtomicPacked16Bit =
7301 (BaseOpcode->BaseOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16 ||
7302 BaseOpcode->BaseOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16);
7303
7304 // Check for 16 bit addresses and pack if true.
7305 LLT GradTy =
7306 MRI->getType(Reg: MI.getOperand(i: ArgOffset + Intr->GradientStart).getReg());
7307 LLT AddrTy =
7308 MRI->getType(Reg: MI.getOperand(i: ArgOffset + Intr->CoordStart).getReg());
7309 const bool GradTyIs16 = GradTy == I16 || GradTy == F16;
7310 const bool AddrTyIs16 = AddrTy == I16 || AddrTy == F16;
7311 const bool DataTyIs16 =
7312 Ty.getScalarType() == I16 || Ty.getScalarType() == F16;
7313 const bool IsG16 =
7314 ST.hasG16() ? (BaseOpcode->Gradients && GradTyIs16) : GradTyIs16;
7315 const bool IsA16 = AddrTyIs16;
7316 const bool IsD16 = !IsAtomicPacked16Bit && DataTyIs16;
7317
7318 int DMaskLanes = 0;
7319 if (!BaseOpcode->Atomic) {
7320 DMask = MI.getOperand(i: ArgOffset + Intr->DMaskIndex).getImm();
7321 if (BaseOpcode->Gather4) {
7322 DMaskLanes = 4;
7323 } else if (DMask != 0) {
7324 DMaskLanes = llvm::popcount(Value: DMask);
7325 } else if (!IsTFE && !BaseOpcode->Store) {
7326 // If dmask is 0, this is a no-op load. This can be eliminated.
7327 B.buildUndef(Res: MI.getOperand(i: 0));
7328 MI.eraseFromParent();
7329 return true;
7330 }
7331 }
7332
7333 Observer.changingInstr(MI);
7334 scope_exit ChangedInstr([&] { Observer.changedInstr(MI); });
7335
7336 const unsigned StoreOpcode = IsD16 ? AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16
7337 : AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE;
7338 const unsigned LoadOpcode = IsD16 ? AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16
7339 : AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD;
7340 unsigned NewOpcode = LoadOpcode;
7341 if (BaseOpcode->Store)
7342 NewOpcode = StoreOpcode;
7343 else if (BaseOpcode->NoReturn)
7344 NewOpcode = AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET;
7345
7346 // Track that we legalized this
7347 MI.setDesc(B.getTII().get(Opcode: NewOpcode));
7348
7349 // Expecting to get an error flag since TFC is on - and dmask is 0 Force
7350 // dmask to be at least 1 otherwise the instruction will fail
7351 if (IsTFE && DMask == 0) {
7352 DMask = 0x1;
7353 DMaskLanes = 1;
7354 MI.getOperand(i: ArgOffset + Intr->DMaskIndex).setImm(DMask);
7355 }
7356
7357 if (BaseOpcode->Atomic) {
7358 Register VData0 = MI.getOperand(i: 2).getReg();
7359 LLT Ty = MRI->getType(Reg: VData0);
7360
7361 // TODO: Allow atomic swap and bit ops for v2f16/v4f16
7362 if (Ty.isVector() && !IsAtomicPacked16Bit)
7363 return false;
7364
7365 if (BaseOpcode->AtomicX2) {
7366 Register VData1 = MI.getOperand(i: 3).getReg();
7367 // The two values are packed in one register.
7368 LLT PackedTy = LLT::fixed_vector(NumElements: 2, ScalarTy: Ty);
7369 auto Concat = B.buildBuildVector(Res: PackedTy, Ops: {VData0, VData1});
7370 MI.getOperand(i: 2).setReg(Concat.getReg(Idx: 0));
7371 MI.getOperand(i: 3).setReg(AMDGPU::NoRegister);
7372 }
7373 }
7374
7375 unsigned CorrectedNumVAddrs = Intr->NumVAddrs;
7376
7377 // Rewrite the addressing register layout before doing anything else.
7378 if (BaseOpcode->Gradients && !ST.hasG16() && (IsA16 != IsG16)) {
7379 // 16 bit gradients are supported, but are tied to the A16 control
7380 // so both gradients and addresses must be 16 bit
7381 return false;
7382 }
7383
7384 if (IsA16 && !ST.hasA16()) {
7385 // A16 not supported
7386 return false;
7387 }
7388
7389 const unsigned NSAMaxSize = ST.getNSAMaxSize(HasSampler: BaseOpcode->Sampler);
7390 const unsigned HasPartialNSA = ST.hasPartialNSAEncoding();
7391
7392 if (IsA16 || IsG16) {
7393 // Even if NumVAddrs == 1 we should pack it into a 32-bit value, because the
7394 // instructions expect VGPR_32
7395 SmallVector<Register, 4> PackedRegs;
7396
7397 packImage16bitOpsToDwords(B, MI, PackedAddrs&: PackedRegs, ArgOffset, Intr, IsA16, IsG16);
7398
7399 // See also below in the non-a16 branch
7400 const bool UseNSA = ST.hasNSAEncoding() &&
7401 PackedRegs.size() >= ST.getNSAThreshold(MF) &&
7402 (PackedRegs.size() <= NSAMaxSize || HasPartialNSA);
7403 const bool UsePartialNSA =
7404 UseNSA && HasPartialNSA && PackedRegs.size() > NSAMaxSize;
7405
7406 if (UsePartialNSA) {
7407 // Pack registers that would go over NSAMaxSize into last VAddr register
7408 LLT PackedAddrTy =
7409 LLT::fixed_vector(NumElements: 2 * (PackedRegs.size() - NSAMaxSize + 1), ScalarTy: F16);
7410 auto Concat = B.buildConcatVectors(
7411 Res: PackedAddrTy, Ops: ArrayRef(PackedRegs).slice(N: NSAMaxSize - 1));
7412 PackedRegs[NSAMaxSize - 1] = Concat.getReg(Idx: 0);
7413 PackedRegs.resize(N: NSAMaxSize);
7414 } else if (!UseNSA && PackedRegs.size() > 1) {
7415 LLT PackedAddrTy = LLT::fixed_vector(NumElements: 2 * PackedRegs.size(), ScalarTy: F16);
7416 auto Concat = B.buildConcatVectors(Res: PackedAddrTy, Ops: PackedRegs);
7417 PackedRegs[0] = Concat.getReg(Idx: 0);
7418 PackedRegs.resize(N: 1);
7419 }
7420
7421 const unsigned NumPacked = PackedRegs.size();
7422 for (unsigned I = Intr->VAddrStart; I < Intr->VAddrEnd; I++) {
7423 MachineOperand &SrcOp = MI.getOperand(i: ArgOffset + I);
7424 if (!SrcOp.isReg()) {
7425 assert(SrcOp.isImm() && SrcOp.getImm() == 0);
7426 continue;
7427 }
7428
7429 assert(SrcOp.getReg() != AMDGPU::NoRegister);
7430
7431 if (I - Intr->VAddrStart < NumPacked)
7432 SrcOp.setReg(PackedRegs[I - Intr->VAddrStart]);
7433 else
7434 SrcOp.setReg(AMDGPU::NoRegister);
7435 }
7436 } else {
7437 // If the register allocator cannot place the address registers contiguously
7438 // without introducing moves, then using the non-sequential address encoding
7439 // is always preferable, since it saves VALU instructions and is usually a
7440 // wash in terms of code size or even better.
7441 //
7442 // However, we currently have no way of hinting to the register allocator
7443 // that MIMG addresses should be placed contiguously when it is possible to
7444 // do so, so force non-NSA for the common 2-address case as a heuristic.
7445 //
7446 // SIShrinkInstructions will convert NSA encodings to non-NSA after register
7447 // allocation when possible.
7448 //
7449 // Partial NSA is allowed on GFX11+ where the final register is a contiguous
7450 // set of the remaining addresses.
7451 const bool UseNSA = ST.hasNSAEncoding() &&
7452 CorrectedNumVAddrs >= ST.getNSAThreshold(MF) &&
7453 (CorrectedNumVAddrs <= NSAMaxSize || HasPartialNSA);
7454 const bool UsePartialNSA =
7455 UseNSA && HasPartialNSA && CorrectedNumVAddrs > NSAMaxSize;
7456
7457 if (UsePartialNSA) {
7458 convertImageAddrToPacked(B, MI,
7459 DimIdx: ArgOffset + Intr->VAddrStart + NSAMaxSize - 1,
7460 NumVAddrs: Intr->NumVAddrs - NSAMaxSize + 1);
7461 } else if (!UseNSA && Intr->NumVAddrs > 1) {
7462 convertImageAddrToPacked(B, MI, DimIdx: ArgOffset + Intr->VAddrStart,
7463 NumVAddrs: Intr->NumVAddrs);
7464 }
7465 }
7466
7467 int Flags = 0;
7468 if (IsA16)
7469 Flags |= 1;
7470 if (IsG16)
7471 Flags |= 2;
7472 MI.addOperand(Op: MachineOperand::CreateImm(Val: Flags));
7473
7474 if (BaseOpcode->NoReturn) { // No TFE for stores?
7475 // TODO: Handle dmask trim
7476 if (!Ty.isVector() || !IsD16)
7477 return true;
7478
7479 Register RepackedReg = handleD16VData(B, MRI&: *MRI, Reg: VData, ImageStore: true);
7480 if (RepackedReg != VData) {
7481 MI.getOperand(i: 1).setReg(RepackedReg);
7482 }
7483
7484 return true;
7485 }
7486
7487 Register DstReg = MI.getOperand(i: 0).getReg();
7488 const LLT EltTy = Ty.getScalarType();
7489 const int NumElts = Ty.isVector() ? Ty.getNumElements() : 1;
7490
7491 // Confirm that the return type is large enough for the dmask specified
7492 if (NumElts < DMaskLanes)
7493 return false;
7494
7495 if (NumElts > 4 || DMaskLanes > 4)
7496 return false;
7497
7498 // Image atomic instructions are using DMask to specify how many bits
7499 // input/output data will have. 32-bits (i32, f32, v2f16) or 64-bits (i64,
7500 // f64, v4f16).
7501 // DMaskLanes for image atomic has default value '0'.
7502 // We must be sure that atomic variants (especially packed) will not be
7503 // truncated from v2f16 or v4f16 to f16 type.
7504 //
7505 // ChangeElementCount will be needed for image load where Ty is always scalar.
7506 const unsigned AdjustedNumElts = DMaskLanes == 0 ? 1 : DMaskLanes;
7507 const LLT AdjustedTy =
7508 DMaskLanes == 0
7509 ? Ty
7510 : Ty.changeElementCount(EC: ElementCount::getFixed(MinVal: AdjustedNumElts));
7511
7512 // The raw dword aligned data component of the load. The only legal cases
7513 // where this matters should be when using the packed D16 format, for
7514 // f16 -> <2 x f16>, and <3 x f16> -> <4 x f16>,
7515 LLT RoundedTy;
7516
7517 // I32 vector to cover all data, plus TFE result element.
7518 LLT TFETy;
7519
7520 // Register type to use for each loaded component. Will be I32 or V2I16.
7521 LLT RegTy;
7522
7523 if (IsD16 && ST.hasUnpackedD16VMem()) {
7524 RoundedTy =
7525 LLT::scalarOrVector(EC: ElementCount::getFixed(MinVal: AdjustedNumElts), ScalarTy: I32);
7526 TFETy = LLT::fixed_vector(NumElements: AdjustedNumElts + 1, ScalarTy: I32);
7527 RegTy = I32;
7528 } else {
7529 unsigned EltSize = EltTy.getSizeInBits();
7530 unsigned RoundedElts = (AdjustedTy.getSizeInBits() + 31) / 32;
7531 unsigned RoundedSize = 32 * RoundedElts;
7532 RoundedTy = LLT::scalarOrVector(
7533 EC: ElementCount::getFixed(MinVal: RoundedSize / EltSize), ScalarTy: EltTy);
7534 TFETy = LLT::fixed_vector(NumElements: RoundedSize / 32 + 1, ScalarTy: I32);
7535 RegTy = !IsTFE && EltSize == 16 ? V2I16 : I32;
7536 }
7537
7538 // The return type does not need adjustment.
7539 // TODO: Should we change f16 case to i32 or <2 x f16>?
7540 if (!IsTFE && (RoundedTy == Ty || !Ty.isVector()))
7541 return true;
7542
7543 Register Dst1Reg;
7544
7545 // Insert after the instruction.
7546 B.setInsertPt(MBB&: *MI.getParent(), II: ++MI.getIterator());
7547
7548 // TODO: For TFE with d16, if we used a TFE type that was a multiple of <2 x
7549 // f16> instead of i32, we would only need 1 bitcast instead of multiple.
7550 const LLT LoadResultTy = IsTFE ? TFETy : RoundedTy;
7551 const int ResultNumRegs = LoadResultTy.getSizeInBits() / 32;
7552
7553 Register NewResultReg = MRI->createGenericVirtualRegister(Ty: LoadResultTy);
7554
7555 MI.getOperand(i: 0).setReg(NewResultReg);
7556
7557 // In the IR, TFE is supposed to be used with a 2 element struct return
7558 // type. The instruction really returns these two values in one contiguous
7559 // register, with one additional dword beyond the loaded data. Rewrite the
7560 // return type to use a single register result.
7561
7562 if (IsTFE) {
7563 Dst1Reg = MI.getOperand(i: 1).getReg();
7564 if (MRI->getType(Reg: Dst1Reg) != I32)
7565 return false;
7566
7567 // TODO: Make sure the TFE operand bit is set.
7568 MI.removeOperand(OpNo: 1);
7569
7570 // Handle the easy case that requires no repack instructions.
7571 if (!Ty.isVector() && Ty.getSizeInBits() == 32) {
7572 auto Unmerge = B.buildUnmerge(Res: {I32, I32}, Op: NewResultReg);
7573 B.buildBitcast(Dst: DstReg, Src: Unmerge.getReg(Idx: 0));
7574 B.buildCopy(Res: Dst1Reg, Op: Unmerge.getReg(Idx: 1));
7575 return true;
7576 }
7577 }
7578
7579 // Now figure out how to copy the new result register back into the old
7580 // result.
7581 SmallVector<Register, 5> ResultRegs(ResultNumRegs, Dst1Reg);
7582
7583 const int NumDataRegs = IsTFE ? ResultNumRegs - 1 : ResultNumRegs;
7584
7585 if (ResultNumRegs == 1) {
7586 assert(!IsTFE);
7587 ResultRegs[0] = NewResultReg;
7588 } else {
7589 // We have to repack into a new vector of some kind.
7590 for (int I = 0; I != NumDataRegs; ++I)
7591 ResultRegs[I] = MRI->createGenericVirtualRegister(Ty: RegTy);
7592 B.buildUnmerge(Res: ResultRegs, Op: NewResultReg);
7593
7594 // Drop the final TFE element to get the data part. The TFE result is
7595 // directly written to the right place already.
7596 if (IsTFE)
7597 ResultRegs.resize(N: NumDataRegs);
7598 }
7599
7600 // For an f16 scalar result, we form an i32 result with a truncate regardless
7601 // of packed vs. unpacked.
7602 if (IsD16 && !Ty.isVector()) {
7603 B.buildTrunc(Res: DstReg, Op: ResultRegs[0]);
7604 return true;
7605 }
7606
7607 // Avoid a build/concat_vector of 1 entry.
7608 if ((Ty == V2I16 || Ty == V2F16) && NumDataRegs == 1 &&
7609 !ST.hasUnpackedD16VMem()) {
7610 B.buildBitcast(Dst: DstReg, Src: ResultRegs[0]);
7611 return true;
7612 }
7613
7614 assert(Ty.isVector());
7615
7616 if (IsD16) {
7617 // For packed D16 results with TFE enabled, all the data components are
7618 // I32. Cast back to the expected type.
7619 //
7620 // TODO: We don't really need to use load i32 elements. We would only need
7621 // one cast for the TFE result if a multiple of v2f16 was used.
7622 if (RegTy != V2I16 && !ST.hasUnpackedD16VMem()) {
7623 for (Register &Reg : ResultRegs)
7624 Reg = B.buildBitcast(Dst: V2I16, Src: Reg).getReg(Idx: 0);
7625 } else if (ST.hasUnpackedD16VMem()) {
7626 for (Register &Reg : ResultRegs)
7627 Reg = B.buildTrunc(Res: I16, Op: Reg).getReg(Idx: 0);
7628 }
7629 }
7630
7631 auto padWithUndef = [&](LLT Ty, int NumElts) {
7632 if (NumElts == 0)
7633 return;
7634 Register Undef = B.buildUndef(Res: Ty).getReg(Idx: 0);
7635 for (int I = 0; I != NumElts; ++I)
7636 ResultRegs.push_back(Elt: Undef);
7637 };
7638
7639 // Pad out any elements eliminated due to the dmask.
7640 LLT ResTy = MRI->getType(Reg: ResultRegs[0]);
7641 if (!ResTy.isVector()) {
7642 padWithUndef(ResTy, NumElts - ResultRegs.size());
7643 B.buildBuildVector(Res: DstReg, Ops: ResultRegs);
7644 return true;
7645 }
7646
7647 assert(!ST.hasUnpackedD16VMem() && (ResTy == V2I16 || ResTy == V2F16));
7648 const int RegsToCover = (Ty.getSizeInBits() + 31) / 32;
7649
7650 // Deal with the one annoying legal case.
7651 const LLT V3I16 = LLT::fixed_vector(NumElements: 3, ScalarTy: I16);
7652 const LLT V3F16 = LLT::fixed_vector(NumElements: 3, ScalarTy: F16);
7653 if (Ty == V3I16 || Ty == V3F16) {
7654 if (IsTFE) {
7655 if (ResultRegs.size() == 1) {
7656 NewResultReg = ResultRegs[0];
7657 } else if (ResultRegs.size() == 2) {
7658 LLT V4I16 = LLT::fixed_vector(NumElements: 4, ScalarTy: I16);
7659 NewResultReg = B.buildConcatVectors(Res: V4I16, Ops: ResultRegs).getReg(Idx: 0);
7660 } else {
7661 return false;
7662 }
7663 }
7664
7665 LLT DstTy = MRI->getType(Reg: DstReg);
7666 LLT NewResTy = MRI->getType(Reg: NewResultReg);
7667 LLT ResEltTy = NewResTy.getElementType();
7668 Register ResizeDst = DstTy.getElementType() == ResEltTy
7669 ? DstReg
7670 : MRI->createGenericVirtualRegister(
7671 Ty: DstTy.changeElementType(NewEltTy: ResEltTy));
7672
7673 if (DstTy.getNumElements() < NewResTy.getNumElements()) {
7674 B.buildDeleteTrailingVectorElements(Res: ResizeDst, Op0: NewResultReg);
7675 } else {
7676 B.buildPadVectorWithUndefElements(Res: ResizeDst, Op0: NewResultReg);
7677 }
7678 if (ResizeDst != DstReg)
7679 B.buildBitcast(Dst: DstReg, Src: ResizeDst);
7680 return true;
7681 }
7682
7683 padWithUndef(ResTy, RegsToCover - ResultRegs.size());
7684 B.buildConcatVectors(Res: DstReg, Ops: ResultRegs);
7685 return true;
7686}
7687
7688bool AMDGPULegalizerInfo::legalizeSBufferLoad(LegalizerHelper &Helper,
7689 MachineInstr &MI) const {
7690 MachineIRBuilder &B = Helper.MIRBuilder;
7691 GISelChangeObserver &Observer = Helper.Observer;
7692
7693 Register OrigDst = MI.getOperand(i: 0).getReg();
7694 Register Dst;
7695 LLT Ty = B.getMRI()->getType(Reg: OrigDst);
7696 unsigned Size = Ty.getSizeInBits();
7697 MachineFunction &MF = B.getMF();
7698 unsigned Opc = 0;
7699 if (Size < 32 && ST.hasScalarSubwordLoads()) {
7700 assert(Size == 8 || Size == 16);
7701 Opc = Size == 8 ? AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE
7702 : AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT;
7703 // The 8-bit and 16-bit scalar buffer load instructions have 32-bit
7704 // destination register.
7705 Dst = B.getMRI()->createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: 32));
7706 } else {
7707 Opc = AMDGPU::G_AMDGPU_S_BUFFER_LOAD;
7708 Dst = OrigDst;
7709 }
7710
7711 Observer.changingInstr(MI);
7712
7713 // Handle needing to s.buffer.load() a p8 value.
7714 if (hasBufferRsrcWorkaround(Ty)) {
7715 Ty = castBufferRsrcFromV4I32(MI, B, MRI&: *B.getMRI(), Idx: 0);
7716 B.setInsertPt(MBB&: B.getMBB(), II: MI);
7717 }
7718 if (shouldBitcastLoadStoreType(ST, Ty, MemTy: LLT::integer(SizeInBits: Size))) {
7719 Ty = getBitcastRegisterType(Ty);
7720 Helper.bitcastDst(MI, CastTy: Ty, OpIdx: 0);
7721 B.setInsertPt(MBB&: B.getMBB(), II: MI);
7722 }
7723
7724 // FIXME: We don't really need this intermediate instruction. The intrinsic
7725 // should be fixed to have a memory operand. Since it's readnone, we're not
7726 // allowed to add one.
7727 MI.setDesc(B.getTII().get(Opcode: Opc));
7728 MI.removeOperand(OpNo: 1); // Remove intrinsic ID
7729
7730 // FIXME: When intrinsic definition is fixed, this should have an MMO already.
7731 const unsigned MemSize = (Size + 7) / 8;
7732 const Align MemAlign = B.getDataLayout().getABITypeAlign(
7733 Ty: getTypeForLLT(Ty, C&: MF.getFunction().getContext()));
7734 MachineMemOperand *MMO = MF.getMachineMemOperand(
7735 PtrInfo: MachinePointerInfo(),
7736 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
7737 MachineMemOperand::MOInvariant,
7738 Size: MemSize, BaseAlignment: MemAlign);
7739 MI.addMemOperand(MF, MO: MMO);
7740 if (Dst != OrigDst) {
7741 MI.getOperand(i: 0).setReg(Dst);
7742 B.setInsertPt(MBB&: B.getMBB(), II: ++B.getInsertPt());
7743 B.buildTrunc(Res: OrigDst, Op: Dst);
7744 }
7745
7746 // If we don't have 96-bit result scalar loads, widening to 128-bit should
7747 // always be legal. We may need to restore this to a 96-bit result if it turns
7748 // out this needs to be converted to a vector load during RegBankSelect.
7749 if (!isPowerOf2_32(Value: Size) && (Size != 96 || !ST.hasScalarDwordx3Loads())) {
7750 if (Ty.isVector())
7751 Helper.moreElementsVectorDst(MI, MoreTy: getPow2VectorType(Ty), OpIdx: 0);
7752 else
7753 Helper.widenScalarDst(MI, WideTy: getPow2ScalarType(Ty), OpIdx: 0);
7754 }
7755
7756 Observer.changedInstr(MI);
7757 return true;
7758}
7759
7760bool AMDGPULegalizerInfo::legalizeSBufferPrefetch(LegalizerHelper &Helper,
7761 MachineInstr &MI) const {
7762 MachineIRBuilder &B = Helper.MIRBuilder;
7763 GISelChangeObserver &Observer = Helper.Observer;
7764 Observer.changingInstr(MI);
7765 MI.setDesc(B.getTII().get(Opcode: AMDGPU::G_AMDGPU_S_BUFFER_PREFETCH));
7766 MI.removeOperand(OpNo: 0); // Remove intrinsic ID
7767 castBufferRsrcArgToV4I32(MI, B, Idx: 0);
7768 Observer.changedInstr(MI);
7769 return true;
7770}
7771
7772// TODO: Move to selection
7773bool AMDGPULegalizerInfo::legalizeTrap(MachineInstr &MI,
7774 MachineRegisterInfo &MRI,
7775 MachineIRBuilder &B) const {
7776 if (!ST.hasTrapHandler() ||
7777 ST.getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA)
7778 return legalizeTrapEndpgm(MI, MRI, B);
7779
7780 return ST.supportsGetDoorbellID() ?
7781 legalizeTrapHsa(MI, MRI, B) : legalizeTrapHsaQueuePtr(MI, MRI, B);
7782}
7783
7784bool AMDGPULegalizerInfo::legalizeTrapEndpgm(
7785 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
7786 const DebugLoc &DL = MI.getDebugLoc();
7787 MachineBasicBlock &BB = B.getMBB();
7788 MachineFunction *MF = BB.getParent();
7789
7790 if (BB.succ_empty() && std::next(x: MI.getIterator()) == BB.end()) {
7791 BuildMI(BB, I: BB.end(), MIMD: DL, MCID: B.getTII().get(Opcode: AMDGPU::S_ENDPGM))
7792 .addImm(Val: 0);
7793 MI.eraseFromParent();
7794 return true;
7795 }
7796
7797 // We need a block split to make the real endpgm a terminator. We also don't
7798 // want to break phis in successor blocks, so we can't just delete to the
7799 // end of the block.
7800 BB.splitAt(SplitInst&: MI, UpdateLiveIns: false /*UpdateLiveIns*/);
7801 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
7802 MF->push_back(MBB: TrapBB);
7803 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: B.getTII().get(Opcode: AMDGPU::S_ENDPGM))
7804 .addImm(Val: 0);
7805 BuildMI(BB, I: &MI, MIMD: DL, MCID: B.getTII().get(Opcode: AMDGPU::S_CBRANCH_EXECNZ))
7806 .addMBB(MBB: TrapBB);
7807
7808 BB.addSuccessor(Succ: TrapBB);
7809 MI.eraseFromParent();
7810 return true;
7811}
7812
7813bool AMDGPULegalizerInfo::legalizeTrapHsaQueuePtr(
7814 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
7815 MachineFunction &MF = B.getMF();
7816 const LLT I64 = LLT::integer(SizeInBits: 64);
7817
7818 Register SGPR01(AMDGPU::SGPR0_SGPR1);
7819 // For code object version 5, queue_ptr is passed through implicit kernarg.
7820 if (AMDGPU::getAMDHSACodeObjectVersion(M: *MF.getFunction().getParent()) >=
7821 AMDGPU::AMDHSA_COV5) {
7822 AMDGPUTargetLowering::ImplicitParameter Param =
7823 AMDGPUTargetLowering::QUEUE_PTR;
7824 uint64_t Offset =
7825 ST.getTargetLowering()->getImplicitParameterOffset(MF: B.getMF(), Param);
7826
7827 Register KernargPtrReg = MRI.createGenericVirtualRegister(
7828 Ty: LLT::pointer(AddressSpace: AMDGPUAS::CONSTANT_ADDRESS, SizeInBits: 64));
7829
7830 if (!loadInputValue(DstReg: KernargPtrReg, B,
7831 ArgType: AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR))
7832 return false;
7833
7834 // TODO: can we be smarter about machine pointer info?
7835 MachinePointerInfo PtrInfo = getKernargSegmentPtrInfo(MF);
7836 MachineMemOperand *MMO = MF.getMachineMemOperand(
7837 PtrInfo: PtrInfo.getWithOffset(O: Offset),
7838 f: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
7839 MachineMemOperand::MOInvariant,
7840 MemTy: LLT::integer(SizeInBits: 64), base_alignment: commonAlignment(A: Align(64), Offset));
7841
7842 // Pointer address
7843 Register LoadAddr = MRI.createGenericVirtualRegister(
7844 Ty: LLT::pointer(AddressSpace: AMDGPUAS::CONSTANT_ADDRESS, SizeInBits: 64));
7845 B.buildObjectPtrOffset(Res: LoadAddr, Op0: KernargPtrReg,
7846 Op1: B.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: Offset).getReg(Idx: 0));
7847 // Load address
7848 Register Temp = B.buildLoad(Res: I64, Addr: LoadAddr, MMO&: *MMO).getReg(Idx: 0);
7849 B.buildCopy(Res: SGPR01, Op: Temp);
7850 B.buildInstr(Opcode: AMDGPU::S_TRAP)
7851 .addImm(Val: static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap))
7852 .addReg(RegNo: SGPR01, Flags: RegState::Implicit);
7853 MI.eraseFromParent();
7854 return true;
7855 }
7856
7857 // Pass queue pointer to trap handler as input, and insert trap instruction
7858 // Reference: https://llvm.org/docs/AMDGPUUsage.html#trap-handler-abi
7859 Register LiveIn =
7860 MRI.createGenericVirtualRegister(Ty: LLT::pointer(AddressSpace: AMDGPUAS::CONSTANT_ADDRESS, SizeInBits: 64));
7861 if (!loadInputValue(DstReg: LiveIn, B, ArgType: AMDGPUFunctionArgInfo::QUEUE_PTR))
7862 return false;
7863
7864 B.buildCopy(Res: SGPR01, Op: LiveIn);
7865 B.buildInstr(Opcode: AMDGPU::S_TRAP)
7866 .addImm(Val: static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap))
7867 .addReg(RegNo: SGPR01, Flags: RegState::Implicit);
7868
7869 MI.eraseFromParent();
7870 return true;
7871}
7872
7873bool AMDGPULegalizerInfo::legalizeTrapHsa(MachineInstr &MI,
7874 MachineRegisterInfo &MRI,
7875 MachineIRBuilder &B) const {
7876 // We need to simulate the 's_trap 2' instruction on targets that run in
7877 // PRIV=1 (where it is treated as a nop).
7878 if (ST.hasPrivEnabledTrap2NopBug()) {
7879 ST.getInstrInfo()->insertSimulatedTrap(MRI, MBB&: B.getMBB(), MI,
7880 DL: MI.getDebugLoc());
7881 MI.eraseFromParent();
7882 return true;
7883 }
7884
7885 B.buildInstr(Opcode: AMDGPU::S_TRAP)
7886 .addImm(Val: static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap));
7887 MI.eraseFromParent();
7888 return true;
7889}
7890
7891bool AMDGPULegalizerInfo::legalizeDebugTrap(MachineInstr &MI,
7892 MachineRegisterInfo &MRI,
7893 MachineIRBuilder &B) const {
7894 // Is non-HSA path or trap-handler disabled? Then, report a warning
7895 // accordingly
7896 if (!ST.hasTrapHandler() ||
7897 ST.getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) {
7898 Function &Fn = B.getMF().getFunction();
7899 Fn.getContext().diagnose(DI: DiagnosticInfoUnsupported(
7900 Fn, "debugtrap handler not supported", MI.getDebugLoc(), DS_Warning));
7901 } else {
7902 // Insert debug-trap instruction
7903 B.buildInstr(Opcode: AMDGPU::S_TRAP)
7904 .addImm(Val: static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSADebugTrap));
7905 }
7906
7907 MI.eraseFromParent();
7908 return true;
7909}
7910
7911bool AMDGPULegalizerInfo::legalizeBVHIntersectRayIntrinsic(
7912 MachineInstr &MI, MachineIRBuilder &B) const {
7913 MachineRegisterInfo &MRI = *B.getMRI();
7914 const LLT I16 = LLT::integer(SizeInBits: 16);
7915 const LLT I32 = LLT::integer(SizeInBits: 32);
7916 const LLT V2I16 = LLT::fixed_vector(NumElements: 2, ScalarTy: I16);
7917 const LLT V3I32 = LLT::fixed_vector(NumElements: 3, ScalarTy: I32);
7918 const LLT V3I16 = LLT::fixed_vector(NumElements: 3, ScalarTy: I16);
7919
7920 Register DstReg = MI.getOperand(i: 0).getReg();
7921 Register NodePtr = MI.getOperand(i: 2).getReg();
7922 Register RayExtent = MI.getOperand(i: 3).getReg();
7923 Register RayOrigin = MI.getOperand(i: 4).getReg();
7924 Register RayDir = MI.getOperand(i: 5).getReg();
7925 Register RayInvDir = MI.getOperand(i: 6).getReg();
7926 Register TDescr = MI.getOperand(i: 7).getReg();
7927
7928 RayExtent = B.buildBitcast(Dst: I32, Src: RayExtent).getReg(Idx: 0);
7929
7930 const bool IsGFX11 = AMDGPU::isGFX11(STI: ST);
7931 const bool IsGFX11Plus = AMDGPU::isGFX11Plus(STI: ST);
7932 const bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI: ST);
7933 const bool IsA16 = MRI.getType(Reg: RayDir).getElementType().getSizeInBits() == 16;
7934 const bool Is64 = MRI.getType(Reg: NodePtr).getSizeInBits() == 64;
7935 const unsigned NumVDataDwords = 4;
7936 const unsigned NumVAddrDwords = IsA16 ? (Is64 ? 9 : 8) : (Is64 ? 12 : 11);
7937 const unsigned NumVAddrs = IsGFX11Plus ? (IsA16 ? 4 : 5) : NumVAddrDwords;
7938 const bool UseNSA =
7939 IsGFX12Plus || (ST.hasNSAEncoding() && NumVAddrs <= ST.getNSAMaxSize());
7940
7941 const unsigned BaseOpcodes[2][2] = {
7942 {AMDGPU::IMAGE_BVH_INTERSECT_RAY, AMDGPU::IMAGE_BVH_INTERSECT_RAY_a16},
7943 {AMDGPU::IMAGE_BVH64_INTERSECT_RAY,
7944 AMDGPU::IMAGE_BVH64_INTERSECT_RAY_a16}};
7945 int Opcode;
7946 if (UseNSA) {
7947 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: BaseOpcodes[Is64][IsA16],
7948 MIMGEncoding: IsGFX12Plus ? AMDGPU::MIMGEncGfx12
7949 : IsGFX11 ? AMDGPU::MIMGEncGfx11NSA
7950 : AMDGPU::MIMGEncGfx10NSA,
7951 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
7952 } else {
7953 assert(!IsGFX12Plus);
7954 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: BaseOpcodes[Is64][IsA16],
7955 MIMGEncoding: IsGFX11 ? AMDGPU::MIMGEncGfx11Default
7956 : AMDGPU::MIMGEncGfx10Default,
7957 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
7958 }
7959 assert(Opcode != -1);
7960
7961 SmallVector<Register, 12> Ops;
7962 if (UseNSA && IsGFX11Plus) {
7963 auto packLanes = [&Ops, &I32, &V3I32, &B](Register Src) {
7964 auto SrcInt = B.buildBitcast(Dst: V3I32, Src);
7965 auto Unmerge = B.buildUnmerge(Res: {I32, I32, I32}, Op: SrcInt);
7966 auto Merged = B.buildMergeLikeInstr(
7967 Res: V3I32, Ops: {Unmerge.getReg(Idx: 0), Unmerge.getReg(Idx: 1), Unmerge.getReg(Idx: 2)});
7968 Ops.push_back(Elt: Merged.getReg(Idx: 0));
7969 };
7970
7971 Ops.push_back(Elt: NodePtr);
7972 Ops.push_back(Elt: RayExtent);
7973 packLanes(RayOrigin);
7974
7975 if (IsA16) {
7976 auto UnmergeRayDir =
7977 B.buildUnmerge(Res: {I16, I16, I16}, Op: B.buildBitcast(Dst: V3I16, Src: RayDir));
7978 auto UnmergeRayInvDir =
7979 B.buildUnmerge(Res: {I16, I16, I16}, Op: B.buildBitcast(Dst: V3I16, Src: RayInvDir));
7980 auto MergedDir = B.buildMergeLikeInstr(
7981 Res: V3I32,
7982 Ops: {B.buildBitcast(
7983 Dst: I32, Src: B.buildMergeLikeInstr(Res: V2I16, Ops: {UnmergeRayInvDir.getReg(Idx: 0),
7984 UnmergeRayDir.getReg(Idx: 0)}))
7985 .getReg(Idx: 0),
7986 B.buildBitcast(
7987 Dst: I32, Src: B.buildMergeLikeInstr(Res: V2I16, Ops: {UnmergeRayInvDir.getReg(Idx: 1),
7988 UnmergeRayDir.getReg(Idx: 1)}))
7989 .getReg(Idx: 0),
7990 B.buildBitcast(
7991 Dst: I32, Src: B.buildMergeLikeInstr(Res: V2I16, Ops: {UnmergeRayInvDir.getReg(Idx: 2),
7992 UnmergeRayDir.getReg(Idx: 2)}))
7993 .getReg(Idx: 0)});
7994 Ops.push_back(Elt: MergedDir.getReg(Idx: 0));
7995 } else {
7996 packLanes(RayDir);
7997 packLanes(RayInvDir);
7998 }
7999 } else {
8000 if (Is64) {
8001 auto Unmerge = B.buildUnmerge(Res: {I32, I32}, Op: NodePtr);
8002 Ops.push_back(Elt: Unmerge.getReg(Idx: 0));
8003 Ops.push_back(Elt: Unmerge.getReg(Idx: 1));
8004 } else {
8005 Ops.push_back(Elt: NodePtr);
8006 }
8007 Ops.push_back(Elt: RayExtent);
8008
8009 auto packLanes = [&Ops, &I32, &V3I32, &B](Register Src) {
8010 auto SrcInt = B.buildBitcast(Dst: V3I32, Src);
8011 auto Unmerge = B.buildUnmerge(Res: {I32, I32, I32}, Op: SrcInt);
8012 Ops.push_back(Elt: Unmerge.getReg(Idx: 0));
8013 Ops.push_back(Elt: Unmerge.getReg(Idx: 1));
8014 Ops.push_back(Elt: Unmerge.getReg(Idx: 2));
8015 };
8016
8017 packLanes(RayOrigin);
8018 if (IsA16) {
8019 auto UnmergeRayDir =
8020 B.buildUnmerge(Res: {I16, I16, I16}, Op: B.buildBitcast(Dst: V3I16, Src: RayDir));
8021 auto UnmergeRayInvDir =
8022 B.buildUnmerge(Res: {I16, I16, I16}, Op: B.buildBitcast(Dst: V3I16, Src: RayInvDir));
8023 Register R1 = MRI.createGenericVirtualRegister(Ty: I32);
8024 Register R2 = MRI.createGenericVirtualRegister(Ty: I32);
8025 Register R3 = MRI.createGenericVirtualRegister(Ty: I32);
8026 B.buildMergeLikeInstr(Res: R1,
8027 Ops: {UnmergeRayDir.getReg(Idx: 0), UnmergeRayDir.getReg(Idx: 1)});
8028 B.buildMergeLikeInstr(
8029 Res: R2, Ops: {UnmergeRayDir.getReg(Idx: 2), UnmergeRayInvDir.getReg(Idx: 0)});
8030 B.buildMergeLikeInstr(
8031 Res: R3, Ops: {UnmergeRayInvDir.getReg(Idx: 1), UnmergeRayInvDir.getReg(Idx: 2)});
8032 Ops.push_back(Elt: R1);
8033 Ops.push_back(Elt: R2);
8034 Ops.push_back(Elt: R3);
8035 } else {
8036 packLanes(RayDir);
8037 packLanes(RayInvDir);
8038 }
8039 }
8040
8041 if (!UseNSA) {
8042 // Build a single vector containing all the operands so far prepared.
8043 LLT OpTy = LLT::fixed_vector(NumElements: Ops.size(), ScalarTy: I32);
8044 Register MergedOps = B.buildMergeLikeInstr(Res: OpTy, Ops).getReg(Idx: 0);
8045 Ops.clear();
8046 Ops.push_back(Elt: MergedOps);
8047 }
8048
8049 auto MIB = B.buildInstr(Opcode: AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY)
8050 .addDef(RegNo: DstReg)
8051 .addImm(Val: Opcode);
8052
8053 for (Register R : Ops) {
8054 MIB.addUse(RegNo: R);
8055 }
8056
8057 MIB.addUse(RegNo: TDescr)
8058 .addImm(Val: IsA16 ? 1 : 0)
8059 .cloneMemRefs(OtherMI: MI);
8060
8061 MI.eraseFromParent();
8062 return true;
8063}
8064
8065bool AMDGPULegalizerInfo::legalizeBVHDualOrBVH8IntersectRayIntrinsic(
8066 MachineInstr &MI, MachineIRBuilder &B) const {
8067 const LLT I32 = LLT::integer(SizeInBits: 32);
8068 const LLT V2I32 = LLT::fixed_vector(NumElements: 2, ScalarTy: I32);
8069
8070 Register DstReg = MI.getOperand(i: 0).getReg();
8071 Register DstOrigin = MI.getOperand(i: 1).getReg();
8072 Register DstDir = MI.getOperand(i: 2).getReg();
8073 Register NodePtr = MI.getOperand(i: 4).getReg();
8074 Register RayExtent = MI.getOperand(i: 5).getReg();
8075 Register InstanceMask = MI.getOperand(i: 6).getReg();
8076 Register RayOrigin = MI.getOperand(i: 7).getReg();
8077 Register RayDir = MI.getOperand(i: 8).getReg();
8078 Register Offsets = MI.getOperand(i: 9).getReg();
8079 Register TDescr = MI.getOperand(i: 10).getReg();
8080
8081 bool IsBVH8 = cast<GIntrinsic>(Val&: MI).getIntrinsicID() ==
8082 Intrinsic::amdgcn_image_bvh8_intersect_ray;
8083 const unsigned NumVDataDwords = 10;
8084 const unsigned NumVAddrDwords = IsBVH8 ? 11 : 12;
8085 int Opcode = AMDGPU::getMIMGOpcode(
8086 BaseOpcode: IsBVH8 ? AMDGPU::IMAGE_BVH8_INTERSECT_RAY
8087 : AMDGPU::IMAGE_BVH_DUAL_INTERSECT_RAY,
8088 MIMGEncoding: AMDGPU::MIMGEncGfx12, VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
8089 assert(Opcode != -1);
8090
8091 auto RayExtentInstanceMaskVec =
8092 B.buildMergeLikeInstr(Res: V2I32, Ops: {B.buildBitcast(Dst: I32, Src: RayExtent),
8093 B.buildAnyExt(Res: I32, Op: InstanceMask)});
8094
8095 B.buildInstr(Opcode: IsBVH8 ? AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY
8096 : AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY)
8097 .addDef(RegNo: DstReg)
8098 .addDef(RegNo: DstOrigin)
8099 .addDef(RegNo: DstDir)
8100 .addImm(Val: Opcode)
8101 .addUse(RegNo: NodePtr)
8102 .addUse(RegNo: RayExtentInstanceMaskVec.getReg(Idx: 0))
8103 .addUse(RegNo: RayOrigin)
8104 .addUse(RegNo: RayDir)
8105 .addUse(RegNo: Offsets)
8106 .addUse(RegNo: TDescr)
8107 .cloneMemRefs(OtherMI: MI);
8108
8109 MI.eraseFromParent();
8110 return true;
8111}
8112
8113bool AMDGPULegalizerInfo::legalizeStackSave(MachineInstr &MI,
8114 MachineIRBuilder &B) const {
8115 const SITargetLowering *TLI = ST.getTargetLowering();
8116 Register StackPtr = TLI->getStackPointerRegisterToSaveRestore();
8117 Register DstReg = MI.getOperand(i: 0).getReg();
8118 B.buildInstr(Opc: AMDGPU::G_AMDGPU_WAVE_ADDRESS, DstOps: {DstReg}, SrcOps: {StackPtr});
8119 MI.eraseFromParent();
8120 return true;
8121}
8122
8123bool AMDGPULegalizerInfo::legalizeWaveID(MachineInstr &MI,
8124 MachineIRBuilder &B) const {
8125 // With architected SGPRs, waveIDinGroup is in TTMP8[29:25].
8126 if (!ST.hasArchitectedSGPRs())
8127 return false;
8128 LLT I32 = LLT::integer(SizeInBits: 32);
8129 Register DstReg = MI.getOperand(i: 0).getReg();
8130 auto TTMP8 = B.buildCopy(Res: I32, Op: Register(AMDGPU::TTMP8));
8131 auto LSB = B.buildConstant(Res: I32, Val: 25);
8132 auto Width = B.buildConstant(Res: I32, Val: 5);
8133 B.buildUbfx(Dst: DstReg, Src: TTMP8, LSB, Width);
8134 MI.eraseFromParent();
8135 return true;
8136}
8137
8138bool AMDGPULegalizerInfo::legalizeConstHwRegRead(MachineInstr &MI,
8139 MachineIRBuilder &B,
8140 AMDGPU::Hwreg::Id HwReg,
8141 unsigned LowBit,
8142 unsigned Width) const {
8143 MachineRegisterInfo &MRI = *B.getMRI();
8144 Register DstReg = MI.getOperand(i: 0).getReg();
8145 if (!MRI.getRegClassOrNull(Reg: DstReg))
8146 MRI.setRegClass(Reg: DstReg, RC: &AMDGPU::SReg_32RegClass);
8147 B.buildInstr(Opcode: AMDGPU::S_GETREG_B32_const)
8148 .addDef(RegNo: DstReg)
8149 .addImm(Val: AMDGPU::Hwreg::HwregEncoding::encode(Values: HwReg, Values: LowBit, Values: Width));
8150 MI.eraseFromParent();
8151 return true;
8152}
8153
8154static constexpr unsigned FPEnvModeBitField =
8155 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 23);
8156
8157static constexpr unsigned FPEnvTrapBitField =
8158 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_TRAPSTS, Values: 0, Values: 5);
8159
8160bool AMDGPULegalizerInfo::legalizeGetFPEnv(MachineInstr &MI,
8161 MachineRegisterInfo &MRI,
8162 MachineIRBuilder &B) const {
8163 const LLT I32 = LLT::integer(SizeInBits: 32);
8164 const LLT I64 = LLT::integer(SizeInBits: 64);
8165 Register Src = MI.getOperand(i: 0).getReg();
8166 if (MRI.getType(Reg: Src) != I64)
8167 return false;
8168
8169 auto ModeReg =
8170 B.buildIntrinsic(ID: Intrinsic::amdgcn_s_getreg, Res: {I32},
8171 /*HasSideEffects=*/true, /*isConvergent=*/false)
8172 .addImm(Val: FPEnvModeBitField);
8173 auto TrapReg =
8174 B.buildIntrinsic(ID: Intrinsic::amdgcn_s_getreg, Res: {I32},
8175 /*HasSideEffects=*/true, /*isConvergent=*/false)
8176 .addImm(Val: FPEnvTrapBitField);
8177 B.buildMergeLikeInstr(Res: Src, Ops: {ModeReg, TrapReg});
8178 MI.eraseFromParent();
8179 return true;
8180}
8181
8182bool AMDGPULegalizerInfo::legalizeSetFPEnv(MachineInstr &MI,
8183 MachineRegisterInfo &MRI,
8184 MachineIRBuilder &B) const {
8185 const LLT I32 = LLT::integer(SizeInBits: 32);
8186 const LLT I64 = LLT::integer(SizeInBits: 64);
8187 Register Src = MI.getOperand(i: 0).getReg();
8188 if (MRI.getType(Reg: Src) != I64)
8189 return false;
8190
8191 auto Unmerge = B.buildUnmerge(Res: {I32, I32}, Op: MI.getOperand(i: 0));
8192 B.buildIntrinsic(ID: Intrinsic::amdgcn_s_setreg, Res: ArrayRef<DstOp>(),
8193 /*HasSideEffects=*/true, /*isConvergent=*/false)
8194 .addImm(Val: static_cast<int16_t>(FPEnvModeBitField))
8195 .addReg(RegNo: Unmerge.getReg(Idx: 0));
8196 B.buildIntrinsic(ID: Intrinsic::amdgcn_s_setreg, Res: ArrayRef<DstOp>(),
8197 /*HasSideEffects=*/true, /*isConvergent=*/false)
8198 .addImm(Val: static_cast<int16_t>(FPEnvTrapBitField))
8199 .addReg(RegNo: Unmerge.getReg(Idx: 1));
8200 MI.eraseFromParent();
8201 return true;
8202}
8203
8204bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
8205 MachineInstr &MI) const {
8206 MachineIRBuilder &B = Helper.MIRBuilder;
8207 MachineRegisterInfo &MRI = *B.getMRI();
8208
8209 // Replace the use G_BRCOND with the exec manipulate and branch pseudos.
8210 auto IntrID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
8211 switch (IntrID) {
8212 case Intrinsic::amdgcn_icmp: {
8213 // amdgcn.icmp(i1 src0, i1 0, NE) -> ballot(src0)
8214 // This is the only valid form of amdgcn.icmp with i1 inputs.
8215 Register Src0 = MI.getOperand(i: 2).getReg();
8216 LLT SrcTy = MRI.getType(Reg: Src0);
8217 if (SrcTy != LLT::scalar(SizeInBits: 1))
8218 return true; // Not i1, leave for default handling.
8219
8220 // Check that src1 is constant 0.
8221 Register Src1 = MI.getOperand(i: 3).getReg();
8222 auto Src1Const = getIConstantVRegValWithLookThrough(VReg: Src1, MRI);
8223 if (!Src1Const || Src1Const->Value != 0)
8224 return false; // Invalid i1 icmp form.
8225
8226 // Check that predicate is ICMP_NE.
8227 int64_t Pred = MI.getOperand(i: 4).getImm();
8228 if (Pred != CmpInst::ICMP_NE)
8229 return false; // Invalid i1 icmp form.
8230
8231 // Convert to ballot.
8232 Register Dst = MI.getOperand(i: 0).getReg();
8233 B.buildIntrinsic(ID: Intrinsic::amdgcn_ballot, Res: Dst).addUse(RegNo: Src0);
8234 MI.eraseFromParent();
8235 return true;
8236 }
8237 case Intrinsic::sponentry:
8238 if (B.getMF().getInfo<SIMachineFunctionInfo>()->isBottomOfStack()) {
8239 // FIXME: The imported pattern checks for i32 instead of p5; if we fix
8240 // that we can remove this cast.
8241 const LLT I32 = LLT::integer(SizeInBits: 32);
8242 Register TmpReg = MRI.createGenericVirtualRegister(Ty: I32);
8243 B.buildInstr(Opcode: AMDGPU::G_AMDGPU_SPONENTRY).addDef(RegNo: TmpReg);
8244
8245 Register DstReg = MI.getOperand(i: 0).getReg();
8246 B.buildIntToPtr(Dst: DstReg, Src: TmpReg);
8247 MI.eraseFromParent();
8248 } else {
8249 int FI = B.getMF().getFrameInfo().CreateFixedObject(
8250 Size: 1, SPOffset: 0, /*IsImmutable=*/false);
8251 B.buildFrameIndex(Res: MI.getOperand(i: 0), Idx: FI);
8252 MI.eraseFromParent();
8253 }
8254 return true;
8255 case Intrinsic::amdgcn_if:
8256 case Intrinsic::amdgcn_else: {
8257 MachineInstr *Br = nullptr;
8258 MachineBasicBlock *UncondBrTarget = nullptr;
8259 bool Negated = false;
8260 if (MachineInstr *BrCond =
8261 verifyCFIntrinsic(MI, MRI, Br, UncondBrTarget, Negated)) {
8262 const SIRegisterInfo *TRI
8263 = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
8264
8265 Register Def = MI.getOperand(i: 1).getReg();
8266 Register Use = MI.getOperand(i: 3).getReg();
8267
8268 MachineBasicBlock *CondBrTarget = BrCond->getOperand(i: 1).getMBB();
8269
8270 if (Negated)
8271 std::swap(a&: CondBrTarget, b&: UncondBrTarget);
8272
8273 B.setInsertPt(MBB&: B.getMBB(), II: BrCond->getIterator());
8274 if (IntrID == Intrinsic::amdgcn_if) {
8275 B.buildInstr(Opcode: AMDGPU::SI_IF)
8276 .addDef(RegNo: Def)
8277 .addUse(RegNo: Use)
8278 .addMBB(MBB: UncondBrTarget);
8279 } else {
8280 B.buildInstr(Opcode: AMDGPU::SI_ELSE)
8281 .addDef(RegNo: Def)
8282 .addUse(RegNo: Use)
8283 .addMBB(MBB: UncondBrTarget);
8284 }
8285
8286 if (Br) {
8287 Br->getOperand(i: 0).setMBB(CondBrTarget);
8288 } else {
8289 // The IRTranslator skips inserting the G_BR for fallthrough cases, but
8290 // since we're swapping branch targets it needs to be reinserted.
8291 // FIXME: IRTranslator should probably not do this
8292 B.buildBr(Dest&: *CondBrTarget);
8293 }
8294
8295 MRI.setRegClass(Reg: Def, RC: TRI->getWaveMaskRegClass());
8296 MRI.setRegClass(Reg: Use, RC: TRI->getWaveMaskRegClass());
8297 MI.eraseFromParent();
8298 BrCond->eraseFromParent();
8299 return true;
8300 }
8301
8302 return false;
8303 }
8304 case Intrinsic::amdgcn_loop: {
8305 MachineInstr *Br = nullptr;
8306 MachineBasicBlock *UncondBrTarget = nullptr;
8307 bool Negated = false;
8308 if (MachineInstr *BrCond =
8309 verifyCFIntrinsic(MI, MRI, Br, UncondBrTarget, Negated)) {
8310 const SIRegisterInfo *TRI
8311 = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
8312
8313 MachineBasicBlock *CondBrTarget = BrCond->getOperand(i: 1).getMBB();
8314 Register Reg = MI.getOperand(i: 2).getReg();
8315
8316 if (Negated)
8317 std::swap(a&: CondBrTarget, b&: UncondBrTarget);
8318
8319 B.setInsertPt(MBB&: B.getMBB(), II: BrCond->getIterator());
8320 B.buildInstr(Opcode: AMDGPU::SI_LOOP)
8321 .addUse(RegNo: Reg)
8322 .addMBB(MBB: UncondBrTarget);
8323
8324 if (Br)
8325 Br->getOperand(i: 0).setMBB(CondBrTarget);
8326 else
8327 B.buildBr(Dest&: *CondBrTarget);
8328
8329 MI.eraseFromParent();
8330 BrCond->eraseFromParent();
8331 MRI.setRegClass(Reg, RC: TRI->getWaveMaskRegClass());
8332 return true;
8333 }
8334
8335 return false;
8336 }
8337 case Intrinsic::amdgcn_wave_reduce_min:
8338 case Intrinsic::amdgcn_wave_reduce_umin:
8339 case Intrinsic::amdgcn_wave_reduce_fmin:
8340 case Intrinsic::amdgcn_wave_reduce_max:
8341 case Intrinsic::amdgcn_wave_reduce_umax:
8342 case Intrinsic::amdgcn_wave_reduce_fmax:
8343 case Intrinsic::amdgcn_wave_reduce_add:
8344 case Intrinsic::amdgcn_wave_reduce_fadd:
8345 case Intrinsic::amdgcn_wave_reduce_sub:
8346 case Intrinsic::amdgcn_wave_reduce_fsub:
8347 case Intrinsic::amdgcn_wave_reduce_and:
8348 case Intrinsic::amdgcn_wave_reduce_or:
8349 case Intrinsic::amdgcn_wave_reduce_xor: {
8350 Register SrcReg = MI.getOperand(i: 2).getReg();
8351 if (MRI.getType(Reg: SrcReg).getSizeInBits() != 16)
8352 return true;
8353 Register DstReg = MI.getOperand(i: 0).getReg();
8354 bool IsFPOp = IntrID == Intrinsic::amdgcn_wave_reduce_fmin ||
8355 IntrID == Intrinsic::amdgcn_wave_reduce_fmax ||
8356 IntrID == Intrinsic::amdgcn_wave_reduce_fadd ||
8357 IntrID == Intrinsic::amdgcn_wave_reduce_fsub;
8358 bool NeedsSignExt = IntrID == Intrinsic::amdgcn_wave_reduce_min ||
8359 IntrID == Intrinsic::amdgcn_wave_reduce_max ||
8360 IntrID == Intrinsic::amdgcn_wave_reduce_add ||
8361 IntrID == Intrinsic::amdgcn_wave_reduce_sub;
8362 auto Ext = IsFPOp ? B.buildFPExt(Res: LLT::float32(), Op: SrcReg)
8363 : NeedsSignExt ? B.buildSExt(Res: LLT::integer(SizeInBits: 32), Op: SrcReg)
8364 : B.buildZExt(Res: LLT::integer(SizeInBits: 32), Op: SrcReg);
8365 auto NewDst = MRI.createGenericVirtualRegister(Ty: IsFPOp ? LLT::float32()
8366 : LLT::integer(SizeInBits: 32));
8367 B.buildIntrinsic(ID: IntrID, Res: ArrayRef<Register>{NewDst},
8368 /*hasSideEffects=*/HasSideEffects: false, /*isConvergent=*/true)
8369 .addUse(RegNo: Ext.getReg(Idx: 0))
8370 .addImm(Val: MI.getOperand(i: 3).getImm()); // strategy
8371 if (IsFPOp)
8372 B.buildFPTrunc(Res: DstReg, Op: NewDst);
8373 else
8374 B.buildTrunc(Res: DstReg, Op: NewDst);
8375 MI.eraseFromParent();
8376 return true;
8377 }
8378 case Intrinsic::amdgcn_addrspacecast_nonnull:
8379 return legalizeAddrSpaceCast(MI, MRI, B);
8380 case Intrinsic::amdgcn_make_buffer_rsrc:
8381 return legalizePointerAsRsrcIntrin(MI, MRI, B);
8382 case Intrinsic::amdgcn_kernarg_segment_ptr:
8383 if (!AMDGPU::isKernel(F: B.getMF().getFunction())) {
8384 // This only makes sense to call in a kernel, so just lower to null.
8385 B.buildConstant(Res: MI.getOperand(i: 0).getReg(), Val: 0);
8386 MI.eraseFromParent();
8387 return true;
8388 }
8389
8390 return legalizePreloadedArgIntrin(
8391 MI, MRI, B, ArgType: AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
8392 case Intrinsic::amdgcn_implicitarg_ptr:
8393 return legalizeImplicitArgPtr(MI, MRI, B);
8394 case Intrinsic::amdgcn_workitem_id_x:
8395 return legalizeWorkitemIDIntrinsic(MI, MRI, B, Dim: 0,
8396 ArgType: AMDGPUFunctionArgInfo::WORKITEM_ID_X);
8397 case Intrinsic::amdgcn_workitem_id_y:
8398 return legalizeWorkitemIDIntrinsic(MI, MRI, B, Dim: 1,
8399 ArgType: AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
8400 case Intrinsic::amdgcn_workitem_id_z:
8401 return legalizeWorkitemIDIntrinsic(MI, MRI, B, Dim: 2,
8402 ArgType: AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
8403 case Intrinsic::amdgcn_workgroup_id_x:
8404 return legalizeWorkGroupId(
8405 MI, B, WorkGroupIdPV: AMDGPUFunctionArgInfo::WORKGROUP_ID_X,
8406 ClusterMaxIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X,
8407 ClusterWorkGroupIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X);
8408 case Intrinsic::amdgcn_workgroup_id_y:
8409 return legalizeWorkGroupId(
8410 MI, B, WorkGroupIdPV: AMDGPUFunctionArgInfo::WORKGROUP_ID_Y,
8411 ClusterMaxIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y,
8412 ClusterWorkGroupIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y);
8413 case Intrinsic::amdgcn_workgroup_id_z:
8414 return legalizeWorkGroupId(
8415 MI, B, WorkGroupIdPV: AMDGPUFunctionArgInfo::WORKGROUP_ID_Z,
8416 ClusterMaxIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z,
8417 ClusterWorkGroupIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z);
8418 case Intrinsic::amdgcn_cluster_id_x:
8419 return ST.hasClusters() &&
8420 legalizePreloadedArgIntrin(MI, MRI, B,
8421 ArgType: AMDGPUFunctionArgInfo::WORKGROUP_ID_X);
8422 case Intrinsic::amdgcn_cluster_id_y:
8423 return ST.hasClusters() &&
8424 legalizePreloadedArgIntrin(MI, MRI, B,
8425 ArgType: AMDGPUFunctionArgInfo::WORKGROUP_ID_Y);
8426 case Intrinsic::amdgcn_cluster_id_z:
8427 return ST.hasClusters() &&
8428 legalizePreloadedArgIntrin(MI, MRI, B,
8429 ArgType: AMDGPUFunctionArgInfo::WORKGROUP_ID_Z);
8430 case Intrinsic::amdgcn_cluster_workgroup_id_x:
8431 return ST.hasClusters() &&
8432 legalizePreloadedArgIntrin(
8433 MI, MRI, B, ArgType: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X);
8434 case Intrinsic::amdgcn_cluster_workgroup_id_y:
8435 return ST.hasClusters() &&
8436 legalizePreloadedArgIntrin(
8437 MI, MRI, B, ArgType: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y);
8438 case Intrinsic::amdgcn_cluster_workgroup_id_z:
8439 return ST.hasClusters() &&
8440 legalizePreloadedArgIntrin(
8441 MI, MRI, B, ArgType: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z);
8442 case Intrinsic::amdgcn_cluster_workgroup_flat_id:
8443 return ST.hasClusters() &&
8444 legalizeConstHwRegRead(MI, B, HwReg: AMDGPU::Hwreg::ID_IB_STS2, LowBit: 21, Width: 4);
8445 case Intrinsic::amdgcn_cluster_workgroup_max_id_x:
8446 return ST.hasClusters() &&
8447 legalizePreloadedArgIntrin(
8448 MI, MRI, B, ArgType: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X);
8449 case Intrinsic::amdgcn_cluster_workgroup_max_id_y:
8450 return ST.hasClusters() &&
8451 legalizePreloadedArgIntrin(
8452 MI, MRI, B, ArgType: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y);
8453 case Intrinsic::amdgcn_cluster_workgroup_max_id_z:
8454 return ST.hasClusters() &&
8455 legalizePreloadedArgIntrin(
8456 MI, MRI, B, ArgType: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z);
8457 case Intrinsic::amdgcn_cluster_workgroup_max_flat_id:
8458 return ST.hasClusters() &&
8459 legalizePreloadedArgIntrin(
8460 MI, MRI, B,
8461 ArgType: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_FLAT_ID);
8462 case Intrinsic::amdgcn_wave_id:
8463 return legalizeWaveID(MI, B);
8464 case Intrinsic::amdgcn_lds_kernel_id:
8465 return legalizePreloadedArgIntrin(MI, MRI, B,
8466 ArgType: AMDGPUFunctionArgInfo::LDS_KERNEL_ID);
8467 case Intrinsic::amdgcn_dispatch_ptr:
8468 return legalizePreloadedArgIntrin(MI, MRI, B,
8469 ArgType: AMDGPUFunctionArgInfo::DISPATCH_PTR);
8470 case Intrinsic::amdgcn_queue_ptr:
8471 return legalizePreloadedArgIntrin(MI, MRI, B,
8472 ArgType: AMDGPUFunctionArgInfo::QUEUE_PTR);
8473 case Intrinsic::amdgcn_implicit_buffer_ptr:
8474 return legalizePreloadedArgIntrin(
8475 MI, MRI, B, ArgType: AMDGPUFunctionArgInfo::IMPLICIT_BUFFER_PTR);
8476 case Intrinsic::amdgcn_dispatch_id:
8477 return legalizePreloadedArgIntrin(MI, MRI, B,
8478 ArgType: AMDGPUFunctionArgInfo::DISPATCH_ID);
8479 case Intrinsic::r600_read_ngroups_x:
8480 // TODO: Emit error for hsa
8481 return legalizeKernargMemParameter(MI, B,
8482 Offset: SI::KernelInputOffsets::NGROUPS_X);
8483 case Intrinsic::r600_read_ngroups_y:
8484 return legalizeKernargMemParameter(MI, B,
8485 Offset: SI::KernelInputOffsets::NGROUPS_Y);
8486 case Intrinsic::r600_read_ngroups_z:
8487 return legalizeKernargMemParameter(MI, B,
8488 Offset: SI::KernelInputOffsets::NGROUPS_Z);
8489 case Intrinsic::r600_read_local_size_x:
8490 // TODO: Could insert G_ASSERT_ZEXT from i16
8491 return legalizeKernargMemParameter(MI, B, Offset: SI::KernelInputOffsets::LOCAL_SIZE_X);
8492 case Intrinsic::r600_read_local_size_y:
8493 // TODO: Could insert G_ASSERT_ZEXT from i16
8494 return legalizeKernargMemParameter(MI, B, Offset: SI::KernelInputOffsets::LOCAL_SIZE_Y);
8495 // TODO: Could insert G_ASSERT_ZEXT from i16
8496 case Intrinsic::r600_read_local_size_z:
8497 return legalizeKernargMemParameter(MI, B,
8498 Offset: SI::KernelInputOffsets::LOCAL_SIZE_Z);
8499 case Intrinsic::amdgcn_fdiv_fast:
8500 return legalizeFDIVFastIntrin(MI, MRI, B);
8501 case Intrinsic::amdgcn_is_shared:
8502 return legalizeIsAddrSpace(MI, MRI, B, AddrSpace: AMDGPUAS::LOCAL_ADDRESS);
8503 case Intrinsic::amdgcn_is_private:
8504 return legalizeIsAddrSpace(MI, MRI, B, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS);
8505 case Intrinsic::amdgcn_wavefrontsize: {
8506 B.buildConstant(Res: MI.getOperand(i: 0), Val: ST.getWavefrontSize());
8507 MI.eraseFromParent();
8508 return true;
8509 }
8510 case Intrinsic::amdgcn_s_buffer_load:
8511 return legalizeSBufferLoad(Helper, MI);
8512 case Intrinsic::amdgcn_raw_buffer_store:
8513 case Intrinsic::amdgcn_raw_ptr_buffer_store:
8514 case Intrinsic::amdgcn_struct_buffer_store:
8515 case Intrinsic::amdgcn_struct_ptr_buffer_store:
8516 return legalizeBufferStore(MI, Helper, IsTyped: false, IsFormat: false);
8517 case Intrinsic::amdgcn_raw_buffer_store_format:
8518 case Intrinsic::amdgcn_raw_ptr_buffer_store_format:
8519 case Intrinsic::amdgcn_struct_buffer_store_format:
8520 case Intrinsic::amdgcn_struct_ptr_buffer_store_format:
8521 return legalizeBufferStore(MI, Helper, IsTyped: false, IsFormat: true);
8522 case Intrinsic::amdgcn_raw_tbuffer_store:
8523 case Intrinsic::amdgcn_raw_ptr_tbuffer_store:
8524 case Intrinsic::amdgcn_struct_tbuffer_store:
8525 case Intrinsic::amdgcn_struct_ptr_tbuffer_store:
8526 return legalizeBufferStore(MI, Helper, IsTyped: true, IsFormat: true);
8527 case Intrinsic::amdgcn_raw_buffer_load:
8528 case Intrinsic::amdgcn_raw_ptr_buffer_load:
8529 case Intrinsic::amdgcn_raw_atomic_buffer_load:
8530 case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
8531 case Intrinsic::amdgcn_struct_buffer_load:
8532 case Intrinsic::amdgcn_struct_ptr_buffer_load:
8533 case Intrinsic::amdgcn_struct_atomic_buffer_load:
8534 case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load:
8535 return legalizeBufferLoad(MI, Helper, IsFormat: false, IsTyped: false);
8536 case Intrinsic::amdgcn_raw_buffer_load_format:
8537 case Intrinsic::amdgcn_raw_ptr_buffer_load_format:
8538 case Intrinsic::amdgcn_struct_buffer_load_format:
8539 case Intrinsic::amdgcn_struct_ptr_buffer_load_format:
8540 return legalizeBufferLoad(MI, Helper, IsFormat: true, IsTyped: false);
8541 case Intrinsic::amdgcn_raw_tbuffer_load:
8542 case Intrinsic::amdgcn_raw_ptr_tbuffer_load:
8543 case Intrinsic::amdgcn_struct_tbuffer_load:
8544 case Intrinsic::amdgcn_struct_ptr_tbuffer_load:
8545 return legalizeBufferLoad(MI, Helper, IsFormat: true, IsTyped: true);
8546 case Intrinsic::amdgcn_raw_buffer_atomic_swap:
8547 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
8548 case Intrinsic::amdgcn_struct_buffer_atomic_swap:
8549 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
8550 case Intrinsic::amdgcn_raw_buffer_atomic_add:
8551 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
8552 case Intrinsic::amdgcn_struct_buffer_atomic_add:
8553 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
8554 case Intrinsic::amdgcn_raw_buffer_atomic_sub:
8555 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
8556 case Intrinsic::amdgcn_struct_buffer_atomic_sub:
8557 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
8558 case Intrinsic::amdgcn_raw_buffer_atomic_smin:
8559 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
8560 case Intrinsic::amdgcn_struct_buffer_atomic_smin:
8561 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
8562 case Intrinsic::amdgcn_raw_buffer_atomic_umin:
8563 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
8564 case Intrinsic::amdgcn_struct_buffer_atomic_umin:
8565 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
8566 case Intrinsic::amdgcn_raw_buffer_atomic_smax:
8567 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
8568 case Intrinsic::amdgcn_struct_buffer_atomic_smax:
8569 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
8570 case Intrinsic::amdgcn_raw_buffer_atomic_umax:
8571 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
8572 case Intrinsic::amdgcn_struct_buffer_atomic_umax:
8573 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
8574 case Intrinsic::amdgcn_raw_buffer_atomic_and:
8575 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
8576 case Intrinsic::amdgcn_struct_buffer_atomic_and:
8577 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
8578 case Intrinsic::amdgcn_raw_buffer_atomic_or:
8579 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
8580 case Intrinsic::amdgcn_struct_buffer_atomic_or:
8581 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
8582 case Intrinsic::amdgcn_raw_buffer_atomic_xor:
8583 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
8584 case Intrinsic::amdgcn_struct_buffer_atomic_xor:
8585 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
8586 case Intrinsic::amdgcn_raw_buffer_atomic_inc:
8587 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
8588 case Intrinsic::amdgcn_struct_buffer_atomic_inc:
8589 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
8590 case Intrinsic::amdgcn_raw_buffer_atomic_dec:
8591 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
8592 case Intrinsic::amdgcn_struct_buffer_atomic_dec:
8593 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
8594 case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
8595 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap:
8596 case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
8597 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap:
8598 case Intrinsic::amdgcn_raw_buffer_atomic_fmin:
8599 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
8600 case Intrinsic::amdgcn_struct_buffer_atomic_fmin:
8601 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
8602 case Intrinsic::amdgcn_raw_buffer_atomic_fmax:
8603 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
8604 case Intrinsic::amdgcn_struct_buffer_atomic_fmax:
8605 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
8606 case Intrinsic::amdgcn_raw_buffer_atomic_sub_clamp_u32:
8607 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub_clamp_u32:
8608 case Intrinsic::amdgcn_struct_buffer_atomic_sub_clamp_u32:
8609 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub_clamp_u32:
8610 case Intrinsic::amdgcn_raw_buffer_atomic_cond_sub_u32:
8611 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cond_sub_u32:
8612 case Intrinsic::amdgcn_struct_buffer_atomic_cond_sub_u32:
8613 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cond_sub_u32:
8614 case Intrinsic::amdgcn_raw_buffer_atomic_fadd:
8615 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
8616 case Intrinsic::amdgcn_struct_buffer_atomic_fadd:
8617 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
8618 return legalizeBufferAtomic(MI, B, IID: IntrID);
8619 case Intrinsic::amdgcn_rsq_clamp:
8620 return legalizeRsqClampIntrinsic(MI, MRI, B);
8621 case Intrinsic::amdgcn_image_bvh_intersect_ray:
8622 return legalizeBVHIntersectRayIntrinsic(MI, B);
8623 case Intrinsic::amdgcn_image_bvh_dual_intersect_ray:
8624 case Intrinsic::amdgcn_image_bvh8_intersect_ray:
8625 return legalizeBVHDualOrBVH8IntersectRayIntrinsic(MI, B);
8626 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_fp8:
8627 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_bf8:
8628 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_fp8:
8629 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_bf8:
8630 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_fp8:
8631 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_bf8:
8632 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_fp8:
8633 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_bf8: {
8634 Register Index = MI.getOperand(i: 5).getReg();
8635 LLT I64 = LLT::integer(SizeInBits: 64);
8636 LLT IndexArgTy = MRI.getType(Reg: Index);
8637 if (IndexArgTy != I64) {
8638 auto NewIndex = IndexArgTy.isVector() ? B.buildBitcast(Dst: I64, Src: Index)
8639 : B.buildAnyExt(Res: I64, Op: Index);
8640 MI.getOperand(i: 5).setReg(NewIndex.getReg(Idx: 0));
8641 }
8642 return true;
8643 }
8644 case Intrinsic::amdgcn_swmmac_f16_16x16x32_f16:
8645 case Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16:
8646 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16:
8647 case Intrinsic::amdgcn_swmmac_f32_16x16x32_f16:
8648 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8:
8649 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8:
8650 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8:
8651 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8: {
8652 Register Index = MI.getOperand(i: 5).getReg();
8653 LLT I32 = LLT::integer(SizeInBits: 32);
8654 if (MRI.getType(Reg: Index) != I32)
8655 MI.getOperand(i: 5).setReg(B.buildAnyExt(Res: I32, Op: Index).getReg(Idx: 0));
8656 return true;
8657 }
8658 case Intrinsic::amdgcn_swmmac_f16_16x16x64_f16:
8659 case Intrinsic::amdgcn_swmmac_bf16_16x16x64_bf16:
8660 case Intrinsic::amdgcn_swmmac_f32_16x16x64_bf16:
8661 case Intrinsic::amdgcn_swmmac_bf16f32_16x16x64_bf16:
8662 case Intrinsic::amdgcn_swmmac_f32_16x16x64_f16:
8663 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
8664 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4:
8665 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8:
8666 case Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4: {
8667 Register Index = MI.getOperand(i: 7).getReg();
8668 LLT IdxTy = IntrID == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8
8669 ? LLT::integer(SizeInBits: 64)
8670 : LLT::integer(SizeInBits: 32);
8671 LLT IndexArgTy = MRI.getType(Reg: Index);
8672 if (IndexArgTy != IdxTy) {
8673 auto NewIndex = IndexArgTy.isVector() ? B.buildBitcast(Dst: IdxTy, Src: Index)
8674 : B.buildAnyExt(Res: IdxTy, Op: Index);
8675 MI.getOperand(i: 7).setReg(NewIndex.getReg(Idx: 0));
8676 }
8677 return true;
8678 }
8679
8680 case Intrinsic::amdgcn_fmed3: {
8681 GISelChangeObserver &Observer = Helper.Observer;
8682
8683 // FIXME: This is to workaround the inability of tablegen match combiners to
8684 // match intrinsics in patterns.
8685 Observer.changingInstr(MI);
8686 MI.setDesc(B.getTII().get(Opcode: AMDGPU::G_AMDGPU_FMED3));
8687 MI.removeOperand(OpNo: 1);
8688 Observer.changedInstr(MI);
8689 return true;
8690 }
8691 case Intrinsic::amdgcn_readlane:
8692 case Intrinsic::amdgcn_writelane:
8693 case Intrinsic::amdgcn_readfirstlane:
8694 case Intrinsic::amdgcn_permlane16:
8695 case Intrinsic::amdgcn_permlanex16:
8696 case Intrinsic::amdgcn_permlane64:
8697 case Intrinsic::amdgcn_set_inactive:
8698 case Intrinsic::amdgcn_set_inactive_chain_arg:
8699 case Intrinsic::amdgcn_mov_dpp8:
8700 case Intrinsic::amdgcn_update_dpp:
8701 case Intrinsic::amdgcn_permlane_bcast:
8702 case Intrinsic::amdgcn_permlane_up:
8703 case Intrinsic::amdgcn_permlane_down:
8704 case Intrinsic::amdgcn_permlane_xor:
8705 return legalizeLaneOp(Helper, MI, IID: IntrID);
8706 case Intrinsic::amdgcn_s_buffer_prefetch_data:
8707 return legalizeSBufferPrefetch(Helper, MI);
8708 case Intrinsic::amdgcn_dead: {
8709 // TODO: Use poison instead of undef
8710 for (const MachineOperand &Def : MI.defs())
8711 B.buildUndef(Res: Def);
8712 MI.eraseFromParent();
8713 return true;
8714 }
8715 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
8716 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
8717 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B:
8718 assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
8719 B.buildLoad(Res: MI.getOperand(i: 0), Addr: MI.getOperand(i: 2), MMO&: **MI.memoperands_begin());
8720 MI.eraseFromParent();
8721 return true;
8722 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
8723 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
8724 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B:
8725 assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
8726 B.buildStore(Val: MI.getOperand(i: 2), Addr: MI.getOperand(i: 1), MMO&: **MI.memoperands_begin());
8727 MI.eraseFromParent();
8728 return true;
8729 case Intrinsic::amdgcn_av_load_b128:
8730 case Intrinsic::amdgcn_av_store_b128: {
8731 assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
8732 if (IntrID == Intrinsic::amdgcn_av_load_b128)
8733 B.buildLoad(Res: MI.getOperand(i: 0), Addr: MI.getOperand(i: 2), MMO&: **MI.memoperands_begin());
8734 else
8735 B.buildStore(Val: MI.getOperand(i: 2), Addr: MI.getOperand(i: 1),
8736 MMO&: **MI.memoperands_begin());
8737 MI.eraseFromParent();
8738 return true;
8739 }
8740 case Intrinsic::amdgcn_flat_load_monitor_b32:
8741 case Intrinsic::amdgcn_flat_load_monitor_b64:
8742 case Intrinsic::amdgcn_flat_load_monitor_b128:
8743 assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
8744 B.buildInstr(Opcode: AMDGPU::G_AMDGPU_FLAT_LOAD_MONITOR)
8745 .add(MO: MI.getOperand(i: 0))
8746 .add(MO: MI.getOperand(i: 2))
8747 .addMemOperand(MMO: *MI.memoperands_begin());
8748 MI.eraseFromParent();
8749 return true;
8750 case Intrinsic::amdgcn_global_load_monitor_b32:
8751 case Intrinsic::amdgcn_global_load_monitor_b64:
8752 case Intrinsic::amdgcn_global_load_monitor_b128:
8753 assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
8754 B.buildInstr(Opcode: AMDGPU::G_AMDGPU_GLOBAL_LOAD_MONITOR)
8755 .add(MO: MI.getOperand(i: 0))
8756 .add(MO: MI.getOperand(i: 2))
8757 .addMemOperand(MMO: *MI.memoperands_begin());
8758 MI.eraseFromParent();
8759 return true;
8760 default: {
8761 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
8762 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrID))
8763 return legalizeImageIntrinsic(MI, B, Observer&: Helper.Observer, Intr: ImageDimIntr);
8764 return true;
8765 }
8766 }
8767
8768 return true;
8769}
8770