1//===-- NVPTXISelLowering.cpp - NVPTX DAG Lowering Implementation ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that NVPTX uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "NVPTXISelLowering.h"
15#include "MCTargetDesc/NVPTXBaseInfo.h"
16#include "NVPTX.h"
17#include "NVPTXMachineFunctionInfo.h"
18#include "NVPTXSelectionDAGInfo.h"
19#include "NVPTXSubtarget.h"
20#include "NVPTXTargetMachine.h"
21#include "NVPTXTargetObjectFile.h"
22#include "NVPTXUtilities.h"
23#include "NVVMProperties.h"
24#include "llvm/ADT/APFloat.h"
25#include "llvm/ADT/APInt.h"
26#include "llvm/ADT/STLExtras.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/StringRef.h"
29#include "llvm/CodeGen/Analysis.h"
30#include "llvm/CodeGen/ISDOpcodes.h"
31#include "llvm/CodeGen/MachineFrameInfo.h"
32#include "llvm/CodeGen/MachineFunction.h"
33#include "llvm/CodeGen/MachineJumpTableInfo.h"
34#include "llvm/CodeGen/MachineMemOperand.h"
35#include "llvm/CodeGen/SDPatternMatch.h"
36#include "llvm/CodeGen/SelectionDAG.h"
37#include "llvm/CodeGen/SelectionDAGNodes.h"
38#include "llvm/CodeGen/TargetCallingConv.h"
39#include "llvm/CodeGen/TargetLowering.h"
40#include "llvm/CodeGen/ValueTypes.h"
41#include "llvm/CodeGenTypes/MachineValueType.h"
42#include "llvm/IR/Argument.h"
43#include "llvm/IR/Attributes.h"
44#include "llvm/IR/Constants.h"
45#include "llvm/IR/DataLayout.h"
46#include "llvm/IR/DerivedTypes.h"
47#include "llvm/IR/DiagnosticInfo.h"
48#include "llvm/IR/FPEnv.h"
49#include "llvm/IR/Function.h"
50#include "llvm/IR/GlobalValue.h"
51#include "llvm/IR/IRBuilder.h"
52#include "llvm/IR/Instruction.h"
53#include "llvm/IR/Instructions.h"
54#include "llvm/IR/IntrinsicsNVPTX.h"
55#include "llvm/IR/Module.h"
56#include "llvm/IR/NVVMIntrinsicUtils.h"
57#include "llvm/IR/Type.h"
58#include "llvm/IR/Value.h"
59#include "llvm/MC/MCContext.h"
60#include "llvm/MC/MCSymbol.h"
61#include "llvm/Support/Alignment.h"
62#include "llvm/Support/AtomicOrdering.h"
63#include "llvm/Support/Casting.h"
64#include "llvm/Support/CodeGen.h"
65#include "llvm/Support/CommandLine.h"
66#include "llvm/Support/ErrorHandling.h"
67#include "llvm/Support/KnownBits.h"
68#include "llvm/Support/NVPTXAddrSpace.h"
69#include "llvm/Target/TargetMachine.h"
70#include "llvm/Target/TargetOptions.h"
71#include <algorithm>
72#include <cassert>
73#include <cmath>
74#include <cstdint>
75#include <iterator>
76#include <optional>
77#include <tuple>
78#include <utility>
79#include <vector>
80
81#define DEBUG_TYPE "nvptx-lower"
82
83using namespace llvm;
84
85static cl::opt<bool> sched4reg(
86 "nvptx-sched4reg",
87 cl::desc("NVPTX Specific: schedule for register pressue"), cl::init(Val: false));
88
89static cl::opt<unsigned> FMAContractLevelOpt(
90 "nvptx-fma-level", cl::Hidden,
91 cl::desc("NVPTX Specific: FMA contraction (0: don't do it"
92 " 1: do it 2: do it aggressively"),
93 cl::init(Val: 2));
94
95static cl::opt<NVPTX::DivPrecisionLevel> UsePrecDivF32(
96 "nvptx-prec-divf32", cl::Hidden,
97 cl::desc(
98 "NVPTX Specific: Override the precision of the lowering for f32 fdiv"),
99 cl::values(
100 clEnumValN(NVPTX::DivPrecisionLevel::Approx, "0", "Use div.approx"),
101 clEnumValN(NVPTX::DivPrecisionLevel::Full, "1", "Use div.full"),
102 clEnumValN(NVPTX::DivPrecisionLevel::IEEE754, "2",
103 "Use IEEE Compliant F32 div.rnd if available (default)"),
104 clEnumValN(NVPTX::DivPrecisionLevel::IEEE754_NoFTZ, "3",
105 "Use IEEE Compliant F32 div.rnd if available, no FTZ")),
106 cl::init(Val: NVPTX::DivPrecisionLevel::IEEE754));
107
108static cl::opt<bool> UsePrecSqrtF32(
109 "nvptx-prec-sqrtf32", cl::Hidden,
110 cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
111 cl::init(Val: true));
112
113// PTX atom.add.f32 has fixed FTZ behavior that may not match the function's
114// (see shouldExpandAtomicRMWInIR), so we'd normally fall back to a CAS loop
115// when they disagree. This option (enabled by default) allows using atom.add
116// anyway, trading correct denormal handling for the speed of the native
117// instruction.
118static cl::opt<bool> AllowFTZAtomics(
119 "nvptx-allow-ftz-atomics", cl::Hidden,
120 cl::desc("NVPTX Specific: Lower atomicrmw fadd to atom.add even when its "
121 "FTZ behavior does not match the function's denormal mode."),
122 cl::init(Val: true));
123
124/// Whereas CUDA's implementation (see libdevice) uses ex2.approx for exp2(), it
125/// does NOT use lg2.approx for log2, so this is disabled by default.
126static cl::opt<bool> UseApproxLog2F32(
127 "nvptx-approx-log2f32",
128 cl::desc("NVPTX Specific: whether to use lg2.approx for log2"),
129 cl::init(Val: false));
130
131NVPTX::DivPrecisionLevel
132NVPTXTargetLowering::getDivF32Level(const MachineFunction &MF,
133 const SDNode &N) const {
134 // If nvptx-prec-div32=N is used on the command-line, always honor it
135 if (UsePrecDivF32.getNumOccurrences() > 0)
136 return UsePrecDivF32;
137
138 const SDNodeFlags Flags = N.getFlags();
139 if (Flags.hasApproximateFuncs())
140 return NVPTX::DivPrecisionLevel::Approx;
141
142 return NVPTX::DivPrecisionLevel::IEEE754;
143}
144
145bool NVPTXTargetLowering::usePrecSqrtF32(const SDNode *N) const {
146 // If nvptx-prec-sqrtf32 is used on the command-line, always honor it
147 if (UsePrecSqrtF32.getNumOccurrences() > 0)
148 return UsePrecSqrtF32;
149
150 if (N) {
151 const SDNodeFlags Flags = N->getFlags();
152 if (Flags.hasApproximateFuncs())
153 return false;
154 }
155
156 return true;
157}
158
159bool NVPTXTargetLowering::useF32FTZ(const MachineFunction &MF) const {
160 return MF.getDenormalMode(FPType: APFloat::IEEEsingle()).Output ==
161 DenormalMode::PreserveSign;
162}
163
164static bool IsPTXVectorType(MVT VT) {
165 switch (VT.SimpleTy) {
166 default:
167 return false;
168 case MVT::v2i1:
169 case MVT::v4i1:
170 case MVT::v2i8:
171 case MVT::v4i8:
172 case MVT::v8i8: // <2 x i8x4>
173 case MVT::v16i8: // <4 x i8x4>
174 case MVT::v2i16:
175 case MVT::v4i16:
176 case MVT::v8i16: // <4 x i16x2>
177 case MVT::v2i32:
178 case MVT::v4i32:
179 case MVT::v2i64:
180 case MVT::v2f16:
181 case MVT::v4f16:
182 case MVT::v8f16: // <4 x f16x2>
183 case MVT::v2bf16:
184 case MVT::v4bf16:
185 case MVT::v8bf16: // <4 x bf16x2>
186 case MVT::v2f32:
187 case MVT::v4f32:
188 case MVT::v2f64:
189 case MVT::v4i64:
190 case MVT::v4f64:
191 case MVT::v8i32:
192 case MVT::v8f32:
193 case MVT::v16f16: // <8 x f16x2>
194 case MVT::v16bf16: // <8 x bf16x2>
195 case MVT::v16i16: // <8 x i16x2>
196 case MVT::v32i8: // <8 x i8x4>
197 return true;
198 }
199}
200
201// When legalizing vector loads/stores, this function is called, which does two
202// things:
203// 1. Determines Whether the vector is something we want to custom lower,
204// std::nullopt is returned if we do not want to custom lower it.
205// 2. If we do want to handle it, returns two parameters:
206// - unsigned int NumElts - The number of elements in the final vector
207// - EVT EltVT - The type of the elements in the final vector
208static std::optional<std::pair<unsigned int, MVT>>
209getVectorLoweringShape(EVT VectorEVT, const NVPTXSubtarget &STI,
210 unsigned AddressSpace) {
211 const bool CanLowerTo256Bit = STI.has256BitVectorLoadStore(AS: AddressSpace);
212
213 if (CanLowerTo256Bit && VectorEVT.isScalarInteger() &&
214 VectorEVT.getSizeInBits() == 256)
215 return {{4, MVT::i64}};
216
217 if (!VectorEVT.isSimple())
218 return std::nullopt;
219 const MVT VectorVT = VectorEVT.getSimpleVT();
220
221 if (!VectorVT.isVector()) {
222 if (VectorVT == MVT::i128 || VectorVT == MVT::f128)
223 return {{2, MVT::i64}};
224 return std::nullopt;
225 }
226
227 const MVT EltVT = VectorVT.getVectorElementType();
228 const unsigned NumElts = VectorVT.getVectorNumElements();
229
230 // The size of the PTX virtual register that holds a packed type.
231 unsigned PackRegSize;
232
233 // We only handle "native" vector sizes for now, e.g. <4 x double> is not
234 // legal. We can (and should) split that into 2 stores of <2 x double> here
235 // but I'm leaving that as a TODO for now.
236 switch (VectorVT.SimpleTy) {
237 default:
238 return std::nullopt;
239
240 case MVT::v4i64:
241 case MVT::v4f64:
242 // This is a "native" vector type iff the address space is global and the
243 // target supports 256-bit loads/stores
244 if (!CanLowerTo256Bit)
245 return std::nullopt;
246 [[fallthrough]];
247 case MVT::v2i8:
248 case MVT::v2i64:
249 case MVT::v2f64:
250 // This is a "native" vector type
251 return std::pair(NumElts, EltVT);
252
253 case MVT::v16f16: // <8 x f16x2>
254 case MVT::v16bf16: // <8 x bf16x2>
255 case MVT::v16i16: // <8 x i16x2>
256 case MVT::v32i8: // <8 x i8x4>
257 // This can be upsized into a "native" vector type iff the address space is
258 // global and the target supports 256-bit loads/stores.
259 if (!CanLowerTo256Bit)
260 return std::nullopt;
261 [[fallthrough]];
262 case MVT::v2i16: // <1 x i16x2>
263 case MVT::v2f16: // <1 x f16x2>
264 case MVT::v2bf16: // <1 x bf16x2>
265 case MVT::v4i8: // <1 x i8x4>
266 case MVT::v4i16: // <2 x i16x2>
267 case MVT::v4f16: // <2 x f16x2>
268 case MVT::v4bf16: // <2 x bf16x2>
269 case MVT::v8i8: // <2 x i8x4>
270 case MVT::v8f16: // <4 x f16x2>
271 case MVT::v8bf16: // <4 x bf16x2>
272 case MVT::v8i16: // <4 x i16x2>
273 case MVT::v16i8: // <4 x i8x4>
274 PackRegSize = 32;
275 break;
276
277 case MVT::v8f32: // <4 x f32x2>
278 case MVT::v8i32: // <4 x i32x2>
279 // This is a "native" vector type iff the address space is global and the
280 // target supports 256-bit loads/stores
281 if (!CanLowerTo256Bit)
282 return std::nullopt;
283 [[fallthrough]];
284 case MVT::v2f32: // <1 x f32x2>
285 case MVT::v4f32: // <2 x f32x2>
286 case MVT::v2i32: // <1 x i32x2>
287 case MVT::v4i32: // <2 x i32x2>
288 if (!STI.hasF32x2Instructions())
289 return std::pair(NumElts, EltVT);
290 PackRegSize = 64;
291 break;
292 }
293
294 // If we reach here, then we can pack 2 or more elements into a single 32-bit
295 // or 64-bit PTX register and treat the vector as a new vector containing
296 // packed elements.
297
298 // Number of elements to pack in one word.
299 const unsigned NPerReg = PackRegSize / EltVT.getSizeInBits();
300
301 return std::pair(NumElts / NPerReg, MVT::getVectorVT(VT: EltVT, NumElements: NPerReg));
302}
303
304/// ComputePTXValueVTs - For the given Type \p Ty, returns the set of primitive
305/// legal-ish MVTs that compose it. Unlike ComputeValueVTs, this will legalize
306/// the types as required by the calling convention (with special handling for
307/// i8s).
308/// NOTE: This is a band-aid for code that expects ComputeValueVTs to return the
309/// same number of types as the Ins/Outs arrays in LowerFormalArguments,
310/// LowerCall, and LowerReturn.
311static void ComputePTXValueVTs(const TargetLowering &TLI, const DataLayout &DL,
312 LLVMContext &Ctx, CallingConv::ID CallConv,
313 Type *Ty, SmallVectorImpl<EVT> &ValueVTs,
314 SmallVectorImpl<uint64_t> &Offsets,
315 uint64_t StartingOffset = 0) {
316 SmallVector<EVT, 16> TempVTs;
317 SmallVector<uint64_t, 16> TempOffsets;
318 ComputeValueVTs(TLI, DL, Ty, ValueVTs&: TempVTs, /*MemVTs=*/nullptr, FixedOffsets: &TempOffsets,
319 StartingOffset);
320
321 for (const auto [VT, Off] : zip(t&: TempVTs, u&: TempOffsets)) {
322 MVT RegisterVT = TLI.getRegisterTypeForCallingConv(Context&: Ctx, CC: CallConv, VT);
323 unsigned NumRegs = TLI.getNumRegistersForCallingConv(Context&: Ctx, CC: CallConv, VT);
324
325 // Since we actually can load/store b8, we need to ensure that we'll use
326 // the original sized type for any i8s or i8 vectors.
327 if (VT.getScalarType() == MVT::i8) {
328 if (RegisterVT == MVT::i16)
329 RegisterVT = MVT::i8;
330 else if (RegisterVT == MVT::v2i16)
331 RegisterVT = MVT::v2i8;
332 else
333 assert(RegisterVT == MVT::v4i8 &&
334 "Expected v4i8, v2i16, or i16 for i8 RegisterVT");
335 }
336
337 // TODO: This is horribly incorrect for cases where the vector elements are
338 // not a multiple of bytes (ex i1) and legal or i8. However, this problem
339 // has existed for as long as NVPTX has and no one has complained, so we'll
340 // leave it for now.
341 for (unsigned I : seq(Size: NumRegs)) {
342 ValueVTs.push_back(Elt: RegisterVT);
343 Offsets.push_back(Elt: Off + I * RegisterVT.getStoreSize());
344 }
345 }
346}
347
348// We return an EVT that can hold N VTs
349// If the VT is a vector, the resulting EVT is a flat vector with the same
350// element type as VT's element type.
351static EVT getVectorizedVT(EVT VT, unsigned N, LLVMContext &C) {
352 if (N == 1)
353 return VT;
354
355 return VT.isVector() ? EVT::getVectorVT(Context&: C, VT: VT.getScalarType(),
356 NumElements: VT.getVectorNumElements() * N)
357 : EVT::getVectorVT(Context&: C, VT, NumElements: N);
358}
359
360static SDValue getExtractVectorizedValue(SDValue V, unsigned I, EVT VT,
361 const SDLoc &dl, SelectionDAG &DAG) {
362 if (V.getValueType() == VT) {
363 assert(I == 0 && "Index must be 0 for scalar value");
364 return V;
365 }
366
367 if (!VT.isVector())
368 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT, N1: V,
369 N2: DAG.getVectorIdxConstant(Val: I, DL: dl));
370
371 return DAG.getNode(
372 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT, N1: V,
373 N2: DAG.getVectorIdxConstant(Val: I * VT.getVectorNumElements(), DL: dl));
374}
375
376template <typename T>
377static inline SDValue getBuildVectorizedValue(unsigned N, const SDLoc &dl,
378 SelectionDAG &DAG, T GetElement) {
379 if (N == 1)
380 return GetElement(0);
381
382 SmallVector<SDValue, 8> Values;
383 for (const unsigned I : llvm::seq(Size: N)) {
384 SDValue Val = GetElement(I);
385 if (Val.getValueType().isVector())
386 DAG.ExtractVectorElements(Op: Val, Args&: Values);
387 else
388 Values.push_back(Elt: Val);
389 }
390
391 EVT VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: Values[0].getValueType(),
392 NumElements: Values.size());
393 return DAG.getBuildVector(VT, DL: dl, Ops: Values);
394}
395
396/// PromoteScalarIntegerPTX
397/// Used to make sure the arguments/returns are suitable for passing
398/// and promote them to a larger size if they're not.
399///
400/// The promoted type is placed in \p PromoteVT if the function returns true.
401static EVT promoteScalarIntegerPTX(const EVT VT) {
402 if (VT.isScalarInteger()) {
403 switch (PowerOf2Ceil(A: VT.getFixedSizeInBits())) {
404 default:
405 llvm_unreachable(
406 "Promotion is not suitable for scalars of size larger than 64-bits");
407 case 1:
408 return MVT::i1;
409 case 2:
410 case 4:
411 case 8:
412 return MVT::i8;
413 case 16:
414 return MVT::i16;
415 case 32:
416 return MVT::i32;
417 case 64:
418 return MVT::i64;
419 }
420 }
421 return VT;
422}
423
424// Check whether we can merge loads/stores of some of the pieces of a
425// flattened function parameter or return value into a single vector
426// load/store.
427//
428// The flattened parameter is represented as a list of EVTs and
429// offsets, and the whole structure is aligned to ParamAlignment. This
430// function determines whether we can load/store pieces of the
431// parameter starting at index Idx using a single vectorized op of
432// size AccessSize. If so, it returns the number of param pieces
433// covered by the vector op. Otherwise, it returns 1.
434template <typename T>
435static unsigned canMergeParamLoadStoresStartingAt(
436 unsigned Idx, uint32_t AccessSize, const SmallVectorImpl<EVT> &ValueVTs,
437 const SmallVectorImpl<T> &Offsets, Align ParamAlignment) {
438
439 // Can't vectorize if param alignment is not sufficient.
440 if (ParamAlignment < AccessSize)
441 return 1;
442 // Can't vectorize if offset is not aligned.
443 if (Offsets[Idx] & (AccessSize - 1))
444 return 1;
445
446 EVT EltVT = ValueVTs[Idx];
447 unsigned EltSize = EltVT.getStoreSize();
448
449 // Element is too large to vectorize.
450 if (EltSize >= AccessSize)
451 return 1;
452
453 unsigned NumElts = AccessSize / EltSize;
454 // Can't vectorize if AccessBytes if not a multiple of EltSize.
455 if (AccessSize != EltSize * NumElts)
456 return 1;
457
458 // We don't have enough elements to vectorize.
459 if (Idx + NumElts > ValueVTs.size())
460 return 1;
461
462 // PTX ISA can only deal with 2- and 4-element vector ops.
463 if (NumElts != 4 && NumElts != 2)
464 return 1;
465
466 for (unsigned j = Idx + 1; j < Idx + NumElts; ++j) {
467 // Types do not match.
468 if (ValueVTs[j] != EltVT)
469 return 1;
470
471 // Elements are not contiguous.
472 if (Offsets[j] - Offsets[j - 1] != EltSize)
473 return 1;
474 }
475 // OK. We can vectorize ValueVTs[i..i+NumElts)
476 return NumElts;
477}
478
479// Computes whether and how we can vectorize the loads/stores of a
480// flattened function parameter or return value.
481//
482// The flattened parameter is represented as the list of ValueVTs and
483// Offsets, and is aligned to ParamAlignment bytes. We return a vector
484// of the same size as ValueVTs indicating how each piece should be
485// loaded/stored (i.e. as a scalar, or as part of a vector
486// load/store).
487template <typename T>
488static SmallVector<unsigned, 16>
489VectorizePTXValueVTs(const SmallVectorImpl<EVT> &ValueVTs,
490 const SmallVectorImpl<T> &Offsets, Align ParamAlignment,
491 bool IsVAArg = false) {
492 // Set vector size to match ValueVTs and mark all elements as
493 // scalars by default.
494
495 if (IsVAArg)
496 return SmallVector<unsigned>(ValueVTs.size(), 1);
497
498 SmallVector<unsigned, 16> VectorInfo;
499
500 const auto GetNumElts = [&](unsigned I) -> unsigned {
501 for (const unsigned AccessSize : {16, 8, 4, 2}) {
502 const unsigned NumElts = canMergeParamLoadStoresStartingAt(
503 I, AccessSize, ValueVTs, Offsets, ParamAlignment);
504 assert((NumElts == 1 || NumElts == 2 || NumElts == 4) &&
505 "Unexpected vectorization size");
506 if (NumElts != 1)
507 return NumElts;
508 }
509 return 1;
510 };
511
512 // Check what we can vectorize using 128/64/32-bit accesses.
513 for (unsigned I = 0, E = ValueVTs.size(); I != E;) {
514 const unsigned NumElts = GetNumElts(I);
515 VectorInfo.push_back(Elt: NumElts);
516 I += NumElts;
517 }
518 assert(std::accumulate(VectorInfo.begin(), VectorInfo.end(), 0u) ==
519 ValueVTs.size());
520 return VectorInfo;
521}
522
523// NVPTXTargetLowering Constructor.
524NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
525 const NVPTXSubtarget &STI)
526 : TargetLowering(TM, STI), STI(STI), GlobalUniqueCallSite(0) {
527 // always lower memset, memcpy, and memmove intrinsics to load/store
528 // instructions, rather
529 // then generating calls to memset, mempcy or memmove.
530 MaxStoresPerMemset = MaxStoresPerMemsetOptSize = (unsigned)0xFFFFFFFF;
531 MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = (unsigned) 0xFFFFFFFF;
532 MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = (unsigned) 0xFFFFFFFF;
533
534 setBooleanContents(ZeroOrNegativeOneBooleanContent);
535 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
536
537 // Jump is Expensive. Don't create extra control flow for 'and', 'or'
538 // condition branches.
539 setJumpIsExpensive(true);
540
541 // Wide divides are _very_ slow. Try to reduce the width of the divide if
542 // possible.
543 addBypassSlowDiv(SlowBitWidth: 64, FastBitWidth: 32);
544
545 // By default, use the Source scheduling
546 if (sched4reg)
547 setSchedulingPreference(Sched::RegPressure);
548 else
549 setSchedulingPreference(Sched::Source);
550
551 auto setFP16OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action,
552 LegalizeAction NoF16Action) {
553 bool IsOpSupported = STI.allowFP16Math();
554 switch (Op) {
555 // Several FP16 instructions are available on sm_80 only.
556 case ISD::FMINNUM:
557 case ISD::FMAXNUM:
558 case ISD::FMAXNUM_IEEE:
559 case ISD::FMINNUM_IEEE:
560 case ISD::FMAXIMUM:
561 case ISD::FMINIMUM:
562 case ISD::FMAXIMUMNUM:
563 case ISD::FMINIMUMNUM:
564 IsOpSupported &= STI.hasFeature(Feature: NVPTX::SM80);
565 break;
566 case ISD::FEXP2:
567 case ISD::FTANH:
568 IsOpSupported &=
569 STI.hasFeature(Feature: NVPTX::SM75) && STI.hasFeature(Feature: NVPTX::PTX70);
570 break;
571 }
572 setOperationAction(Op, VT, Action: IsOpSupported ? Action : NoF16Action);
573 };
574
575 auto setBF16OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action,
576 LegalizeAction NoBF16Action) {
577 bool IsOpSupported = STI.hasNativeBF16Support(Opcode: Op);
578 setOperationAction(
579 Op, VT, Action: IsOpSupported ? Action : NoBF16Action);
580 };
581
582 auto setI16x2OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action,
583 LegalizeAction NoI16x2Action) {
584 bool IsOpSupported = false;
585 // instructions are available on sm_90 only
586 switch (Op) {
587 case ISD::ADD:
588 case ISD::SMAX:
589 case ISD::SMIN:
590 case ISD::UMIN:
591 case ISD::UMAX:
592 IsOpSupported =
593 STI.hasFeature(Feature: NVPTX::SM90) && STI.hasFeature(Feature: NVPTX::PTX80);
594 break;
595 }
596 setOperationAction(Op, VT, Action: IsOpSupported ? Action : NoI16x2Action);
597 };
598
599 addRegisterClass(VT: MVT::i1, RC: &NVPTX::B1RegClass);
600 addRegisterClass(VT: MVT::i16, RC: &NVPTX::B16RegClass);
601 addRegisterClass(VT: MVT::v2i16, RC: &NVPTX::B32RegClass);
602 addRegisterClass(VT: MVT::v4i8, RC: &NVPTX::B32RegClass);
603 addRegisterClass(VT: MVT::i32, RC: &NVPTX::B32RegClass);
604 addRegisterClass(VT: MVT::i64, RC: &NVPTX::B64RegClass);
605 addRegisterClass(VT: MVT::f32, RC: &NVPTX::B32RegClass);
606 addRegisterClass(VT: MVT::f64, RC: &NVPTX::B64RegClass);
607 addRegisterClass(VT: MVT::f16, RC: &NVPTX::B16RegClass);
608 addRegisterClass(VT: MVT::v2f16, RC: &NVPTX::B32RegClass);
609 addRegisterClass(VT: MVT::bf16, RC: &NVPTX::B16RegClass);
610 addRegisterClass(VT: MVT::v2bf16, RC: &NVPTX::B32RegClass);
611
612 if (STI.hasF32x2Instructions()) {
613 addRegisterClass(VT: MVT::v2f32, RC: &NVPTX::B64RegClass);
614 addRegisterClass(VT: MVT::v2i32, RC: &NVPTX::B64RegClass);
615 }
616
617 // Conversion to/from FP16/FP16x2 is always legal.
618 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v2f16, Action: Custom);
619 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2f16, Action: Custom);
620 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v2f16, Action: Expand);
621 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v2f16, Action: Expand);
622
623 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: Legal);
624 if (STI.hasFeature(Feature: NVPTX::SM30))
625 setOperationAction(Op: ISD::READSTEADYCOUNTER, VT: MVT::i64, Action: Legal);
626
627 setFP16OperationAction(ISD::SETCC, MVT::f16, Legal, Promote);
628 setFP16OperationAction(ISD::SETCC, MVT::v2f16, Legal, Expand);
629
630 // Conversion to/from BFP16/BFP16x2 is always legal.
631 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v2bf16, Action: Custom);
632 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2bf16, Action: Custom);
633 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v2bf16, Action: Expand);
634 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v2bf16, Action: Expand);
635
636 setBF16OperationAction(ISD::SETCC, MVT::v2bf16, Legal, Expand);
637 setBF16OperationAction(ISD::SETCC, MVT::bf16, Legal, Promote);
638 if (getOperationAction(Op: ISD::SETCC, VT: MVT::bf16) == Promote)
639 AddPromotedToType(Opc: ISD::SETCC, OrigVT: MVT::bf16, DestVT: MVT::f32);
640
641 // Conversion to/from i16/i16x2 is always legal.
642 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v2i16, Action: Custom);
643 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2i16, Action: Custom);
644 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v2i16, Action: Expand);
645 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v2i16, Action: Expand);
646
647 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v4i8, Action: Custom);
648 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v4i8, Action: Custom);
649 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v4i8, Action: Custom);
650 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v4i8, Action: Custom);
651
652 // No support for these operations with v2f32/v2i32
653 setOperationAction(Ops: ISD::INSERT_VECTOR_ELT, VTs: {MVT::v2f32, MVT::v2i32}, Action: Expand);
654 setOperationAction(Ops: ISD::VECTOR_SHUFFLE, VTs: {MVT::v2f32, MVT::v2i32}, Action: Expand);
655
656 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v2i16, Action: Expand);
657 setOperationAction(Ops: {ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
658 VT: MVT::v2i32, Action: Expand);
659
660 // Need custom lowering in case the index is dynamic.
661 if (STI.hasF32x2Instructions())
662 setOperationAction(Ops: ISD::EXTRACT_VECTOR_ELT, VTs: {MVT::v2f32, MVT::v2i32},
663 Action: Custom);
664
665 // Custom conversions to/from v2i8.
666 setOperationAction(Op: ISD::BITCAST, VT: MVT::v2i8, Action: Custom);
667
668 // Only logical ops can be done on v4i8/v2i32 directly, others must be done
669 // elementwise.
670 setOperationAction(
671 Ops: {ISD::ABS, ISD::ADD, ISD::ADDC, ISD::ADDE,
672 ISD::BITREVERSE, ISD::CTLZ, ISD::CTPOP, ISD::CTTZ,
673 ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::FSHL, ISD::FSHR,
674 ISD::MUL, ISD::MULHS, ISD::MULHU, ISD::PARITY,
675 ISD::ROTL, ISD::ROTR, ISD::SADDO, ISD::SADDO_CARRY,
676 ISD::SADDSAT, ISD::SDIV, ISD::SDIVREM, ISD::SELECT_CC,
677 ISD::SETCC, ISD::SHL, ISD::SINT_TO_FP, ISD::SMAX,
678 ISD::SMIN, ISD::SMULO, ISD::SMUL_LOHI, ISD::SRA,
679 ISD::SREM, ISD::SRL, ISD::SSHLSAT, ISD::SSUBO,
680 ISD::SSUBO_CARRY, ISD::SSUBSAT, ISD::SUB, ISD::SUBC,
681 ISD::SUBE, ISD::UADDO, ISD::UADDO_CARRY, ISD::UADDSAT,
682 ISD::UDIV, ISD::UDIVREM, ISD::UINT_TO_FP, ISD::UMAX,
683 ISD::UMIN, ISD::UMULO, ISD::UMUL_LOHI, ISD::UREM,
684 ISD::USHLSAT, ISD::USUBO, ISD::USUBO_CARRY, ISD::VSELECT,
685 ISD::USUBSAT},
686 VTs: {MVT::v4i8, MVT::v2i32}, Action: Expand);
687
688 // Operations not directly supported by NVPTX.
689 for (MVT VT : {MVT::bf16, MVT::f16, MVT::v2bf16, MVT::v2f16, MVT::f32,
690 MVT::v2f32, MVT::f64, MVT::i1, MVT::i8, MVT::i16, MVT::v2i16,
691 MVT::v4i8, MVT::i32, MVT::v2i32, MVT::i64}) {
692 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
693 setOperationAction(Op: ISD::BR_CC, VT, Action: Expand);
694 }
695
696 setOperationAction(Ops: ISD::SDIVREM, VTs: {MVT::i32, MVT::i64}, Action: Expand);
697 setOperationAction(Ops: ISD::UDIVREM, VTs: {MVT::i32, MVT::i64}, Action: Expand);
698
699 // We don't want ops like FMINIMUM or UMAX to be lowered to SETCC+VSELECT.
700 setOperationAction(Ops: ISD::VSELECT, VTs: {MVT::v2f32, MVT::v2i32}, Action: Expand);
701
702 // Some SIGN_EXTEND_INREG can be done using cvt instruction.
703 // For others we will expand to a SHL/SRA pair.
704 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i64, Action: Legal);
705 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i32, Action: Legal);
706 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i16, Action: Legal);
707 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i8 , Action: Legal);
708 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
709 setOperationAction(Ops: ISD::SIGN_EXTEND_INREG, VTs: {MVT::v2i16, MVT::v2i32}, Action: Expand);
710
711 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i32 , Action: Custom);
712 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i32 , Action: Custom);
713 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i32 , Action: Custom);
714 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i64 , Action: Custom);
715 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i64 , Action: Custom);
716 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i64 , Action: Custom);
717
718 if (STI.hasCLMAD())
719 setOperationAction(Ops: {ISD::CLMUL, ISD::CLMULH}, VT: MVT::i64, Action: Legal);
720 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i32, Action: Legal);
721 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i64, Action: Legal);
722
723 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR},
724 VTs: {MVT::i8, MVT::i16, MVT::v2i16, MVT::i32, MVT::i64},
725 Action: Expand);
726
727 if (STI.hasHWROT32()) {
728 setOperationAction(Ops: {ISD::FSHL, ISD::FSHR}, VT: MVT::i32, Action: Legal);
729 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR, ISD::FSHL, ISD::FSHR}, VT: MVT::i64,
730 Action: Custom);
731 }
732
733 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: STI.hasBrx() ? Legal : Expand);
734 setOperationAction(Op: ISD::BRIND, VT: MVT::Other, Action: Expand);
735
736 // We want to legalize constant related memmove and memcopy
737 // intrinsics.
738 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::Other, Action: Custom);
739
740 // FP extload/truncstore is not legal in PTX. We need to expand all these.
741 for (auto FloatVTs :
742 {MVT::fp_valuetypes(), MVT::fp_fixedlen_vector_valuetypes()}) {
743 for (MVT ValVT : FloatVTs) {
744 for (MVT MemVT : FloatVTs) {
745 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT, MemVT, Action: Expand);
746 setTruncStoreAction(ValVT, MemVT, Action: Expand);
747 }
748 }
749 }
750
751 // To improve CodeGen we'll legalize any-extend loads to zext loads. This is
752 // how they'll be lowered in ISel anyway, and by doing this a little earlier
753 // we allow for more DAG combine opportunities.
754 for (auto IntVTs :
755 {MVT::integer_valuetypes(), MVT::integer_fixedlen_vector_valuetypes()})
756 for (MVT ValVT : IntVTs)
757 for (MVT MemVT : IntVTs)
758 if (isTypeLegal(VT: ValVT))
759 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT, MemVT, Action: Custom);
760
761 // PTX does not support load / store predicate registers
762 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VT: MVT::i1, Action: Custom);
763 for (MVT VT : MVT::integer_valuetypes()) {
764 setLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}, ValVT: VT, MemVT: MVT::i1,
765 Action: Promote);
766 setTruncStoreAction(ValVT: VT, MemVT: MVT::i1, Action: Expand);
767 }
768
769 // Disable generations of extload/truncstore for v2i32/v2i16/v2i8. The generic
770 // expansion for these nodes when they are unaligned is incorrect if the
771 // type is a vector.
772 //
773 // TODO: Fix the generic expansion for these nodes found in
774 // TargetLowering::expandUnalignedLoad/Store.
775 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::v2i16,
776 MemVT: MVT::v2i8, Action: Expand);
777 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::v2i32,
778 MemVTs: {MVT::v2i8, MVT::v2i16}, Action: Expand);
779 setTruncStoreAction(ValVT: MVT::v2i16, MemVT: MVT::v2i8, Action: Expand);
780 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i16, Action: Expand);
781 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i8, Action: Expand);
782
783 // Register custom handling for illegal type loads/stores. We'll try to custom
784 // lower almost all illegal types and logic in the lowering will discard cases
785 // we can't handle.
786 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VTs: {MVT::i128, MVT::i256, MVT::f128},
787 Action: Custom);
788 for (MVT VT : MVT::fixedlen_vector_valuetypes())
789 if (!isTypeLegal(VT) && VT.getStoreSizeInBits() <= 256)
790 setOperationAction(Ops: {ISD::STORE, ISD::LOAD, ISD::MSTORE, ISD::MLOAD}, VT,
791 Action: Custom);
792
793 // Custom legalization for LDU intrinsics.
794 // TODO: The logic to lower these is not very robust and we should rewrite it.
795 // Perhaps LDU should not be represented as an intrinsic at all.
796 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::i8, Action: Custom);
797 for (MVT VT : MVT::fixedlen_vector_valuetypes())
798 if (IsPTXVectorType(VT))
799 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT, Action: Custom);
800
801 setCondCodeAction(CCs: {ISD::SETNE, ISD::SETEQ, ISD::SETUGE, ISD::SETULE,
802 ISD::SETUGT, ISD::SETULT, ISD::SETGT, ISD::SETLT,
803 ISD::SETGE, ISD::SETLE},
804 VT: MVT::i1, Action: Expand);
805
806 // This is legal in NVPTX
807 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f64, Action: Legal);
808 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f32, Action: Legal);
809 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f16, Action: Legal);
810 setOperationAction(Op: ISD::ConstantFP, VT: MVT::bf16, Action: Legal);
811
812 setOperationAction(Ops: ISD::DYNAMIC_STACKALLOC, VTs: {MVT::i32, MVT::i64}, Action: Custom);
813 setOperationAction(Ops: {ISD::STACKRESTORE, ISD::STACKSAVE}, VT: MVT::Other, Action: Custom);
814
815 // TRAP can be lowered to PTX trap
816 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
817 // DEBUGTRAP can be lowered to PTX brkpt
818 setOperationAction(Op: ISD::DEBUGTRAP, VT: MVT::Other, Action: Legal);
819
820 // Support varargs.
821 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
822 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Custom);
823 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Expand);
824 setOperationAction(Op: ISD::VAEND, VT: MVT::Other, Action: Expand);
825
826 setOperationAction(Ops: {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX},
827 VTs: {MVT::i16, MVT::i32, MVT::i64}, Action: Legal);
828 // PTX abs.s is undefined for INT_MIN, so ISD::ABS (which requires
829 // abs(INT_MIN) == INT_MIN) must be expanded. ABS_MIN_POISON matches
830 // PTX abs semantics since INT_MIN input is poison/undefined.
831 setOperationAction(Ops: ISD::ABS, VTs: {MVT::i16, MVT::i32, MVT::i64}, Action: Expand);
832 setOperationAction(Ops: ISD::ABS_MIN_POISON, VTs: {MVT::i16, MVT::i32, MVT::i64},
833 Action: Legal);
834
835 setOperationAction(Ops: {ISD::CTPOP, ISD::CTLZ, ISD::CTLZ_ZERO_POISON}, VT: MVT::i16,
836 Action: Promote);
837 setOperationAction(Ops: {ISD::CTPOP, ISD::CTLZ}, VT: MVT::i32, Action: Legal);
838 setOperationAction(Ops: {ISD::CTPOP, ISD::CTLZ}, VT: MVT::i64, Action: Custom);
839
840 setI16x2OperationAction(ISD::ABS_MIN_POISON, MVT::v2i16, Legal, Custom);
841 setI16x2OperationAction(ISD::SMIN, MVT::v2i16, Legal, Custom);
842 setI16x2OperationAction(ISD::SMAX, MVT::v2i16, Legal, Custom);
843 setI16x2OperationAction(ISD::UMIN, MVT::v2i16, Legal, Custom);
844 setI16x2OperationAction(ISD::UMAX, MVT::v2i16, Legal, Custom);
845 setI16x2OperationAction(ISD::CTPOP, MVT::v2i16, Legal, Expand);
846 setI16x2OperationAction(ISD::CTLZ, MVT::v2i16, Legal, Expand);
847
848 setI16x2OperationAction(ISD::ADD, MVT::v2i16, Legal, Custom);
849 setI16x2OperationAction(ISD::SUB, MVT::v2i16, Legal, Custom);
850 setI16x2OperationAction(ISD::MUL, MVT::v2i16, Legal, Custom);
851 setI16x2OperationAction(ISD::SHL, MVT::v2i16, Legal, Custom);
852 setI16x2OperationAction(ISD::SREM, MVT::v2i16, Legal, Custom);
853 setI16x2OperationAction(ISD::UREM, MVT::v2i16, Legal, Custom);
854
855 // Other arithmetic and logic ops are unsupported.
856 setOperationAction(Ops: {ISD::SDIV, ISD::UDIV, ISD::SRA, ISD::SRL, ISD::MULHS,
857 ISD::MULHU, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
858 ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::SETCC},
859 VTs: {MVT::v2i16, MVT::v2i32}, Action: Expand);
860
861 // v2i32 is not supported for any arithmetic operations
862 setOperationAction(Ops: {ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX,
863 ISD::CTPOP, ISD::CTLZ, ISD::ADD, ISD::SUB, ISD::MUL,
864 ISD::SHL, ISD::SRA, ISD::SRL, ISD::OR, ISD::AND, ISD::XOR,
865 ISD::SREM, ISD::UREM},
866 VT: MVT::v2i32, Action: Expand);
867
868 setOperationAction(Op: ISD::ADDC, VT: MVT::i32, Action: Legal);
869 setOperationAction(Op: ISD::ADDE, VT: MVT::i32, Action: Legal);
870 setOperationAction(Op: ISD::SUBC, VT: MVT::i32, Action: Legal);
871 setOperationAction(Op: ISD::SUBE, VT: MVT::i32, Action: Legal);
872 if (STI.hasFeature(Feature: NVPTX::PTX43)) {
873 setOperationAction(Op: ISD::ADDC, VT: MVT::i64, Action: Legal);
874 setOperationAction(Op: ISD::ADDE, VT: MVT::i64, Action: Legal);
875 setOperationAction(Op: ISD::SUBC, VT: MVT::i64, Action: Legal);
876 setOperationAction(Op: ISD::SUBE, VT: MVT::i64, Action: Legal);
877 }
878
879 setOperationAction(Op: ISD::CTTZ, VT: MVT::i16, Action: Expand);
880 setOperationAction(Ops: ISD::CTTZ, VTs: {MVT::v2i16, MVT::v2i32}, Action: Expand);
881 setOperationAction(Op: ISD::CTTZ, VT: MVT::i32, Action: Expand);
882 setOperationAction(Op: ISD::CTTZ, VT: MVT::i64, Action: Expand);
883
884 // PTX does not directly support SELP of i1, so promote to i32 first
885 setOperationAction(Op: ISD::SELECT, VT: MVT::i1, Action: Custom);
886
887 // PTX cannot multiply two i64s in a single instruction.
888 setOperationAction(Op: ISD::SMUL_LOHI, VT: MVT::i64, Action: Expand);
889 setOperationAction(Op: ISD::UMUL_LOHI, VT: MVT::i64, Action: Expand);
890
891 // We have some custom DAG combine patterns for these nodes
892 setTargetDAGCombine({ISD::ADD,
893 ISD::AND,
894 ISD::EXTRACT_VECTOR_ELT,
895 ISD::FADD,
896 ISD::FMAXNUM,
897 ISD::FMINNUM,
898 ISD::FMAXIMUM,
899 ISD::FMINIMUM,
900 ISD::FMAXIMUMNUM,
901 ISD::FMINIMUMNUM,
902 ISD::MUL,
903 ISD::SELECT,
904 ISD::SHL,
905 ISD::SREM,
906 ISD::UREM,
907 ISD::VSELECT,
908 ISD::BUILD_VECTOR,
909 ISD::ADDRSPACECAST,
910 ISD::LOAD,
911 ISD::STORE,
912 ISD::ZERO_EXTEND,
913 ISD::SIGN_EXTEND,
914 ISD::INTRINSIC_WO_CHAIN});
915
916 // If the vector operands require register coalescing, scalarize instead
917 if (STI.hasF32x2Instructions())
918 setTargetDAGCombine({ISD::FMA, ISD::FMUL, ISD::FSUB});
919
920 // setcc for f16x2 and bf16x2 needs special handling to prevent
921 // legalizer's attempt to scalarize it due to v2i1 not being legal.
922 if (STI.allowFP16Math() || STI.hasBF16Math())
923 setTargetDAGCombine(ISD::SETCC);
924
925 // Vector reduction operations. These may be turned into shuffle or tree
926 // reductions depending on what instructions are available for each type.
927 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
928 MVT EltVT = VT.getVectorElementType();
929 if (EltVT == MVT::f32 || EltVT == MVT::f64) {
930 setOperationAction(Ops: {ISD::VECREDUCE_FMAX, ISD::VECREDUCE_FMIN,
931 ISD::VECREDUCE_FMAXIMUM, ISD::VECREDUCE_FMINIMUM},
932 VT, Action: Custom);
933 }
934 }
935
936 // Promote fp16 arithmetic if fp16 hardware isn't available or the
937 // user passed --nvptx-no-fp16-math. The flag is useful because,
938 // although sm_53+ GPUs have some sort of FP16 support in
939 // hardware, only sm_53 and sm_60 have full implementation. Others
940 // only have token amount of hardware and are likely to run faster
941 // by using fp32 units instead.
942 for (const auto &Op : {ISD::FADD, ISD::FMUL, ISD::FSUB, ISD::FMA}) {
943 setFP16OperationAction(Op, MVT::f16, Legal, Promote);
944 setFP16OperationAction(Op, MVT::v2f16, Legal, Expand);
945 setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);
946 // bf16 must be promoted to f32.
947 setBF16OperationAction(Op, MVT::bf16, Legal, Promote);
948 if (getOperationAction(Op, VT: MVT::bf16) == Promote)
949 AddPromotedToType(Opc: Op, OrigVT: MVT::bf16, DestVT: MVT::f32);
950 setOperationAction(Op, VT: MVT::v2f32,
951 Action: STI.hasF32x2Instructions() ? Legal : Expand);
952 }
953
954 // On SM80, we select add/mul/sub as fma to avoid promotion to float
955 for (const auto &Op : {ISD::FADD, ISD::FMUL, ISD::FSUB}) {
956 for (const auto &VT : {MVT::bf16, MVT::v2bf16}) {
957 if (!STI.hasNativeBF16Support(Opcode: Op) && STI.hasNativeBF16Support(Opcode: ISD::FMA)) {
958 setOperationAction(Op, VT, Action: Custom);
959 }
960 }
961 }
962
963 // f16/f16x2 neg was introduced in PTX 60, SM_53.
964 const bool IsFP16FP16x2NegAvailable = STI.hasFeature(Feature: NVPTX::SM53) &&
965 STI.hasFeature(Feature: NVPTX::PTX60) &&
966 STI.allowFP16Math();
967 for (const auto &VT : {MVT::f16, MVT::v2f16})
968 setOperationAction(Op: ISD::FNEG, VT,
969 Action: IsFP16FP16x2NegAvailable ? Legal : Expand);
970
971 setBF16OperationAction(ISD::FNEG, MVT::bf16, Legal, Expand);
972 setBF16OperationAction(ISD::FNEG, MVT::v2bf16, Legal, Expand);
973 setOperationAction(Op: ISD::FNEG, VT: MVT::v2f32, Action: Expand);
974 // (would be) Library functions.
975
976 // These map to conversion instructions for scalar FP types.
977 for (const auto &Op : {ISD::FCEIL, ISD::FFLOOR, ISD::FNEARBYINT, ISD::FRINT,
978 ISD::FROUNDEVEN, ISD::FTRUNC}) {
979 setOperationAction(Op, VT: MVT::f16, Action: Legal);
980 setOperationAction(Op, VT: MVT::f32, Action: Legal);
981 setOperationAction(Op, VT: MVT::f64, Action: Legal);
982 setOperationAction(Op, VT: MVT::v2f16, Action: Expand);
983 setOperationAction(Op, VT: MVT::v2bf16, Action: Expand);
984 setOperationAction(Op, VT: MVT::v2f32, Action: Expand);
985 setBF16OperationAction(Op, MVT::bf16, Legal, Promote);
986 if (getOperationAction(Op, VT: MVT::bf16) == Promote)
987 AddPromotedToType(Opc: Op, OrigVT: MVT::bf16, DestVT: MVT::f32);
988 }
989
990 if (!STI.hasFeature(Feature: NVPTX::SM80) || !STI.hasFeature(Feature: NVPTX::PTX71)) {
991 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f32, Action: Expand);
992 }
993 if (!STI.hasFeature(Feature: NVPTX::SM90)) {
994 for (MVT VT : {MVT::bf16, MVT::f32, MVT::f64}) {
995 setOperationAction(Op: ISD::FP_EXTEND, VT, Action: Custom);
996 setOperationAction(Op: ISD::FP_ROUND, VT, Action: Custom);
997 }
998 }
999
1000 // Expand v2f32 = fp_extend
1001 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v2f32, Action: Expand);
1002 // Expand v2[b]f16 = fp_round v2f32
1003 setOperationAction(Ops: ISD::FP_ROUND, VTs: {MVT::v2bf16, MVT::v2f16}, Action: Expand);
1004
1005 // sm_80 only has conversions between f32 and bf16. Custom lower all other
1006 // bf16 conversions.
1007 if (!STI.hasFeature(Feature: NVPTX::SM90)) {
1008 for (MVT VT : {MVT::i1, MVT::i16, MVT::i32, MVT::i64}) {
1009 setOperationAction(
1010 Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT},
1011 VT, Action: Custom);
1012 }
1013 setOperationAction(
1014 Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT},
1015 VT: MVT::bf16, Action: Custom);
1016 }
1017
1018 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT: MVT::i1, Action: Custom);
1019 setOperationAction(Op: ISD::FROUND, VT: MVT::f16, Action: Promote);
1020 setOperationAction(Op: ISD::FROUND, VT: MVT::v2f16, Action: Expand);
1021 setOperationAction(Op: ISD::FROUND, VT: MVT::v2bf16, Action: Expand);
1022 setOperationAction(Op: ISD::FROUND, VT: MVT::f32, Action: Custom);
1023 setOperationAction(Op: ISD::FROUND, VT: MVT::f64, Action: Custom);
1024 setOperationAction(Op: ISD::FROUND, VT: MVT::bf16, Action: Promote);
1025 AddPromotedToType(Opc: ISD::FROUND, OrigVT: MVT::bf16, DestVT: MVT::f32);
1026
1027 setOperationAction(Ops: {ISD::LROUND, ISD::LLROUND}, VTs: {MVT::f32, MVT::f64}, Action: Expand);
1028
1029 // 'Expand' implements FCOPYSIGN without calling an external library.
1030 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f16, Action: Expand);
1031 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::v2f16, Action: Expand);
1032 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::bf16, Action: Expand);
1033 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::v2bf16, Action: Expand);
1034 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f32, Action: Custom);
1035 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f64, Action: Custom);
1036
1037 // These map to corresponding instructions for f32/f64. f16 must be
1038 // promoted to f32. v2f16 is expanded to f16, which is then promoted
1039 // to f32.
1040 for (const auto &Op :
1041 {ISD::FDIV, ISD::FREM, ISD::FSQRT, ISD::FSIN, ISD::FCOS}) {
1042 setOperationAction(Op, VT: MVT::f16, Action: Promote);
1043 setOperationAction(Op, VT: MVT::f32, Action: Legal);
1044 // only div/rem/sqrt are legal for f64
1045 if (Op == ISD::FDIV || Op == ISD::FREM || Op == ISD::FSQRT) {
1046 setOperationAction(Op, VT: MVT::f64, Action: Legal);
1047 }
1048 setOperationAction(Ops: Op, VTs: {MVT::v2f16, MVT::v2bf16, MVT::v2f32}, Action: Expand);
1049 setOperationAction(Op, VT: MVT::bf16, Action: Promote);
1050 AddPromotedToType(Opc: Op, OrigVT: MVT::bf16, DestVT: MVT::f32);
1051 }
1052 setOperationAction(Ops: ISD::FREM, VTs: {MVT::f32, MVT::f64}, Action: Custom);
1053
1054 // FTANH support:
1055 // - f32 (sm_75+, PTX 7.0+)
1056 // - f16/f16x2 (sm_75+, PTX 7.0+)
1057 // - bf16/bf16x2 (sm_90+, PTX 7.8+)
1058 // When f16/bf16 types aren't supported, they are promoted/expanded to f32.
1059 if (STI.hasFeature(Feature: NVPTX::SM75) && STI.hasFeature(Feature: NVPTX::PTX70))
1060 setOperationAction(Op: ISD::FTANH, VT: MVT::f32, Action: Legal);
1061 setOperationAction(Op: ISD::FTANH, VT: MVT::v2f32, Action: Expand);
1062
1063 // Scalar f16/bf16: promote to f32 when not natively supported.
1064 setFP16OperationAction(ISD::FTANH, MVT::f16, Legal, Promote);
1065 setBF16OperationAction(ISD::FTANH, MVT::bf16, Legal, Promote);
1066 if (getOperationAction(Op: ISD::FTANH, VT: MVT::bf16) == Promote)
1067 AddPromotedToType(Opc: ISD::FTANH, OrigVT: MVT::bf16, DestVT: MVT::f32);
1068
1069 // Vector v2f16/v2bf16: expand when not natively supported.
1070 setFP16OperationAction(ISD::FTANH, MVT::v2f16, Legal, Expand);
1071 setBF16OperationAction(ISD::FTANH, MVT::v2bf16, Legal, Expand);
1072
1073 setOperationAction(Ops: ISD::FABS, VTs: {MVT::f32, MVT::f64}, Action: Legal);
1074 setOperationAction(Op: ISD::FABS, VT: MVT::v2f32, Action: Expand);
1075 if (STI.hasFeature(Feature: NVPTX::PTX65)) {
1076 setFP16OperationAction(ISD::FABS, MVT::f16, Legal, Promote);
1077 setFP16OperationAction(ISD::FABS, MVT::v2f16, Legal, Expand);
1078 } else {
1079 setOperationAction(Op: ISD::FABS, VT: MVT::f16, Action: Promote);
1080 setOperationAction(Op: ISD::FABS, VT: MVT::v2f16, Action: Expand);
1081 }
1082 setBF16OperationAction(ISD::FABS, MVT::v2bf16, Legal, Expand);
1083 setBF16OperationAction(ISD::FABS, MVT::bf16, Legal, Promote);
1084 if (getOperationAction(Op: ISD::FABS, VT: MVT::bf16) == Promote)
1085 AddPromotedToType(Opc: ISD::FABS, OrigVT: MVT::bf16, DestVT: MVT::f32);
1086
1087 for (const auto &Op :
1088 {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM}) {
1089 setOperationAction(Op, VT: MVT::f32, Action: Legal);
1090 setOperationAction(Op, VT: MVT::f64, Action: Legal);
1091 setFP16OperationAction(Op, MVT::f16, Legal, Promote);
1092 setFP16OperationAction(Op, MVT::v2f16, Legal, Expand);
1093 setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);
1094 setBF16OperationAction(Op, MVT::bf16, Legal, Promote);
1095 if (getOperationAction(Op, VT: MVT::bf16) == Promote)
1096 AddPromotedToType(Opc: Op, OrigVT: MVT::bf16, DestVT: MVT::f32);
1097 setOperationAction(Op, VT: MVT::v2f32, Action: Expand);
1098 }
1099 bool SupportsF32MinMaxNaN = STI.hasFeature(Feature: NVPTX::SM80);
1100 for (const auto &Op : {ISD::FMINIMUM, ISD::FMAXIMUM}) {
1101 setOperationAction(Op, VT: MVT::f32, Action: SupportsF32MinMaxNaN ? Legal : Expand);
1102 setFP16OperationAction(Op, MVT::f16, Legal, Expand);
1103 setFP16OperationAction(Op, MVT::v2f16, Legal, Expand);
1104 setBF16OperationAction(Op, MVT::bf16, Legal, Expand);
1105 setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);
1106 setOperationAction(Op, VT: MVT::v2f32, Action: Expand);
1107 }
1108
1109 // Custom lowering for inline asm with 128-bit operands
1110 setOperationAction(Op: ISD::CopyToReg, VT: MVT::i128, Action: Custom);
1111 setOperationAction(Op: ISD::CopyFromReg, VT: MVT::i128, Action: Custom);
1112
1113 // FEXP2 support:
1114 // - f32
1115 // - f16/f16x2 (sm_70+, PTX 7.0+)
1116 // - bf16/bf16x2 (sm_90+, PTX 7.8+)
1117 // When f16/bf16 types aren't supported, they are promoted/expanded to f32.
1118 setOperationAction(Op: ISD::FEXP2, VT: MVT::f32, Action: Legal);
1119 setOperationAction(Op: ISD::FEXP2, VT: MVT::v2f32, Action: Expand);
1120 setFP16OperationAction(ISD::FEXP2, MVT::f16, Legal, Promote);
1121 setFP16OperationAction(ISD::FEXP2, MVT::v2f16, Legal, Expand);
1122 setBF16OperationAction(ISD::FEXP2, MVT::bf16, Legal, Promote);
1123 setBF16OperationAction(ISD::FEXP2, MVT::v2bf16, Legal, Expand);
1124
1125 // FLOG2 supports f32 only
1126 // f16/bf16 types aren't supported, but they are promoted/expanded to f32.
1127 if (UseApproxLog2F32) {
1128 setOperationAction(Op: ISD::FLOG2, VT: MVT::f32, Action: Legal);
1129 setOperationPromotedToType(Opc: ISD::FLOG2, OrigVT: MVT::f16, DestVT: MVT::f32);
1130 setOperationPromotedToType(Opc: ISD::FLOG2, OrigVT: MVT::bf16, DestVT: MVT::f32);
1131 setOperationAction(Ops: ISD::FLOG2, VTs: {MVT::v2f16, MVT::v2bf16, MVT::v2f32},
1132 Action: Expand);
1133 }
1134
1135 setOperationAction(Ops: ISD::ADDRSPACECAST, VTs: {MVT::i32, MVT::i64}, Action: Custom);
1136
1137 setOperationAction(Ops: ISD::ATOMIC_LOAD_SUB, VTs: {MVT::i32, MVT::i64}, Action: Expand);
1138
1139 // atom.b128 is legal in PTX but since we don't represent i128 as a legal
1140 // type, we need to custom lower it.
1141 setOperationAction(Ops: {ISD::ATOMIC_CMP_SWAP, ISD::ATOMIC_SWAP}, VT: MVT::i128,
1142 Action: Custom);
1143
1144 // Now deduce the information based on the above mentioned
1145 // actions
1146 computeRegisterProperties(TRI: STI.getRegisterInfo());
1147
1148 // PTX support for 16-bit CAS is emulated. Only use 32+
1149 setMinCmpXchgSizeInBits(STI.getMinCmpXchgSizeInBits());
1150 setMaxAtomicSizeInBitsSupported(STI.hasAtomSwap128() ? 128 : 64);
1151 setMaxDivRemBitWidthSupported(64);
1152 setMaxLargeFPConvertBitWidthSupported(64);
1153
1154 // Custom lowering for tcgen05.ld vector operands
1155 setOperationAction(Ops: ISD::INTRINSIC_W_CHAIN,
1156 VTs: {MVT::v1i32, MVT::v2i32, MVT::v4i32, MVT::v8i32,
1157 MVT::v16i32, MVT::v32i32, MVT::v64i32, MVT::v128i32,
1158 MVT::v2f32, MVT::v4f32, MVT::v8f32, MVT::v16f32,
1159 MVT::v32f32, MVT::v64f32, MVT::v128f32},
1160 Action: Custom);
1161
1162 // Custom lowering for tcgen05.st vector operands and the st.async
1163 // i128 (.b128) operand. MVT::i8 is needed for the st.async.{sys,gpu} b8
1164 // variant.
1165 setOperationAction(Ops: ISD::INTRINSIC_VOID,
1166 VTs: {MVT::i8, MVT::v1i32, MVT::v2i32, MVT::v4i32, MVT::v8i32,
1167 MVT::v16i32, MVT::v32i32, MVT::v64i32, MVT::v128i32,
1168 MVT::i128, MVT::Other},
1169 Action: Custom);
1170
1171 // Enable custom lowering for the following:
1172 // * MVT::i128 - clusterlaunchcontrol
1173 // * MVT::i32 - prmt
1174 // * MVT::v4f32 - cvt_rs fp{4/6/8}x4 intrinsics
1175 // * MVT::Other - internal.addrspace.wrap
1176 setOperationAction(Ops: ISD::INTRINSIC_WO_CHAIN,
1177 VTs: {MVT::i32, MVT::i128, MVT::v4f32, MVT::Other}, Action: Custom);
1178
1179 // Custom lowering for bswap
1180 setOperationAction(Ops: ISD::BSWAP, VTs: {MVT::i16, MVT::i32, MVT::i64, MVT::v2i16},
1181 Action: Custom);
1182}
1183
1184TargetLoweringBase::LegalizeTypeAction
1185NVPTXTargetLowering::getPreferredVectorAction(MVT VT) const {
1186 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
1187 VT.getScalarType() == MVT::i1)
1188 return TypeSplitVector;
1189 return TargetLoweringBase::getPreferredVectorAction(VT);
1190}
1191
1192SDValue NVPTXTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,
1193 int Enabled, int &ExtraSteps,
1194 bool &UseOneConst,
1195 bool Reciprocal) const {
1196 if (!(Enabled == ReciprocalEstimate::Enabled ||
1197 (Enabled == ReciprocalEstimate::Unspecified && !usePrecSqrtF32())))
1198 return SDValue();
1199
1200 if (ExtraSteps == ReciprocalEstimate::Unspecified)
1201 ExtraSteps = 0;
1202
1203 SDLoc DL(Operand);
1204 EVT VT = Operand.getValueType();
1205 bool Ftz = useF32FTZ(MF: DAG.getMachineFunction());
1206
1207 auto MakeIntrinsicCall = [&](Intrinsic::ID IID) {
1208 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT,
1209 N1: DAG.getConstant(Val: IID, DL, VT: MVT::i32), N2: Operand);
1210 };
1211
1212 // The sqrt and rsqrt refinement processes assume we always start out with an
1213 // approximation of the rsqrt. Therefore, if we're going to do any refinement
1214 // (i.e. ExtraSteps > 0), we must return an rsqrt. But if we're *not* doing
1215 // any refinement, we must return a regular sqrt.
1216 if (Reciprocal || ExtraSteps > 0) {
1217 if (VT == MVT::f32)
1218 return MakeIntrinsicCall(Ftz ? Intrinsic::nvvm_rsqrt_approx_ftz_f
1219 : Intrinsic::nvvm_rsqrt_approx_f);
1220 else if (VT == MVT::f64)
1221 return MakeIntrinsicCall(Intrinsic::nvvm_rsqrt_approx_d);
1222 else
1223 return SDValue();
1224 } else {
1225 if (VT == MVT::f32)
1226 return MakeIntrinsicCall(Ftz ? Intrinsic::nvvm_sqrt_approx_ftz_f
1227 : Intrinsic::nvvm_sqrt_approx_f);
1228 else {
1229 // There's no sqrt.approx.f64 instruction, so we emit
1230 // reciprocal(rsqrt(x)). This is faster than
1231 // select(x == 0, 0, x * rsqrt(x)). (In fact, it's faster than plain
1232 // x * rsqrt(x).)
1233 return DAG.getNode(
1234 Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT,
1235 N1: DAG.getConstant(Val: Intrinsic::nvvm_rcp_approx_ftz_d, DL, VT: MVT::i32),
1236 N2: MakeIntrinsicCall(Intrinsic::nvvm_rsqrt_approx_d));
1237 }
1238 }
1239}
1240
1241static MachinePointerInfo refinePtrAS(SDValue &Ptr, SelectionDAG &DAG) {
1242 // Load directly from the source address space of a cast to generic.
1243 unsigned SrcAS = ADDRESS_SPACE_GENERIC;
1244 if (Ptr->getOpcode() == ISD::ADDRSPACECAST) {
1245 const auto *ASC = cast<AddrSpaceCastSDNode>(Val&: Ptr);
1246 if (ASC->getDestAddressSpace() == ADDRESS_SPACE_GENERIC) {
1247 Ptr = ASC->getOperand(Num: 0);
1248 SrcAS = ASC->getSrcAddressSpace();
1249 }
1250 }
1251
1252 // Preserve the alloca's address space through frame-index inference.
1253 if (const auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Ptr))
1254 if (const AllocaInst *AI =
1255 DAG.getMachineFunction().getFrameInfo().getObjectAllocation(
1256 ObjectIdx: FIN->getIndex()))
1257 return MachinePointerInfo(AI);
1258
1259 return MachinePointerInfo(SrcAS);
1260}
1261
1262static ISD::NodeType getExtOpcode(const ISD::ArgFlagsTy &Flags) {
1263 if (Flags.isSExt())
1264 return ISD::SIGN_EXTEND;
1265 if (Flags.isZExt())
1266 return ISD::ZERO_EXTEND;
1267 return ISD::ANY_EXTEND;
1268}
1269
1270static SDValue correctParamType(SDValue V, EVT ExpectedVT,
1271 ISD::ArgFlagsTy Flags, SelectionDAG &DAG,
1272 SDLoc dl) {
1273 const EVT ActualVT = V.getValueType();
1274 assert((ActualVT == ExpectedVT ||
1275 (ExpectedVT.isInteger() && ActualVT.isInteger())) &&
1276 "Non-integer argument type size mismatch");
1277 if (ExpectedVT.bitsGT(VT: ActualVT))
1278 return DAG.getNode(Opcode: getExtOpcode(Flags), DL: dl, VT: ExpectedVT, Operand: V);
1279 if (ExpectedVT.bitsLT(VT: ActualVT))
1280 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ExpectedVT, Operand: V);
1281
1282 return V;
1283}
1284
1285static SDValue getSymbolNode(SelectionDAG &DAG, MCSymbol *Sym, EVT T) {
1286 return DAG.getNode(Opcode: NVPTXISD::Symbol, DL: SDLoc(), VT: T, Operand: DAG.getMCSymbol(Sym, VT: T));
1287}
1288
1289static SDValue getSymbolNode(SelectionDAG &DAG, const Twine &Name, EVT T) {
1290 MCContext &Ctx = DAG.getMachineFunction().getContext();
1291 return getSymbolNode(DAG, Sym: Ctx.getOrCreateSymbol(Name), T);
1292}
1293
1294SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
1295 SmallVectorImpl<SDValue> &InVals) const {
1296
1297 if (CLI.IsVarArg &&
1298 (!STI.hasFeature(Feature: NVPTX::PTX60) || !STI.hasFeature(Feature: NVPTX::SM30)))
1299 report_fatal_error(
1300 reason: "Support for variadic functions (unsized array parameter) introduced "
1301 "in PTX ISA version 6.0 and requires target sm_30.");
1302
1303 SelectionDAG &DAG = CLI.DAG;
1304 SDLoc dl = CLI.DL;
1305 const SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
1306 SDValue Callee = CLI.Callee;
1307 ArgListTy &Args = CLI.getArgs();
1308 Type *RetTy = CLI.RetTy;
1309 const CallBase *CB = CLI.CB;
1310 const DataLayout &DL = DAG.getDataLayout();
1311 LLVMContext &Ctx = *DAG.getContext();
1312
1313 const auto GetI32 = [&](const unsigned I) {
1314 return DAG.getConstant(Val: I, DL: dl, VT: MVT::i32);
1315 };
1316
1317 const unsigned UniqueCallSite = GlobalUniqueCallSite++;
1318 const SDValue CallChain = CLI.Chain;
1319 const SDValue StartChain =
1320 DAG.getCALLSEQ_START(Chain: CallChain, InSize: UniqueCallSite, OutSize: 0, DL: dl);
1321 SDValue DeclareGlue = StartChain.getValue(R: 1);
1322
1323 SmallVector<SDValue, 16> CallPrereqs{StartChain};
1324
1325 const auto MakeDeclareScalarParam = [&](SDValue Symbol, unsigned Size) {
1326 // PTX ABI requires integral types to be at least 32 bits in size. FP16 is
1327 // loaded/stored using i16, so it's handled here as well.
1328 const unsigned SizeBits = promoteScalarArgumentSize(size: Size * 8);
1329 SDValue Declare =
1330 DAG.getNode(Opcode: NVPTXISD::DeclareScalarParam, DL: dl, ResultTys: {MVT::Other, MVT::Glue},
1331 Ops: {StartChain, Symbol, GetI32(SizeBits), DeclareGlue});
1332 CallPrereqs.push_back(Elt: Declare);
1333 DeclareGlue = Declare.getValue(R: 1);
1334 return Declare;
1335 };
1336
1337 const auto MakeDeclareArrayParam = [&](SDValue Symbol, Align Align,
1338 unsigned Size) {
1339 SDValue Declare = DAG.getNode(
1340 Opcode: NVPTXISD::DeclareArrayParam, DL: dl, ResultTys: {MVT::Other, MVT::Glue},
1341 Ops: {StartChain, Symbol, GetI32(Align.value()), GetI32(Size), DeclareGlue});
1342 CallPrereqs.push_back(Elt: Declare);
1343 DeclareGlue = Declare.getValue(R: 1);
1344 return Declare;
1345 };
1346
1347 // Variadic arguments.
1348 //
1349 // Normally, for each argument, we declare a param scalar or a param
1350 // byte array in the .param space, and store the argument value to that
1351 // param scalar or array starting at offset 0.
1352 //
1353 // In the case of the first variadic argument, we declare a vararg byte array
1354 // with size 0. The exact size of this array isn't known at this point, so
1355 // it'll be patched later. All the variadic arguments will be stored to this
1356 // array at a certain offset (which gets tracked by 'VAOffset'). The offset is
1357 // initially set to 0, so it can be used for non-variadic arguments (which use
1358 // 0 offset) to simplify the code.
1359 //
1360 // After all vararg is processed, 'VAOffset' holds the size of the
1361 // vararg byte array.
1362 assert((CLI.IsVarArg || CLI.Args.size() <= CLI.NumFixedArgs) &&
1363 "Non-VarArg function with extra arguments");
1364
1365 const unsigned FirstVAArg = CLI.NumFixedArgs; // position of first variadic
1366 unsigned VAOffset = 0; // current offset in the param array
1367
1368 const SDValue VADeclareParam =
1369 CLI.Args.size() > FirstVAArg
1370 ? MakeDeclareArrayParam(
1371 getCallParamSymbolNode(DAG, I: FirstVAArg, T: MVT::i32),
1372 Align(STI.getMaxRequiredAlignment()), 0)
1373 : SDValue();
1374
1375 // Args.size() and Outs.size() need not match.
1376 // Outs.size() will be larger
1377 // * if there is an aggregate argument with multiple fields (each field
1378 // showing up separately in Outs)
1379 // * if there is a vector argument with more than typical vector-length
1380 // elements (generally if more than 4) where each vector element is
1381 // individually present in Outs.
1382 // So a different index should be used for indexing into Outs/OutVals.
1383 // See similar issue in LowerFormalArguments.
1384 auto AllOuts = ArrayRef(CLI.Outs);
1385 auto AllOutVals = ArrayRef(CLI.OutVals);
1386 assert(AllOuts.size() == AllOutVals.size() &&
1387 "Outs and OutVals must be the same size");
1388 // Declare the .params or .reg need to pass values
1389 // to the function
1390 for (const auto E : llvm::enumerate(First&: Args)) {
1391 const auto ArgI = E.index();
1392 const auto Arg = E.value();
1393 const auto ArgOuts =
1394 AllOuts.take_while(Pred: [&](auto O) { return O.OrigArgIndex == ArgI; });
1395 const auto ArgOutVals = AllOutVals.take_front(N: ArgOuts.size());
1396 AllOuts = AllOuts.drop_front(N: ArgOuts.size());
1397 AllOutVals = AllOutVals.drop_front(N: ArgOuts.size());
1398
1399 const bool IsVAArg = (ArgI >= FirstVAArg);
1400 const bool IsByVal = Arg.IsByVal;
1401
1402 const SDValue ParamSymbol =
1403 getCallParamSymbolNode(DAG, I: IsVAArg ? FirstVAArg : ArgI, T: MVT::i32);
1404
1405 assert((!IsByVal || Arg.IndirectType) &&
1406 "byval arg must have indirect type");
1407 Type *ETy = (IsByVal ? Arg.IndirectType : Arg.Ty);
1408
1409 const Align ArgAlign = [&]() {
1410 const unsigned ParamIdx = ArgI + AttributeList::FirstArgIndex;
1411 if (IsByVal)
1412 return getDeviceByValParamAlign(CB, ArgTy: ETy, AttrIdx: ParamIdx, DL);
1413 return getPTXParamAlign(CB, Ty: Arg.Ty, AttrIdx: ParamIdx, DL);
1414 }();
1415
1416 const unsigned TySize = DL.getTypeAllocSize(Ty: ETy);
1417 assert((!IsByVal || TySize == ArgOuts[0].Flags.getByValSize()) &&
1418 "type size mismatch");
1419
1420 const SDValue ArgDeclare = [&]() {
1421 if (IsVAArg)
1422 return VADeclareParam;
1423
1424 if (IsByVal || shouldPassAsArray(Ty: Arg.Ty))
1425 return MakeDeclareArrayParam(ParamSymbol, ArgAlign, TySize);
1426
1427 assert(ArgOuts.size() == 1 && "We must pass only one value as non-array");
1428 assert((ArgOuts[0].VT.isInteger() || ArgOuts[0].VT.isFloatingPoint()) &&
1429 "Only int and float types are supported as non-array arguments");
1430
1431 return MakeDeclareScalarParam(ParamSymbol, TySize);
1432 }();
1433
1434 if (IsByVal) {
1435 assert(ArgOutVals.size() == 1 && "We must pass only one value as byval");
1436 SDValue SrcPtr = ArgOutVals[0];
1437 const MachinePointerInfo SrcPtrInfo = refinePtrAS(Ptr&: SrcPtr, DAG);
1438 // Don't use Flags.getNonZeroByValAlign as this includes the stackalign,
1439 // which does not apply to the source pointer.
1440 const Align BaseSrcAlign = [&]() {
1441 // The align attribute on a byval argument indicates the known alignment
1442 // of the pointer passed to the function.
1443 if (CB)
1444 if (const MaybeAlign A = CB->getParamAlign(ArgNo: ArgI))
1445 return *A;
1446 // Fall back to the default alignment for the type.
1447 // TODO: This might be too aggressive but we haven't had a problem with
1448 // it yet.
1449 return getPTXParamTypeAlign(ArgTy: ETy, DL);
1450 }();
1451
1452 if (IsVAArg)
1453 VAOffset = alignTo(Size: VAOffset, A: ArgAlign);
1454
1455 SmallVector<EVT, 4> ValueVTs, MemVTs;
1456 SmallVector<TypeSize, 4> Offsets;
1457 ComputeValueVTs(TLI: *this, DL, Ty: ETy, ValueVTs, MemVTs: &MemVTs, Offsets: &Offsets);
1458
1459 unsigned J = 0;
1460 const auto VI = VectorizePTXValueVTs(ValueVTs: MemVTs, Offsets, ParamAlignment: ArgAlign, IsVAArg);
1461 for (const unsigned NumElts : VI) {
1462 EVT LoadVT = getVectorizedVT(VT: MemVTs[J], N: NumElts, C&: Ctx);
1463 Align SrcAlign = commonAlignment(A: BaseSrcAlign, Offset: Offsets[J]);
1464 SDValue SrcAddr = DAG.getObjectPtrOffset(SL: dl, Ptr: SrcPtr, Offset: Offsets[J]);
1465 SDValue SrcLoad =
1466 DAG.getLoad(VT: LoadVT, dl, Chain: CallChain, Ptr: SrcAddr,
1467 PtrInfo: SrcPtrInfo.getWithOffset(O: Offsets[J]), Alignment: SrcAlign);
1468
1469 TypeSize ParamOffset = Offsets[J].getWithIncrement(RHS: VAOffset);
1470 Align ParamAlign = commonAlignment(A: ArgAlign, Offset: ParamOffset);
1471 SDValue ParamAddr =
1472 DAG.getObjectPtrOffset(SL: dl, Ptr: ParamSymbol, Offset: ParamOffset);
1473 SDValue StoreParam = DAG.getStore(
1474 Chain: ArgDeclare, dl, Val: SrcLoad, Ptr: ParamAddr,
1475 PtrInfo: MachinePointerInfo(NVPTX::AddressSpace::DeviceParam), Alignment: ParamAlign);
1476 CallPrereqs.push_back(Elt: StoreParam);
1477
1478 J += NumElts;
1479 }
1480 if (IsVAArg)
1481 VAOffset += TySize;
1482 } else {
1483 SmallVector<EVT, 16> VTs;
1484 SmallVector<uint64_t, 16> Offsets;
1485 ComputePTXValueVTs(TLI: *this, DL, Ctx, CallConv: CLI.CallConv, Ty: Arg.Ty, ValueVTs&: VTs, Offsets,
1486 StartingOffset: VAOffset);
1487 assert(VTs.size() == Offsets.size() && "Size mismatch");
1488 assert(VTs.size() == ArgOuts.size() && "Size mismatch");
1489
1490 // PTX Interoperability Guide 3.3(A): [Integer] Values shorter
1491 // than 32-bits are sign extended or zero extended, depending on
1492 // whether they are signed or unsigned types. This case applies
1493 // only to scalar parameters and not to aggregate values.
1494 const bool ExtendIntegerParam =
1495 Arg.Ty->isIntegerTy() && DL.getTypeAllocSizeInBits(Ty: Arg.Ty) < 32;
1496
1497 const auto GetStoredValue = [&](const unsigned I) {
1498 SDValue StVal = ArgOutVals[I];
1499 assert(promoteScalarIntegerPTX(StVal.getValueType()) ==
1500 StVal.getValueType() &&
1501 "OutVal type should always be legal");
1502
1503 const EVT VTI = promoteScalarIntegerPTX(VT: VTs[I]);
1504 const EVT StoreVT =
1505 ExtendIntegerParam ? MVT::i32 : (VTI == MVT::i1 ? MVT::i8 : VTI);
1506
1507 return correctParamType(V: StVal, ExpectedVT: StoreVT, Flags: ArgOuts[I].Flags, DAG, dl);
1508 };
1509
1510 unsigned J = 0;
1511 const auto VI = VectorizePTXValueVTs(ValueVTs: VTs, Offsets, ParamAlignment: ArgAlign, IsVAArg);
1512 for (const unsigned NumElts : VI) {
1513 const EVT EltVT = promoteScalarIntegerPTX(VT: VTs[J]);
1514
1515 unsigned Offset;
1516 if (IsVAArg) {
1517 // TODO: We may need to support vector types that can be passed
1518 // as scalars in variadic arguments.
1519 assert(NumElts == 1 &&
1520 "Vectorization should be disabled for vaargs.");
1521
1522 // Align each part of the variadic argument to their type.
1523 VAOffset = alignTo(Size: VAOffset, A: DAG.getEVTAlign(MemoryVT: EltVT));
1524 Offset = VAOffset;
1525
1526 const EVT TheStoreType = ExtendIntegerParam ? MVT::i32 : EltVT;
1527 VAOffset += DL.getTypeAllocSize(Ty: TheStoreType.getTypeForEVT(Context&: Ctx));
1528 } else {
1529 assert(VAOffset == 0 && "VAOffset must be 0 for non-VA args");
1530 Offset = Offsets[J];
1531 }
1532
1533 SDValue Ptr =
1534 DAG.getObjectPtrOffset(SL: dl, Ptr: ParamSymbol, Offset: TypeSize::getFixed(ExactSize: Offset));
1535
1536 const MaybeAlign CurrentAlign = ExtendIntegerParam
1537 ? MaybeAlign(std::nullopt)
1538 : commonAlignment(A: ArgAlign, Offset);
1539
1540 SDValue Val =
1541 getBuildVectorizedValue(N: NumElts, dl, DAG, GetElement: [&](unsigned K) {
1542 return GetStoredValue(J + K);
1543 });
1544
1545 SDValue StoreParam = DAG.getStore(
1546 Chain: ArgDeclare, dl, Val, Ptr,
1547 PtrInfo: MachinePointerInfo(NVPTX::AddressSpace::DeviceParam), Alignment: CurrentAlign);
1548 CallPrereqs.push_back(Elt: StoreParam);
1549
1550 J += NumElts;
1551 }
1552 }
1553 }
1554
1555 // Handle Result
1556 if (!Ins.empty()) {
1557 const SDValue RetSymbol = getSymbolNode(DAG, Name: "retval0", T: MVT::i32);
1558 const unsigned ResultSize = DL.getTypeAllocSize(Ty: RetTy);
1559 if (shouldPassAsArray(Ty: RetTy)) {
1560 const Align RetAlign =
1561 getPTXParamAlign(CB, Ty: RetTy, AttrIdx: AttributeList::ReturnIndex, DL);
1562 MakeDeclareArrayParam(RetSymbol, RetAlign, ResultSize);
1563 } else {
1564 MakeDeclareScalarParam(RetSymbol, ResultSize);
1565 }
1566 }
1567
1568 // Set the size of the vararg param byte array if the callee is a variadic
1569 // function and the variadic part is not empty.
1570 if (VADeclareParam) {
1571 SDValue DeclareParamOps[] = {VADeclareParam.getOperand(i: 0),
1572 VADeclareParam.getOperand(i: 1),
1573 VADeclareParam.getOperand(i: 2), GetI32(VAOffset),
1574 VADeclareParam.getOperand(i: 4)};
1575 DAG.MorphNodeTo(N: VADeclareParam.getNode(), Opc: VADeclareParam.getOpcode(),
1576 VTs: VADeclareParam->getVTList(), Ops: DeclareParamOps);
1577 }
1578
1579 const auto *Func = dyn_cast<GlobalAddressSDNode>(Val: Callee.getNode());
1580 const auto *CalleeF = Func ? dyn_cast<Function>(Val: Func->getGlobal()) : nullptr;
1581
1582 // If the type of the callsite does not match that of the function, convert
1583 // the callsite to an indirect call.
1584 const bool ConvertToIndirectCall =
1585 CalleeF && CB->getFunctionType() != CalleeF->getFunctionType();
1586
1587 // Both indirect calls and libcalls have nullptr Func. In order to distinguish
1588 // between them we must rely on the call site value which is valid for
1589 // indirect calls but is always null for libcalls.
1590 const bool IsIndirectCall = (!Func && CB) || ConvertToIndirectCall;
1591
1592 if (isa<ExternalSymbolSDNode>(Val: Callee)) {
1593 Function* CalleeFunc = nullptr;
1594
1595 // Try to find the callee in the current module.
1596 Callee = DAG.getSymbolFunctionGlobalAddress(Op: Callee, TargetFunction: &CalleeFunc);
1597 assert(CalleeFunc != nullptr && "Libcall callee must be set.");
1598
1599 // Set the "libcall callee" attribute to indicate that the function
1600 // must always have a declaration.
1601 CalleeFunc->addFnAttr(Kind: "nvptx-libcall-callee", Val: "true");
1602 }
1603
1604 // In the indirect function call case, PTX requires a prototype of the form:
1605 // proto_0 : .callprototype(.param .b32 _) _ (.param .b32 _);
1606 // Where the label is to be used as the last arg of the call instruction.
1607 // We record the call site here and emit all prototypes at the
1608 // start of the function in the AsmPrinter.
1609 if (IsIndirectCall)
1610 DAG.getMachineFunction()
1611 .getInfo<NVPTXMachineFunctionInfo>()
1612 ->addCallPrototype(Id: UniqueCallSite, CB);
1613
1614 const bool IsUnknownIntrinsic =
1615 CalleeF && CalleeF->isIntrinsic() &&
1616 CalleeF->getIntrinsicID() == Intrinsic::not_intrinsic;
1617 if (IsUnknownIntrinsic) {
1618 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1619 DAG.getMachineFunction().getFunction(),
1620 "call to unknown intrinsic '" + CalleeF->getName() +
1621 "' cannot be lowered by the NVPTX backend",
1622 dl.getDebugLoc()));
1623 }
1624
1625 const unsigned Proto = IsIndirectCall ? UniqueCallSite : 0;
1626 const unsigned NumArgs =
1627 std::min<unsigned>(a: CLI.NumFixedArgs + 1, b: Args.size());
1628 /// CALL(Chain, IsConvergent, IsIndirectCall/IsUniform, NumReturns,
1629 /// NumParams, Callee, Proto)
1630 const SDValue CallToken = DAG.getTokenFactor(DL: dl, Vals&: CallPrereqs);
1631 const SDValue Call = DAG.getNode(
1632 Opcode: NVPTXISD::CALL, DL: dl, VT: MVT::Other,
1633 Ops: {CallToken, GetI32(CLI.IsConvergent), GetI32(IsIndirectCall),
1634 GetI32(Ins.empty() ? 0 : 1), GetI32(NumArgs), Callee, GetI32(Proto)});
1635
1636 SmallVector<SDValue, 16> LoadChains{Call};
1637 SmallVector<SDValue, 16> ProxyRegOps;
1638 if (!Ins.empty()) {
1639 SmallVector<EVT, 16> VTs;
1640 SmallVector<uint64_t, 16> Offsets;
1641 ComputePTXValueVTs(TLI: *this, DL, Ctx, CallConv: CLI.CallConv, Ty: RetTy, ValueVTs&: VTs, Offsets);
1642 assert(VTs.size() == Ins.size() && "Bad value decomposition");
1643
1644 const Align RetAlign =
1645 getPTXParamAlign(CB, Ty: RetTy, AttrIdx: AttributeList::ReturnIndex, DL);
1646 const SDValue RetSymbol = getSymbolNode(DAG, Name: "retval0", T: MVT::i32);
1647
1648 // PTX Interoperability Guide 3.3(A): [Integer] Values shorter than
1649 // 32-bits are sign extended or zero extended, depending on whether
1650 // they are signed or unsigned types.
1651 const bool ExtendIntegerRetVal =
1652 RetTy->isIntegerTy() && DL.getTypeAllocSizeInBits(Ty: RetTy) < 32;
1653
1654 unsigned I = 0;
1655 const auto VI = VectorizePTXValueVTs(ValueVTs: VTs, Offsets, ParamAlignment: RetAlign);
1656 for (const unsigned NumElts : VI) {
1657 const MaybeAlign CurrentAlign =
1658 ExtendIntegerRetVal ? MaybeAlign(std::nullopt)
1659 : commonAlignment(A: RetAlign, Offset: Offsets[I]);
1660
1661 const EVT VTI = promoteScalarIntegerPTX(VT: VTs[I]);
1662 const EVT LoadVT =
1663 ExtendIntegerRetVal ? MVT::i32 : (VTI == MVT::i1 ? MVT::i8 : VTI);
1664 const EVT VecVT = getVectorizedVT(VT: LoadVT, N: NumElts, C&: Ctx);
1665 SDValue Ptr =
1666 DAG.getObjectPtrOffset(SL: dl, Ptr: RetSymbol, Offset: TypeSize::getFixed(ExactSize: Offsets[I]));
1667
1668 SDValue R = DAG.getLoad(
1669 VT: VecVT, dl, Chain: Call, Ptr,
1670 PtrInfo: MachinePointerInfo(NVPTX::AddressSpace::DeviceParam), Alignment: CurrentAlign);
1671
1672 LoadChains.push_back(Elt: R.getValue(R: 1));
1673 for (const unsigned J : llvm::seq(Size: NumElts))
1674 ProxyRegOps.push_back(Elt: getExtractVectorizedValue(V: R, I: J, VT: LoadVT, dl, DAG));
1675 I += NumElts;
1676 }
1677 }
1678
1679 const SDValue EndToken = DAG.getTokenFactor(DL: dl, Vals&: LoadChains);
1680 const SDValue CallEnd = DAG.getCALLSEQ_END(Chain: EndToken, Size1: UniqueCallSite,
1681 Size2: UniqueCallSite + 1, Glue: SDValue(), DL: dl);
1682
1683 // Append ProxyReg instructions to the chain to make sure that `callseq_end`
1684 // will not get lost. Otherwise, during libcalls expansion, the nodes can become
1685 // dangling.
1686 for (const auto [I, Reg] : llvm::enumerate(First&: ProxyRegOps)) {
1687 SDValue Proxy =
1688 DAG.getNode(Opcode: NVPTXISD::ProxyReg, DL: dl, VT: Reg.getValueType(), Ops: {CallEnd, Reg});
1689 SDValue Ret = correctParamType(V: Proxy, ExpectedVT: Ins[I].VT, Flags: Ins[I].Flags, DAG, dl);
1690 InVals.push_back(Elt: Ret);
1691 }
1692
1693 // set IsTailCall to false for now, until we figure out how to express
1694 // tail call optimization in PTX
1695 CLI.IsTailCall = false;
1696 return CallEnd;
1697}
1698
1699SDValue NVPTXTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
1700 SelectionDAG &DAG) const {
1701
1702 if (!STI.hasFeature(Feature: NVPTX::PTX73) || !STI.hasFeature(Feature: NVPTX::SM52)) {
1703 const Function &Fn = DAG.getMachineFunction().getFunction();
1704
1705 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1706 Fn,
1707 "Support for dynamic alloca introduced in PTX ISA version 7.3 and "
1708 "requires target sm_52.",
1709 SDLoc(Op).getDebugLoc()));
1710 auto Ops = {DAG.getConstant(Val: 0, DL: SDLoc(), VT: Op.getValueType()),
1711 Op.getOperand(i: 0)};
1712 return DAG.getMergeValues(Ops, dl: SDLoc());
1713 }
1714
1715 SDLoc DL(Op.getNode());
1716 SDValue Chain = Op.getOperand(i: 0);
1717 SDValue Size = Op.getOperand(i: 1);
1718 uint64_t Align = Op.getConstantOperandVal(i: 2);
1719
1720 // The alignment on a ISD::DYNAMIC_STACKALLOC node may be 0 to indicate that
1721 // the default stack alignment should be used.
1722 if (Align == 0)
1723 Align = DAG.getSubtarget().getFrameLowering()->getStackAlign().value();
1724
1725 // The size for ptx alloca instruction is 64-bit for m64 and 32-bit for m32.
1726 const MVT LocalVT = getPointerTy(DL: DAG.getDataLayout(), AS: ADDRESS_SPACE_LOCAL);
1727
1728 SDValue Alloc =
1729 DAG.getNode(Opcode: NVPTXISD::DYNAMIC_STACKALLOC, DL, ResultTys: {LocalVT, MVT::Other},
1730 Ops: {Chain, DAG.getZExtOrTrunc(Op: Size, DL, VT: LocalVT),
1731 DAG.getTargetConstant(Val: Align, DL, VT: MVT::i32)});
1732
1733 // NVPTXLowerAlloca puts allocas in the local address space, so a local
1734 // pointer is requested here; escapes are explicit addrspacecasts in the IR.
1735 assert(Op.getValueType() == LocalVT && "Unexpected alloca pointer size");
1736
1737 return DAG.getMergeValues(Ops: {Alloc, SDValue(Alloc.getNode(), 1)}, dl: DL);
1738}
1739
1740SDValue NVPTXTargetLowering::LowerSTACKRESTORE(SDValue Op,
1741 SelectionDAG &DAG) const {
1742 SDLoc DL(Op.getNode());
1743 if (!STI.hasFeature(Feature: NVPTX::PTX73) || !STI.hasFeature(Feature: NVPTX::SM52)) {
1744 const Function &Fn = DAG.getMachineFunction().getFunction();
1745
1746 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1747 Fn,
1748 "Support for stackrestore requires PTX ISA version >= 7.3 and target "
1749 ">= sm_52.",
1750 DL.getDebugLoc()));
1751 return Op.getOperand(i: 0);
1752 }
1753
1754 const MVT LocalVT = getPointerTy(DL: DAG.getDataLayout(), AS: ADDRESS_SPACE_LOCAL);
1755 SDValue Chain = Op.getOperand(i: 0);
1756 SDValue Ptr = Op.getOperand(i: 1);
1757 SDValue ASC = DAG.getAddrSpaceCast(dl: DL, VT: LocalVT, Ptr, SrcAS: ADDRESS_SPACE_GENERIC,
1758 DestAS: ADDRESS_SPACE_LOCAL);
1759 return DAG.getNode(Opcode: NVPTXISD::STACKRESTORE, DL, VT: MVT::Other, Ops: {Chain, ASC});
1760}
1761
1762SDValue NVPTXTargetLowering::LowerSTACKSAVE(SDValue Op,
1763 SelectionDAG &DAG) const {
1764 SDLoc DL(Op.getNode());
1765 if (!STI.hasFeature(Feature: NVPTX::PTX73) || !STI.hasFeature(Feature: NVPTX::SM52)) {
1766 const Function &Fn = DAG.getMachineFunction().getFunction();
1767
1768 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1769 Fn,
1770 "Support for stacksave requires PTX ISA version >= 7.3 and target >= "
1771 "sm_52.",
1772 DL.getDebugLoc()));
1773 auto Ops = {DAG.getConstant(Val: 0, DL, VT: Op.getValueType()), Op.getOperand(i: 0)};
1774 return DAG.getMergeValues(Ops, dl: DL);
1775 }
1776
1777 const MVT LocalVT = getPointerTy(DL: DAG.getDataLayout(), AS: ADDRESS_SPACE_LOCAL);
1778 SDValue Chain = Op.getOperand(i: 0);
1779 SDValue SS =
1780 DAG.getNode(Opcode: NVPTXISD::STACKSAVE, DL, ResultTys: {LocalVT, MVT::Other}, Ops: Chain);
1781 SDValue ASC = DAG.getAddrSpaceCast(
1782 dl: DL, VT: Op.getValueType(), Ptr: SS, SrcAS: ADDRESS_SPACE_LOCAL, DestAS: ADDRESS_SPACE_GENERIC);
1783 return DAG.getMergeValues(Ops: {ASC, SDValue(SS.getNode(), 1)}, dl: DL);
1784}
1785
1786// By default CONCAT_VECTORS is lowered by ExpandVectorBuildThroughStack()
1787// (see LegalizeDAG.cpp). This is slow and uses local memory.
1788// We use extract/insert/build vector just as what LegalizeOp() does in llvm 2.5
1789SDValue
1790NVPTXTargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const {
1791 SDNode *Node = Op.getNode();
1792 SDLoc dl(Node);
1793 SmallVector<SDValue, 8> Ops;
1794 unsigned NumOperands = Node->getNumOperands();
1795 for (unsigned i = 0; i < NumOperands; ++i) {
1796 SDValue SubOp = Node->getOperand(Num: i);
1797 EVT VVT = SubOp.getNode()->getValueType(ResNo: 0);
1798 EVT EltVT = VVT.getVectorElementType();
1799 unsigned NumSubElem = VVT.getVectorNumElements();
1800 for (unsigned j = 0; j < NumSubElem; ++j) {
1801 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: SubOp,
1802 N2: DAG.getIntPtrConstant(Val: j, DL: dl)));
1803 }
1804 }
1805 return DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL: dl, Ops);
1806}
1807
1808static SDValue getPRMT(SDValue A, SDValue B, SDValue Selector, SDLoc DL,
1809 SelectionDAG &DAG,
1810 unsigned Mode = NVPTX::PTXPrmtMode::NONE) {
1811 assert(A.getValueType() == MVT::i32 && B.getValueType() == MVT::i32 &&
1812 Selector.getValueType() == MVT::i32 && "PRMT must have i32 operands");
1813 return DAG.getNode(Opcode: NVPTXISD::PRMT, DL, VT: MVT::i32,
1814 Ops: {A, B, Selector, DAG.getConstant(Val: Mode, DL, VT: MVT::i32)});
1815}
1816
1817static SDValue getPRMT(SDValue A, SDValue B, uint64_t Selector, SDLoc DL,
1818 SelectionDAG &DAG,
1819 unsigned Mode = NVPTX::PTXPrmtMode::NONE) {
1820 return getPRMT(A, B, Selector: DAG.getConstant(Val: Selector, DL, VT: MVT::i32), DL, DAG, Mode);
1821}
1822
1823/// Reduces the elements using the scalar operations provided. The operations
1824/// are sorted descending in number of inputs they take. The flags on the
1825/// original reduction operation will be propagated to each scalar operation.
1826/// Nearby elements are grouped in tree reduction, unlike the shuffle reduction
1827/// used in ExpandReductions and SelectionDAG.
1828static SDValue buildTreeReduction(
1829 const SmallVector<SDValue> &Elements, EVT EltTy,
1830 ArrayRef<std::pair<unsigned /*NodeType*/, unsigned /*NumInputs*/>> Ops,
1831 const SDLoc &DL, const SDNodeFlags Flags, SelectionDAG &DAG) {
1832 // Build the reduction tree at each level, starting with all the elements.
1833 SmallVector<SDValue> Level = Elements;
1834
1835 unsigned OpIdx = 0;
1836 while (Level.size() > 1) {
1837 // Try to reduce this level using the current operator.
1838 const auto [Op, NumInputs] = Ops[OpIdx];
1839
1840 // Build the next level by partially reducing all elements.
1841 SmallVector<SDValue> ReducedLevel;
1842 unsigned I = 0, E = Level.size();
1843 for (; I + NumInputs <= E; I += NumInputs) {
1844 // Reduce elements in groups of [NumInputs], as much as possible.
1845 ReducedLevel.push_back(Elt: DAG.getNode(
1846 Opcode: Op, DL, VT: EltTy, Ops: ArrayRef<SDValue>(Level).slice(N: I, M: NumInputs), Flags));
1847 }
1848
1849 if (I < E) {
1850 // Handle leftover elements.
1851
1852 if (ReducedLevel.empty()) {
1853 // We didn't reduce anything at this level. We need to pick a smaller
1854 // operator.
1855 ++OpIdx;
1856 assert(OpIdx < Ops.size() && "no smaller operators for reduction");
1857 continue;
1858 }
1859
1860 // We reduced some things but there's still more left, meaning the
1861 // operator's number of inputs doesn't evenly divide this level size. Move
1862 // these elements to the next level.
1863 for (; I < E; ++I)
1864 ReducedLevel.push_back(Elt: Level[I]);
1865 }
1866
1867 // Process the next level.
1868 Level = ReducedLevel;
1869 }
1870
1871 return *Level.begin();
1872}
1873
1874// Get scalar reduction opcode
1875static ISD::NodeType getScalarOpcodeForReduction(unsigned ReductionOpcode) {
1876 switch (ReductionOpcode) {
1877 case ISD::VECREDUCE_FMAX:
1878 return ISD::FMAXNUM;
1879 case ISD::VECREDUCE_FMIN:
1880 return ISD::FMINNUM;
1881 case ISD::VECREDUCE_FMAXIMUM:
1882 return ISD::FMAXIMUM;
1883 case ISD::VECREDUCE_FMINIMUM:
1884 return ISD::FMINIMUM;
1885 default:
1886 llvm_unreachable("unhandled reduction opcode");
1887 }
1888}
1889
1890/// Get 3-input scalar reduction opcode
1891static std::optional<unsigned>
1892getScalar3OpcodeForReduction(unsigned ReductionOpcode) {
1893 switch (ReductionOpcode) {
1894 case ISD::VECREDUCE_FMAX:
1895 return NVPTXISD::FMAXNUM3;
1896 case ISD::VECREDUCE_FMIN:
1897 return NVPTXISD::FMINNUM3;
1898 case ISD::VECREDUCE_FMAXIMUM:
1899 return NVPTXISD::FMAXIMUM3;
1900 case ISD::VECREDUCE_FMINIMUM:
1901 return NVPTXISD::FMINIMUM3;
1902 default:
1903 return std::nullopt;
1904 }
1905}
1906
1907/// Lower reductions to either a sequence of operations or a tree if
1908/// reassociations are allowed. This method will use larger operations like
1909/// max3/min3 when the target supports them.
1910SDValue NVPTXTargetLowering::LowerVECREDUCE(SDValue Op,
1911 SelectionDAG &DAG) const {
1912 SDLoc DL(Op);
1913 const SDNodeFlags Flags = Op->getFlags();
1914 SDValue Vector = Op.getOperand(i: 0);
1915
1916 const unsigned Opcode = Op->getOpcode();
1917 const EVT EltTy = Vector.getValueType().getVectorElementType();
1918
1919 // Whether we can use 3-input min/max when expanding the reduction.
1920 const bool CanUseMinMax3 =
1921 EltTy == MVT::f32 && STI.hasFeature(Feature: NVPTX::SM100) &&
1922 STI.hasFeature(Feature: NVPTX::PTX88) &&
1923 (Opcode == ISD::VECREDUCE_FMAX || Opcode == ISD::VECREDUCE_FMIN ||
1924 Opcode == ISD::VECREDUCE_FMAXIMUM || Opcode == ISD::VECREDUCE_FMINIMUM);
1925
1926 // A list of SDNode opcodes with equivalent semantics, sorted descending by
1927 // number of inputs they take.
1928 SmallVector<std::pair<unsigned /*Op*/, unsigned /*NumIn*/>, 2> ScalarOps;
1929
1930 if (auto Opcode3Elem = getScalar3OpcodeForReduction(ReductionOpcode: Opcode);
1931 CanUseMinMax3 && Opcode3Elem)
1932 ScalarOps.push_back(Elt: {*Opcode3Elem, 3});
1933 ScalarOps.push_back(Elt: {getScalarOpcodeForReduction(ReductionOpcode: Opcode), 2});
1934
1935 SmallVector<SDValue> Elements;
1936 DAG.ExtractVectorElements(Op: Vector, Args&: Elements);
1937
1938 return buildTreeReduction(Elements, EltTy, Ops: ScalarOps, DL, Flags, DAG);
1939}
1940
1941SDValue NVPTXTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
1942 // Handle bitcasting from v2i8 without hitting the default promotion
1943 // strategy which goes through stack memory.
1944 EVT FromVT = Op->getOperand(Num: 0)->getValueType(ResNo: 0);
1945 if (FromVT != MVT::v2i8) {
1946 return Op;
1947 }
1948
1949 // Pack vector elements into i16 and bitcast to final type
1950 SDLoc DL(Op);
1951 SDValue Vec0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i8,
1952 N1: Op->getOperand(Num: 0), N2: DAG.getIntPtrConstant(Val: 0, DL));
1953 SDValue Vec1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i8,
1954 N1: Op->getOperand(Num: 0), N2: DAG.getIntPtrConstant(Val: 1, DL));
1955 SDValue Extend0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i16, Operand: Vec0);
1956 SDValue Extend1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i16, Operand: Vec1);
1957 SDValue Const8 = DAG.getConstant(Val: 8, DL, VT: MVT::i16);
1958 SDValue AsInt = DAG.getNode(
1959 Opcode: ISD::OR, DL, VT: MVT::i16,
1960 Ops: {Extend0, DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i16, Ops: {Extend1, Const8})});
1961 EVT ToVT = Op->getValueType(ResNo: 0);
1962 return DAG.getBitcast(VT: ToVT, V: AsInt);
1963}
1964
1965// We can init constant f16x2/v2i16/v4i8 with a single .b32 move. Normally it
1966// would get lowered as two constant loads and vector-packing move.
1967// Instead we want just a constant move:
1968// mov.b32 %r2, 0x40003C00
1969SDValue NVPTXTargetLowering::LowerBUILD_VECTOR(SDValue Op,
1970 SelectionDAG &DAG) const {
1971 EVT VT = Op->getValueType(ResNo: 0);
1972 if (!(NVPTX::isPackedVectorTy(VT) && VT.is32BitVector()))
1973 return Op;
1974 SDLoc DL(Op);
1975
1976 if (!llvm::all_of(Range: Op->ops(), P: [](SDValue Operand) {
1977 return Operand->isUndef() || isa<ConstantSDNode>(Val: Operand) ||
1978 isa<ConstantFPSDNode>(Val: Operand);
1979 })) {
1980 if (VT != MVT::v4i8)
1981 return Op;
1982 // Lower non-const v4i8 vector as byte-wise constructed i32, which allows us
1983 // to optimize calculation of constant parts.
1984 auto GetPRMT = [&](const SDValue Left, const SDValue Right, bool Cast,
1985 uint64_t SelectionValue) -> SDValue {
1986 SDValue L = Left;
1987 SDValue R = Right;
1988 if (Cast) {
1989 L = DAG.getAnyExtOrTrunc(Op: L, DL, VT: MVT::i32);
1990 R = DAG.getAnyExtOrTrunc(Op: R, DL, VT: MVT::i32);
1991 }
1992 return getPRMT(A: L, B: R, Selector: SelectionValue, DL, DAG);
1993 };
1994 auto PRMT__10 = GetPRMT(Op->getOperand(Num: 0), Op->getOperand(Num: 1), true, 0x3340);
1995 auto PRMT__32 = GetPRMT(Op->getOperand(Num: 2), Op->getOperand(Num: 3), true, 0x3340);
1996 auto PRMT3210 = GetPRMT(PRMT__10, PRMT__32, false, 0x5410);
1997 return DAG.getBitcast(VT, V: PRMT3210);
1998 }
1999
2000 // Get value or the Nth operand as an APInt(32). Undef values treated as 0.
2001 auto GetOperand = [](SDValue Op, int N) -> APInt {
2002 const SDValue &Operand = Op->getOperand(Num: N);
2003 EVT VT = Op->getValueType(ResNo: 0);
2004 if (Operand->isUndef())
2005 return APInt(32, 0);
2006 APInt Value;
2007 if (VT == MVT::v2f16 || VT == MVT::v2bf16)
2008 Value = cast<ConstantFPSDNode>(Val: Operand)->getValueAPF().bitcastToAPInt();
2009 else if (VT == MVT::v2i16 || VT == MVT::v4i8)
2010 Value = Operand->getAsAPIntVal();
2011 else
2012 llvm_unreachable("Unsupported type");
2013 // i8 values are carried around as i16, so we need to zero out upper bits,
2014 // so they do not get in the way of combining individual byte values
2015 if (VT == MVT::v4i8)
2016 Value = Value.trunc(width: 8);
2017 return Value.zext(width: 32);
2018 };
2019
2020 // Construct a 32-bit constant by shifting into place smaller values
2021 // (elements of the vector type VT).
2022 // For example, if VT has 2 elements, then N == 2:
2023 // ShiftAmount = 32 / N = 16
2024 // Value |= Op0 (b16) << 0
2025 // Value |= Op1 (b16) << 16
2026 // If N == 4:
2027 // ShiftAmount = 32 / N = 8
2028 // Value |= Op0 (b8) << 0
2029 // Value |= Op1 (b8) << 8
2030 // Value |= Op2 (b8) << 16
2031 // Value |= Op3 (b8) << 24
2032 // ...etc
2033 APInt Value(32, 0);
2034 const unsigned NumElements = VT.getVectorNumElements();
2035 assert(32 % NumElements == 0 && "must evenly divide bit length");
2036 const unsigned ShiftAmount = 32 / NumElements;
2037 for (unsigned ElementNo : seq(Size: NumElements))
2038 Value |= GetOperand(Op, ElementNo).shl(shiftAmt: ElementNo * ShiftAmount);
2039 SDValue Const = DAG.getConstant(Val: Value, DL, VT: MVT::i32);
2040 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Op->getValueType(ResNo: 0), Operand: Const);
2041}
2042
2043SDValue NVPTXTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
2044 SelectionDAG &DAG) const {
2045 SDValue Index = Op->getOperand(Num: 1);
2046 SDValue Vector = Op->getOperand(Num: 0);
2047 SDLoc DL(Op);
2048 EVT VectorVT = Vector.getValueType();
2049
2050 if (VectorVT == MVT::v4i8) {
2051 SDValue Selector = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32,
2052 N1: DAG.getZExtOrTrunc(Op: Index, DL, VT: MVT::i32),
2053 N2: DAG.getConstant(Val: 0x7770, DL, VT: MVT::i32));
2054 SDValue PRMT = getPRMT(A: DAG.getBitcast(VT: MVT::i32, V: Vector),
2055 B: DAG.getConstant(Val: 0, DL, VT: MVT::i32), Selector, DL, DAG);
2056 SDValue Ext = DAG.getAnyExtOrTrunc(Op: PRMT, DL, VT: Op->getValueType(ResNo: 0));
2057 SDNodeFlags Flags;
2058 Flags.setNoSignedWrap(Ext.getScalarValueSizeInBits() > 8);
2059 Flags.setNoUnsignedWrap(Ext.getScalarValueSizeInBits() >= 8);
2060 Ext->setFlags(Flags);
2061 return Ext;
2062 }
2063
2064 // Constant index will be matched by tablegen.
2065 if (isa<ConstantSDNode>(Val: Index.getNode()))
2066 return Op;
2067
2068 // Extract individual elements and select one of them.
2069 assert(NVPTX::isPackedVectorTy(VectorVT) &&
2070 VectorVT.getVectorNumElements() == 2 && "Unexpected vector type.");
2071 EVT EltVT = VectorVT.getVectorElementType();
2072
2073 SDLoc dl(Op.getNode());
2074 SDValue E0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Vector,
2075 N2: DAG.getIntPtrConstant(Val: 0, DL: dl));
2076 SDValue E1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Vector,
2077 N2: DAG.getIntPtrConstant(Val: 1, DL: dl));
2078 return DAG.getSelectCC(DL: dl, LHS: Index, RHS: DAG.getIntPtrConstant(Val: 0, DL: dl), True: E0, False: E1,
2079 Cond: ISD::CondCode::SETEQ);
2080}
2081
2082SDValue NVPTXTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
2083 SelectionDAG &DAG) const {
2084 SDValue Vector = Op->getOperand(Num: 0);
2085 EVT VectorVT = Vector.getValueType();
2086
2087 if (VectorVT != MVT::v4i8)
2088 return Op;
2089 SDLoc DL(Op);
2090 SDValue Value = Op->getOperand(Num: 1);
2091 if (Value->isUndef())
2092 return Vector;
2093
2094 SDValue Index = Op->getOperand(Num: 2);
2095
2096 SDValue BFI =
2097 DAG.getNode(Opcode: NVPTXISD::BFI, DL, VT: MVT::i32,
2098 Ops: {DAG.getZExtOrTrunc(Op: Value, DL, VT: MVT::i32), Vector,
2099 DAG.getNode(Opcode: ISD::MUL, DL, VT: MVT::i32,
2100 N1: DAG.getZExtOrTrunc(Op: Index, DL, VT: MVT::i32),
2101 N2: DAG.getConstant(Val: 8, DL, VT: MVT::i32)),
2102 DAG.getConstant(Val: 8, DL, VT: MVT::i32)});
2103 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Op->getValueType(ResNo: 0), Operand: BFI);
2104}
2105
2106SDValue NVPTXTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
2107 SelectionDAG &DAG) const {
2108 SDValue V1 = Op.getOperand(i: 0);
2109 EVT VectorVT = V1.getValueType();
2110 if (VectorVT != MVT::v4i8 || Op.getValueType() != MVT::v4i8)
2111 return Op;
2112
2113 // Lower shuffle to PRMT instruction.
2114 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: Op.getNode());
2115 SDValue V2 = Op.getOperand(i: 1);
2116 uint32_t Selector = 0;
2117 for (auto I : llvm::enumerate(First: SVN->getMask())) {
2118 if (I.value() != -1) // -1 is a placeholder for undef.
2119 Selector |= (I.value() << (I.index() * 4));
2120 }
2121
2122 SDLoc DL(Op);
2123 SDValue PRMT = getPRMT(A: DAG.getBitcast(VT: MVT::i32, V: V1),
2124 B: DAG.getBitcast(VT: MVT::i32, V: V2), Selector, DL, DAG);
2125 return DAG.getBitcast(VT: Op.getValueType(), V: PRMT);
2126}
2127/// LowerShiftRightParts - Lower SRL_PARTS, SRA_PARTS, which
2128/// 1) returns two i32 values and take a 2 x i32 value to shift plus a shift
2129/// amount, or
2130/// 2) returns two i64 values and take a 2 x i64 value to shift plus a shift
2131/// amount.
2132SDValue NVPTXTargetLowering::LowerShiftRightParts(SDValue Op,
2133 SelectionDAG &DAG) const {
2134 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
2135 assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS);
2136
2137 EVT VT = Op.getValueType();
2138 unsigned VTBits = VT.getSizeInBits();
2139 SDLoc dl(Op);
2140 SDValue ShOpLo = Op.getOperand(i: 0);
2141 SDValue ShOpHi = Op.getOperand(i: 1);
2142 SDValue ShAmt = Op.getOperand(i: 2);
2143 unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;
2144
2145 if (VTBits == 32 && STI.hasFeature(Feature: NVPTX::SM35)) {
2146 // For 32bit and sm35, we can use the funnel shift 'shf' instruction.
2147 // {dHi, dLo} = {aHi, aLo} >> Amt
2148 // dHi = aHi >> Amt
2149 // dLo = shf.r.clamp aLo, aHi, Amt
2150
2151 SDValue Hi = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: ShOpHi, N2: ShAmt);
2152 SDValue Lo =
2153 DAG.getNode(Opcode: NVPTXISD::FSHR_CLAMP, DL: dl, VT, N1: ShOpHi, N2: ShOpLo, N3: ShAmt);
2154
2155 SDValue Ops[2] = { Lo, Hi };
2156 return DAG.getMergeValues(Ops, dl);
2157 } else {
2158 // {dHi, dLo} = {aHi, aLo} >> Amt
2159 // - if (Amt>=size) then
2160 // dLo = aHi >> (Amt-size)
2161 // dHi = aHi >> Amt (this is either all 0 or all 1)
2162 // else
2163 // dLo = (aLo >>logic Amt) | (aHi << (size-Amt))
2164 // dHi = aHi >> Amt
2165
2166 SDValue RevShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
2167 N1: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32),
2168 N2: ShAmt);
2169 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: ShOpLo, N2: ShAmt);
2170 SDValue ExtraShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: ShAmt,
2171 N2: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32));
2172 SDValue Tmp2 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpHi, N2: RevShAmt);
2173 SDValue FalseVal = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp1, N2: Tmp2);
2174 SDValue TrueVal = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: ShOpHi, N2: ExtraShAmt);
2175
2176 SDValue Cmp = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: ShAmt,
2177 RHS: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32),
2178 Cond: ISD::SETGE);
2179 SDValue Hi = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: ShOpHi, N2: ShAmt);
2180 SDValue Lo = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cmp, N2: TrueVal, N3: FalseVal);
2181
2182 SDValue Ops[2] = { Lo, Hi };
2183 return DAG.getMergeValues(Ops, dl);
2184 }
2185}
2186
2187/// LowerShiftLeftParts - Lower SHL_PARTS, which
2188/// 1) returns two i32 values and take a 2 x i32 value to shift plus a shift
2189/// amount, or
2190/// 2) returns two i64 values and take a 2 x i64 value to shift plus a shift
2191/// amount.
2192SDValue NVPTXTargetLowering::LowerShiftLeftParts(SDValue Op,
2193 SelectionDAG &DAG) const {
2194 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
2195 assert(Op.getOpcode() == ISD::SHL_PARTS);
2196
2197 EVT VT = Op.getValueType();
2198 unsigned VTBits = VT.getSizeInBits();
2199 SDLoc dl(Op);
2200 SDValue ShOpLo = Op.getOperand(i: 0);
2201 SDValue ShOpHi = Op.getOperand(i: 1);
2202 SDValue ShAmt = Op.getOperand(i: 2);
2203
2204 if (VTBits == 32 && STI.hasFeature(Feature: NVPTX::SM35)) {
2205 // For 32bit and sm35, we can use the funnel shift 'shf' instruction.
2206 // {dHi, dLo} = {aHi, aLo} << Amt
2207 // dHi = shf.l.clamp aLo, aHi, Amt
2208 // dLo = aLo << Amt
2209
2210 SDValue Hi =
2211 DAG.getNode(Opcode: NVPTXISD::FSHL_CLAMP, DL: dl, VT, N1: ShOpHi, N2: ShOpLo, N3: ShAmt);
2212 SDValue Lo = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpLo, N2: ShAmt);
2213
2214 SDValue Ops[2] = { Lo, Hi };
2215 return DAG.getMergeValues(Ops, dl);
2216 } else {
2217 // {dHi, dLo} = {aHi, aLo} << Amt
2218 // - if (Amt>=size) then
2219 // dLo = aLo << Amt (all 0)
2220 // dLo = aLo << (Amt-size)
2221 // else
2222 // dLo = aLo << Amt
2223 // dHi = (aHi << Amt) | (aLo >> (size-Amt))
2224
2225 SDValue RevShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
2226 N1: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32),
2227 N2: ShAmt);
2228 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpHi, N2: ShAmt);
2229 SDValue ExtraShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: ShAmt,
2230 N2: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32));
2231 SDValue Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: ShOpLo, N2: RevShAmt);
2232 SDValue FalseVal = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp1, N2: Tmp2);
2233 SDValue TrueVal = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpLo, N2: ExtraShAmt);
2234
2235 SDValue Cmp = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: ShAmt,
2236 RHS: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32),
2237 Cond: ISD::SETGE);
2238 SDValue Lo = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpLo, N2: ShAmt);
2239 SDValue Hi = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cmp, N2: TrueVal, N3: FalseVal);
2240
2241 SDValue Ops[2] = { Lo, Hi };
2242 return DAG.getMergeValues(Ops, dl);
2243 }
2244}
2245
2246/// If the types match, convert the generic copysign to the NVPTXISD version,
2247/// otherwise bail ensuring that mismatched cases are properly expaned.
2248SDValue NVPTXTargetLowering::LowerFCOPYSIGN(SDValue Op,
2249 SelectionDAG &DAG) const {
2250 EVT VT = Op.getValueType();
2251 SDLoc DL(Op);
2252
2253 SDValue In1 = Op.getOperand(i: 0);
2254 SDValue In2 = Op.getOperand(i: 1);
2255 EVT SrcVT = In2.getValueType();
2256
2257 if (!SrcVT.bitsEq(VT))
2258 return SDValue();
2259
2260 return DAG.getNode(Opcode: NVPTXISD::FCOPYSIGN, DL, VT, N1: In1, N2: In2);
2261}
2262
2263SDValue NVPTXTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const {
2264 EVT VT = Op.getValueType();
2265
2266 if (VT == MVT::f32)
2267 return LowerFROUND32(Op, DAG);
2268
2269 if (VT == MVT::f64)
2270 return LowerFROUND64(Op, DAG);
2271
2272 llvm_unreachable("unhandled type");
2273}
2274
2275// This is the the rounding method used in CUDA libdevice in C like code:
2276// float roundf(float A)
2277// {
2278// float RoundedA = (float) (int) ( A > 0 ? (A + 0.5f) : (A - 0.5f));
2279// RoundedA = abs(A) > 0x1.0p23 ? A : RoundedA;
2280// return abs(A) < 0.5 ? (float)(int)A : RoundedA;
2281// }
2282SDValue NVPTXTargetLowering::LowerFROUND32(SDValue Op,
2283 SelectionDAG &DAG) const {
2284 SDLoc SL(Op);
2285 SDValue A = Op.getOperand(i: 0);
2286 EVT VT = Op.getValueType();
2287
2288 SDValue AbsA = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT, Operand: A);
2289
2290 // RoundedA = (float) (int) ( A > 0 ? (A + 0.5f) : (A - 0.5f))
2291 SDValue Bitcast = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: A);
2292 const unsigned SignBitMask = 0x80000000;
2293 SDValue Sign = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: Bitcast,
2294 N2: DAG.getConstant(Val: SignBitMask, DL: SL, VT: MVT::i32));
2295 const unsigned PointFiveInBits = 0x3F000000;
2296 SDValue PointFiveWithSignRaw =
2297 DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: Sign,
2298 N2: DAG.getConstant(Val: PointFiveInBits, DL: SL, VT: MVT::i32));
2299 SDValue PointFiveWithSign =
2300 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: PointFiveWithSignRaw);
2301 SDValue AdjustedA = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: A, N2: PointFiveWithSign);
2302 SDValue RoundedA = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: AdjustedA);
2303
2304 // RoundedA = abs(A) > 0x1.0p23 ? A : RoundedA;
2305 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Ctx&: *DAG.getContext(), VT);
2306 SDValue IsLarge =
2307 DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: AbsA, RHS: DAG.getConstantFP(Val: pow(x: 2.0, y: 23.0), DL: SL, VT),
2308 Cond: ISD::SETOGT);
2309 RoundedA = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsLarge, N2: A, N3: RoundedA);
2310
2311 // return abs(A) < 0.5 ? (float)(int)A : RoundedA;
2312 SDValue IsSmall =DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: AbsA,
2313 RHS: DAG.getConstantFP(Val: 0.5, DL: SL, VT), Cond: ISD::SETOLT);
2314 SDValue RoundedAForSmallA = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: A);
2315 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsSmall, N2: RoundedAForSmallA, N3: RoundedA);
2316}
2317
2318// The implementation of round(double) is similar to that of round(float) in
2319// that they both separate the value range into three regions and use a method
2320// specific to the region to round the values. However, round(double) first
2321// calculates the round of the absolute value and then adds the sign back while
2322// round(float) directly rounds the value with sign.
2323SDValue NVPTXTargetLowering::LowerFROUND64(SDValue Op,
2324 SelectionDAG &DAG) const {
2325 SDLoc SL(Op);
2326 SDValue A = Op.getOperand(i: 0);
2327 EVT VT = Op.getValueType();
2328
2329 SDValue AbsA = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT, Operand: A);
2330
2331 // double RoundedA = (double) (int) (abs(A) + 0.5f);
2332 SDValue AdjustedA = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: AbsA,
2333 N2: DAG.getConstantFP(Val: 0.5, DL: SL, VT));
2334 SDValue RoundedA = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: AdjustedA);
2335
2336 // RoundedA = abs(A) < 0.5 ? (double)0 : RoundedA;
2337 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Ctx&: *DAG.getContext(), VT);
2338 SDValue IsSmall =DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: AbsA,
2339 RHS: DAG.getConstantFP(Val: 0.5, DL: SL, VT), Cond: ISD::SETOLT);
2340 RoundedA = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsSmall,
2341 N2: DAG.getConstantFP(Val: 0, DL: SL, VT),
2342 N3: RoundedA);
2343
2344 // Add sign to rounded_A
2345 RoundedA = DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT, N1: RoundedA, N2: A);
2346 DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: A);
2347
2348 // RoundedA = abs(A) > 0x1.0p52 ? A : RoundedA;
2349 SDValue IsLarge =
2350 DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: AbsA, RHS: DAG.getConstantFP(Val: pow(x: 2.0, y: 52.0), DL: SL, VT),
2351 Cond: ISD::SETOGT);
2352 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsLarge, N2: A, N3: RoundedA);
2353}
2354
2355static SDValue PromoteBinOpToF32(SDNode *N, SelectionDAG &DAG) {
2356 EVT VT = N->getValueType(ResNo: 0);
2357 EVT NVT = MVT::f32;
2358 if (VT.isVector()) {
2359 NVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NVT, EC: VT.getVectorElementCount());
2360 }
2361 SDLoc DL(N);
2362 SDValue Tmp0 = DAG.getFPExtendOrRound(Op: N->getOperand(Num: 0), DL, VT: NVT);
2363 SDValue Tmp1 = DAG.getFPExtendOrRound(Op: N->getOperand(Num: 1), DL, VT: NVT);
2364 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL, VT: NVT, N1: Tmp0, N2: Tmp1, Flags: N->getFlags());
2365 return DAG.getFPExtendOrRound(Op: Res, DL, VT);
2366}
2367
2368SDValue NVPTXTargetLowering::PromoteBinOpIfF32FTZ(SDValue Op,
2369 SelectionDAG &DAG) const {
2370 if (useF32FTZ(MF: DAG.getMachineFunction())) {
2371 return PromoteBinOpToF32(N: Op.getNode(), DAG);
2372 }
2373 return Op;
2374}
2375
2376SDValue NVPTXTargetLowering::LowerINT_TO_FP(SDValue Op,
2377 SelectionDAG &DAG) const {
2378 assert(!STI.hasFeature(NVPTX::SM90));
2379
2380 if (Op.getValueType() == MVT::bf16) {
2381 SDLoc Loc(Op);
2382 return DAG.getNode(
2383 Opcode: ISD::FP_ROUND, DL: Loc, VT: MVT::bf16,
2384 N1: DAG.getNode(Opcode: Op.getOpcode(), DL: Loc, VT: MVT::f32, Operand: Op.getOperand(i: 0)),
2385 N2: DAG.getIntPtrConstant(Val: 0, DL: Loc, /*isTarget=*/true));
2386 }
2387
2388 // Everything else is considered legal.
2389 return Op;
2390}
2391
2392SDValue NVPTXTargetLowering::LowerFP_TO_INT(SDValue Op,
2393 SelectionDAG &DAG) const {
2394 assert(!STI.hasFeature(NVPTX::SM90));
2395
2396 if (Op.getOperand(i: 0).getValueType() == MVT::bf16) {
2397 SDLoc Loc(Op);
2398 return DAG.getNode(
2399 Opcode: Op.getOpcode(), DL: Loc, VT: Op.getValueType(),
2400 Operand: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: Loc, VT: MVT::f32, Operand: Op.getOperand(i: 0)));
2401 }
2402
2403 // Everything else is considered legal.
2404 return Op;
2405}
2406
2407SDValue NVPTXTargetLowering::LowerFP_ROUND(SDValue Op,
2408 SelectionDAG &DAG) const {
2409 EVT NarrowVT = Op.getValueType();
2410 SDValue Wide = Op.getOperand(i: 0);
2411 EVT WideVT = Wide.getValueType();
2412 if (NarrowVT.getScalarType() == MVT::bf16) {
2413 const TargetLowering *TLI = STI.getTargetLowering();
2414 if (!STI.hasFeature(Feature: NVPTX::SM80)) {
2415 return TLI->expandFP_ROUND(Node: Op.getNode(), DAG);
2416 }
2417 if (!STI.hasFeature(Feature: NVPTX::SM90)) {
2418 // sm_80 was the first architecture to support f32 -> bf16.
2419 if (WideVT.getScalarType() == MVT::f32) {
2420 return Op;
2421 }
2422 if (WideVT.getScalarType() == MVT::f64) {
2423 SDLoc Loc(Op);
2424 // Round-inexact-to-odd f64 to f32, then do the final rounding using
2425 // the hardware f32 -> bf16 instruction.
2426 SDValue rod = TLI->expandRoundInexactToOdd(
2427 ResultVT: WideVT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::f32), Op: Wide, DL: Loc,
2428 DAG);
2429 return DAG.getFPExtendOrRound(Op: rod, DL: Loc, VT: NarrowVT);
2430 }
2431 return TLI->expandFP_ROUND(Node: Op.getNode(), DAG);
2432 }
2433 }
2434
2435 // Everything else is considered legal.
2436 return Op;
2437}
2438
2439SDValue NVPTXTargetLowering::LowerFP_EXTEND(SDValue Op,
2440 SelectionDAG &DAG) const {
2441 SDValue Narrow = Op.getOperand(i: 0);
2442 EVT NarrowVT = Narrow.getValueType();
2443 EVT WideVT = Op.getValueType();
2444 if (NarrowVT.getScalarType() == MVT::bf16) {
2445 if (WideVT.getScalarType() == MVT::f32 &&
2446 (!STI.hasFeature(Feature: NVPTX::SM80) || !STI.hasFeature(Feature: NVPTX::PTX71))) {
2447 SDLoc Loc(Op);
2448 return DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: Loc, VT: WideVT, Operand: Narrow);
2449 }
2450 if (WideVT.getScalarType() == MVT::f64 && !STI.hasFeature(Feature: NVPTX::SM90)) {
2451 EVT F32 = NarrowVT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::f32);
2452 SDLoc Loc(Op);
2453 if (STI.hasFeature(Feature: NVPTX::SM80) && STI.hasFeature(Feature: NVPTX::PTX71)) {
2454 Op = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: Loc, VT: F32, Operand: Narrow);
2455 } else {
2456 Op = DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: Loc, VT: F32, Operand: Narrow);
2457 }
2458 return DAG.getNode(Opcode: ISD::FP_EXTEND, DL: Loc, VT: WideVT, Operand: Op);
2459 }
2460 }
2461
2462 // Everything else is considered legal.
2463 return Op;
2464}
2465
2466static SDValue LowerVectorArith(SDValue Op, SelectionDAG &DAG) {
2467 SDLoc DL(Op);
2468 if (Op.getValueType() != MVT::v2i16)
2469 return Op;
2470 EVT EltVT = Op.getValueType().getVectorElementType();
2471 SmallVector<SDValue> VecElements;
2472 for (int I = 0, E = Op.getValueType().getVectorNumElements(); I < E; I++) {
2473 SmallVector<SDValue> ScalarArgs;
2474 llvm::transform(Range: Op->ops(), d_first: std::back_inserter(x&: ScalarArgs),
2475 F: [&](const SDUse &O) {
2476 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT,
2477 N1: O.get(), N2: DAG.getIntPtrConstant(Val: I, DL));
2478 });
2479 VecElements.push_back(Elt: DAG.getNode(Opcode: Op.getOpcode(), DL, VT: EltVT, Ops: ScalarArgs));
2480 }
2481 SDValue V =
2482 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: Op.getValueType(), Ops: VecElements);
2483 return V;
2484}
2485
2486static SDValue lowerTcgen05St(SDValue Op, SelectionDAG &DAG,
2487 bool hasOffset = false) {
2488 // skip lowering if the vector operand is already legalized
2489 if (!Op->getOperand(Num: hasOffset ? 4 : 3).getValueType().isVector())
2490 return Op;
2491
2492 SDNode *N = Op.getNode();
2493 SDLoc DL(N);
2494 SmallVector<SDValue, 32> Ops;
2495
2496 // split the vector argument
2497 for (size_t I = 0; I < N->getNumOperands(); I++) {
2498 SDValue Val = N->getOperand(Num: I);
2499 EVT ValVT = Val.getValueType();
2500 if (ValVT.isVector()) {
2501 EVT EltVT = ValVT.getVectorElementType();
2502 for (unsigned J = 0, NElts = ValVT.getVectorNumElements(); J < NElts; J++)
2503 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Val,
2504 N2: DAG.getIntPtrConstant(Val: J, DL)));
2505 } else
2506 Ops.push_back(Elt: Val);
2507 }
2508
2509 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(Val: N);
2510 SDValue Tcgen05StNode =
2511 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl: DL, VTList: N->getVTList(), Ops,
2512 MemVT: MemSD->getMemoryVT(), MMO: MemSD->getMemOperand());
2513
2514 return Tcgen05StNode;
2515}
2516
2517static SDValue lowerBSWAP(SDValue Op, SelectionDAG &DAG) {
2518 SDLoc DL(Op);
2519 SDValue Src = Op.getOperand(i: 0);
2520 EVT VT = Op.getValueType();
2521
2522 switch (VT.getSimpleVT().SimpleTy) {
2523 case MVT::i16: {
2524 SDValue Extended = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Src);
2525 SDValue Swapped =
2526 getPRMT(A: Extended, B: DAG.getConstant(Val: 0, DL, VT: MVT::i32), Selector: 0x7701, DL, DAG);
2527 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: Swapped);
2528 }
2529 case MVT::i32: {
2530 return getPRMT(A: Src, B: DAG.getConstant(Val: 0, DL, VT: MVT::i32), Selector: 0x0123, DL, DAG);
2531 }
2532 case MVT::v2i16: {
2533 SDValue Converted = DAG.getBitcast(VT: MVT::i32, V: Src);
2534 SDValue Swapped =
2535 getPRMT(A: Converted, B: DAG.getConstant(Val: 0, DL, VT: MVT::i32), Selector: 0x2301, DL, DAG);
2536 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i16, Operand: Swapped);
2537 }
2538 case MVT::i64: {
2539 SDValue UnpackSrc =
2540 DAG.getNode(Opcode: NVPTXISD::UNPACK_VECTOR, DL, ResultTys: {MVT::i32, MVT::i32}, Ops: Src);
2541 SDValue SwappedLow =
2542 getPRMT(A: UnpackSrc.getValue(R: 0), B: DAG.getConstant(Val: 0, DL, VT: MVT::i32), Selector: 0x0123,
2543 DL, DAG);
2544 SDValue SwappedHigh =
2545 getPRMT(A: UnpackSrc.getValue(R: 1), B: DAG.getConstant(Val: 0, DL, VT: MVT::i32), Selector: 0x0123,
2546 DL, DAG);
2547 return DAG.getNode(Opcode: NVPTXISD::BUILD_VECTOR, DL, VT: MVT::i64,
2548 Ops: {SwappedHigh, SwappedLow});
2549 }
2550 default:
2551 llvm_unreachable("unsupported type for bswap");
2552 }
2553}
2554
2555static SDValue lowerStAsyncWithMbarrier(SDValue Op, SelectionDAG &DAG) {
2556 const Function &Fn = DAG.getMachineFunction().getFunction();
2557 SDNode *N = Op.getNode();
2558 SDLoc DL(N);
2559 Intrinsic::ID IntrinsicID = N->getConstantOperandVal(Num: 1);
2560 SDValue DestAddr = N->getOperand(Num: 2);
2561 SDValue Value = N->getOperand(Num: 3);
2562 SDValue MbarAddr = N->getOperand(Num: 4);
2563
2564 MVT ValueVT = Value.getSimpleValueType();
2565
2566 if (ValueVT == MVT::i32 || ValueVT == MVT::i64)
2567 return Op;
2568
2569 if (ValueVT == MVT::i128) {
2570 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: Value);
2571 SDValue ValueLo = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i64, N1: Cast,
2572 N2: DAG.getIntPtrConstant(Val: 0, DL));
2573 SDValue ValueHi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i64, N1: Cast,
2574 N2: DAG.getIntPtrConstant(Val: 1, DL));
2575 SDValue Ops[] = {N->getOperand(Num: 0), DestAddr, ValueLo, ValueHi, MbarAddr};
2576 return DAG.getNode(Opcode: NVPTXISD::ST_ASYNC_MBARRIER_B128, DL, VT: MVT::Other, Ops);
2577 }
2578
2579 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
2580 Fn,
2581 Twine("unsupported argument type ") + llvm::EVT(ValueVT).getEVTString() +
2582 " for " + llvm::Intrinsic::getName(id: IntrinsicID) + " intrinsic",
2583 DiagnosticLocation(DL.getDebugLoc())));
2584 return Op.getOperand(i: 0); // Return only the chain
2585}
2586
2587static SDValue lowerStAsyncRelease(SDValue Op, SelectionDAG &DAG) {
2588 const Function &Fn = DAG.getMachineFunction().getFunction();
2589 SDNode *N = Op.getNode();
2590 SDLoc DL(N);
2591 Intrinsic::ID IntrinsicID = N->getConstantOperandVal(Num: 1);
2592 SDValue DestAddr = N->getOperand(Num: 2);
2593 SDValue Value = N->getOperand(Num: 3);
2594
2595 MVT ValueVT = Value.getSimpleValueType();
2596
2597 if (ValueVT == MVT::i16 || ValueVT == MVT::i32 || ValueVT == MVT::i64)
2598 return Op;
2599
2600 if (ValueVT == MVT::i8) {
2601 unsigned OpCode;
2602 switch (IntrinsicID) {
2603 case Intrinsic::nvvm_st_async_sys:
2604 OpCode = NVPTXISD::ST_ASYNC_SYS_B8;
2605 break;
2606 case Intrinsic::nvvm_st_async_gpu:
2607 OpCode = NVPTXISD::ST_ASYNC_GPU_B8;
2608 break;
2609 case Intrinsic::nvvm_st_async_mmio_sys:
2610 OpCode = NVPTXISD::ST_ASYNC_MMIO_SYS_B8;
2611 break;
2612 default:
2613 llvm_unreachable("unexpected intrinsic ID for st.async.release");
2614 }
2615
2616 Value = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i16, Operand: Value);
2617
2618 // The `.mmio` variant has no multimem form and therefore no `isMultimem`
2619 // operand.
2620 if (IntrinsicID == Intrinsic::nvvm_st_async_mmio_sys) {
2621 SDValue Ops[] = {N->getOperand(Num: 0), DestAddr, Value};
2622 return DAG.getNode(Opcode: OpCode, DL, VT: MVT::Other, Ops);
2623 }
2624
2625 SDValue IsMultimem =
2626 DAG.getTargetConstant(Val: N->getConstantOperandVal(Num: 4), DL, VT: MVT::i1);
2627 SDValue Ops[] = {N->getOperand(Num: 0), DestAddr, Value, IsMultimem};
2628 return DAG.getNode(Opcode: OpCode, DL, VT: MVT::Other, Ops);
2629 }
2630
2631 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
2632 Fn,
2633 Twine("unsupported argument type ") + llvm::EVT(ValueVT).getEVTString() +
2634 " for " + llvm::Intrinsic::getName(id: IntrinsicID) + " intrinsic",
2635 DiagnosticLocation(DL.getDebugLoc())));
2636 return Op.getOperand(i: 0); // Return only the chain
2637}
2638
2639static unsigned getTcgen05MMADisableOutputLane(unsigned IID) {
2640 switch (IID) {
2641 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg1:
2642 return NVPTXISD::TCGEN05_MMA_SHARED_DISABLE_OUTPUT_LANE_CG1;
2643 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg2:
2644 return NVPTXISD::TCGEN05_MMA_SHARED_DISABLE_OUTPUT_LANE_CG2;
2645 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg1:
2646 return NVPTXISD::TCGEN05_MMA_SHARED_SCALE_D_DISABLE_OUTPUT_LANE_CG1;
2647 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg2:
2648 return NVPTXISD::TCGEN05_MMA_SHARED_SCALE_D_DISABLE_OUTPUT_LANE_CG2;
2649 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1:
2650 return NVPTXISD::TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG1;
2651 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2:
2652 return NVPTXISD::TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG2;
2653 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1:
2654 return NVPTXISD::TCGEN05_MMA_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG1;
2655 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2:
2656 return NVPTXISD::TCGEN05_MMA_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG2;
2657 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1_ashift:
2658 return NVPTXISD::TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG1_ASHIFT;
2659 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2_ashift:
2660 return NVPTXISD::TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG2_ASHIFT;
2661 case Intrinsic::
2662 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1_ashift:
2663 return NVPTXISD::TCGEN05_MMA_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG1_ASHIFT;
2664 case Intrinsic::
2665 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2_ashift:
2666 return NVPTXISD::TCGEN05_MMA_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG2_ASHIFT;
2667 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg1:
2668 return NVPTXISD::TCGEN05_MMA_SP_SHARED_DISABLE_OUTPUT_LANE_CG1;
2669 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg2:
2670 return NVPTXISD::TCGEN05_MMA_SP_SHARED_DISABLE_OUTPUT_LANE_CG2;
2671 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg1:
2672 return NVPTXISD::TCGEN05_MMA_SP_SHARED_SCALE_D_DISABLE_OUTPUT_LANE_CG1;
2673 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg2:
2674 return NVPTXISD::TCGEN05_MMA_SP_SHARED_SCALE_D_DISABLE_OUTPUT_LANE_CG2;
2675 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1:
2676 return NVPTXISD::TCGEN05_MMA_SP_TENSOR_DISABLE_OUTPUT_LANE_CG1;
2677 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2:
2678 return NVPTXISD::TCGEN05_MMA_SP_TENSOR_DISABLE_OUTPUT_LANE_CG2;
2679 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1_ashift:
2680 return NVPTXISD::TCGEN05_MMA_SP_TENSOR_DISABLE_OUTPUT_LANE_CG1_ASHIFT;
2681 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2_ashift:
2682 return NVPTXISD::TCGEN05_MMA_SP_TENSOR_DISABLE_OUTPUT_LANE_CG2_ASHIFT;
2683 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1:
2684 return NVPTXISD::TCGEN05_MMA_SP_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG1;
2685 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2:
2686 return NVPTXISD::TCGEN05_MMA_SP_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG2;
2687 case Intrinsic::
2688 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1_ashift:
2689 return NVPTXISD::
2690 TCGEN05_MMA_SP_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG1_ASHIFT;
2691 case Intrinsic::
2692 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2_ashift:
2693 return NVPTXISD::
2694 TCGEN05_MMA_SP_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG2_ASHIFT;
2695 case Intrinsic::
2696 nvvm_tcgen05_mma_shared_f8f6f4_disable_output_lane_cg1_decompress_b:
2697 return NVPTXISD::TCGEN05_MMA_SHARED_DISABLE_OUTPUT_LANE_CG1_DECOMPRESS_B;
2698 case Intrinsic::
2699 nvvm_tcgen05_mma_shared_f8f6f4_disable_output_lane_cg2_decompress_b:
2700 return NVPTXISD::TCGEN05_MMA_SHARED_DISABLE_OUTPUT_LANE_CG2_DECOMPRESS_B;
2701 case Intrinsic::
2702 nvvm_tcgen05_mma_tensor_f8f6f4_disable_output_lane_cg1_decompress_b:
2703 return NVPTXISD::TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG1_DECOMPRESS_B;
2704 case Intrinsic::
2705 nvvm_tcgen05_mma_tensor_f8f6f4_disable_output_lane_cg2_decompress_b:
2706 return NVPTXISD::TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG2_DECOMPRESS_B;
2707 };
2708 llvm_unreachable("unhandled tcgen05.mma.disable_output_lane intrinsic");
2709}
2710
2711static SDValue LowerTcgen05MMADisableOutputLane(SDValue Op, SelectionDAG &DAG) {
2712 SDNode *N = Op.getNode();
2713 SDLoc DL(N);
2714 unsigned IID = cast<ConstantSDNode>(Val: N->getOperand(Num: 1))->getZExtValue();
2715
2716 SmallVector<SDValue, 16> Ops;
2717 // split the vector argument
2718 for (size_t I = 0; I < N->getNumOperands(); I++) {
2719 if (I == 1)
2720 continue; // skip IID
2721 SDValue Val = N->getOperand(Num: I);
2722 EVT ValVT = Val.getValueType();
2723 if (ValVT.isVector()) {
2724 EVT EltVT = ValVT.getVectorElementType();
2725 for (unsigned J = 0, NElts = ValVT.getVectorNumElements(); J < NElts; J++)
2726 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Val,
2727 N2: DAG.getIntPtrConstant(Val: J, DL)));
2728 } else
2729 Ops.push_back(Elt: Val);
2730 }
2731
2732 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(Val: N);
2733 SDValue Tcgen05MMANode = DAG.getMemIntrinsicNode(
2734 Opcode: getTcgen05MMADisableOutputLane(IID), dl: DL, VTList: N->getVTList(), Ops,
2735 MemVT: MemSD->getMemoryVT(), MMO: MemSD->getMemOperand());
2736
2737 return Tcgen05MMANode;
2738}
2739
2740// Lower vector return type of tcgen05.ld intrinsics
2741static std::optional<std::pair<SDValue, SDValue>>
2742lowerTcgen05Ld(SDNode *N, SelectionDAG &DAG, bool HasOffset = false) {
2743 SDLoc DL(N);
2744 EVT ResVT = N->getValueType(ResNo: 0);
2745 if (!ResVT.isVector())
2746 return {}; // already legalized.
2747
2748 const unsigned NumElts = ResVT.getVectorNumElements();
2749
2750 // Create the return type of the instructions
2751 SmallVector<EVT, 5> ListVTs;
2752 for (unsigned i = 0; i < NumElts; ++i)
2753 ListVTs.push_back(Elt: MVT::i32);
2754
2755 ListVTs.push_back(Elt: N->getValueType(ResNo: 1)); // Chain
2756
2757 SDVTList ResVTs = DAG.getVTList(VTs: ListVTs);
2758
2759 SmallVector<SDValue, 8> Ops{N->getOperand(Num: 0), N->getOperand(Num: 1),
2760 N->getOperand(Num: 2)};
2761
2762 if (HasOffset) {
2763 Ops.push_back(Elt: N->getOperand(Num: 3)); // offset
2764 Ops.push_back(Elt: N->getOperand(Num: 4)); // Pack flag
2765 } else
2766 Ops.push_back(Elt: N->getOperand(Num: 3)); // Pack flag
2767
2768 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(Val: N);
2769 SDValue NewNode =
2770 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: ResVTs, Ops,
2771 MemVT: MemSD->getMemoryVT(), MMO: MemSD->getMemOperand());
2772
2773 // split the vector result
2774 SmallVector<SDValue, 4> ScalarRes;
2775 for (unsigned i = 0; i < NumElts; ++i) {
2776 SDValue Res = NewNode.getValue(R: i);
2777 ScalarRes.push_back(Elt: Res);
2778 }
2779
2780 SDValue Chain = NewNode.getValue(R: NumElts);
2781 SDValue BuildVector = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: ResVT, Ops: ScalarRes);
2782 return {{BuildVector, Chain}};
2783}
2784
2785static SDValue reportInvalidTensormapReplaceUsage(SDValue Op, SelectionDAG &DAG,
2786 unsigned Val) {
2787 SDNode *N = Op.getNode();
2788 SDLoc DL(N);
2789
2790 const Function &Fn = DAG.getMachineFunction().getFunction();
2791
2792 unsigned AS = 0;
2793 if (auto *MemN = dyn_cast<MemIntrinsicSDNode>(Val: N))
2794 AS = MemN->getAddressSpace();
2795 Type *PtrTy = PointerType::get(C&: *DAG.getContext(), AddressSpace: AS);
2796 Module *M = DAG.getMachineFunction().getFunction().getParent();
2797
2798 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
2799 Fn,
2800 "Intrinsic " +
2801 Intrinsic::getName(Id: N->getConstantOperandVal(Num: 1), OverloadTys: {PtrTy}, M) +
2802 " with value " + Twine(Val) +
2803 " is not supported on the given target.",
2804 DL.getDebugLoc()));
2805 return Op.getOperand(i: 0);
2806}
2807
2808static SDValue lowerTensormapReplaceElemtype(SDValue Op, SelectionDAG &DAG) {
2809 SDNode *N = Op.getNode();
2810 SDLoc DL(N);
2811
2812 // immediate argument representing elemtype
2813 unsigned Val = N->getConstantOperandVal(Num: 3);
2814
2815 if (!DAG.getSubtarget<NVPTXSubtarget>().hasTensormapReplaceElemtypeSupport(
2816 ElemType: Val))
2817 return reportInvalidTensormapReplaceUsage(Op, DAG, Val);
2818
2819 return Op;
2820}
2821
2822static SDValue lowerTensormapReplaceSwizzleMode(SDValue Op, SelectionDAG &DAG) {
2823 SDNode *N = Op.getNode();
2824 SDLoc DL(N);
2825
2826 // immediate argument representing swizzle mode
2827 unsigned Val = N->getConstantOperandVal(Num: 3);
2828
2829 if (!DAG.getSubtarget<NVPTXSubtarget>().hasTensormapReplaceSwizzleModeSupport(
2830 SwizzleMode: Val))
2831 return reportInvalidTensormapReplaceUsage(Op, DAG, Val);
2832
2833 return Op;
2834}
2835
2836static SDValue lowerIntrinsicVoid(SDValue Op, SelectionDAG &DAG) {
2837 SDNode *N = Op.getNode();
2838 SDValue Intrin = N->getOperand(Num: 1);
2839
2840 // Get the intrinsic ID
2841 unsigned IntrinNo = cast<ConstantSDNode>(Val: Intrin.getNode())->getZExtValue();
2842 switch (IntrinNo) {
2843 default:
2844 break;
2845 case Intrinsic::nvvm_st_async:
2846 return lowerStAsyncWithMbarrier(Op, DAG);
2847 case Intrinsic::nvvm_st_async_sys:
2848 case Intrinsic::nvvm_st_async_gpu:
2849 case Intrinsic::nvvm_st_async_mmio_sys:
2850 return lowerStAsyncRelease(Op, DAG);
2851
2852 case Intrinsic::nvvm_tcgen05_st_16x64b_x1:
2853 case Intrinsic::nvvm_tcgen05_st_16x64b_x2:
2854 case Intrinsic::nvvm_tcgen05_st_16x64b_x4:
2855 case Intrinsic::nvvm_tcgen05_st_16x64b_x8:
2856 case Intrinsic::nvvm_tcgen05_st_16x64b_x16:
2857 case Intrinsic::nvvm_tcgen05_st_16x64b_x32:
2858 case Intrinsic::nvvm_tcgen05_st_16x64b_x128:
2859 case Intrinsic::nvvm_tcgen05_st_16x128b_x1:
2860 case Intrinsic::nvvm_tcgen05_st_16x128b_x2:
2861 case Intrinsic::nvvm_tcgen05_st_16x128b_x4:
2862 case Intrinsic::nvvm_tcgen05_st_16x128b_x8:
2863 case Intrinsic::nvvm_tcgen05_st_16x128b_x16:
2864 case Intrinsic::nvvm_tcgen05_st_16x128b_x32:
2865 case Intrinsic::nvvm_tcgen05_st_16x128b_x64:
2866 case Intrinsic::nvvm_tcgen05_st_16x256b_x1:
2867 case Intrinsic::nvvm_tcgen05_st_16x256b_x2:
2868 case Intrinsic::nvvm_tcgen05_st_16x256b_x4:
2869 case Intrinsic::nvvm_tcgen05_st_16x256b_x8:
2870 case Intrinsic::nvvm_tcgen05_st_16x256b_x16:
2871 case Intrinsic::nvvm_tcgen05_st_16x256b_x32:
2872 case Intrinsic::nvvm_tcgen05_st_32x32b_x1:
2873 case Intrinsic::nvvm_tcgen05_st_32x32b_x2:
2874 case Intrinsic::nvvm_tcgen05_st_32x32b_x4:
2875 case Intrinsic::nvvm_tcgen05_st_32x32b_x8:
2876 case Intrinsic::nvvm_tcgen05_st_32x32b_x16:
2877 case Intrinsic::nvvm_tcgen05_st_32x32b_x32:
2878 case Intrinsic::nvvm_tcgen05_st_16x64b_x64:
2879 case Intrinsic::nvvm_tcgen05_st_32x32b_x64:
2880 case Intrinsic::nvvm_tcgen05_st_32x32b_x128:
2881 return lowerTcgen05St(Op, DAG);
2882 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x1:
2883 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x2:
2884 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x4:
2885 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x8:
2886 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x16:
2887 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x32:
2888 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x64:
2889 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x128:
2890 return lowerTcgen05St(Op, DAG, /* hasOffset */ true);
2891 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg1:
2892 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg2:
2893 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg1:
2894 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg2:
2895 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg1:
2896 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg2:
2897 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg1:
2898 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg2:
2899 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1:
2900 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2:
2901 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1:
2902 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2:
2903 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1:
2904 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2:
2905 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1:
2906 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2:
2907 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1_ashift:
2908 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2_ashift:
2909 case Intrinsic::
2910 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1_ashift:
2911 case Intrinsic::
2912 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2_ashift:
2913 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1_ashift:
2914 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2_ashift:
2915 case Intrinsic::
2916 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1_ashift:
2917 case Intrinsic::
2918 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2_ashift:
2919 case Intrinsic::
2920 nvvm_tcgen05_mma_shared_f8f6f4_disable_output_lane_cg1_decompress_b:
2921 case Intrinsic::
2922 nvvm_tcgen05_mma_shared_f8f6f4_disable_output_lane_cg2_decompress_b:
2923 case Intrinsic::
2924 nvvm_tcgen05_mma_tensor_f8f6f4_disable_output_lane_cg1_decompress_b:
2925 case Intrinsic::
2926 nvvm_tcgen05_mma_tensor_f8f6f4_disable_output_lane_cg2_decompress_b:
2927 return LowerTcgen05MMADisableOutputLane(Op, DAG);
2928 case Intrinsic::nvvm_tensormap_replace_elemtype:
2929 return lowerTensormapReplaceElemtype(Op, DAG);
2930 case Intrinsic::nvvm_tensormap_replace_swizzle_mode:
2931 return lowerTensormapReplaceSwizzleMode(Op, DAG);
2932 }
2933 return Op;
2934}
2935
2936static SDValue LowerClusterLaunchControlQueryCancel(SDValue Op,
2937 SelectionDAG &DAG) {
2938
2939 SDNode *N = Op.getNode();
2940 if (N->getOperand(Num: 1).getValueType() != MVT::i128) {
2941 // return, if the operand is already lowered
2942 return SDValue();
2943 }
2944
2945 unsigned IID =
2946 cast<ConstantSDNode>(Val: N->getOperand(Num: 0).getNode())->getZExtValue();
2947 auto Opcode = [&]() {
2948 switch (IID) {
2949 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_is_canceled:
2950 return NVPTXISD::CLUSTERLAUNCHCONTROL_QUERY_CANCEL_IS_CANCELED;
2951 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_x:
2952 return NVPTXISD::CLUSTERLAUNCHCONTROL_QUERY_CANCEL_GET_FIRST_CTAID_X;
2953 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_y:
2954 return NVPTXISD::CLUSTERLAUNCHCONTROL_QUERY_CANCEL_GET_FIRST_CTAID_Y;
2955 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_z:
2956 return NVPTXISD::CLUSTERLAUNCHCONTROL_QUERY_CANCEL_GET_FIRST_CTAID_Z;
2957 default:
2958 llvm_unreachable("unsupported/unhandled intrinsic");
2959 }
2960 }();
2961
2962 SDLoc DL(N);
2963 SDValue TryCancelResponse = N->getOperand(Num: 1);
2964 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: TryCancelResponse);
2965 SDValue TryCancelResponse0 =
2966 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i64, N1: Cast,
2967 N2: DAG.getIntPtrConstant(Val: 0, DL));
2968 SDValue TryCancelResponse1 =
2969 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i64, N1: Cast,
2970 N2: DAG.getIntPtrConstant(Val: 1, DL));
2971
2972 return DAG.getNode(Opcode, DL, VTList: N->getVTList(),
2973 Ops: {TryCancelResponse0, TryCancelResponse1});
2974}
2975
2976static SDValue lowerCvtRSIntrinsics(SDValue Op, SelectionDAG &DAG) {
2977 SDNode *N = Op.getNode();
2978 SDLoc DL(N);
2979 SDValue F32Vec = N->getOperand(Num: 1);
2980 SDValue RBits = N->getOperand(Num: 2);
2981
2982 unsigned IntrinsicID = N->getConstantOperandVal(Num: 0);
2983
2984 // Extract the 4 float elements from the vector
2985 SmallVector<SDValue, 6> Ops;
2986 for (unsigned i = 0; i < 4; ++i)
2987 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: F32Vec,
2988 N2: DAG.getIntPtrConstant(Val: i, DL)));
2989
2990 using NVPTX::PTXCvtMode::CvtMode;
2991
2992 auto [OpCode, RetTy, CvtModeFlag] =
2993 [&]() -> std::tuple<unsigned, MVT::SimpleValueType, uint32_t> {
2994 switch (IntrinsicID) {
2995 case Intrinsic::nvvm_f32x4_to_e4m3x4_rs_relu_satfinite:
2996 return {NVPTXISD::CVT_E4M3X4_F32X4_RS_SF, MVT::v4i8,
2997 CvtMode::RS | CvtMode::RELU_FLAG};
2998 case Intrinsic::nvvm_f32x4_to_e4m3x4_rs_satfinite:
2999 return {NVPTXISD::CVT_E4M3X4_F32X4_RS_SF, MVT::v4i8, CvtMode::RS};
3000 case Intrinsic::nvvm_f32x4_to_e5m2x4_rs_relu_satfinite:
3001 return {NVPTXISD::CVT_E5M2X4_F32X4_RS_SF, MVT::v4i8,
3002 CvtMode::RS | CvtMode::RELU_FLAG};
3003 case Intrinsic::nvvm_f32x4_to_e5m2x4_rs_satfinite:
3004 return {NVPTXISD::CVT_E5M2X4_F32X4_RS_SF, MVT::v4i8, CvtMode::RS};
3005 case Intrinsic::nvvm_f32x4_to_e2m3x4_rs_relu_satfinite:
3006 return {NVPTXISD::CVT_E2M3X4_F32X4_RS_SF, MVT::v4i8,
3007 CvtMode::RS | CvtMode::RELU_FLAG};
3008 case Intrinsic::nvvm_f32x4_to_e2m3x4_rs_satfinite:
3009 return {NVPTXISD::CVT_E2M3X4_F32X4_RS_SF, MVT::v4i8, CvtMode::RS};
3010 case Intrinsic::nvvm_f32x4_to_e3m2x4_rs_relu_satfinite:
3011 return {NVPTXISD::CVT_E3M2X4_F32X4_RS_SF, MVT::v4i8,
3012 CvtMode::RS | CvtMode::RELU_FLAG};
3013 case Intrinsic::nvvm_f32x4_to_e3m2x4_rs_satfinite:
3014 return {NVPTXISD::CVT_E3M2X4_F32X4_RS_SF, MVT::v4i8, CvtMode::RS};
3015 case Intrinsic::nvvm_f32x4_to_e2m1x4_rs_relu_satfinite:
3016 return {NVPTXISD::CVT_E2M1X4_F32X4_RS_SF, MVT::i16,
3017 CvtMode::RS | CvtMode::RELU_FLAG};
3018 case Intrinsic::nvvm_f32x4_to_e2m1x4_rs_satfinite:
3019 return {NVPTXISD::CVT_E2M1X4_F32X4_RS_SF, MVT::i16, CvtMode::RS};
3020 default:
3021 llvm_unreachable("unsupported/unhandled intrinsic");
3022 }
3023 }();
3024
3025 Ops.push_back(Elt: RBits);
3026 Ops.push_back(Elt: DAG.getConstant(Val: CvtModeFlag, DL, VT: MVT::i32));
3027
3028 return DAG.getNode(Opcode: OpCode, DL, VT: RetTy, Ops);
3029}
3030
3031static SDValue lowerPrmtIntrinsic(SDValue Op, SelectionDAG &DAG) {
3032 const unsigned Mode = [&]() {
3033 switch (Op->getConstantOperandVal(Num: 0)) {
3034 case Intrinsic::nvvm_prmt:
3035 return NVPTX::PTXPrmtMode::NONE;
3036 case Intrinsic::nvvm_prmt_b4e:
3037 return NVPTX::PTXPrmtMode::B4E;
3038 case Intrinsic::nvvm_prmt_ecl:
3039 return NVPTX::PTXPrmtMode::ECL;
3040 case Intrinsic::nvvm_prmt_ecr:
3041 return NVPTX::PTXPrmtMode::ECR;
3042 case Intrinsic::nvvm_prmt_f4e:
3043 return NVPTX::PTXPrmtMode::F4E;
3044 case Intrinsic::nvvm_prmt_rc16:
3045 return NVPTX::PTXPrmtMode::RC16;
3046 case Intrinsic::nvvm_prmt_rc8:
3047 return NVPTX::PTXPrmtMode::RC8;
3048 default:
3049 llvm_unreachable("unsupported/unhandled intrinsic");
3050 }
3051 }();
3052 SDLoc DL(Op);
3053 SDValue A = Op->getOperand(Num: 1);
3054 SDValue B = Op.getNumOperands() == 4 ? Op.getOperand(i: 2)
3055 : DAG.getConstant(Val: 0, DL, VT: MVT::i32);
3056 SDValue Selector = (Op->op_end() - 1)->get();
3057 return getPRMT(A, B, Selector, DL, DAG, Mode);
3058}
3059
3060#define TCGEN05_LD_RED_INTR(SHAPE, NUM, TYPE) \
3061 Intrinsic::nvvm_tcgen05_ld_red_##SHAPE##_x##NUM##_##TYPE
3062
3063#define TCGEN05_LD_RED_INST(SHAPE, NUM, TYPE) \
3064 NVPTXISD::TCGEN05_LD_RED_##SHAPE##_X##NUM##_##TYPE
3065
3066static unsigned getTcgen05LdRedID(Intrinsic::ID IID) {
3067 switch (IID) {
3068 case TCGEN05_LD_RED_INTR(32x32b, 2, f32):
3069 return TCGEN05_LD_RED_INST(32x32b, 2, F32);
3070 case TCGEN05_LD_RED_INTR(32x32b, 4, f32):
3071 return TCGEN05_LD_RED_INST(32x32b, 4, F32);
3072 case TCGEN05_LD_RED_INTR(32x32b, 8, f32):
3073 return TCGEN05_LD_RED_INST(32x32b, 8, F32);
3074 case TCGEN05_LD_RED_INTR(32x32b, 16, f32):
3075 return TCGEN05_LD_RED_INST(32x32b, 16, F32);
3076 case TCGEN05_LD_RED_INTR(32x32b, 32, f32):
3077 return TCGEN05_LD_RED_INST(32x32b, 32, F32);
3078 case TCGEN05_LD_RED_INTR(32x32b, 64, f32):
3079 return TCGEN05_LD_RED_INST(32x32b, 64, F32);
3080 case TCGEN05_LD_RED_INTR(32x32b, 128, f32):
3081 return TCGEN05_LD_RED_INST(32x32b, 128, F32);
3082 case TCGEN05_LD_RED_INTR(16x32bx2, 2, f32):
3083 return TCGEN05_LD_RED_INST(16x32bx2, 2, F32);
3084 case TCGEN05_LD_RED_INTR(16x32bx2, 4, f32):
3085 return TCGEN05_LD_RED_INST(16x32bx2, 4, F32);
3086 case TCGEN05_LD_RED_INTR(16x32bx2, 8, f32):
3087 return TCGEN05_LD_RED_INST(16x32bx2, 8, F32);
3088 case TCGEN05_LD_RED_INTR(16x32bx2, 16, f32):
3089 return TCGEN05_LD_RED_INST(16x32bx2, 16, F32);
3090 case TCGEN05_LD_RED_INTR(16x32bx2, 32, f32):
3091 return TCGEN05_LD_RED_INST(16x32bx2, 32, F32);
3092 case TCGEN05_LD_RED_INTR(16x32bx2, 64, f32):
3093 return TCGEN05_LD_RED_INST(16x32bx2, 64, F32);
3094 case TCGEN05_LD_RED_INTR(16x32bx2, 128, f32):
3095 return TCGEN05_LD_RED_INST(16x32bx2, 128, F32);
3096 case TCGEN05_LD_RED_INTR(32x32b, 2, i32):
3097 return TCGEN05_LD_RED_INST(32x32b, 2, I32);
3098 case TCGEN05_LD_RED_INTR(32x32b, 4, i32):
3099 return TCGEN05_LD_RED_INST(32x32b, 4, I32);
3100 case TCGEN05_LD_RED_INTR(32x32b, 8, i32):
3101 return TCGEN05_LD_RED_INST(32x32b, 8, I32);
3102 case TCGEN05_LD_RED_INTR(32x32b, 16, i32):
3103 return TCGEN05_LD_RED_INST(32x32b, 16, I32);
3104 case TCGEN05_LD_RED_INTR(32x32b, 32, i32):
3105 return TCGEN05_LD_RED_INST(32x32b, 32, I32);
3106 case TCGEN05_LD_RED_INTR(32x32b, 64, i32):
3107 return TCGEN05_LD_RED_INST(32x32b, 64, I32);
3108 case TCGEN05_LD_RED_INTR(32x32b, 128, i32):
3109 return TCGEN05_LD_RED_INST(32x32b, 128, I32);
3110 case TCGEN05_LD_RED_INTR(16x32bx2, 2, i32):
3111 return TCGEN05_LD_RED_INST(16x32bx2, 2, I32);
3112 case TCGEN05_LD_RED_INTR(16x32bx2, 4, i32):
3113 return TCGEN05_LD_RED_INST(16x32bx2, 4, I32);
3114 case TCGEN05_LD_RED_INTR(16x32bx2, 8, i32):
3115 return TCGEN05_LD_RED_INST(16x32bx2, 8, I32);
3116 case TCGEN05_LD_RED_INTR(16x32bx2, 16, i32):
3117 return TCGEN05_LD_RED_INST(16x32bx2, 16, I32);
3118 case TCGEN05_LD_RED_INTR(16x32bx2, 32, i32):
3119 return TCGEN05_LD_RED_INST(16x32bx2, 32, I32);
3120 case TCGEN05_LD_RED_INTR(16x32bx2, 64, i32):
3121 return TCGEN05_LD_RED_INST(16x32bx2, 64, I32);
3122 case TCGEN05_LD_RED_INTR(16x32bx2, 128, i32):
3123 return TCGEN05_LD_RED_INST(16x32bx2, 128, I32);
3124 default:
3125 llvm_unreachable("Invalid tcgen05.ld.red intrinsic ID");
3126 }
3127}
3128
3129// Lower vector return type of tcgen05.ld intrinsics
3130static std::optional<std::tuple<SDValue, SDValue, SDValue>>
3131lowerTcgen05LdRed(SDNode *N, SelectionDAG &DAG) {
3132 SDLoc DL(N);
3133 EVT ResVT = N->getValueType(ResNo: 0);
3134 if (!ResVT.isVector())
3135 return {}; // already legalized.
3136
3137 const unsigned NumElts = ResVT.getVectorNumElements();
3138
3139 // Create the return type of the instructions
3140 // +1 represents the reduction value
3141 SmallVector<EVT, 132> ListVTs{
3142 NumElts + 1,
3143 ResVT.getVectorElementType().isFloatingPoint() ? MVT::f32 : MVT::i32};
3144
3145 ListVTs.push_back(Elt: MVT::Other); // Chain
3146
3147 SDVTList ResVTs = DAG.getVTList(VTs: ListVTs);
3148
3149 // Prepare the Operands
3150 SmallVector<SDValue, 8> Ops{N->getOperand(Num: 0)}; // Chain
3151
3152 // skip IID at index 1
3153 for (unsigned i = 2; i < N->getNumOperands(); i++)
3154 Ops.push_back(Elt: N->getOperand(Num: i));
3155
3156 unsigned IID = cast<ConstantSDNode>(Val: N->getOperand(Num: 1))->getZExtValue();
3157 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(Val: N);
3158 SDValue NewNode =
3159 DAG.getMemIntrinsicNode(Opcode: getTcgen05LdRedID(IID), dl: DL, VTList: ResVTs, Ops,
3160 MemVT: MemSD->getMemoryVT(), MMO: MemSD->getMemOperand());
3161
3162 // Split vector result
3163 SmallVector<SDValue, 132> ScalarRes;
3164 for (unsigned i = 0; i < NumElts; ++i) {
3165 SDValue Res = NewNode.getValue(R: i);
3166 ScalarRes.push_back(Elt: Res);
3167 }
3168
3169 SDValue BuildVector = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: ResVT, Ops: ScalarRes);
3170 SDValue RedResult = NewNode.getValue(R: NumElts);
3171 SDValue Chain = NewNode.getValue(R: NumElts + 1);
3172 return {{BuildVector, RedResult, Chain}};
3173}
3174
3175static SDValue lowerIntrinsicWChain(SDValue Op, SelectionDAG &DAG) {
3176 switch (Op->getConstantOperandVal(Num: 1)) {
3177 default:
3178 return Op;
3179
3180 // These tcgen05 intrinsics return a v2i32, which is legal, so we have to
3181 // lower them through LowerOperation() instead of ReplaceNodeResults().
3182 case Intrinsic::nvvm_tcgen05_ld_16x64b_x2:
3183 case Intrinsic::nvvm_tcgen05_ld_16x128b_x1:
3184 case Intrinsic::nvvm_tcgen05_ld_32x32b_x2:
3185 if (auto Res = lowerTcgen05Ld(N: Op.getNode(), DAG))
3186 return DAG.getMergeValues(Ops: {Res->first, Res->second}, dl: SDLoc(Op));
3187 return SDValue();
3188
3189 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x2:
3190 if (auto Res = lowerTcgen05Ld(N: Op.getNode(), DAG, /*HasOffset=*/true))
3191 return DAG.getMergeValues(Ops: {Res->first, Res->second}, dl: SDLoc(Op));
3192 return SDValue();
3193
3194 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x2_f32:
3195 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x2_i32:
3196 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x2_f32:
3197 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x2_i32:
3198 if (auto Res = lowerTcgen05LdRed(N: Op.getNode(), DAG))
3199 return DAG.getMergeValues(
3200 Ops: {std::get<0>(t&: *Res), std::get<1>(t&: *Res), std::get<2>(t&: *Res)}, dl: SDLoc(Op));
3201 return SDValue();
3202 }
3203}
3204
3205static SDValue lowerIntrinsicWOChain(SDValue Op, SelectionDAG &DAG) {
3206 switch (Op->getConstantOperandVal(Num: 0)) {
3207 default:
3208 return Op;
3209 case Intrinsic::nvvm_prmt:
3210 case Intrinsic::nvvm_prmt_b4e:
3211 case Intrinsic::nvvm_prmt_ecl:
3212 case Intrinsic::nvvm_prmt_ecr:
3213 case Intrinsic::nvvm_prmt_f4e:
3214 case Intrinsic::nvvm_prmt_rc16:
3215 case Intrinsic::nvvm_prmt_rc8:
3216 return lowerPrmtIntrinsic(Op, DAG);
3217 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_is_canceled:
3218 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_x:
3219 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_y:
3220 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_z:
3221 return LowerClusterLaunchControlQueryCancel(Op, DAG);
3222 case Intrinsic::nvvm_f32x4_to_e4m3x4_rs_satfinite:
3223 case Intrinsic::nvvm_f32x4_to_e4m3x4_rs_relu_satfinite:
3224 case Intrinsic::nvvm_f32x4_to_e5m2x4_rs_satfinite:
3225 case Intrinsic::nvvm_f32x4_to_e5m2x4_rs_relu_satfinite:
3226 case Intrinsic::nvvm_f32x4_to_e2m3x4_rs_satfinite:
3227 case Intrinsic::nvvm_f32x4_to_e2m3x4_rs_relu_satfinite:
3228 case Intrinsic::nvvm_f32x4_to_e3m2x4_rs_satfinite:
3229 case Intrinsic::nvvm_f32x4_to_e3m2x4_rs_relu_satfinite:
3230 case Intrinsic::nvvm_f32x4_to_e2m1x4_rs_satfinite:
3231 case Intrinsic::nvvm_f32x4_to_e2m1x4_rs_relu_satfinite:
3232 return lowerCvtRSIntrinsics(Op, DAG);
3233 }
3234}
3235
3236// In PTX 64-bit CTLZ and CTPOP are supported, but they return a 32-bit value.
3237// Lower these into a node returning the correct type which is zero-extended
3238// back to the correct size.
3239static SDValue lowerCTLZCTPOP(SDValue Op, SelectionDAG &DAG) {
3240 SDValue V = Op->getOperand(Num: 0);
3241 assert(V.getValueType() == MVT::i64 &&
3242 "Unexpected CTLZ/CTPOP type to legalize");
3243
3244 SDLoc DL(Op);
3245 SDValue CT = DAG.getNode(Opcode: Op->getOpcode(), DL, VT: MVT::i32, Operand: V);
3246 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: CT, Flags: SDNodeFlags::NonNeg);
3247}
3248
3249static SDValue expandFSH64(SDValue A, SDValue B, SDValue ShiftAmount, SDLoc DL,
3250 unsigned Opcode, SelectionDAG &DAG) {
3251 assert(A.getValueType() == MVT::i64 && B.getValueType() == MVT::i64);
3252
3253 const auto *AmtConst = dyn_cast<ConstantSDNode>(Val&: ShiftAmount);
3254 if (!AmtConst)
3255 return SDValue();
3256 const auto Amt = AmtConst->getZExtValue() & 63;
3257
3258 SDValue UnpackA =
3259 DAG.getNode(Opcode: NVPTXISD::UNPACK_VECTOR, DL, ResultTys: {MVT::i32, MVT::i32}, Ops: A);
3260 SDValue UnpackB =
3261 DAG.getNode(Opcode: NVPTXISD::UNPACK_VECTOR, DL, ResultTys: {MVT::i32, MVT::i32}, Ops: B);
3262
3263 // Arch is Little endiain: 0 = low bits, 1 = high bits
3264 SDValue ALo = UnpackA.getValue(R: 0);
3265 SDValue AHi = UnpackA.getValue(R: 1);
3266 SDValue BLo = UnpackB.getValue(R: 0);
3267 SDValue BHi = UnpackB.getValue(R: 1);
3268
3269 // The bitfeild consists of { AHi : ALo : BHi : BLo }
3270 //
3271 // * FSHL, Amt < 32 - The window will contain { AHi : ALo : BHi }
3272 // * FSHL, Amt >= 32 - The window will contain { ALo : BHi : BLo }
3273 // * FSHR, Amt < 32 - The window will contain { ALo : BHi : BLo }
3274 // * FSHR, Amt >= 32 - The window will contain { AHi : ALo : BHi }
3275 //
3276 // Note that Amt = 0 and Amt = 32 are special cases where 32-bit funnel shifts
3277 // are not needed at all. Amt = 0 is a no-op producing either A or B depending
3278 // on the direction. Amt = 32 can be implemented by a packing and unpacking
3279 // move to select and arrange the 32bit values. For simplicity, these cases
3280 // are not handled here explicitly and instead we rely on DAGCombiner to
3281 // remove the no-op funnel shifts we insert.
3282 auto [High, Mid, Low] = ((Opcode == ISD::FSHL) == (Amt < 32))
3283 ? std::make_tuple(args&: AHi, args&: ALo, args&: BHi)
3284 : std::make_tuple(args&: ALo, args&: BHi, args&: BLo);
3285
3286 SDValue NewAmt = DAG.getConstant(Val: Amt & 31, DL, VT: MVT::i32);
3287 SDValue RHi = DAG.getNode(Opcode, DL, VT: MVT::i32, Ops: {High, Mid, NewAmt});
3288 SDValue RLo = DAG.getNode(Opcode, DL, VT: MVT::i32, Ops: {Mid, Low, NewAmt});
3289
3290 return DAG.getNode(Opcode: NVPTXISD::BUILD_VECTOR, DL, VT: MVT::i64, Ops: {RLo, RHi});
3291}
3292
3293static SDValue lowerFSH(SDValue Op, SelectionDAG &DAG) {
3294 return expandFSH64(A: Op->getOperand(Num: 0), B: Op->getOperand(Num: 1), ShiftAmount: Op->getOperand(Num: 2),
3295 DL: SDLoc(Op), Opcode: Op->getOpcode(), DAG);
3296}
3297
3298static SDValue lowerROT(SDValue Op, SelectionDAG &DAG) {
3299 unsigned Opcode = Op->getOpcode() == ISD::ROTL ? ISD::FSHL : ISD::FSHR;
3300 return expandFSH64(A: Op->getOperand(Num: 0), B: Op->getOperand(Num: 0), ShiftAmount: Op->getOperand(Num: 1),
3301 DL: SDLoc(Op), Opcode, DAG);
3302}
3303
3304static SDValue lowerFREM(SDValue Op, SelectionDAG &DAG) {
3305 // Lower (frem x, y) into (sub x, (mul (ftrunc (div x, y)) y)),
3306 // i.e. "poor man's fmod()". When y is infinite, x is returned. This matches
3307 // the semantics of LLVM's frem.
3308 SDLoc DL(Op);
3309 SDValue X = Op->getOperand(Num: 0);
3310 SDValue Y = Op->getOperand(Num: 1);
3311 EVT Ty = Op.getValueType();
3312 SDNodeFlags Flags = Op->getFlags();
3313
3314 SDValue Div = DAG.getNode(Opcode: ISD::FDIV, DL, VT: Ty, N1: X, N2: Y, Flags);
3315 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL, VT: Ty, Operand: Div, Flags);
3316 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL, VT: Ty, N1: Trunc, N2: Y,
3317 Flags: Flags | SDNodeFlags::AllowContract);
3318 SDValue Sub = DAG.getNode(Opcode: ISD::FSUB, DL, VT: Ty, N1: X, N2: Mul,
3319 Flags: Flags | SDNodeFlags::AllowContract);
3320
3321 if (Flags.hasNoInfs())
3322 return Sub;
3323
3324 // If Y is infinite, return X
3325 SDValue AbsY = DAG.getNode(Opcode: ISD::FABS, DL, VT: Ty, Operand: Y);
3326 SDValue Inf =
3327 DAG.getConstantFP(Val: APFloat::getInf(Sem: Ty.getFltSemantics()), DL, VT: Ty);
3328 SDValue IsInf = DAG.getSetCC(DL, VT: MVT::i1, LHS: AbsY, RHS: Inf, Cond: ISD::SETEQ);
3329 return DAG.getSelect(DL, VT: Ty, Cond: IsInf, LHS: X, RHS: Sub);
3330}
3331
3332static SDValue lowerSELECT(SDValue Op, SelectionDAG &DAG) {
3333 assert(Op.getValueType() == MVT::i1 && "Custom lowering enabled only for i1");
3334
3335 SDValue Cond = Op->getOperand(Num: 0);
3336 SDValue TrueVal = Op->getOperand(Num: 1);
3337 SDValue FalseVal = Op->getOperand(Num: 2);
3338 SDLoc DL(Op);
3339
3340 // If both operands are truncated, we push the select through the truncates.
3341 if (TrueVal.getOpcode() == ISD::TRUNCATE &&
3342 FalseVal.getOpcode() == ISD::TRUNCATE) {
3343 TrueVal = TrueVal.getOperand(i: 0);
3344 FalseVal = FalseVal.getOperand(i: 0);
3345
3346 EVT VT = TrueVal.getSimpleValueType().bitsLE(VT: FalseVal.getSimpleValueType())
3347 ? TrueVal.getValueType()
3348 : FalseVal.getValueType();
3349 TrueVal = DAG.getAnyExtOrTrunc(Op: TrueVal, DL, VT);
3350 FalseVal = DAG.getAnyExtOrTrunc(Op: FalseVal, DL, VT);
3351 SDValue Select = DAG.getSelect(DL, VT, Cond, LHS: TrueVal, RHS: FalseVal);
3352 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Select);
3353 }
3354
3355 // Otherwise, expand the select into a series of logical operations. These
3356 // often can be folded into other operations either by us or ptxas.
3357 TrueVal = DAG.getFreeze(V: TrueVal);
3358 FalseVal = DAG.getFreeze(V: FalseVal);
3359 SDValue And1 = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i1, N1: Cond, N2: TrueVal);
3360 SDValue NotCond = DAG.getNOT(DL, Val: Cond, VT: MVT::i1);
3361 SDValue And2 = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i1, N1: NotCond, N2: FalseVal);
3362 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i1, N1: And1, N2: And2);
3363 return Or;
3364}
3365
3366static SDValue lowerMSTORE(SDValue Op, SelectionDAG &DAG) {
3367 SDNode *N = Op.getNode();
3368
3369 SDValue Chain = N->getOperand(Num: 0);
3370 SDValue Val = N->getOperand(Num: 1);
3371 SDValue BasePtr = N->getOperand(Num: 2);
3372 SDValue Offset = N->getOperand(Num: 3);
3373 SDValue Mask = N->getOperand(Num: 4);
3374
3375 SDLoc DL(N);
3376 EVT ValVT = Val.getValueType();
3377 MemSDNode *MemSD = cast<MemSDNode>(Val: N);
3378 assert(ValVT.isVector() && "Masked vector store must have vector type");
3379 assert(MemSD->getAlign() >= DAG.getEVTAlign(ValVT) &&
3380 "Unexpected alignment for masked store");
3381
3382 unsigned Opcode = 0;
3383 switch (ValVT.getSimpleVT().SimpleTy) {
3384 default:
3385 llvm_unreachable("Unexpected masked vector store type");
3386 case MVT::v4i64:
3387 case MVT::v4f64: {
3388 Opcode = NVPTXISD::StoreV4;
3389 break;
3390 }
3391 case MVT::v8i32:
3392 case MVT::v8f32: {
3393 Opcode = NVPTXISD::StoreV8;
3394 break;
3395 }
3396 }
3397
3398 SmallVector<SDValue, 8> Ops;
3399
3400 // Construct the new SDNode. First operand is the chain.
3401 Ops.push_back(Elt: Chain);
3402
3403 // The next N operands are the values to store. Encode the mask into the
3404 // values using the sentinel register 0 to represent a masked-off element.
3405 assert(Mask.getValueType().isVector() &&
3406 Mask.getValueType().getVectorElementType() == MVT::i1 &&
3407 "Mask must be a vector of i1");
3408 assert(Mask.getOpcode() == ISD::BUILD_VECTOR &&
3409 "Mask expected to be a BUILD_VECTOR");
3410 assert(Mask.getValueType().getVectorNumElements() ==
3411 ValVT.getVectorNumElements() &&
3412 "Mask size must be the same as the vector size");
3413 for (auto [I, Op] : enumerate(First: Mask->ops())) {
3414 // Mask elements must be constants.
3415 if (Op.getNode()->getAsZExtVal() == 0) {
3416 // Append a sentinel register 0 to the Ops vector to represent a masked
3417 // off element, this will be handled in tablegen
3418 Ops.push_back(Elt: DAG.getRegister(Reg: MCRegister::NoRegister,
3419 VT: ValVT.getVectorElementType()));
3420 } else {
3421 // Extract the element from the vector to store
3422 SDValue ExtVal =
3423 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ValVT.getVectorElementType(),
3424 N1: Val, N2: DAG.getIntPtrConstant(Val: I, DL));
3425 Ops.push_back(Elt: ExtVal);
3426 }
3427 }
3428
3429 // Next, the pointer operand.
3430 Ops.push_back(Elt: BasePtr);
3431
3432 // Finally, the offset operand. We expect this to always be undef, and it will
3433 // be ignored in lowering, but to mirror the handling of the other vector
3434 // store instructions we include it in the new SDNode.
3435 assert(Offset.isUndef() && "Offset operand expected to be undef or poison");
3436 Ops.push_back(Elt: Offset);
3437
3438 SDValue NewSt =
3439 DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops,
3440 MemVT: MemSD->getMemoryVT(), MMO: MemSD->getMemOperand());
3441
3442 return NewSt;
3443}
3444
3445SDValue
3446NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
3447 switch (Op.getOpcode()) {
3448 case ISD::RETURNADDR:
3449 return SDValue();
3450 case ISD::FRAMEADDR:
3451 return SDValue();
3452 case ISD::ADDRSPACECAST:
3453 return LowerADDRSPACECAST(Op, DAG);
3454 case ISD::INTRINSIC_W_CHAIN:
3455 return lowerIntrinsicWChain(Op, DAG);
3456 case ISD::INTRINSIC_WO_CHAIN:
3457 return lowerIntrinsicWOChain(Op, DAG);
3458 case ISD::INTRINSIC_VOID:
3459 return lowerIntrinsicVoid(Op, DAG);
3460 case ISD::BUILD_VECTOR:
3461 return LowerBUILD_VECTOR(Op, DAG);
3462 case ISD::BITCAST:
3463 return LowerBITCAST(Op, DAG);
3464 case ISD::EXTRACT_SUBVECTOR:
3465 return Op;
3466 case ISD::EXTRACT_VECTOR_ELT:
3467 return LowerEXTRACT_VECTOR_ELT(Op, DAG);
3468 case ISD::INSERT_VECTOR_ELT:
3469 return LowerINSERT_VECTOR_ELT(Op, DAG);
3470 case ISD::VECTOR_SHUFFLE:
3471 return LowerVECTOR_SHUFFLE(Op, DAG);
3472 case ISD::CONCAT_VECTORS:
3473 return LowerCONCAT_VECTORS(Op, DAG);
3474 case ISD::VECREDUCE_FMAX:
3475 case ISD::VECREDUCE_FMIN:
3476 case ISD::VECREDUCE_FMAXIMUM:
3477 case ISD::VECREDUCE_FMINIMUM:
3478 return LowerVECREDUCE(Op, DAG);
3479 case ISD::STORE:
3480 return LowerSTORE(Op, DAG);
3481 case ISD::MSTORE: {
3482 assert(STI.has256BitVectorLoadStore(
3483 cast<MemSDNode>(Op.getNode())->getAddressSpace()) &&
3484 "Masked store vector not supported on subtarget.");
3485 return lowerMSTORE(Op, DAG);
3486 }
3487 case ISD::LOAD:
3488 return LowerLOAD(Op, DAG);
3489 case ISD::MLOAD:
3490 return LowerMLOAD(Op, DAG);
3491 case ISD::SHL_PARTS:
3492 return LowerShiftLeftParts(Op, DAG);
3493 case ISD::SRA_PARTS:
3494 case ISD::SRL_PARTS:
3495 return LowerShiftRightParts(Op, DAG);
3496 case ISD::SELECT:
3497 return lowerSELECT(Op, DAG);
3498 case ISD::FROUND:
3499 return LowerFROUND(Op, DAG);
3500 case ISD::FCOPYSIGN:
3501 return LowerFCOPYSIGN(Op, DAG);
3502 case ISD::SINT_TO_FP:
3503 case ISD::UINT_TO_FP:
3504 return LowerINT_TO_FP(Op, DAG);
3505 case ISD::FP_TO_SINT:
3506 case ISD::FP_TO_UINT:
3507 // fptosi/fptoui to i1 truncate toward zero, so the only defined results
3508 // are {0,-1} (signed) and {0,1} (unsigned); every other input results in
3509 // poison. Thus we can simply lower to `x <= -1.0` or `x >= 1.0`.
3510 if (Op.getValueType() == MVT::i1) {
3511 SDLoc DL(Op);
3512 SDValue X = Op.getOperand(i: 0);
3513 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT;
3514 return DAG.getSetCC(
3515 DL, VT: MVT::i1, LHS: X,
3516 RHS: DAG.getConstantFP(Val: IsSigned ? -1.0 : 1.0, DL, VT: X.getValueType()),
3517 Cond: IsSigned ? ISD::SETOLE : ISD::SETOGE);
3518 }
3519 return LowerFP_TO_INT(Op, DAG);
3520 case ISD::FP_ROUND:
3521 return LowerFP_ROUND(Op, DAG);
3522 case ISD::FP_EXTEND:
3523 return LowerFP_EXTEND(Op, DAG);
3524 case ISD::VAARG:
3525 return LowerVAARG(Op, DAG);
3526 case ISD::VASTART:
3527 return LowerVASTART(Op, DAG);
3528 case ISD::FSHL:
3529 case ISD::FSHR:
3530 return lowerFSH(Op, DAG);
3531 case ISD::ROTL:
3532 case ISD::ROTR:
3533 return lowerROT(Op, DAG);
3534 case ISD::ABS:
3535 case ISD::ABS_MIN_POISON:
3536 case ISD::SMIN:
3537 case ISD::SMAX:
3538 case ISD::UMIN:
3539 case ISD::UMAX:
3540 case ISD::ADD:
3541 case ISD::SUB:
3542 case ISD::MUL:
3543 case ISD::SHL:
3544 case ISD::SREM:
3545 case ISD::UREM:
3546 return LowerVectorArith(Op, DAG);
3547 case ISD::DYNAMIC_STACKALLOC:
3548 return LowerDYNAMIC_STACKALLOC(Op, DAG);
3549 case ISD::STACKRESTORE:
3550 return LowerSTACKRESTORE(Op, DAG);
3551 case ISD::STACKSAVE:
3552 return LowerSTACKSAVE(Op, DAG);
3553 case ISD::CopyToReg:
3554 return LowerCopyToReg_128(Op, DAG);
3555 case ISD::FADD:
3556 case ISD::FSUB:
3557 case ISD::FMUL:
3558 // Used only for bf16 on SM80, where we select fma for non-ftz operation
3559 return PromoteBinOpIfF32FTZ(Op, DAG);
3560 case ISD::CTPOP:
3561 case ISD::CTLZ:
3562 return lowerCTLZCTPOP(Op, DAG);
3563 case ISD::FREM:
3564 return lowerFREM(Op, DAG);
3565 case ISD::BSWAP:
3566 return lowerBSWAP(Op, DAG);
3567 default:
3568 llvm_unreachable("Custom lowering not defined for operation");
3569 }
3570}
3571
3572// This will prevent AsmPrinter from trying to print the jump tables itself.
3573unsigned NVPTXTargetLowering::getJumpTableEncoding() const {
3574 return MachineJumpTableInfo::EK_Inline;
3575}
3576
3577SDValue NVPTXTargetLowering::LowerADDRSPACECAST(SDValue Op,
3578 SelectionDAG &DAG) const {
3579 AddrSpaceCastSDNode *N = cast<AddrSpaceCastSDNode>(Val: Op.getNode());
3580 unsigned SrcAS = N->getSrcAddressSpace();
3581 unsigned DestAS = N->getDestAddressSpace();
3582 if (SrcAS != llvm::ADDRESS_SPACE_GENERIC &&
3583 DestAS != llvm::ADDRESS_SPACE_GENERIC) {
3584 // Shared and SharedCluster can be converted to each other through generic
3585 // space
3586 if ((SrcAS == llvm::ADDRESS_SPACE_SHARED &&
3587 DestAS == llvm::ADDRESS_SPACE_SHARED_CLUSTER) ||
3588 (SrcAS == llvm::ADDRESS_SPACE_SHARED_CLUSTER &&
3589 DestAS == llvm::ADDRESS_SPACE_SHARED)) {
3590 SDLoc DL(Op.getNode());
3591 const MVT GenerictVT =
3592 getPointerTy(DL: DAG.getDataLayout(), AS: ADDRESS_SPACE_GENERIC);
3593 SDValue GenericConversion = DAG.getAddrSpaceCast(
3594 dl: DL, VT: GenerictVT, Ptr: Op.getOperand(i: 0), SrcAS, DestAS: ADDRESS_SPACE_GENERIC);
3595 SDValue SharedClusterConversion =
3596 DAG.getAddrSpaceCast(dl: DL, VT: Op.getValueType(), Ptr: GenericConversion,
3597 SrcAS: ADDRESS_SPACE_GENERIC, DestAS);
3598 return SharedClusterConversion;
3599 }
3600
3601 return DAG.getUNDEF(VT: Op.getValueType());
3602 }
3603
3604 return Op;
3605}
3606
3607// This function is almost a copy of SelectionDAG::expandVAArg().
3608// The only diff is that this one produces loads from local address space.
3609SDValue NVPTXTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
3610 const TargetLowering *TLI = STI.getTargetLowering();
3611 SDLoc DL(Op);
3612
3613 SDNode *Node = Op.getNode();
3614 const Value *V = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 2))->getValue();
3615 EVT VT = Node->getValueType(ResNo: 0);
3616 auto *Ty = VT.getTypeForEVT(Context&: *DAG.getContext());
3617 SDValue Tmp1 = Node->getOperand(Num: 0);
3618 SDValue Tmp2 = Node->getOperand(Num: 1);
3619 const MaybeAlign MA(Node->getConstantOperandVal(Num: 3));
3620
3621 SDValue VAListLoad = DAG.getLoad(VT: TLI->getPointerTy(DL: DAG.getDataLayout()), dl: DL,
3622 Chain: Tmp1, Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
3623 SDValue VAList = VAListLoad;
3624
3625 if (MA && *MA > TLI->getMinStackArgumentAlignment()) {
3626 VAList = DAG.getNode(
3627 Opcode: ISD::ADD, DL, VT: VAList.getValueType(), N1: VAList,
3628 N2: DAG.getConstant(Val: MA->value() - 1, DL, VT: VAList.getValueType()));
3629
3630 VAList = DAG.getNode(Opcode: ISD::AND, DL, VT: VAList.getValueType(), N1: VAList,
3631 N2: DAG.getSignedConstant(Val: -(int64_t)MA->value(), DL,
3632 VT: VAList.getValueType()));
3633 }
3634
3635 // Increment the pointer, VAList, to the next vaarg
3636 Tmp1 = DAG.getNode(Opcode: ISD::ADD, DL, VT: VAList.getValueType(), N1: VAList,
3637 N2: DAG.getConstant(Val: DAG.getDataLayout().getTypeAllocSize(Ty),
3638 DL, VT: VAList.getValueType()));
3639
3640 // Store the incremented VAList to the legalized pointer
3641 Tmp1 = DAG.getStore(Chain: VAListLoad.getValue(R: 1), dl: DL, Val: Tmp1, Ptr: Tmp2,
3642 PtrInfo: MachinePointerInfo(V));
3643
3644 const Value *SrcV = Constant::getNullValue(
3645 Ty: PointerType::get(C&: *DAG.getContext(), AddressSpace: ADDRESS_SPACE_LOCAL));
3646
3647 // Load the actual argument out of the pointer VAList
3648 return DAG.getLoad(VT, dl: DL, Chain: Tmp1, Ptr: VAList, PtrInfo: MachinePointerInfo(SrcV));
3649}
3650
3651SDValue NVPTXTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
3652 const TargetLowering *TLI = STI.getTargetLowering();
3653 SDLoc DL(Op);
3654 EVT PtrVT = TLI->getPointerTy(DL: DAG.getDataLayout());
3655
3656 // Store the address of unsized array <function>_vararg[] in the ap object.
3657 SDValue VAReg = getParamSymbolNode(DAG, /* vararg */ I: -1, T: PtrVT);
3658
3659 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
3660 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL, Val: VAReg, Ptr: Op.getOperand(i: 1),
3661 PtrInfo: MachinePointerInfo(SV));
3662}
3663
3664static std::pair<MemSDNode *, uint32_t>
3665convertMLOADToLoadWithUsedBytesMask(MemSDNode *N, SelectionDAG &DAG,
3666 const NVPTXSubtarget &STI) {
3667 SDValue Chain = N->getOperand(Num: 0);
3668 SDValue BasePtr = N->getOperand(Num: 1);
3669 SDValue Mask = N->getOperand(Num: 3);
3670 [[maybe_unused]] SDValue Passthru = N->getOperand(Num: 4);
3671
3672 SDLoc DL(N);
3673 EVT ResVT = N->getValueType(ResNo: 0);
3674 assert(ResVT.isVector() && "Masked vector load must have vector type");
3675 // While we only expect poison passthru vectors as an input to the backend,
3676 // when the legalization framework splits a poison vector in half, it creates
3677 // two undef vectors, so we can technically expect those too.
3678 assert((Passthru.getOpcode() == ISD::POISON ||
3679 Passthru.getOpcode() == ISD::UNDEF) &&
3680 "Passthru operand expected to be poison or undef");
3681
3682 // Extract the mask and convert it to a uint32_t representing the used bytes
3683 // of the entire vector load
3684 uint32_t UsedBytesMask = 0;
3685 uint32_t ElementSizeInBits = ResVT.getVectorElementType().getSizeInBits();
3686 assert(ElementSizeInBits % 8 == 0 && "Unexpected element size");
3687 uint32_t ElementSizeInBytes = ElementSizeInBits / 8;
3688 uint32_t ElementMask = (1u << ElementSizeInBytes) - 1u;
3689
3690 for (SDValue Op : reverse(C: Mask->ops())) {
3691 // We technically only want to do this shift for every
3692 // iteration *but* the first, but in the first iteration UsedBytesMask is 0,
3693 // so this shift is a no-op.
3694 UsedBytesMask <<= ElementSizeInBytes;
3695
3696 // Mask elements must be constants.
3697 if (Op->getAsZExtVal() != 0)
3698 UsedBytesMask |= ElementMask;
3699 }
3700
3701 assert(UsedBytesMask != 0 && UsedBytesMask != UINT32_MAX &&
3702 "Unexpected masked load with elements masked all on or all off");
3703
3704 // Create a new load sd node to be handled normally by ReplaceLoadVector.
3705 MemSDNode *NewLD = cast<MemSDNode>(
3706 Val: DAG.getLoad(VT: ResVT, dl: DL, Chain, Ptr: BasePtr, MMO: N->getMemOperand()).getNode());
3707
3708 // If our subtarget does not support the used bytes mask pragma, "drop" the
3709 // mask by setting it to UINT32_MAX
3710 if (!STI.hasUsedBytesMaskPragma())
3711 UsedBytesMask = UINT32_MAX;
3712
3713 return {NewLD, UsedBytesMask};
3714}
3715
3716/// replaceLoadVector - Convert vector loads into multi-output scalar loads.
3717static std::optional<std::pair<SDValue, SDValue>>
3718replaceLoadVector(SDNode *N, SelectionDAG &DAG, const NVPTXSubtarget &STI) {
3719 MemSDNode *LD = cast<MemSDNode>(Val: N);
3720 const EVT ResVT = LD->getValueType(ResNo: 0);
3721 const EVT MemVT = LD->getMemoryVT();
3722
3723 // If we're doing sign/zero extension as part of the load, avoid lowering to
3724 // a LoadV node. TODO: consider relaxing this restriction.
3725 if (ResVT != MemVT)
3726 return std::nullopt;
3727
3728 const auto NumEltsAndEltVT =
3729 getVectorLoweringShape(VectorEVT: ResVT, STI, AddressSpace: LD->getAddressSpace());
3730 if (!NumEltsAndEltVT)
3731 return std::nullopt;
3732 const auto [NumElts, EltVT] = NumEltsAndEltVT.value();
3733
3734 Align Alignment = LD->getAlign();
3735 const auto &TD = DAG.getDataLayout();
3736 Align PrefAlign = TD.getPrefTypeAlign(Ty: MemVT.getTypeForEVT(Context&: *DAG.getContext()));
3737 if (Alignment < PrefAlign) {
3738 // This load is not sufficiently aligned, so bail out and let this vector
3739 // load be scalarized. Note that we may still be able to emit smaller
3740 // vector loads. For example, if we are loading a <4 x float> with an
3741 // alignment of 8, this check will fail but the legalizer will try again
3742 // with 2 x <2 x float>, which will succeed with an alignment of 8.
3743 return std::nullopt;
3744 }
3745
3746 // If we have a masked load, convert it to a normal load now
3747 std::optional<uint32_t> UsedBytesMask = std::nullopt;
3748 if (LD->getOpcode() == ISD::MLOAD)
3749 std::tie(args&: LD, args&: UsedBytesMask) =
3750 convertMLOADToLoadWithUsedBytesMask(N: LD, DAG, STI);
3751
3752 // Since LoadV2 is a target node, we cannot rely on DAG type legalization.
3753 // Therefore, we must ensure the type is legal. For i1 and i8, we set the
3754 // loaded type to i16 and propagate the "real" type as the memory type.
3755 const MVT LoadEltVT = (EltVT.getSizeInBits() < 16) ? MVT::i16 : EltVT;
3756
3757 unsigned Opcode;
3758 switch (NumElts) {
3759 default:
3760 return std::nullopt;
3761 case 2:
3762 Opcode = NVPTXISD::LoadV2;
3763 break;
3764 case 4:
3765 Opcode = NVPTXISD::LoadV4;
3766 break;
3767 case 8:
3768 Opcode = NVPTXISD::LoadV8;
3769 break;
3770 }
3771 auto ListVTs = SmallVector<EVT, 9>(NumElts, LoadEltVT);
3772 ListVTs.push_back(Elt: MVT::Other);
3773 SDVTList LdResVTs = DAG.getVTList(VTs: ListVTs);
3774
3775 SDLoc DL(LD);
3776
3777 // Copy regular operands
3778 SmallVector<SDValue, 8> OtherOps(LD->ops());
3779
3780 OtherOps.push_back(
3781 Elt: DAG.getConstant(Val: UsedBytesMask.value_or(UINT32_MAX), DL, VT: MVT::i32));
3782
3783 // The select routine does not have access to the LoadSDNode instance, so
3784 // pass along the extension information
3785 OtherOps.push_back(
3786 Elt: DAG.getIntPtrConstant(Val: cast<LoadSDNode>(Val: LD)->getExtensionType(), DL));
3787
3788 SDValue NewLD = DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList: LdResVTs, Ops: OtherOps, MemVT,
3789 MMO: LD->getMemOperand());
3790
3791 SmallVector<SDValue> ScalarRes;
3792 if (EltVT.isVector()) {
3793 assert(EVT(EltVT.getVectorElementType()) == ResVT.getVectorElementType());
3794 assert(NumElts * EltVT.getVectorNumElements() ==
3795 ResVT.getVectorNumElements());
3796 // Generate EXTRACT_VECTOR_ELTs to split v2[i,f,bf]16/v4i8 subvectors back
3797 // into individual elements.
3798 for (const unsigned I : llvm::seq(Size: NumElts)) {
3799 SDValue SubVector = NewLD.getValue(R: I);
3800 DAG.ExtractVectorElements(Op: SubVector, Args&: ScalarRes);
3801 }
3802 } else {
3803 for (const unsigned I : llvm::seq(Size: NumElts)) {
3804 SDValue Res = NewLD.getValue(R: I);
3805 if (LoadEltVT != EltVT)
3806 Res = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Res);
3807 ScalarRes.push_back(Elt: Res);
3808 }
3809 }
3810
3811 SDValue LoadChain = NewLD.getValue(R: NumElts);
3812
3813 const MVT BuildVecVT =
3814 MVT::getVectorVT(VT: EltVT.getScalarType(), NumElements: ScalarRes.size());
3815 SDValue BuildVec = DAG.getBuildVector(VT: BuildVecVT, DL, Ops: ScalarRes);
3816 SDValue LoadValue = DAG.getBitcast(VT: ResVT, V: BuildVec);
3817
3818 return {{LoadValue, LoadChain}};
3819}
3820
3821static void replaceLoadVector(SDNode *N, SelectionDAG &DAG,
3822 SmallVectorImpl<SDValue> &Results,
3823 const NVPTXSubtarget &STI) {
3824 if (auto Res = replaceLoadVector(N, DAG, STI))
3825 Results.append(IL: {Res->first, Res->second});
3826}
3827
3828static SDValue lowerLoadVector(SDNode *N, SelectionDAG &DAG,
3829 const NVPTXSubtarget &STI) {
3830 if (auto Res = replaceLoadVector(N, DAG, STI))
3831 return DAG.getMergeValues(Ops: {Res->first, Res->second}, dl: SDLoc(N));
3832 return SDValue();
3833}
3834
3835// v = ld i1* addr
3836// =>
3837// v1 = ld i8* addr (-> i16)
3838// v = trunc i16 to i1
3839static SDValue lowerLOADi1(LoadSDNode *LD, SelectionDAG &DAG) {
3840 SDLoc dl(LD);
3841 assert(LD->getExtensionType() == ISD::NON_EXTLOAD);
3842 assert(LD->getValueType(0) == MVT::i1 && "Custom lowering for i1 load only");
3843 SDValue newLD = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: MVT::i16, Chain: LD->getChain(),
3844 Ptr: LD->getBasePtr(), PtrInfo: LD->getPointerInfo(),
3845 MemVT: MVT::i8, Alignment: LD->getAlign(),
3846 MMOFlags: LD->getMemOperand()->getFlags());
3847 SDValue result = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i1, Operand: newLD);
3848 // The legalizer (the caller) is expecting two values from the legalized
3849 // load, so we build a MergeValues node for it. See ExpandUnalignedLoad()
3850 // in LegalizeDAG.cpp which also uses MergeValues.
3851 return DAG.getMergeValues(Ops: {result, LD->getChain()}, dl);
3852}
3853
3854SDValue NVPTXTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
3855 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
3856
3857 if (Op.getValueType() == MVT::i1)
3858 return lowerLOADi1(LD, DAG);
3859
3860 // To improve CodeGen we'll legalize any-extend loads to zext loads. This is
3861 // how they'll be lowered in ISel anyway, and by doing this a little earlier
3862 // we allow for more DAG combine opportunities.
3863 if (LD->getExtensionType() == ISD::EXTLOAD) {
3864 assert(LD->getValueType(0).isInteger() && LD->getMemoryVT().isInteger() &&
3865 "Unexpected fpext-load");
3866 return DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl: SDLoc(Op), VT: Op.getValueType(),
3867 Chain: LD->getChain(), Ptr: LD->getBasePtr(), MemVT: LD->getMemoryVT(),
3868 MMO: LD->getMemOperand());
3869 }
3870
3871 llvm_unreachable("Unexpected custom lowering for load");
3872}
3873
3874SDValue NVPTXTargetLowering::LowerMLOAD(SDValue Op, SelectionDAG &DAG) const {
3875 // v2f16/v2bf16/v2i16/v4i8 are legal, so we can't rely on legalizer to handle
3876 // masked loads of these types and have to handle them here.
3877 // v2f32 also needs to be handled here if the subtarget has f32x2
3878 // instructions, making it legal.
3879 //
3880 // Note: misaligned masked loads should never reach this point
3881 // because the override of isLegalMaskedLoad in NVPTXTargetTransformInfo.cpp
3882 // will validate alignment. Therefore, we do not need to special case handle
3883 // them here.
3884 EVT VT = Op.getValueType();
3885 if (NVPTX::isPackedVectorTy(VT)) {
3886 auto Result = convertMLOADToLoadWithUsedBytesMask(
3887 N: cast<MemSDNode>(Val: Op.getNode()), DAG, STI);
3888 MemSDNode *LD = std::get<0>(in&: Result);
3889 uint32_t UsedBytesMask = std::get<1>(in&: Result);
3890
3891 SDLoc DL(LD);
3892
3893 // Copy regular operands
3894 SmallVector<SDValue, 8> OtherOps(LD->ops());
3895
3896 OtherOps.push_back(Elt: DAG.getConstant(Val: UsedBytesMask, DL, VT: MVT::i32));
3897
3898 // We currently are not lowering extending loads, but pass the extension
3899 // type anyway as later handling expects it.
3900 OtherOps.push_back(
3901 Elt: DAG.getIntPtrConstant(Val: cast<LoadSDNode>(Val: LD)->getExtensionType(), DL));
3902 SDValue NewLD =
3903 DAG.getMemIntrinsicNode(Opcode: NVPTXISD::MLoad, dl: DL, VTList: LD->getVTList(), Ops: OtherOps,
3904 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
3905 return NewLD;
3906 }
3907 return SDValue();
3908}
3909
3910static SDValue lowerSTOREVector(SDValue Op, SelectionDAG &DAG,
3911 const NVPTXSubtarget &STI) {
3912 MemSDNode *N = cast<MemSDNode>(Val: Op.getNode());
3913 SDValue Val = N->getOperand(Num: 1);
3914 SDLoc DL(N);
3915 const EVT ValVT = Val.getValueType();
3916 const EVT MemVT = N->getMemoryVT();
3917
3918 // If we're truncating as part of the store, avoid lowering to a StoreV node.
3919 // TODO: consider relaxing this restriction.
3920 if (ValVT != MemVT)
3921 return SDValue();
3922
3923 const auto NumEltsAndEltVT =
3924 getVectorLoweringShape(VectorEVT: ValVT, STI, AddressSpace: N->getAddressSpace());
3925 if (!NumEltsAndEltVT)
3926 return SDValue();
3927 const auto [NumElts, EltVT] = NumEltsAndEltVT.value();
3928
3929 const DataLayout &TD = DAG.getDataLayout();
3930
3931 Align Alignment = N->getAlign();
3932 Align PrefAlign = TD.getPrefTypeAlign(Ty: ValVT.getTypeForEVT(Context&: *DAG.getContext()));
3933 if (Alignment < PrefAlign) {
3934 // This store is not sufficiently aligned, so bail out and let this vector
3935 // store be scalarized. Note that we may still be able to emit smaller
3936 // vector stores. For example, if we are storing a <4 x float> with an
3937 // alignment of 8, this check will fail but the legalizer will try again
3938 // with 2 x <2 x float>, which will succeed with an alignment of 8.
3939 return SDValue();
3940 }
3941
3942 unsigned Opcode;
3943 switch (NumElts) {
3944 default:
3945 return SDValue();
3946 case 2:
3947 Opcode = NVPTXISD::StoreV2;
3948 break;
3949 case 4:
3950 Opcode = NVPTXISD::StoreV4;
3951 break;
3952 case 8:
3953 Opcode = NVPTXISD::StoreV8;
3954 break;
3955 }
3956
3957 SmallVector<SDValue, 8> Ops;
3958
3959 // First is the chain
3960 Ops.push_back(Elt: N->getOperand(Num: 0));
3961
3962 // Then the split values
3963 if (EltVT.isVector()) {
3964 assert(EVT(EltVT.getVectorElementType()) == ValVT.getVectorElementType());
3965 assert(NumElts * EltVT.getVectorNumElements() ==
3966 ValVT.getVectorNumElements());
3967 // Combine individual elements into v2[i,f,bf]16/v4i8 subvectors to be
3968 // stored as b32s
3969 const unsigned NumEltsPerSubVector = EltVT.getVectorNumElements();
3970 for (const unsigned I : llvm::seq(Size: NumElts)) {
3971 SmallVector<SDValue, 4> SubVectorElts;
3972 DAG.ExtractVectorElements(Op: Val, Args&: SubVectorElts, Start: I * NumEltsPerSubVector,
3973 Count: NumEltsPerSubVector);
3974 Ops.push_back(Elt: DAG.getBuildVector(VT: EltVT, DL, Ops: SubVectorElts));
3975 }
3976 } else {
3977 SDValue V = DAG.getBitcast(VT: MVT::getVectorVT(VT: EltVT, NumElements: NumElts), V: Val);
3978 for (const unsigned I : llvm::seq(Size: NumElts)) {
3979 SDValue ExtVal = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: V,
3980 N2: DAG.getIntPtrConstant(Val: I, DL));
3981
3982 // Since StoreV2 is a target node, we cannot rely on DAG type
3983 // legalization. Therefore, we must ensure the type is legal. For i1 and
3984 // i8, we set the stored type to i16 and propagate the "real" type as the
3985 // memory type.
3986 if (EltVT.getSizeInBits() < 16)
3987 ExtVal = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i16, Operand: ExtVal);
3988 Ops.push_back(Elt: ExtVal);
3989 }
3990 }
3991
3992 // Then any remaining arguments
3993 Ops.append(in_start: N->op_begin() + 2, in_end: N->op_end());
3994
3995 SDValue NewSt =
3996 DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops,
3997 MemVT: N->getMemoryVT(), MMO: N->getMemOperand());
3998
3999 // return DCI.CombineTo(N, NewSt, true);
4000 return NewSt;
4001}
4002
4003SDValue NVPTXTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
4004 StoreSDNode *Store = cast<StoreSDNode>(Val&: Op);
4005 EVT VT = Store->getMemoryVT();
4006
4007 if (VT == MVT::i1)
4008 return LowerSTOREi1(Op, DAG);
4009
4010 // Lower store of any other vector type, including v2f32 as we want to break
4011 // it apart since this is not a widely-supported type.
4012 return lowerSTOREVector(Op, DAG, STI);
4013}
4014
4015// st i1 v, addr
4016// =>
4017// v1 = zxt v to i16
4018// st.u8 i16, addr
4019SDValue NVPTXTargetLowering::LowerSTOREi1(SDValue Op, SelectionDAG &DAG) const {
4020 SDNode *Node = Op.getNode();
4021 SDLoc dl(Node);
4022 StoreSDNode *ST = cast<StoreSDNode>(Val: Node);
4023 SDValue Tmp1 = ST->getChain();
4024 SDValue Tmp2 = ST->getBasePtr();
4025 SDValue Tmp3 = ST->getValue();
4026 assert(Tmp3.getValueType() == MVT::i1 && "Custom lowering for i1 store only");
4027 Tmp3 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::i16, Operand: Tmp3);
4028 SDValue Result =
4029 DAG.getTruncStore(Chain: Tmp1, dl, Val: Tmp3, Ptr: Tmp2, PtrInfo: ST->getPointerInfo(), SVT: MVT::i8,
4030 Alignment: ST->getAlign(), MMOFlags: ST->getMemOperand()->getFlags());
4031 return Result;
4032}
4033
4034SDValue NVPTXTargetLowering::LowerCopyToReg_128(SDValue Op,
4035 SelectionDAG &DAG) const {
4036 // Change the CopyToReg to take in two 64-bit operands instead of a 128-bit
4037 // operand so that it can pass the legalization.
4038
4039 assert(Op.getOperand(1).getValueType() == MVT::i128 &&
4040 "Custom lowering for 128-bit CopyToReg only");
4041
4042 SDNode *Node = Op.getNode();
4043 SDLoc DL(Node);
4044
4045 SDValue Cast = DAG.getBitcast(VT: MVT::v2i64, V: Op->getOperand(Num: 2));
4046 SDValue Lo = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i64, N1: Cast,
4047 N2: DAG.getIntPtrConstant(Val: 0, DL));
4048 SDValue Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i64, N1: Cast,
4049 N2: DAG.getIntPtrConstant(Val: 1, DL));
4050
4051 SmallVector<SDValue, 5> NewOps(Op->getNumOperands() + 1);
4052 SmallVector<EVT, 3> ResultsType(Node->values());
4053
4054 NewOps[0] = Op->getOperand(Num: 0); // Chain
4055 NewOps[1] = Op->getOperand(Num: 1); // Dst Reg
4056 NewOps[2] = Lo; // Lower 64-bit
4057 NewOps[3] = Hi; // Higher 64-bit
4058 if (Op.getNumOperands() == 4)
4059 NewOps[4] = Op->getOperand(Num: 3); // Glue if exists
4060
4061 return DAG.getNode(Opcode: ISD::CopyToReg, DL, ResultTys: ResultsType, Ops: NewOps);
4062}
4063
4064unsigned NVPTXTargetLowering::getNumRegisters(
4065 LLVMContext &Context, EVT VT,
4066 std::optional<MVT> RegisterVT = std::nullopt) const {
4067 if (VT == MVT::i128 && RegisterVT == MVT::i128)
4068 return 1;
4069 return TargetLoweringBase::getNumRegisters(Context, VT, RegisterVT);
4070}
4071
4072bool NVPTXTargetLowering::splitValueIntoRegisterParts(
4073 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
4074 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
4075 if (Val.getValueType() == MVT::i128 && NumParts == 1) {
4076 Parts[0] = Val;
4077 return true;
4078 }
4079 return false;
4080}
4081
4082SDValue NVPTXTargetLowering::getParamSymbolNode(SelectionDAG &DAG, int I,
4083 EVT T) const {
4084 const MachineFunction &MF = DAG.getMachineFunction();
4085 return getSymbolNode(
4086 DAG, Sym: getParamSymbol(Ctx&: MF.getContext(), F: &MF.getFunction(), Idx: I), T);
4087}
4088
4089SDValue NVPTXTargetLowering::getCallParamSymbolNode(SelectionDAG &DAG, int I,
4090 EVT T) const {
4091 return getSymbolNode(DAG, Name: "param" + Twine(I), T);
4092}
4093
4094SDValue NVPTXTargetLowering::LowerFormalArguments(
4095 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4096 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
4097 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4098 const DataLayout &DL = DAG.getDataLayout();
4099 LLVMContext &Ctx = *DAG.getContext();
4100
4101 const Function &F = DAG.getMachineFunction().getFunction();
4102 const bool IsKernel = isKernelFunction(F);
4103
4104 const MVT PtrVT = getPointerTy(DL, AS: IsKernel ? ADDRESS_SPACE_ENTRY_PARAM
4105 : ADDRESS_SPACE_LOCAL);
4106
4107 SDValue Root = DAG.getRoot();
4108 SmallVector<SDValue, 16> OutChains;
4109
4110 // argTypes.size() (or theArgs.size()) and Ins.size() need not match.
4111 // Ins.size() will be larger
4112 // * if there is an aggregate argument with multiple fields (each field
4113 // showing up separately in Ins)
4114 // * if there is a vector argument with more than typical vector-length
4115 // elements (generally if more than 4) where each vector element is
4116 // individually present in Ins.
4117 // So a different index should be used for indexing into Ins.
4118 // See similar issue in LowerCall.
4119
4120 auto AllIns = ArrayRef(Ins);
4121 const auto NonEmptyArgs = make_filter_range(
4122 Range: F.args(), Pred: [](const Argument &A) { return !A.getType()->isEmptyTy(); });
4123 for (const auto &[ParamI, Arg] : enumerate(First: NonEmptyArgs)) {
4124 const unsigned ArgNo = Arg.getArgNo();
4125 const auto ArgIns =
4126 AllIns.take_while(Pred: [&](auto I) { return I.OrigArgIndex == ArgNo; });
4127 AllIns = AllIns.drop_front(N: ArgIns.size());
4128
4129 Type *Ty = Arg.getType();
4130 assert(!ArgIns.empty() &&
4131 "Non-empty argument produced no parameter values");
4132
4133 if (Arg.use_empty()) {
4134 // argument is dead
4135 for (const auto &In : ArgIns) {
4136 assert(!In.Used && "Arg.use_empty() is true but Arg is used?");
4137 InVals.push_back(Elt: DAG.getUNDEF(VT: In.VT));
4138 }
4139 continue;
4140 }
4141
4142 SDValue ArgSymbol = getParamSymbolNode(DAG, I: ParamI, T: PtrVT);
4143
4144 // In the following cases, assign a node order of "i+1"
4145 // to newly created nodes. The SDNodes for params have to
4146 // appear in the same order as their order of appearance
4147 // in the original function. "i+1" holds that order.
4148 if (Arg.hasByValAttr()) {
4149 // Param has ByVal attribute
4150 // Return MoveParam(param symbol).
4151 // Ideally, the param symbol can be returned directly,
4152 // but when SDNode builder decides to use it in a CopyToReg(),
4153 // machine instruction fails because TargetExternalSymbol
4154 // (not lowered) is target dependent, and CopyToReg assumes
4155 // the source is lowered.
4156 assert(ArgIns.size() == 1 && "ByVal argument must be a pointer");
4157 const auto &ByvalIn = ArgIns[0];
4158 assert(getValueType(DL, Ty) == ByvalIn.VT &&
4159 "Ins type did not match function type");
4160
4161 SDValue P;
4162 if (IsKernel) {
4163 assert(Ty->getPointerAddressSpace() == ADDRESS_SPACE_ENTRY_PARAM &&
4164 "Kernel ByVal argument must be lowered to the param address "
4165 "space by NVPTXLowerArgs");
4166 P = ArgSymbol;
4167 P.getNode()->setIROrder(Arg.getArgNo() + 1);
4168 } else {
4169 P = DAG.getNode(Opcode: NVPTXISD::MoveParam, DL: dl, VT: ArgSymbol.getValueType(),
4170 Operand: ArgSymbol);
4171 P.getNode()->setIROrder(Arg.getArgNo() + 1);
4172 P = DAG.getAddrSpaceCast(dl, VT: ByvalIn.VT, Ptr: P, SrcAS: ADDRESS_SPACE_LOCAL,
4173 DestAS: ADDRESS_SPACE_GENERIC);
4174 }
4175 InVals.push_back(Elt: P);
4176 } else {
4177 SmallVector<EVT, 16> VTs;
4178 SmallVector<uint64_t, 16> Offsets;
4179 ComputePTXValueVTs(TLI: *this, DL, Ctx, CallConv, Ty, ValueVTs&: VTs, Offsets);
4180 assert(VTs.size() == ArgIns.size() && "Size mismatch");
4181 assert(VTs.size() == Offsets.size() && "Size mismatch");
4182
4183 const Align ArgAlign = getPTXParamAlign(
4184 F: &F, Ty, AttrIdx: Arg.getArgNo() + AttributeList::FirstArgIndex, DL);
4185
4186 unsigned I = 0;
4187 const auto VI = VectorizePTXValueVTs(ValueVTs: VTs, Offsets, ParamAlignment: ArgAlign);
4188 for (const unsigned NumElts : VI) {
4189 // i1 is loaded/stored as i8
4190 const EVT LoadVT = VTs[I] == MVT::i1 ? MVT::i8 : VTs[I];
4191 const EVT VecVT = getVectorizedVT(VT: LoadVT, N: NumElts, C&: Ctx);
4192
4193 SDValue VecAddr = DAG.getObjectPtrOffset(
4194 SL: dl, Ptr: ArgSymbol, Offset: TypeSize::getFixed(ExactSize: Offsets[I]));
4195
4196 const Align PartAlign = commonAlignment(A: ArgAlign, Offset: Offsets[I]);
4197 const unsigned AS = IsKernel ? NVPTX::AddressSpace::EntryParam
4198 : NVPTX::AddressSpace::DeviceParam;
4199 SDValue P = DAG.getLoad(VT: VecVT, dl, Chain: Root, Ptr: VecAddr,
4200 PtrInfo: MachinePointerInfo(AS), Alignment: PartAlign,
4201 MMOFlags: MachineMemOperand::MODereferenceable |
4202 MachineMemOperand::MOInvariant);
4203 P.getNode()->setIROrder(Arg.getArgNo() + 1);
4204 for (const unsigned J : llvm::seq(Size: NumElts)) {
4205 SDValue Elt = getExtractVectorizedValue(V: P, I: J, VT: LoadVT, dl, DAG);
4206
4207 Elt = correctParamType(V: Elt, ExpectedVT: ArgIns[I + J].VT, Flags: ArgIns[I + J].Flags,
4208 DAG, dl);
4209 InVals.push_back(Elt);
4210 }
4211 I += NumElts;
4212 }
4213 }
4214 }
4215
4216 if (!OutChains.empty())
4217 DAG.setRoot(DAG.getTokenFactor(DL: dl, Vals&: OutChains));
4218
4219 return Chain;
4220}
4221
4222SDValue
4223NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
4224 bool isVarArg,
4225 const SmallVectorImpl<ISD::OutputArg> &Outs,
4226 const SmallVectorImpl<SDValue> &OutVals,
4227 const SDLoc &dl, SelectionDAG &DAG) const {
4228 const Function &F = DAG.getMachineFunction().getFunction();
4229 Type *RetTy = F.getReturnType();
4230
4231 if (RetTy->isVoidTy()) {
4232 assert(OutVals.empty() && Outs.empty() && "Return value expected for void");
4233 return DAG.getNode(Opcode: NVPTXISD::RET_GLUE, DL: dl, VT: MVT::Other, Operand: Chain);
4234 }
4235
4236 const DataLayout &DL = DAG.getDataLayout();
4237 LLVMContext &Ctx = *DAG.getContext();
4238
4239 const SDValue RetSymbol = getSymbolNode(DAG, Name: "func_retval0", T: MVT::i32);
4240 const auto RetAlign =
4241 getPTXParamAlign(F: &F, Ty: RetTy, AttrIdx: AttributeList::ReturnIndex, DL);
4242
4243 // PTX Interoperability Guide 3.3(A): [Integer] Values shorter than
4244 // 32-bits are sign extended or zero extended, depending on whether
4245 // they are signed or unsigned types.
4246 const bool ExtendIntegerRetVal =
4247 RetTy->isIntegerTy() && DL.getTypeAllocSizeInBits(Ty: RetTy) < 32;
4248
4249 SmallVector<EVT, 16> VTs;
4250 SmallVector<uint64_t, 16> Offsets;
4251 ComputePTXValueVTs(TLI: *this, DL, Ctx, CallConv, Ty: RetTy, ValueVTs&: VTs, Offsets);
4252 assert(VTs.size() == OutVals.size() && "Bad return value decomposition");
4253
4254 const auto GetRetVal = [&](unsigned I) -> SDValue {
4255 SDValue RetVal = OutVals[I];
4256 assert(promoteScalarIntegerPTX(RetVal.getValueType()) ==
4257 RetVal.getValueType() &&
4258 "OutVal type should always be legal");
4259
4260 const EVT VTI = promoteScalarIntegerPTX(VT: VTs[I]);
4261 const EVT StoreVT =
4262 ExtendIntegerRetVal ? MVT::i32 : (VTI == MVT::i1 ? MVT::i8 : VTI);
4263 return correctParamType(V: RetVal, ExpectedVT: StoreVT, Flags: Outs[I].Flags, DAG, dl);
4264 };
4265
4266 unsigned I = 0;
4267 const auto VI = VectorizePTXValueVTs(ValueVTs: VTs, Offsets, ParamAlignment: RetAlign);
4268 for (const unsigned NumElts : VI) {
4269 const MaybeAlign CurrentAlign = ExtendIntegerRetVal
4270 ? MaybeAlign(std::nullopt)
4271 : commonAlignment(A: RetAlign, Offset: Offsets[I]);
4272
4273 SDValue Val = getBuildVectorizedValue(
4274 N: NumElts, dl, DAG, GetElement: [&](unsigned K) { return GetRetVal(I + K); });
4275
4276 SDValue Ptr =
4277 DAG.getObjectPtrOffset(SL: dl, Ptr: RetSymbol, Offset: TypeSize::getFixed(ExactSize: Offsets[I]));
4278
4279 Chain = DAG.getStore(Chain, dl, Val, Ptr,
4280 PtrInfo: MachinePointerInfo(NVPTX::AddressSpace::DeviceParam),
4281 Alignment: CurrentAlign);
4282
4283 I += NumElts;
4284 }
4285
4286 return DAG.getNode(Opcode: NVPTXISD::RET_GLUE, DL: dl, VT: MVT::Other, Operand: Chain);
4287}
4288
4289void NVPTXTargetLowering::LowerAsmOperandForConstraint(
4290 SDValue Op, StringRef Constraint, std::vector<SDValue> &Ops,
4291 SelectionDAG &DAG) const {
4292 if (Constraint.size() > 1)
4293 return;
4294 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
4295}
4296
4297// llvm.ptx.memcpy.const and llvm.ptx.memmove.const need to be modeled as
4298// TgtMemIntrinsic
4299// because we need the information that is only available in the "Value" type
4300// of destination
4301// pointer. In particular, the address space information.
4302void NVPTXTargetLowering::getTgtMemIntrinsic(
4303 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
4304 MachineFunction &MF, unsigned Intrinsic) const {
4305 IntrinsicInfo Info;
4306 switch (Intrinsic) {
4307 default:
4308 return;
4309 case Intrinsic::nvvm_match_all_sync_i32p:
4310 case Intrinsic::nvvm_match_all_sync_i64p:
4311 Info.opc = ISD::INTRINSIC_W_CHAIN;
4312 // memVT is bogus. These intrinsics have IntrInaccessibleMemOnly attribute
4313 // in order to model data exchange with other threads, but perform no real
4314 // memory accesses.
4315 Info.memVT = MVT::i1;
4316
4317 // Our result depends on both our and other thread's arguments.
4318 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
4319 Infos.push_back(Elt: Info);
4320 return;
4321 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_col:
4322 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_row:
4323 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_col_stride:
4324 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_row_stride:
4325 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_col:
4326 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_row:
4327 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_col_stride:
4328 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_row_stride:
4329 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_col:
4330 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_row:
4331 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_col_stride:
4332 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_row_stride:
4333 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_col:
4334 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_row:
4335 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_col_stride:
4336 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_row_stride:
4337 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_col:
4338 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_row:
4339 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_col_stride:
4340 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_row_stride:
4341 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_col:
4342 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_row:
4343 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_col_stride:
4344 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_row_stride: {
4345 Info.opc = ISD::INTRINSIC_W_CHAIN;
4346 Info.memVT = MVT::v8f16;
4347 Info.ptrVal = I.getArgOperand(i: 0);
4348 Info.offset = 0;
4349 Info.flags = MachineMemOperand::MOLoad;
4350 Info.align = Align(16);
4351 Infos.push_back(Elt: Info);
4352 return;
4353 }
4354 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_col:
4355 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_col_stride:
4356 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_col_stride:
4357 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_col:
4358 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_row:
4359 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_row_stride:
4360 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_row_stride:
4361 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_row:
4362 case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_col:
4363 case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_col_stride:
4364 case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_row:
4365 case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_row_stride:
4366 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_col:
4367 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_col_stride:
4368 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_col_stride:
4369 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_col:
4370 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_row:
4371 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_row_stride:
4372 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_row_stride:
4373 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_row:
4374 case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_col:
4375 case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_col_stride:
4376 case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_row:
4377 case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_row_stride: {
4378 Info.opc = ISD::INTRINSIC_W_CHAIN;
4379 Info.memVT = MVT::v2i32;
4380 Info.ptrVal = I.getArgOperand(i: 0);
4381 Info.offset = 0;
4382 Info.flags = MachineMemOperand::MOLoad;
4383 Info.align = Align(8);
4384 Infos.push_back(Elt: Info);
4385 return;
4386 }
4387
4388 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_col:
4389 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_col_stride:
4390 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_col_stride:
4391 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_col:
4392 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_row:
4393 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_row_stride:
4394 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_row_stride:
4395 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_row:
4396 case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_col:
4397 case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_col_stride:
4398 case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_row:
4399 case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_row_stride:
4400 case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_col:
4401 case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_col_stride:
4402 case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_row:
4403 case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_row_stride:
4404
4405 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_col:
4406 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_col_stride:
4407 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_col_stride:
4408 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_col:
4409 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_row:
4410 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_row_stride:
4411 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_row_stride:
4412 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_row:
4413 case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_col:
4414 case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_col_stride:
4415 case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_row:
4416 case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_row_stride:
4417 case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_col:
4418 case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_col_stride:
4419 case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_row:
4420 case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_row_stride:
4421 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x4_b16:
4422 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x4_trans_b16:
4423 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x2_trans_b8:
4424 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x2_trans_b8x16_b4x16_p64:
4425 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x2_trans_b8x16_b6x16_p32:
4426 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x4_b8x16_b4x16_p64:
4427 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x4_b8x16_b6x16_p32: {
4428 Info.opc = ISD::INTRINSIC_W_CHAIN;
4429 Info.memVT = MVT::v4i32;
4430 Info.ptrVal = I.getArgOperand(i: 0);
4431 Info.offset = 0;
4432 Info.flags = MachineMemOperand::MOLoad;
4433 Info.align = Align(16);
4434 Infos.push_back(Elt: Info);
4435 return;
4436 }
4437
4438 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_col:
4439 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_col_stride:
4440 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_col_stride:
4441 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_col:
4442 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_row:
4443 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_row_stride:
4444 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_row_stride:
4445 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_row:
4446
4447 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_col:
4448 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_col_stride:
4449 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_col_stride:
4450 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_col:
4451 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_row:
4452 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_row_stride:
4453 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_row_stride:
4454 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_row:
4455 case Intrinsic::nvvm_wmma_m8n8k128_load_a_b1_row:
4456 case Intrinsic::nvvm_wmma_m8n8k128_load_a_b1_row_stride:
4457 case Intrinsic::nvvm_wmma_m8n8k128_load_b_b1_col:
4458 case Intrinsic::nvvm_wmma_m8n8k128_load_b_b1_col_stride:
4459 case Intrinsic::nvvm_wmma_m8n8k32_load_a_s4_row:
4460 case Intrinsic::nvvm_wmma_m8n8k32_load_a_s4_row_stride:
4461 case Intrinsic::nvvm_wmma_m8n8k32_load_a_u4_row_stride:
4462 case Intrinsic::nvvm_wmma_m8n8k32_load_a_u4_row:
4463 case Intrinsic::nvvm_wmma_m8n8k32_load_b_s4_col:
4464 case Intrinsic::nvvm_wmma_m8n8k32_load_b_s4_col_stride:
4465 case Intrinsic::nvvm_wmma_m8n8k32_load_b_u4_col_stride:
4466 case Intrinsic::nvvm_wmma_m8n8k32_load_b_u4_col:
4467 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x1_b16:
4468 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x1_trans_b16:
4469 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x1_b8x16_b4x16_p64:
4470 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x1_b8x16_b6x16_p32: {
4471 Info.opc = ISD::INTRINSIC_W_CHAIN;
4472 Info.memVT = MVT::i32;
4473 Info.ptrVal = I.getArgOperand(i: 0);
4474 Info.offset = 0;
4475 Info.flags = MachineMemOperand::MOLoad;
4476 Info.align = Align(4);
4477 Infos.push_back(Elt: Info);
4478 return;
4479 }
4480
4481 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_col:
4482 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_row:
4483 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_col_stride:
4484 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_row_stride:
4485 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_col:
4486 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_row:
4487 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_col_stride:
4488 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_row_stride:
4489 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_col:
4490 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_row:
4491 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_col_stride:
4492 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_row_stride: {
4493 Info.opc = ISD::INTRINSIC_W_CHAIN;
4494 Info.memVT = MVT::v4f16;
4495 Info.ptrVal = I.getArgOperand(i: 0);
4496 Info.offset = 0;
4497 Info.flags = MachineMemOperand::MOLoad;
4498 Info.align = Align(16);
4499 Infos.push_back(Elt: Info);
4500 return;
4501 }
4502
4503 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_col:
4504 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_row:
4505 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_col_stride:
4506 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_row_stride:
4507 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_col:
4508 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_row:
4509 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_col_stride:
4510 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_row_stride:
4511 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_col:
4512 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_row:
4513 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_col_stride:
4514 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_row_stride:
4515 case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_col:
4516 case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_row:
4517 case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_col_stride:
4518 case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_row_stride: {
4519 Info.opc = ISD::INTRINSIC_W_CHAIN;
4520 Info.memVT = MVT::v8f32;
4521 Info.ptrVal = I.getArgOperand(i: 0);
4522 Info.offset = 0;
4523 Info.flags = MachineMemOperand::MOLoad;
4524 Info.align = Align(16);
4525 Infos.push_back(Elt: Info);
4526 return;
4527 }
4528
4529 case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_col:
4530 case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_col_stride:
4531 case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_row:
4532 case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_row_stride:
4533
4534 case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_col:
4535 case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_col_stride:
4536 case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_row:
4537 case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_row_stride:
4538
4539 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_col:
4540 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_col_stride:
4541 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_row:
4542 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_row_stride:
4543 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_col:
4544 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_col_stride:
4545 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_row:
4546 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_row_stride:
4547 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_col:
4548 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_col_stride:
4549 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_row:
4550 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_row_stride: {
4551 Info.opc = ISD::INTRINSIC_W_CHAIN;
4552 Info.memVT = MVT::v8i32;
4553 Info.ptrVal = I.getArgOperand(i: 0);
4554 Info.offset = 0;
4555 Info.flags = MachineMemOperand::MOLoad;
4556 Info.align = Align(16);
4557 Infos.push_back(Elt: Info);
4558 return;
4559 }
4560
4561 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_col:
4562 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_col_stride:
4563 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_row:
4564 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_row_stride:
4565 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_col:
4566 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_col_stride:
4567 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_row:
4568 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_row_stride:
4569 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x2_b16:
4570 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x2_trans_b16:
4571 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x1_trans_b8:
4572 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x1_trans_b8x16_b4x16_p64:
4573 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x1_trans_b8x16_b6x16_p32:
4574 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x2_b8x16_b4x16_p64:
4575 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x2_b8x16_b6x16_p32: {
4576 Info.opc = ISD::INTRINSIC_W_CHAIN;
4577 Info.memVT = MVT::v2i32;
4578 Info.ptrVal = I.getArgOperand(i: 0);
4579 Info.offset = 0;
4580 Info.flags = MachineMemOperand::MOLoad;
4581 Info.align = Align(8);
4582 Infos.push_back(Elt: Info);
4583 return;
4584 }
4585
4586 case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_col:
4587 case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_col_stride:
4588 case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_row:
4589 case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_row_stride:
4590
4591 case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_col:
4592 case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_col_stride:
4593 case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_row:
4594 case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_row_stride: {
4595 Info.opc = ISD::INTRINSIC_W_CHAIN;
4596 Info.memVT = MVT::f64;
4597 Info.ptrVal = I.getArgOperand(i: 0);
4598 Info.offset = 0;
4599 Info.flags = MachineMemOperand::MOLoad;
4600 Info.align = Align(8);
4601 Infos.push_back(Elt: Info);
4602 return;
4603 }
4604
4605 case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_col:
4606 case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_col_stride:
4607 case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_row:
4608 case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_row_stride: {
4609 Info.opc = ISD::INTRINSIC_W_CHAIN;
4610 Info.memVT = MVT::v2f64;
4611 Info.ptrVal = I.getArgOperand(i: 0);
4612 Info.offset = 0;
4613 Info.flags = MachineMemOperand::MOLoad;
4614 Info.align = Align(16);
4615 Infos.push_back(Elt: Info);
4616 return;
4617 }
4618
4619 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_col:
4620 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_row:
4621 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_col_stride:
4622 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_row_stride:
4623 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_col:
4624 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_row:
4625 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_col_stride:
4626 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_row_stride:
4627 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_col:
4628 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_row:
4629 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_col_stride:
4630 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_row_stride: {
4631 Info.opc = ISD::INTRINSIC_VOID;
4632 Info.memVT = MVT::v4f16;
4633 Info.ptrVal = I.getArgOperand(i: 0);
4634 Info.offset = 0;
4635 Info.flags = MachineMemOperand::MOStore;
4636 Info.align = Align(16);
4637 Infos.push_back(Elt: Info);
4638 return;
4639 }
4640
4641 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_col:
4642 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_row:
4643 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_col_stride:
4644 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_row_stride:
4645 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_col:
4646 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_row:
4647 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_col_stride:
4648 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_row_stride:
4649 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_col:
4650 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_row:
4651 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_col_stride:
4652 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_row_stride:
4653 case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_col:
4654 case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_row:
4655 case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_col_stride:
4656 case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_row_stride: {
4657 Info.opc = ISD::INTRINSIC_VOID;
4658 Info.memVT = MVT::v8f32;
4659 Info.ptrVal = I.getArgOperand(i: 0);
4660 Info.offset = 0;
4661 Info.flags = MachineMemOperand::MOStore;
4662 Info.align = Align(16);
4663 Infos.push_back(Elt: Info);
4664 return;
4665 }
4666
4667 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_col:
4668 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_col_stride:
4669 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_row:
4670 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_row_stride:
4671 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_col:
4672 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_col_stride:
4673 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_row:
4674 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_row_stride:
4675 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_col:
4676 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_col_stride:
4677 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_row:
4678 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_row_stride: {
4679 Info.opc = ISD::INTRINSIC_VOID;
4680 Info.memVT = MVT::v8i32;
4681 Info.ptrVal = I.getArgOperand(i: 0);
4682 Info.offset = 0;
4683 Info.flags = MachineMemOperand::MOStore;
4684 Info.align = Align(16);
4685 Infos.push_back(Elt: Info);
4686 return;
4687 }
4688
4689 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_col:
4690 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_col_stride:
4691 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_row:
4692 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_row_stride:
4693 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_col:
4694 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_col_stride:
4695 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_row:
4696 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_row_stride:
4697 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x2_b16:
4698 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x2_trans_b16:
4699 case Intrinsic::nvvm_stmatrix_sync_aligned_m16n8_x2_trans_b8: {
4700 Info.opc = ISD::INTRINSIC_VOID;
4701 Info.memVT = MVT::v2i32;
4702 Info.ptrVal = I.getArgOperand(i: 0);
4703 Info.offset = 0;
4704 Info.flags = MachineMemOperand::MOStore;
4705 Info.align = Align(8);
4706 Infos.push_back(Elt: Info);
4707 return;
4708 }
4709
4710 case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_col:
4711 case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_col_stride:
4712 case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_row:
4713 case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_row_stride: {
4714 Info.opc = ISD::INTRINSIC_VOID;
4715 Info.memVT = MVT::v2f64;
4716 Info.ptrVal = I.getArgOperand(i: 0);
4717 Info.offset = 0;
4718 Info.flags = MachineMemOperand::MOStore;
4719 Info.align = Align(16);
4720 Infos.push_back(Elt: Info);
4721 return;
4722 }
4723
4724 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x1_b16:
4725 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x1_trans_b16:
4726 case Intrinsic::nvvm_stmatrix_sync_aligned_m16n8_x1_trans_b8: {
4727 Info.opc = ISD::INTRINSIC_VOID;
4728 Info.memVT = MVT::i32;
4729 Info.ptrVal = I.getArgOperand(i: 0);
4730 Info.offset = 0;
4731 Info.flags = MachineMemOperand::MOStore;
4732 Info.align = Align(4);
4733 Infos.push_back(Elt: Info);
4734 return;
4735 }
4736
4737 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x4_b16:
4738 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x4_trans_b16:
4739 case Intrinsic::nvvm_stmatrix_sync_aligned_m16n8_x4_trans_b8: {
4740 Info.opc = ISD::INTRINSIC_VOID;
4741 Info.memVT = MVT::v4i32;
4742 Info.ptrVal = I.getArgOperand(i: 0);
4743 Info.offset = 0;
4744 Info.flags = MachineMemOperand::MOStore;
4745 Info.align = Align(16);
4746 Infos.push_back(Elt: Info);
4747 return;
4748 }
4749
4750 case Intrinsic::nvvm_prefetch_tensormap: {
4751 auto &DL = I.getDataLayout();
4752 Info.opc = ISD::INTRINSIC_VOID;
4753 Info.memVT = getPointerTy(DL);
4754 Info.ptrVal = I.getArgOperand(i: 0);
4755 Info.offset = 0;
4756 Info.flags =
4757 MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable;
4758 Info.align.reset();
4759 Infos.push_back(Elt: Info);
4760 return;
4761 }
4762
4763 case Intrinsic::nvvm_mbarrier_init: {
4764 Info.opc = ISD::INTRINSIC_VOID;
4765 Info.memVT = MVT::i64;
4766 Info.ptrVal = I.getArgOperand(i: 0);
4767 Info.offset = 0;
4768 Info.flags = MachineMemOperand::MOStore;
4769 Info.align = Align(8);
4770 Infos.push_back(Elt: Info);
4771 return;
4772 }
4773
4774 case Intrinsic::nvvm_mbarrier_check_layout: {
4775 Info.opc = ISD::INTRINSIC_W_CHAIN;
4776 Info.memVT = MVT::i64;
4777 Info.ptrVal = I.getArgOperand(i: 0);
4778 Info.offset = 0;
4779 Info.flags = MachineMemOperand::MOLoad;
4780 Info.align = Align(8);
4781 Infos.push_back(Elt: Info);
4782 return;
4783 }
4784
4785 case Intrinsic::nvvm_tensormap_replace_global_address:
4786 case Intrinsic::nvvm_tensormap_replace_global_stride: {
4787 Info.opc = ISD::INTRINSIC_VOID;
4788 Info.memVT = MVT::i64;
4789 Info.ptrVal = I.getArgOperand(i: 0);
4790 Info.offset = 0;
4791 Info.flags = MachineMemOperand::MOStore;
4792 Info.align.reset();
4793 Infos.push_back(Elt: Info);
4794 return;
4795 }
4796
4797 case Intrinsic::nvvm_tensormap_replace_rank:
4798 case Intrinsic::nvvm_tensormap_replace_box_dim:
4799 case Intrinsic::nvvm_tensormap_replace_global_dim:
4800 case Intrinsic::nvvm_tensormap_replace_element_stride:
4801 case Intrinsic::nvvm_tensormap_replace_elemtype:
4802 case Intrinsic::nvvm_tensormap_replace_interleave_layout:
4803 case Intrinsic::nvvm_tensormap_replace_swizzle_mode:
4804 case Intrinsic::nvvm_tensormap_replace_swizzle_atomicity:
4805 case Intrinsic::nvvm_tensormap_replace_fill_mode: {
4806 Info.opc = ISD::INTRINSIC_VOID;
4807 Info.memVT = MVT::i32;
4808 Info.ptrVal = I.getArgOperand(i: 0);
4809 Info.offset = 0;
4810 Info.flags = MachineMemOperand::MOStore;
4811 Info.align.reset();
4812 Infos.push_back(Elt: Info);
4813 return;
4814 }
4815
4816 case Intrinsic::nvvm_ldu_global_i:
4817 case Intrinsic::nvvm_ldu_global_f:
4818 case Intrinsic::nvvm_ldu_global_p: {
4819 Info.opc = ISD::INTRINSIC_W_CHAIN;
4820 Info.memVT = getValueType(DL: I.getDataLayout(), Ty: I.getType());
4821 Info.ptrVal = I.getArgOperand(i: 0);
4822 Info.offset = 0;
4823 Info.flags = MachineMemOperand::MOLoad;
4824 Info.align = cast<ConstantInt>(Val: I.getArgOperand(i: 1))->getMaybeAlignValue();
4825
4826 Infos.push_back(Elt: Info);
4827 return;
4828 }
4829 case Intrinsic::nvvm_tex_1d_v4f32_s32:
4830 case Intrinsic::nvvm_tex_1d_v4f32_f32:
4831 case Intrinsic::nvvm_tex_1d_level_v4f32_f32:
4832 case Intrinsic::nvvm_tex_1d_grad_v4f32_f32:
4833 case Intrinsic::nvvm_tex_1d_array_v4f32_s32:
4834 case Intrinsic::nvvm_tex_1d_array_v4f32_f32:
4835 case Intrinsic::nvvm_tex_1d_array_level_v4f32_f32:
4836 case Intrinsic::nvvm_tex_1d_array_grad_v4f32_f32:
4837 case Intrinsic::nvvm_tex_2d_v4f32_s32:
4838 case Intrinsic::nvvm_tex_2d_v4f32_f32:
4839 case Intrinsic::nvvm_tex_2d_level_v4f32_f32:
4840 case Intrinsic::nvvm_tex_2d_grad_v4f32_f32:
4841 case Intrinsic::nvvm_tex_2d_array_v4f32_s32:
4842 case Intrinsic::nvvm_tex_2d_array_v4f32_f32:
4843 case Intrinsic::nvvm_tex_2d_array_level_v4f32_f32:
4844 case Intrinsic::nvvm_tex_2d_array_grad_v4f32_f32:
4845 case Intrinsic::nvvm_tex_3d_v4f32_s32:
4846 case Intrinsic::nvvm_tex_3d_v4f32_f32:
4847 case Intrinsic::nvvm_tex_3d_level_v4f32_f32:
4848 case Intrinsic::nvvm_tex_3d_grad_v4f32_f32:
4849 case Intrinsic::nvvm_tex_cube_v4f32_f32:
4850 case Intrinsic::nvvm_tex_cube_level_v4f32_f32:
4851 case Intrinsic::nvvm_tex_cube_array_v4f32_f32:
4852 case Intrinsic::nvvm_tex_cube_array_level_v4f32_f32:
4853 case Intrinsic::nvvm_tld4_r_2d_v4f32_f32:
4854 case Intrinsic::nvvm_tld4_g_2d_v4f32_f32:
4855 case Intrinsic::nvvm_tld4_b_2d_v4f32_f32:
4856 case Intrinsic::nvvm_tld4_a_2d_v4f32_f32:
4857 case Intrinsic::nvvm_tex_unified_1d_v4f32_s32:
4858 case Intrinsic::nvvm_tex_unified_1d_v4f32_f32:
4859 case Intrinsic::nvvm_tex_unified_1d_level_v4f32_f32:
4860 case Intrinsic::nvvm_tex_unified_1d_grad_v4f32_f32:
4861 case Intrinsic::nvvm_tex_unified_1d_array_v4f32_s32:
4862 case Intrinsic::nvvm_tex_unified_1d_array_v4f32_f32:
4863 case Intrinsic::nvvm_tex_unified_1d_array_level_v4f32_f32:
4864 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4f32_f32:
4865 case Intrinsic::nvvm_tex_unified_2d_v4f32_s32:
4866 case Intrinsic::nvvm_tex_unified_2d_v4f32_f32:
4867 case Intrinsic::nvvm_tex_unified_2d_level_v4f32_f32:
4868 case Intrinsic::nvvm_tex_unified_2d_grad_v4f32_f32:
4869 case Intrinsic::nvvm_tex_unified_2d_array_v4f32_s32:
4870 case Intrinsic::nvvm_tex_unified_2d_array_v4f32_f32:
4871 case Intrinsic::nvvm_tex_unified_2d_array_level_v4f32_f32:
4872 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4f32_f32:
4873 case Intrinsic::nvvm_tex_unified_3d_v4f32_s32:
4874 case Intrinsic::nvvm_tex_unified_3d_v4f32_f32:
4875 case Intrinsic::nvvm_tex_unified_3d_level_v4f32_f32:
4876 case Intrinsic::nvvm_tex_unified_3d_grad_v4f32_f32:
4877 case Intrinsic::nvvm_tex_unified_cube_v4f32_f32:
4878 case Intrinsic::nvvm_tex_unified_cube_level_v4f32_f32:
4879 case Intrinsic::nvvm_tex_unified_cube_array_v4f32_f32:
4880 case Intrinsic::nvvm_tex_unified_cube_array_level_v4f32_f32:
4881 case Intrinsic::nvvm_tex_unified_cube_grad_v4f32_f32:
4882 case Intrinsic::nvvm_tex_unified_cube_array_grad_v4f32_f32:
4883 case Intrinsic::nvvm_tld4_unified_r_2d_v4f32_f32:
4884 case Intrinsic::nvvm_tld4_unified_g_2d_v4f32_f32:
4885 case Intrinsic::nvvm_tld4_unified_b_2d_v4f32_f32:
4886 case Intrinsic::nvvm_tld4_unified_a_2d_v4f32_f32:
4887 Info.opc = ISD::INTRINSIC_W_CHAIN;
4888 Info.memVT = MVT::v4f32;
4889 Info.ptrVal = nullptr;
4890 Info.offset = 0;
4891 Info.flags = MachineMemOperand::MOLoad;
4892 Info.align = Align(16);
4893 Infos.push_back(Elt: Info);
4894 return;
4895
4896 case Intrinsic::nvvm_tex_1d_v4s32_s32:
4897 case Intrinsic::nvvm_tex_1d_v4s32_f32:
4898 case Intrinsic::nvvm_tex_1d_level_v4s32_f32:
4899 case Intrinsic::nvvm_tex_1d_grad_v4s32_f32:
4900 case Intrinsic::nvvm_tex_1d_array_v4s32_s32:
4901 case Intrinsic::nvvm_tex_1d_array_v4s32_f32:
4902 case Intrinsic::nvvm_tex_1d_array_level_v4s32_f32:
4903 case Intrinsic::nvvm_tex_1d_array_grad_v4s32_f32:
4904 case Intrinsic::nvvm_tex_2d_v4s32_s32:
4905 case Intrinsic::nvvm_tex_2d_v4s32_f32:
4906 case Intrinsic::nvvm_tex_2d_level_v4s32_f32:
4907 case Intrinsic::nvvm_tex_2d_grad_v4s32_f32:
4908 case Intrinsic::nvvm_tex_2d_array_v4s32_s32:
4909 case Intrinsic::nvvm_tex_2d_array_v4s32_f32:
4910 case Intrinsic::nvvm_tex_2d_array_level_v4s32_f32:
4911 case Intrinsic::nvvm_tex_2d_array_grad_v4s32_f32:
4912 case Intrinsic::nvvm_tex_3d_v4s32_s32:
4913 case Intrinsic::nvvm_tex_3d_v4s32_f32:
4914 case Intrinsic::nvvm_tex_3d_level_v4s32_f32:
4915 case Intrinsic::nvvm_tex_3d_grad_v4s32_f32:
4916 case Intrinsic::nvvm_tex_cube_v4s32_f32:
4917 case Intrinsic::nvvm_tex_cube_level_v4s32_f32:
4918 case Intrinsic::nvvm_tex_cube_array_v4s32_f32:
4919 case Intrinsic::nvvm_tex_cube_array_level_v4s32_f32:
4920 case Intrinsic::nvvm_tex_cube_v4u32_f32:
4921 case Intrinsic::nvvm_tex_cube_level_v4u32_f32:
4922 case Intrinsic::nvvm_tex_cube_array_v4u32_f32:
4923 case Intrinsic::nvvm_tex_cube_array_level_v4u32_f32:
4924 case Intrinsic::nvvm_tex_1d_v4u32_s32:
4925 case Intrinsic::nvvm_tex_1d_v4u32_f32:
4926 case Intrinsic::nvvm_tex_1d_level_v4u32_f32:
4927 case Intrinsic::nvvm_tex_1d_grad_v4u32_f32:
4928 case Intrinsic::nvvm_tex_1d_array_v4u32_s32:
4929 case Intrinsic::nvvm_tex_1d_array_v4u32_f32:
4930 case Intrinsic::nvvm_tex_1d_array_level_v4u32_f32:
4931 case Intrinsic::nvvm_tex_1d_array_grad_v4u32_f32:
4932 case Intrinsic::nvvm_tex_2d_v4u32_s32:
4933 case Intrinsic::nvvm_tex_2d_v4u32_f32:
4934 case Intrinsic::nvvm_tex_2d_level_v4u32_f32:
4935 case Intrinsic::nvvm_tex_2d_grad_v4u32_f32:
4936 case Intrinsic::nvvm_tex_2d_array_v4u32_s32:
4937 case Intrinsic::nvvm_tex_2d_array_v4u32_f32:
4938 case Intrinsic::nvvm_tex_2d_array_level_v4u32_f32:
4939 case Intrinsic::nvvm_tex_2d_array_grad_v4u32_f32:
4940 case Intrinsic::nvvm_tex_3d_v4u32_s32:
4941 case Intrinsic::nvvm_tex_3d_v4u32_f32:
4942 case Intrinsic::nvvm_tex_3d_level_v4u32_f32:
4943 case Intrinsic::nvvm_tex_3d_grad_v4u32_f32:
4944 case Intrinsic::nvvm_tld4_r_2d_v4s32_f32:
4945 case Intrinsic::nvvm_tld4_g_2d_v4s32_f32:
4946 case Intrinsic::nvvm_tld4_b_2d_v4s32_f32:
4947 case Intrinsic::nvvm_tld4_a_2d_v4s32_f32:
4948 case Intrinsic::nvvm_tld4_r_2d_v4u32_f32:
4949 case Intrinsic::nvvm_tld4_g_2d_v4u32_f32:
4950 case Intrinsic::nvvm_tld4_b_2d_v4u32_f32:
4951 case Intrinsic::nvvm_tld4_a_2d_v4u32_f32:
4952 case Intrinsic::nvvm_tex_unified_1d_v4s32_s32:
4953 case Intrinsic::nvvm_tex_unified_1d_v4s32_f32:
4954 case Intrinsic::nvvm_tex_unified_1d_level_v4s32_f32:
4955 case Intrinsic::nvvm_tex_unified_1d_grad_v4s32_f32:
4956 case Intrinsic::nvvm_tex_unified_1d_array_v4s32_s32:
4957 case Intrinsic::nvvm_tex_unified_1d_array_v4s32_f32:
4958 case Intrinsic::nvvm_tex_unified_1d_array_level_v4s32_f32:
4959 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4s32_f32:
4960 case Intrinsic::nvvm_tex_unified_2d_v4s32_s32:
4961 case Intrinsic::nvvm_tex_unified_2d_v4s32_f32:
4962 case Intrinsic::nvvm_tex_unified_2d_level_v4s32_f32:
4963 case Intrinsic::nvvm_tex_unified_2d_grad_v4s32_f32:
4964 case Intrinsic::nvvm_tex_unified_2d_array_v4s32_s32:
4965 case Intrinsic::nvvm_tex_unified_2d_array_v4s32_f32:
4966 case Intrinsic::nvvm_tex_unified_2d_array_level_v4s32_f32:
4967 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4s32_f32:
4968 case Intrinsic::nvvm_tex_unified_3d_v4s32_s32:
4969 case Intrinsic::nvvm_tex_unified_3d_v4s32_f32:
4970 case Intrinsic::nvvm_tex_unified_3d_level_v4s32_f32:
4971 case Intrinsic::nvvm_tex_unified_3d_grad_v4s32_f32:
4972 case Intrinsic::nvvm_tex_unified_1d_v4u32_s32:
4973 case Intrinsic::nvvm_tex_unified_1d_v4u32_f32:
4974 case Intrinsic::nvvm_tex_unified_1d_level_v4u32_f32:
4975 case Intrinsic::nvvm_tex_unified_1d_grad_v4u32_f32:
4976 case Intrinsic::nvvm_tex_unified_1d_array_v4u32_s32:
4977 case Intrinsic::nvvm_tex_unified_1d_array_v4u32_f32:
4978 case Intrinsic::nvvm_tex_unified_1d_array_level_v4u32_f32:
4979 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4u32_f32:
4980 case Intrinsic::nvvm_tex_unified_2d_v4u32_s32:
4981 case Intrinsic::nvvm_tex_unified_2d_v4u32_f32:
4982 case Intrinsic::nvvm_tex_unified_2d_level_v4u32_f32:
4983 case Intrinsic::nvvm_tex_unified_2d_grad_v4u32_f32:
4984 case Intrinsic::nvvm_tex_unified_2d_array_v4u32_s32:
4985 case Intrinsic::nvvm_tex_unified_2d_array_v4u32_f32:
4986 case Intrinsic::nvvm_tex_unified_2d_array_level_v4u32_f32:
4987 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4u32_f32:
4988 case Intrinsic::nvvm_tex_unified_3d_v4u32_s32:
4989 case Intrinsic::nvvm_tex_unified_3d_v4u32_f32:
4990 case Intrinsic::nvvm_tex_unified_3d_level_v4u32_f32:
4991 case Intrinsic::nvvm_tex_unified_3d_grad_v4u32_f32:
4992 case Intrinsic::nvvm_tex_unified_cube_v4s32_f32:
4993 case Intrinsic::nvvm_tex_unified_cube_level_v4s32_f32:
4994 case Intrinsic::nvvm_tex_unified_cube_array_v4s32_f32:
4995 case Intrinsic::nvvm_tex_unified_cube_array_level_v4s32_f32:
4996 case Intrinsic::nvvm_tex_unified_cube_v4u32_f32:
4997 case Intrinsic::nvvm_tex_unified_cube_level_v4u32_f32:
4998 case Intrinsic::nvvm_tex_unified_cube_array_v4u32_f32:
4999 case Intrinsic::nvvm_tex_unified_cube_array_level_v4u32_f32:
5000 case Intrinsic::nvvm_tex_unified_cube_grad_v4s32_f32:
5001 case Intrinsic::nvvm_tex_unified_cube_grad_v4u32_f32:
5002 case Intrinsic::nvvm_tex_unified_cube_array_grad_v4s32_f32:
5003 case Intrinsic::nvvm_tex_unified_cube_array_grad_v4u32_f32:
5004 case Intrinsic::nvvm_tld4_unified_r_2d_v4s32_f32:
5005 case Intrinsic::nvvm_tld4_unified_g_2d_v4s32_f32:
5006 case Intrinsic::nvvm_tld4_unified_b_2d_v4s32_f32:
5007 case Intrinsic::nvvm_tld4_unified_a_2d_v4s32_f32:
5008 case Intrinsic::nvvm_tld4_unified_r_2d_v4u32_f32:
5009 case Intrinsic::nvvm_tld4_unified_g_2d_v4u32_f32:
5010 case Intrinsic::nvvm_tld4_unified_b_2d_v4u32_f32:
5011 case Intrinsic::nvvm_tld4_unified_a_2d_v4u32_f32:
5012 Info.opc = ISD::INTRINSIC_W_CHAIN;
5013 Info.memVT = MVT::v4i32;
5014 Info.ptrVal = nullptr;
5015 Info.offset = 0;
5016 Info.flags = MachineMemOperand::MOLoad;
5017 Info.align = Align(16);
5018 Infos.push_back(Elt: Info);
5019 return;
5020
5021 case Intrinsic::nvvm_suld_1d_i8_clamp:
5022 case Intrinsic::nvvm_suld_1d_v2i8_clamp:
5023 case Intrinsic::nvvm_suld_1d_v4i8_clamp:
5024 case Intrinsic::nvvm_suld_1d_array_i8_clamp:
5025 case Intrinsic::nvvm_suld_1d_array_v2i8_clamp:
5026 case Intrinsic::nvvm_suld_1d_array_v4i8_clamp:
5027 case Intrinsic::nvvm_suld_2d_i8_clamp:
5028 case Intrinsic::nvvm_suld_2d_v2i8_clamp:
5029 case Intrinsic::nvvm_suld_2d_v4i8_clamp:
5030 case Intrinsic::nvvm_suld_2d_array_i8_clamp:
5031 case Intrinsic::nvvm_suld_2d_array_v2i8_clamp:
5032 case Intrinsic::nvvm_suld_2d_array_v4i8_clamp:
5033 case Intrinsic::nvvm_suld_3d_i8_clamp:
5034 case Intrinsic::nvvm_suld_3d_v2i8_clamp:
5035 case Intrinsic::nvvm_suld_3d_v4i8_clamp:
5036 case Intrinsic::nvvm_suld_1d_i8_trap:
5037 case Intrinsic::nvvm_suld_1d_v2i8_trap:
5038 case Intrinsic::nvvm_suld_1d_v4i8_trap:
5039 case Intrinsic::nvvm_suld_1d_array_i8_trap:
5040 case Intrinsic::nvvm_suld_1d_array_v2i8_trap:
5041 case Intrinsic::nvvm_suld_1d_array_v4i8_trap:
5042 case Intrinsic::nvvm_suld_2d_i8_trap:
5043 case Intrinsic::nvvm_suld_2d_v2i8_trap:
5044 case Intrinsic::nvvm_suld_2d_v4i8_trap:
5045 case Intrinsic::nvvm_suld_2d_array_i8_trap:
5046 case Intrinsic::nvvm_suld_2d_array_v2i8_trap:
5047 case Intrinsic::nvvm_suld_2d_array_v4i8_trap:
5048 case Intrinsic::nvvm_suld_3d_i8_trap:
5049 case Intrinsic::nvvm_suld_3d_v2i8_trap:
5050 case Intrinsic::nvvm_suld_3d_v4i8_trap:
5051 case Intrinsic::nvvm_suld_1d_i8_zero:
5052 case Intrinsic::nvvm_suld_1d_v2i8_zero:
5053 case Intrinsic::nvvm_suld_1d_v4i8_zero:
5054 case Intrinsic::nvvm_suld_1d_array_i8_zero:
5055 case Intrinsic::nvvm_suld_1d_array_v2i8_zero:
5056 case Intrinsic::nvvm_suld_1d_array_v4i8_zero:
5057 case Intrinsic::nvvm_suld_2d_i8_zero:
5058 case Intrinsic::nvvm_suld_2d_v2i8_zero:
5059 case Intrinsic::nvvm_suld_2d_v4i8_zero:
5060 case Intrinsic::nvvm_suld_2d_array_i8_zero:
5061 case Intrinsic::nvvm_suld_2d_array_v2i8_zero:
5062 case Intrinsic::nvvm_suld_2d_array_v4i8_zero:
5063 case Intrinsic::nvvm_suld_3d_i8_zero:
5064 case Intrinsic::nvvm_suld_3d_v2i8_zero:
5065 case Intrinsic::nvvm_suld_3d_v4i8_zero:
5066 Info.opc = ISD::INTRINSIC_W_CHAIN;
5067 Info.memVT = MVT::i8;
5068 Info.ptrVal = nullptr;
5069 Info.offset = 0;
5070 Info.flags = MachineMemOperand::MOLoad;
5071 Info.align = Align(16);
5072 Infos.push_back(Elt: Info);
5073 return;
5074
5075 case Intrinsic::nvvm_suld_1d_i16_clamp:
5076 case Intrinsic::nvvm_suld_1d_v2i16_clamp:
5077 case Intrinsic::nvvm_suld_1d_v4i16_clamp:
5078 case Intrinsic::nvvm_suld_1d_array_i16_clamp:
5079 case Intrinsic::nvvm_suld_1d_array_v2i16_clamp:
5080 case Intrinsic::nvvm_suld_1d_array_v4i16_clamp:
5081 case Intrinsic::nvvm_suld_2d_i16_clamp:
5082 case Intrinsic::nvvm_suld_2d_v2i16_clamp:
5083 case Intrinsic::nvvm_suld_2d_v4i16_clamp:
5084 case Intrinsic::nvvm_suld_2d_array_i16_clamp:
5085 case Intrinsic::nvvm_suld_2d_array_v2i16_clamp:
5086 case Intrinsic::nvvm_suld_2d_array_v4i16_clamp:
5087 case Intrinsic::nvvm_suld_3d_i16_clamp:
5088 case Intrinsic::nvvm_suld_3d_v2i16_clamp:
5089 case Intrinsic::nvvm_suld_3d_v4i16_clamp:
5090 case Intrinsic::nvvm_suld_1d_i16_trap:
5091 case Intrinsic::nvvm_suld_1d_v2i16_trap:
5092 case Intrinsic::nvvm_suld_1d_v4i16_trap:
5093 case Intrinsic::nvvm_suld_1d_array_i16_trap:
5094 case Intrinsic::nvvm_suld_1d_array_v2i16_trap:
5095 case Intrinsic::nvvm_suld_1d_array_v4i16_trap:
5096 case Intrinsic::nvvm_suld_2d_i16_trap:
5097 case Intrinsic::nvvm_suld_2d_v2i16_trap:
5098 case Intrinsic::nvvm_suld_2d_v4i16_trap:
5099 case Intrinsic::nvvm_suld_2d_array_i16_trap:
5100 case Intrinsic::nvvm_suld_2d_array_v2i16_trap:
5101 case Intrinsic::nvvm_suld_2d_array_v4i16_trap:
5102 case Intrinsic::nvvm_suld_3d_i16_trap:
5103 case Intrinsic::nvvm_suld_3d_v2i16_trap:
5104 case Intrinsic::nvvm_suld_3d_v4i16_trap:
5105 case Intrinsic::nvvm_suld_1d_i16_zero:
5106 case Intrinsic::nvvm_suld_1d_v2i16_zero:
5107 case Intrinsic::nvvm_suld_1d_v4i16_zero:
5108 case Intrinsic::nvvm_suld_1d_array_i16_zero:
5109 case Intrinsic::nvvm_suld_1d_array_v2i16_zero:
5110 case Intrinsic::nvvm_suld_1d_array_v4i16_zero:
5111 case Intrinsic::nvvm_suld_2d_i16_zero:
5112 case Intrinsic::nvvm_suld_2d_v2i16_zero:
5113 case Intrinsic::nvvm_suld_2d_v4i16_zero:
5114 case Intrinsic::nvvm_suld_2d_array_i16_zero:
5115 case Intrinsic::nvvm_suld_2d_array_v2i16_zero:
5116 case Intrinsic::nvvm_suld_2d_array_v4i16_zero:
5117 case Intrinsic::nvvm_suld_3d_i16_zero:
5118 case Intrinsic::nvvm_suld_3d_v2i16_zero:
5119 case Intrinsic::nvvm_suld_3d_v4i16_zero:
5120 Info.opc = ISD::INTRINSIC_W_CHAIN;
5121 Info.memVT = MVT::i16;
5122 Info.ptrVal = nullptr;
5123 Info.offset = 0;
5124 Info.flags = MachineMemOperand::MOLoad;
5125 Info.align = Align(16);
5126 Infos.push_back(Elt: Info);
5127 return;
5128
5129 case Intrinsic::nvvm_suld_1d_i32_clamp:
5130 case Intrinsic::nvvm_suld_1d_v2i32_clamp:
5131 case Intrinsic::nvvm_suld_1d_v4i32_clamp:
5132 case Intrinsic::nvvm_suld_1d_array_i32_clamp:
5133 case Intrinsic::nvvm_suld_1d_array_v2i32_clamp:
5134 case Intrinsic::nvvm_suld_1d_array_v4i32_clamp:
5135 case Intrinsic::nvvm_suld_2d_i32_clamp:
5136 case Intrinsic::nvvm_suld_2d_v2i32_clamp:
5137 case Intrinsic::nvvm_suld_2d_v4i32_clamp:
5138 case Intrinsic::nvvm_suld_2d_array_i32_clamp:
5139 case Intrinsic::nvvm_suld_2d_array_v2i32_clamp:
5140 case Intrinsic::nvvm_suld_2d_array_v4i32_clamp:
5141 case Intrinsic::nvvm_suld_3d_i32_clamp:
5142 case Intrinsic::nvvm_suld_3d_v2i32_clamp:
5143 case Intrinsic::nvvm_suld_3d_v4i32_clamp:
5144 case Intrinsic::nvvm_suld_1d_i32_trap:
5145 case Intrinsic::nvvm_suld_1d_v2i32_trap:
5146 case Intrinsic::nvvm_suld_1d_v4i32_trap:
5147 case Intrinsic::nvvm_suld_1d_array_i32_trap:
5148 case Intrinsic::nvvm_suld_1d_array_v2i32_trap:
5149 case Intrinsic::nvvm_suld_1d_array_v4i32_trap:
5150 case Intrinsic::nvvm_suld_2d_i32_trap:
5151 case Intrinsic::nvvm_suld_2d_v2i32_trap:
5152 case Intrinsic::nvvm_suld_2d_v4i32_trap:
5153 case Intrinsic::nvvm_suld_2d_array_i32_trap:
5154 case Intrinsic::nvvm_suld_2d_array_v2i32_trap:
5155 case Intrinsic::nvvm_suld_2d_array_v4i32_trap:
5156 case Intrinsic::nvvm_suld_3d_i32_trap:
5157 case Intrinsic::nvvm_suld_3d_v2i32_trap:
5158 case Intrinsic::nvvm_suld_3d_v4i32_trap:
5159 case Intrinsic::nvvm_suld_1d_i32_zero:
5160 case Intrinsic::nvvm_suld_1d_v2i32_zero:
5161 case Intrinsic::nvvm_suld_1d_v4i32_zero:
5162 case Intrinsic::nvvm_suld_1d_array_i32_zero:
5163 case Intrinsic::nvvm_suld_1d_array_v2i32_zero:
5164 case Intrinsic::nvvm_suld_1d_array_v4i32_zero:
5165 case Intrinsic::nvvm_suld_2d_i32_zero:
5166 case Intrinsic::nvvm_suld_2d_v2i32_zero:
5167 case Intrinsic::nvvm_suld_2d_v4i32_zero:
5168 case Intrinsic::nvvm_suld_2d_array_i32_zero:
5169 case Intrinsic::nvvm_suld_2d_array_v2i32_zero:
5170 case Intrinsic::nvvm_suld_2d_array_v4i32_zero:
5171 case Intrinsic::nvvm_suld_3d_i32_zero:
5172 case Intrinsic::nvvm_suld_3d_v2i32_zero:
5173 case Intrinsic::nvvm_suld_3d_v4i32_zero:
5174 Info.opc = ISD::INTRINSIC_W_CHAIN;
5175 Info.memVT = MVT::i32;
5176 Info.ptrVal = nullptr;
5177 Info.offset = 0;
5178 Info.flags = MachineMemOperand::MOLoad;
5179 Info.align = Align(16);
5180 Infos.push_back(Elt: Info);
5181 return;
5182
5183 case Intrinsic::nvvm_suld_1d_i64_clamp:
5184 case Intrinsic::nvvm_suld_1d_v2i64_clamp:
5185 case Intrinsic::nvvm_suld_1d_array_i64_clamp:
5186 case Intrinsic::nvvm_suld_1d_array_v2i64_clamp:
5187 case Intrinsic::nvvm_suld_2d_i64_clamp:
5188 case Intrinsic::nvvm_suld_2d_v2i64_clamp:
5189 case Intrinsic::nvvm_suld_2d_array_i64_clamp:
5190 case Intrinsic::nvvm_suld_2d_array_v2i64_clamp:
5191 case Intrinsic::nvvm_suld_3d_i64_clamp:
5192 case Intrinsic::nvvm_suld_3d_v2i64_clamp:
5193 case Intrinsic::nvvm_suld_1d_i64_trap:
5194 case Intrinsic::nvvm_suld_1d_v2i64_trap:
5195 case Intrinsic::nvvm_suld_1d_array_i64_trap:
5196 case Intrinsic::nvvm_suld_1d_array_v2i64_trap:
5197 case Intrinsic::nvvm_suld_2d_i64_trap:
5198 case Intrinsic::nvvm_suld_2d_v2i64_trap:
5199 case Intrinsic::nvvm_suld_2d_array_i64_trap:
5200 case Intrinsic::nvvm_suld_2d_array_v2i64_trap:
5201 case Intrinsic::nvvm_suld_3d_i64_trap:
5202 case Intrinsic::nvvm_suld_3d_v2i64_trap:
5203 case Intrinsic::nvvm_suld_1d_i64_zero:
5204 case Intrinsic::nvvm_suld_1d_v2i64_zero:
5205 case Intrinsic::nvvm_suld_1d_array_i64_zero:
5206 case Intrinsic::nvvm_suld_1d_array_v2i64_zero:
5207 case Intrinsic::nvvm_suld_2d_i64_zero:
5208 case Intrinsic::nvvm_suld_2d_v2i64_zero:
5209 case Intrinsic::nvvm_suld_2d_array_i64_zero:
5210 case Intrinsic::nvvm_suld_2d_array_v2i64_zero:
5211 case Intrinsic::nvvm_suld_3d_i64_zero:
5212 case Intrinsic::nvvm_suld_3d_v2i64_zero:
5213 Info.opc = ISD::INTRINSIC_W_CHAIN;
5214 Info.memVT = MVT::i64;
5215 Info.ptrVal = nullptr;
5216 Info.offset = 0;
5217 Info.flags = MachineMemOperand::MOLoad;
5218 Info.align = Align(16);
5219 Infos.push_back(Elt: Info);
5220 return;
5221
5222 case Intrinsic::nvvm_tcgen05_ld_16x64b_x1:
5223 case Intrinsic::nvvm_tcgen05_ld_32x32b_x1:
5224 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x1: {
5225 Info.opc = ISD::INTRINSIC_W_CHAIN;
5226 Info.memVT = MVT::v1i32;
5227 Info.ptrVal = I.getArgOperand(i: 0);
5228 Info.offset = 0;
5229 Info.flags = MachineMemOperand::MOLoad;
5230 Info.align.reset();
5231 Infos.push_back(Elt: Info);
5232 return;
5233 }
5234
5235 case Intrinsic::nvvm_tcgen05_ld_16x64b_x2:
5236 case Intrinsic::nvvm_tcgen05_ld_16x128b_x1:
5237 case Intrinsic::nvvm_tcgen05_ld_32x32b_x2:
5238 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x2:
5239 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x2_i32:
5240 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x2_i32: {
5241 Info.opc = ISD::INTRINSIC_W_CHAIN;
5242 Info.memVT = MVT::v2i32;
5243 Info.ptrVal = I.getArgOperand(i: 0);
5244 Info.offset = 0;
5245 Info.flags = MachineMemOperand::MOLoad;
5246 Info.align.reset();
5247 Infos.push_back(Elt: Info);
5248 return;
5249 }
5250
5251 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x2_f32:
5252 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x2_f32: {
5253 Info.opc = ISD::INTRINSIC_W_CHAIN;
5254 Info.memVT = MVT::v2f32;
5255 Info.ptrVal = I.getArgOperand(i: 0);
5256 Info.offset = 0;
5257 Info.flags = MachineMemOperand::MOLoad;
5258 Info.align.reset();
5259 Infos.push_back(Elt: Info);
5260 return;
5261 }
5262
5263 case Intrinsic::nvvm_tcgen05_ld_16x64b_x4:
5264 case Intrinsic::nvvm_tcgen05_ld_16x128b_x2:
5265 case Intrinsic::nvvm_tcgen05_ld_32x32b_x4:
5266 case Intrinsic::nvvm_tcgen05_ld_16x256b_x1:
5267 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x4:
5268 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x4_i32:
5269 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x4_i32: {
5270 Info.opc = ISD::INTRINSIC_W_CHAIN;
5271 Info.memVT = MVT::v4i32;
5272 Info.ptrVal = I.getArgOperand(i: 0);
5273 Info.offset = 0;
5274 Info.flags = MachineMemOperand::MOLoad;
5275 Info.align.reset();
5276 Infos.push_back(Elt: Info);
5277 return;
5278 }
5279
5280 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x4_f32:
5281 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x4_f32: {
5282 Info.opc = ISD::INTRINSIC_W_CHAIN;
5283 Info.memVT = MVT::v4f32;
5284 Info.ptrVal = I.getArgOperand(i: 0);
5285 Info.offset = 0;
5286 Info.flags = MachineMemOperand::MOLoad;
5287 Info.align.reset();
5288 Infos.push_back(Elt: Info);
5289 return;
5290 }
5291
5292 case Intrinsic::nvvm_tcgen05_ld_16x64b_x8:
5293 case Intrinsic::nvvm_tcgen05_ld_16x128b_x4:
5294 case Intrinsic::nvvm_tcgen05_ld_16x256b_x2:
5295 case Intrinsic::nvvm_tcgen05_ld_32x32b_x8:
5296 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x8:
5297 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x8_i32:
5298 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x8_i32: {
5299 Info.opc = ISD::INTRINSIC_W_CHAIN;
5300 Info.memVT = MVT::v8i32;
5301 Info.ptrVal = I.getArgOperand(i: 0);
5302 Info.offset = 0;
5303 Info.flags = MachineMemOperand::MOLoad;
5304 Info.align.reset();
5305 Infos.push_back(Elt: Info);
5306 return;
5307 }
5308
5309 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x8_f32:
5310 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x8_f32: {
5311 Info.opc = ISD::INTRINSIC_W_CHAIN;
5312 Info.memVT = MVT::v8f32;
5313 Info.ptrVal = I.getArgOperand(i: 0);
5314 Info.offset = 0;
5315 Info.flags = MachineMemOperand::MOLoad;
5316 Info.align.reset();
5317 Infos.push_back(Elt: Info);
5318 return;
5319 }
5320
5321 case Intrinsic::nvvm_tcgen05_ld_16x64b_x16:
5322 case Intrinsic::nvvm_tcgen05_ld_16x128b_x8:
5323 case Intrinsic::nvvm_tcgen05_ld_16x256b_x4:
5324 case Intrinsic::nvvm_tcgen05_ld_32x32b_x16:
5325 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x16:
5326 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x16_i32:
5327 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x16_i32: {
5328 Info.opc = ISD::INTRINSIC_W_CHAIN;
5329 Info.memVT = MVT::v16i32;
5330 Info.ptrVal = I.getArgOperand(i: 0);
5331 Info.offset = 0;
5332 Info.flags = MachineMemOperand::MOLoad;
5333 Info.align.reset();
5334 Infos.push_back(Elt: Info);
5335 return;
5336 }
5337
5338 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x16_f32:
5339 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x16_f32: {
5340 Info.opc = ISD::INTRINSIC_W_CHAIN;
5341 Info.memVT = MVT::v16f32;
5342 Info.ptrVal = I.getArgOperand(i: 0);
5343 Info.offset = 0;
5344 Info.flags = MachineMemOperand::MOLoad;
5345 Info.align.reset();
5346 Infos.push_back(Elt: Info);
5347 return;
5348 }
5349
5350 case Intrinsic::nvvm_tcgen05_ld_16x64b_x32:
5351 case Intrinsic::nvvm_tcgen05_ld_16x128b_x16:
5352 case Intrinsic::nvvm_tcgen05_ld_16x256b_x8:
5353 case Intrinsic::nvvm_tcgen05_ld_32x32b_x32:
5354 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x32:
5355 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x32_i32:
5356 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x32_i32: {
5357 Info.opc = ISD::INTRINSIC_W_CHAIN;
5358 Info.memVT = MVT::v32i32;
5359 Info.ptrVal = I.getArgOperand(i: 0);
5360 Info.offset = 0;
5361 Info.flags = MachineMemOperand::MOLoad;
5362 Info.align.reset();
5363 Infos.push_back(Elt: Info);
5364 return;
5365 }
5366
5367 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x32_f32:
5368 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x32_f32: {
5369 Info.opc = ISD::INTRINSIC_W_CHAIN;
5370 Info.memVT = MVT::v32f32;
5371 Info.ptrVal = I.getArgOperand(i: 0);
5372 Info.offset = 0;
5373 Info.flags = MachineMemOperand::MOLoad;
5374 Info.align.reset();
5375 Infos.push_back(Elt: Info);
5376 return;
5377 }
5378
5379 case Intrinsic::nvvm_tcgen05_ld_16x64b_x64:
5380 case Intrinsic::nvvm_tcgen05_ld_16x128b_x32:
5381 case Intrinsic::nvvm_tcgen05_ld_16x256b_x16:
5382 case Intrinsic::nvvm_tcgen05_ld_32x32b_x64:
5383 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x64:
5384 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x64_i32:
5385 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x64_i32: {
5386 Info.opc = ISD::INTRINSIC_W_CHAIN;
5387 Info.memVT = MVT::v64i32;
5388 Info.ptrVal = I.getArgOperand(i: 0);
5389 Info.offset = 0;
5390 Info.flags = MachineMemOperand::MOLoad;
5391 Info.align.reset();
5392 Infos.push_back(Elt: Info);
5393 return;
5394 }
5395
5396 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x64_f32:
5397 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x64_f32: {
5398 Info.opc = ISD::INTRINSIC_W_CHAIN;
5399 Info.memVT = MVT::v64f32;
5400 Info.ptrVal = I.getArgOperand(i: 0);
5401 Info.offset = 0;
5402 Info.flags = MachineMemOperand::MOLoad;
5403 Info.align.reset();
5404 Infos.push_back(Elt: Info);
5405 return;
5406 }
5407
5408 case Intrinsic::nvvm_tcgen05_ld_16x64b_x128:
5409 case Intrinsic::nvvm_tcgen05_ld_16x128b_x64:
5410 case Intrinsic::nvvm_tcgen05_ld_16x256b_x32:
5411 case Intrinsic::nvvm_tcgen05_ld_32x32b_x128:
5412 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x128:
5413 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x128_i32:
5414 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x128_i32: {
5415 Info.opc = ISD::INTRINSIC_W_CHAIN;
5416 Info.memVT = MVT::v128i32;
5417 Info.ptrVal = I.getArgOperand(i: 0);
5418 Info.offset = 0;
5419 Info.flags = MachineMemOperand::MOLoad;
5420 Info.align.reset();
5421 Infos.push_back(Elt: Info);
5422 return;
5423 }
5424
5425 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x128_f32:
5426 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x128_f32: {
5427 Info.opc = ISD::INTRINSIC_W_CHAIN;
5428 Info.memVT = MVT::v128f32;
5429 Info.ptrVal = I.getArgOperand(i: 0);
5430 Info.offset = 0;
5431 Info.flags = MachineMemOperand::MOLoad;
5432 Info.align.reset();
5433 Infos.push_back(Elt: Info);
5434 return;
5435 }
5436
5437 case Intrinsic::nvvm_tcgen05_st_16x64b_x1:
5438 case Intrinsic::nvvm_tcgen05_st_32x32b_x1:
5439 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x1: {
5440 Info.opc = ISD::INTRINSIC_VOID;
5441 Info.memVT = MVT::v1i32;
5442 Info.ptrVal = I.getArgOperand(i: 0);
5443 Info.offset = 0;
5444 Info.flags = MachineMemOperand::MOStore;
5445 Info.align.reset();
5446 Infos.push_back(Elt: Info);
5447 return;
5448 }
5449
5450 case Intrinsic::nvvm_tcgen05_st_16x64b_x2:
5451 case Intrinsic::nvvm_tcgen05_st_16x128b_x1:
5452 case Intrinsic::nvvm_tcgen05_st_32x32b_x2:
5453 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x2: {
5454 Info.opc = ISD::INTRINSIC_VOID;
5455 Info.memVT = MVT::v2i32;
5456 Info.ptrVal = I.getArgOperand(i: 0);
5457 Info.offset = 0;
5458 Info.flags = MachineMemOperand::MOStore;
5459 Info.align.reset();
5460 Infos.push_back(Elt: Info);
5461 return;
5462 }
5463
5464 case Intrinsic::nvvm_tcgen05_st_16x64b_x4:
5465 case Intrinsic::nvvm_tcgen05_st_16x128b_x2:
5466 case Intrinsic::nvvm_tcgen05_st_16x256b_x1:
5467 case Intrinsic::nvvm_tcgen05_st_32x32b_x4:
5468 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x4: {
5469 Info.opc = ISD::INTRINSIC_VOID;
5470 Info.memVT = MVT::v4i32;
5471 Info.ptrVal = I.getArgOperand(i: 0);
5472 Info.offset = 0;
5473 Info.flags = MachineMemOperand::MOStore;
5474 Info.align.reset();
5475 Infos.push_back(Elt: Info);
5476 return;
5477 }
5478
5479 case Intrinsic::nvvm_tcgen05_st_16x64b_x8:
5480 case Intrinsic::nvvm_tcgen05_st_16x128b_x4:
5481 case Intrinsic::nvvm_tcgen05_st_16x256b_x2:
5482 case Intrinsic::nvvm_tcgen05_st_32x32b_x8:
5483 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x8: {
5484 Info.opc = ISD::INTRINSIC_VOID;
5485 Info.memVT = MVT::v8i32;
5486 Info.ptrVal = I.getArgOperand(i: 0);
5487 Info.offset = 0;
5488 Info.flags = MachineMemOperand::MOStore;
5489 Info.align.reset();
5490 Infos.push_back(Elt: Info);
5491 return;
5492 }
5493
5494 case Intrinsic::nvvm_tcgen05_st_16x64b_x16:
5495 case Intrinsic::nvvm_tcgen05_st_16x128b_x8:
5496 case Intrinsic::nvvm_tcgen05_st_16x256b_x4:
5497 case Intrinsic::nvvm_tcgen05_st_32x32b_x16:
5498 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x16: {
5499 Info.opc = ISD::INTRINSIC_VOID;
5500 Info.memVT = MVT::v16i32;
5501 Info.ptrVal = I.getArgOperand(i: 0);
5502 Info.offset = 0;
5503 Info.flags = MachineMemOperand::MOStore;
5504 Info.align.reset();
5505 Infos.push_back(Elt: Info);
5506 return;
5507 }
5508
5509 case Intrinsic::nvvm_tcgen05_st_16x64b_x32:
5510 case Intrinsic::nvvm_tcgen05_st_16x128b_x16:
5511 case Intrinsic::nvvm_tcgen05_st_16x256b_x8:
5512 case Intrinsic::nvvm_tcgen05_st_32x32b_x32:
5513 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x32: {
5514 Info.opc = ISD::INTRINSIC_VOID;
5515 Info.memVT = MVT::v32i32;
5516 Info.ptrVal = I.getArgOperand(i: 0);
5517 Info.offset = 0;
5518 Info.flags = MachineMemOperand::MOStore;
5519 Info.align.reset();
5520 Infos.push_back(Elt: Info);
5521 return;
5522 }
5523
5524 case Intrinsic::nvvm_tcgen05_st_16x64b_x64:
5525 case Intrinsic::nvvm_tcgen05_st_16x128b_x32:
5526 case Intrinsic::nvvm_tcgen05_st_16x256b_x16:
5527 case Intrinsic::nvvm_tcgen05_st_32x32b_x64:
5528 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x64: {
5529 Info.opc = ISD::INTRINSIC_VOID;
5530 Info.memVT = MVT::v64i32;
5531 Info.ptrVal = I.getArgOperand(i: 0);
5532 Info.offset = 0;
5533 Info.flags = MachineMemOperand::MOStore;
5534 Info.align.reset();
5535 Infos.push_back(Elt: Info);
5536 return;
5537 }
5538
5539 case Intrinsic::nvvm_tcgen05_st_16x64b_x128:
5540 case Intrinsic::nvvm_tcgen05_st_16x128b_x64:
5541 case Intrinsic::nvvm_tcgen05_st_16x256b_x32:
5542 case Intrinsic::nvvm_tcgen05_st_32x32b_x128:
5543 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x128: {
5544 Info.opc = ISD::INTRINSIC_VOID;
5545 Info.memVT = MVT::v128i32;
5546 Info.ptrVal = I.getArgOperand(i: 0);
5547 Info.offset = 0;
5548 Info.flags = MachineMemOperand::MOStore;
5549 Info.align.reset();
5550 Infos.push_back(Elt: Info);
5551 return;
5552 }
5553 case Intrinsic::
5554 nvvm_tcgen05_mma_shared_f8f6f4_disable_output_lane_cg1_decompress_b:
5555 case Intrinsic::
5556 nvvm_tcgen05_mma_tensor_f8f6f4_disable_output_lane_cg1_decompress_b:
5557 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg1:
5558 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg1:
5559 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg1:
5560 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg1:
5561 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1:
5562 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1:
5563 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1_ashift:
5564 case Intrinsic::
5565 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1_ashift:
5566 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1:
5567 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1:
5568 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1_ashift:
5569 case Intrinsic::
5570 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1_ashift: {
5571 // We are reading and writing back to TMem
5572 Info.opc = ISD::INTRINSIC_VOID;
5573 Info.memVT = MVT::v4i32;
5574 Info.ptrVal = I.getArgOperand(i: 0);
5575 Info.offset = 0;
5576 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
5577 Info.align = Align(16);
5578 Infos.push_back(Elt: Info);
5579 return;
5580 }
5581
5582 case Intrinsic::
5583 nvvm_tcgen05_mma_shared_f8f6f4_disable_output_lane_cg2_decompress_b:
5584 case Intrinsic::
5585 nvvm_tcgen05_mma_tensor_f8f6f4_disable_output_lane_cg2_decompress_b:
5586 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg2:
5587 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg2:
5588 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg2:
5589 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg2:
5590 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2:
5591 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2:
5592 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2:
5593 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2:
5594 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2_ashift:
5595 case Intrinsic::
5596 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2_ashift:
5597 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2_ashift:
5598 case Intrinsic::
5599 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2_ashift: {
5600 // We are reading and writing back to TMem
5601 Info.opc = ISD::INTRINSIC_VOID;
5602 Info.memVT = MVT::v8i32;
5603 Info.ptrVal = I.getArgOperand(i: 0);
5604 Info.offset = 0;
5605 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
5606 Info.align = Align(16);
5607 Infos.push_back(Elt: Info);
5608 return;
5609 }
5610 case Intrinsic::nvvm_tcgen05_alloc_cg1:
5611 case Intrinsic::nvvm_tcgen05_alloc_cg2:
5612 Info.opc = ISD::INTRINSIC_VOID;
5613 Info.memVT = MVT::i32;
5614 Info.ptrVal = I.getArgOperand(i: 0);
5615 Info.offset = 0;
5616 Info.flags = MachineMemOperand::MOStore;
5617 Info.align = Align(4);
5618 Infos.push_back(Elt: Info);
5619 return;
5620 }
5621}
5622
5623// Helper for getting a function parameter symbol. Its name is composed from
5624// the function name and the parameter index. Negative index corresponds to the
5625// special parameter (unsized array) used for passing variable arguments.
5626MCSymbol *NVPTXTargetLowering::getParamSymbol(MCContext &Ctx, const Function *F,
5627 int Idx) const {
5628 const StringRef FuncName = getTargetMachine().getSymbol(GV: F)->getName();
5629 if (Idx < 0)
5630 return Ctx.getOrCreateSymbol(Name: FuncName + "_vararg");
5631 return Ctx.getOrCreateSymbol(Name: FuncName + "_param_" + Twine(Idx));
5632}
5633
5634/// isLegalAddressingMode - Return true if the addressing mode represented
5635/// by AM is legal for this target, for a load/store of the specified type.
5636/// Used to guide target specific optimizations, like loop strength reduction
5637/// (LoopStrengthReduce.cpp) and memory optimization for address mode
5638/// (CodeGenPrepare.cpp)
5639bool NVPTXTargetLowering::isLegalAddressingMode(const DataLayout &DL,
5640 const AddrMode &AM, Type *Ty,
5641 unsigned AS, Instruction *I) const {
5642 // AddrMode - This represents an addressing mode of:
5643 // BaseGV + BaseOffs + BaseReg + Scale*ScaleReg
5644 //
5645 // The legal address modes are
5646 // - [avar]
5647 // - [areg]
5648 // - [areg+immoff]
5649 // - [immAddr]
5650
5651 // immoff must fit in a signed 32-bit int
5652 if (!APInt(64, AM.BaseOffs).isSignedIntN(N: 32))
5653 return false;
5654
5655 if (AM.BaseGV)
5656 return !AM.BaseOffs && !AM.HasBaseReg && !AM.Scale;
5657
5658 switch (AM.Scale) {
5659 case 0: // "r", "r+i" or "i" is allowed
5660 break;
5661 case 1:
5662 if (AM.HasBaseReg) // "r+r+i" or "r+r" is not allowed.
5663 return false;
5664 // Otherwise we have r+i.
5665 break;
5666 default:
5667 // No scale > 1 is allowed
5668 return false;
5669 }
5670 return true;
5671}
5672
5673//===----------------------------------------------------------------------===//
5674// NVPTX Inline Assembly Support
5675//===----------------------------------------------------------------------===//
5676
5677/// getConstraintType - Given a constraint letter, return the type of
5678/// constraint it is for this target.
5679NVPTXTargetLowering::ConstraintType
5680NVPTXTargetLowering::getConstraintType(StringRef Constraint) const {
5681 if (Constraint.size() == 1) {
5682 switch (Constraint[0]) {
5683 default:
5684 break;
5685 case 'b':
5686 case 'r':
5687 case 'h':
5688 case 'c':
5689 case 'l':
5690 case 'f':
5691 case 'd':
5692 case 'q':
5693 case '0':
5694 case 'N':
5695 return C_RegisterClass;
5696 }
5697 }
5698 return TargetLowering::getConstraintType(Constraint);
5699}
5700
5701std::pair<unsigned, const TargetRegisterClass *>
5702NVPTXTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
5703 StringRef Constraint,
5704 MVT VT) const {
5705 if (Constraint.size() == 1) {
5706 switch (Constraint[0]) {
5707 case 'b':
5708 return std::make_pair(x: 0U, y: &NVPTX::B1RegClass);
5709 case 'c':
5710 case 'h':
5711 return std::make_pair(x: 0U, y: &NVPTX::B16RegClass);
5712 case 'r':
5713 case 'f':
5714 return std::make_pair(x: 0U, y: &NVPTX::B32RegClass);
5715 case 'l':
5716 case 'N':
5717 case 'd':
5718 return std::make_pair(x: 0U, y: &NVPTX::B64RegClass);
5719 case 'q': {
5720 if (!STI.hasFeature(Feature: NVPTX::SM70))
5721 report_fatal_error(reason: "Inline asm with 128 bit operands is only "
5722 "supported for sm_70 and higher!");
5723 return std::make_pair(x: 0U, y: &NVPTX::B128RegClass);
5724 }
5725 }
5726 }
5727 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
5728}
5729
5730//===----------------------------------------------------------------------===//
5731// NVPTX DAG Combining
5732//===----------------------------------------------------------------------===//
5733
5734bool NVPTXTargetLowering::allowFMA(MachineFunction &MF,
5735 CodeGenOptLevel OptLevel) const {
5736 // Always honor command-line argument
5737 if (FMAContractLevelOpt.getNumOccurrences() > 0)
5738 return FMAContractLevelOpt > 0;
5739
5740 // Do not contract if we're not optimizing the code.
5741 if (OptLevel == CodeGenOptLevel::None)
5742 return false;
5743
5744 return false;
5745}
5746
5747static bool isConstZero(const SDValue &Operand) {
5748 const auto *Const = dyn_cast<ConstantSDNode>(Val: Operand);
5749 return Const && Const->getZExtValue() == 0;
5750}
5751
5752/// PerformADDCombineWithOperands - Try DAG combinations for an ADD with
5753/// operands N0 and N1. This is a helper for PerformADDCombine that is
5754/// called with the default operands, and if that fails, with commuted
5755/// operands.
5756static SDValue
5757PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
5758 TargetLowering::DAGCombinerInfo &DCI) {
5759 EVT VT = N0.getValueType();
5760
5761 // Since integer multiply-add costs the same as integer multiply
5762 // but is more costly than integer add, do the fusion only when
5763 // the mul is only used in the add.
5764 // TODO: this may not be true for later architectures, consider relaxing this
5765 if (!N0.getNode()->hasOneUse())
5766 return SDValue();
5767
5768 // fold (add (select cond, 0, (mul a, b)), c)
5769 // -> (select cond, c, (add (mul a, b), c))
5770 //
5771 if (N0.getOpcode() == ISD::SELECT) {
5772 unsigned ZeroOpNum;
5773 if (isConstZero(Operand: N0->getOperand(Num: 1)))
5774 ZeroOpNum = 1;
5775 else if (isConstZero(Operand: N0->getOperand(Num: 2)))
5776 ZeroOpNum = 2;
5777 else
5778 return SDValue();
5779
5780 SDValue M = N0->getOperand(Num: (ZeroOpNum == 1) ? 2 : 1);
5781 if (M->getOpcode() != ISD::MUL || !M.getNode()->hasOneUse())
5782 return SDValue();
5783
5784 SDLoc DL(N);
5785 SDValue Mul =
5786 DCI.DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: M->getOperand(Num: 0), N2: M->getOperand(Num: 1));
5787 SDValue MAD = DCI.DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Mul, N2: N1);
5788 return DCI.DAG.getSelect(DL: SDLoc(N), VT, Cond: N0->getOperand(Num: 0),
5789 LHS: ((ZeroOpNum == 1) ? N1 : MAD),
5790 RHS: ((ZeroOpNum == 1) ? MAD : N1));
5791 }
5792
5793 return SDValue();
5794}
5795
5796SDValue NVPTXTargetLowering::performFADDCombineWithOperands(
5797 SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI,
5798 CodeGenOptLevel OptLevel) const {
5799 EVT VT = N0.getValueType();
5800 if (N0.getOpcode() == ISD::FMUL) {
5801 if (!(allowFMA(MF&: DCI.DAG.getMachineFunction(), OptLevel) ||
5802 (N->getFlags().hasAllowContract() &&
5803 N0->getFlags().hasAllowContract())))
5804 return SDValue();
5805
5806 // For floating point:
5807 // Do the fusion only when the mul has less than 5 uses and all
5808 // are add.
5809 // The heuristic is that if a use is not an add, then that use
5810 // cannot be fused into fma, therefore mul is still needed anyway.
5811 // If there are more than 4 uses, even if they are all add, fusing
5812 // them will increase register pressue.
5813 //
5814 int numUses = 0;
5815 int nonAddCount = 0;
5816 for (const SDNode *User : N0.getNode()->users()) {
5817 numUses++;
5818 if (User->getOpcode() != ISD::FADD)
5819 ++nonAddCount;
5820 if (numUses >= 5)
5821 return SDValue();
5822 }
5823 if (nonAddCount) {
5824 int orderNo = N->getIROrder();
5825 int orderNo2 = N0.getNode()->getIROrder();
5826 // simple heuristics here for considering potential register
5827 // pressure, the logics here is that the differnce are used
5828 // to measure the distance between def and use, the longer distance
5829 // more likely cause register pressure.
5830 if (orderNo - orderNo2 < 500)
5831 return SDValue();
5832
5833 // Now, check if at least one of the FMUL's operands is live beyond the
5834 // node N, which guarantees that the FMA will not increase register
5835 // pressure at node N.
5836 bool opIsLive = false;
5837 const SDNode *left = N0.getOperand(i: 0).getNode();
5838 const SDNode *right = N0.getOperand(i: 1).getNode();
5839
5840 if (isa<ConstantSDNode>(Val: left) || isa<ConstantSDNode>(Val: right))
5841 opIsLive = true;
5842
5843 if (!opIsLive)
5844 for (const SDNode *User : left->users()) {
5845 int orderNo3 = User->getIROrder();
5846 if (orderNo3 > orderNo) {
5847 opIsLive = true;
5848 break;
5849 }
5850 }
5851
5852 if (!opIsLive)
5853 for (const SDNode *User : right->users()) {
5854 int orderNo3 = User->getIROrder();
5855 if (orderNo3 > orderNo) {
5856 opIsLive = true;
5857 break;
5858 }
5859 }
5860
5861 if (!opIsLive)
5862 return SDValue();
5863 }
5864
5865 return DCI.DAG.getNode(Opcode: ISD::FMA, DL: SDLoc(N), VT, N1: N0.getOperand(i: 0),
5866 N2: N0.getOperand(i: 1), N3: N1);
5867 }
5868
5869 return SDValue();
5870}
5871
5872/// Fold unpacking movs into a load by increasing the number of return values.
5873///
5874/// ex:
5875/// L: v2f16,ch = load <p>
5876/// a: f16 = extractelt L:0, 0
5877/// b: f16 = extractelt L:0, 1
5878/// use(a, b)
5879///
5880/// ...is turned into...
5881///
5882/// L: f16,f16,ch = LoadV2 <p>
5883/// use(L:0, L:1)
5884static SDValue
5885combineUnpackingMovIntoLoad(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
5886 // Don't run this optimization before the legalizer
5887 if (!DCI.isAfterLegalizeDAG())
5888 return SDValue();
5889
5890 EVT ElementVT = N->getValueType(ResNo: 0);
5891 // Avoid non-packed types and v4i8
5892 if (!NVPTX::isPackedVectorTy(VT: ElementVT) || ElementVT == MVT::v4i8)
5893 return SDValue();
5894
5895 // Check whether all outputs are either used by an extractelt or are
5896 // glue/chain nodes
5897 if (!all_of(Range: N->uses(), P: [&](SDUse &U) {
5898 // Skip glue, chain nodes
5899 if (U.getValueType() == MVT::Glue || U.getValueType() == MVT::Other)
5900 return true;
5901 if (U.getUser()->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
5902 if (N->getOpcode() != ISD::LOAD)
5903 return true;
5904 // Since this is an ISD::LOAD, check all extractelts are used. If
5905 // any are not used, we don't want to defeat another optimization that
5906 // will narrow the load.
5907 //
5908 // For example:
5909 //
5910 // L: v2f16,ch = load <p>
5911 // e0: f16 = extractelt L:0, 0
5912 // e1: f16 = extractelt L:0, 1 <-- unused
5913 // store e0
5914 //
5915 // Can be optimized by DAGCombiner to:
5916 //
5917 // L: f16,ch = load <p>
5918 // store L:0
5919 return !U.getUser()->use_empty();
5920 }
5921
5922 // Otherwise, this use prevents us from splitting a value.
5923 return false;
5924 }))
5925 return SDValue();
5926
5927 auto *LD = cast<MemSDNode>(Val: N);
5928 SDLoc DL(LD);
5929
5930 // the new opcode after we double the number of operands
5931 unsigned Opcode;
5932 SmallVector<SDValue> Operands(LD->ops());
5933 unsigned OldNumOutputs; // non-glue, non-chain outputs
5934 switch (LD->getOpcode()) {
5935 case ISD::LOAD:
5936 OldNumOutputs = 1;
5937 // Any packed type is legal, so the legalizer will not have lowered
5938 // ISD::LOAD -> NVPTXISD::Load (unless it's under-aligned). We have to do it
5939 // here.
5940 Opcode = NVPTXISD::LoadV2;
5941 // append a "full" used bytes mask operand right before the extension type
5942 // operand, signifying that all bytes are used.
5943 Operands.push_back(Elt: DCI.DAG.getConstant(UINT32_MAX, DL, VT: MVT::i32));
5944 Operands.push_back(Elt: DCI.DAG.getIntPtrConstant(
5945 Val: cast<LoadSDNode>(Val: LD)->getExtensionType(), DL));
5946 break;
5947 case NVPTXISD::LoadV2:
5948 OldNumOutputs = 2;
5949 Opcode = NVPTXISD::LoadV4;
5950 break;
5951 case NVPTXISD::LoadV4:
5952 // V8 is only supported for f32/i32. Don't forget, we're not changing the
5953 // load size here. This is already a 256-bit load.
5954 if (ElementVT != MVT::v2f32 && ElementVT != MVT::v2i32)
5955 return SDValue();
5956 OldNumOutputs = 4;
5957 Opcode = NVPTXISD::LoadV8;
5958 break;
5959 case NVPTXISD::LoadV8:
5960 // PTX doesn't support the next doubling of outputs
5961 return SDValue();
5962 }
5963
5964 // the non-glue, non-chain outputs in the new load
5965 const unsigned NewNumOutputs = OldNumOutputs * 2;
5966 SmallVector<EVT> NewVTs(NewNumOutputs, ElementVT.getVectorElementType());
5967 // add remaining chain and glue values
5968 NewVTs.append(in_start: LD->value_begin() + OldNumOutputs, in_end: LD->value_end());
5969
5970 // Create the new load
5971 SDValue NewLoad = DCI.DAG.getMemIntrinsicNode(
5972 Opcode, dl: DL, VTList: DCI.DAG.getVTList(VTs: NewVTs), Ops: Operands, MemVT: LD->getMemoryVT(),
5973 MMO: LD->getMemOperand());
5974
5975 // Now we use a combination of BUILD_VECTORs and a MERGE_VALUES node to keep
5976 // the outputs the same. These nodes will be optimized away in later
5977 // DAGCombiner iterations.
5978 SmallVector<SDValue> Results;
5979 for (unsigned I : seq(Size: OldNumOutputs))
5980 Results.push_back(Elt: DCI.DAG.getBuildVector(
5981 VT: ElementVT, DL, Ops: {NewLoad.getValue(R: I * 2), NewLoad.getValue(R: I * 2 + 1)}));
5982 // Add remaining chain and glue nodes
5983 for (unsigned I : seq(Size: NewLoad->getNumValues() - NewNumOutputs))
5984 Results.push_back(Elt: NewLoad.getValue(R: NewNumOutputs + I));
5985
5986 return DCI.DAG.getMergeValues(Ops: Results, dl: DL);
5987}
5988
5989/// Fold packing movs into a store.
5990///
5991/// ex:
5992/// v1: v2f16 = BUILD_VECTOR a:f16, b:f16
5993/// v2: v2f16 = BUILD_VECTOR c:f16, d:f16
5994/// StoreV2 v1, v2
5995///
5996/// ...is turned into...
5997///
5998/// StoreV4 a, b, c, d
5999static SDValue combinePackingMovIntoStore(SDNode *N,
6000 TargetLowering::DAGCombinerInfo &DCI,
6001 unsigned Front, unsigned Back) {
6002 // We want to run this as late as possible since other optimizations may
6003 // eliminate the BUILD_VECTORs.
6004 if (!DCI.isAfterLegalizeDAG())
6005 return SDValue();
6006
6007 // Get the type of the operands being stored.
6008 EVT ElementVT = N->getOperand(Num: Front).getValueType();
6009
6010 // Avoid non-packed types and v4i8
6011 if (!NVPTX::isPackedVectorTy(VT: ElementVT) || ElementVT == MVT::v4i8)
6012 return SDValue();
6013
6014 auto *ST = cast<MemSDNode>(Val: N);
6015
6016 // The new opcode after we double the number of operands.
6017 unsigned Opcode;
6018 switch (N->getOpcode()) {
6019 case ISD::STORE:
6020 // Any packed type is legal, so the legalizer will not have lowered
6021 // ISD::STORE -> NVPTXISD::Store (unless it's under-aligned). We have to do
6022 // it here.
6023 Opcode = NVPTXISD::StoreV2;
6024 break;
6025 case NVPTXISD::StoreV2:
6026 Opcode = NVPTXISD::StoreV4;
6027 break;
6028 case NVPTXISD::StoreV4:
6029 // V8 is only supported for f32/i32. Don't forget, we're not changing the
6030 // store size here. This is already a 256-bit store.
6031 if (ElementVT != MVT::v2f32 && ElementVT != MVT::v2i32)
6032 return SDValue();
6033 Opcode = NVPTXISD::StoreV8;
6034 break;
6035 case NVPTXISD::StoreV8:
6036 // PTX doesn't support the next doubling of operands
6037 return SDValue();
6038 default:
6039 llvm_unreachable("Unhandled store opcode");
6040 }
6041
6042 // Scan the operands and if they're all BUILD_VECTORs, we'll have gathered
6043 // their elements.
6044 SmallVector<SDValue, 4> Operands(N->ops().take_front(N: Front));
6045 for (SDValue BV : N->ops().drop_front(N: Front).drop_back(N: Back)) {
6046 if (BV.getOpcode() != ISD::BUILD_VECTOR)
6047 return SDValue();
6048
6049 // If the operand has multiple uses, this optimization can increase register
6050 // pressure.
6051 if (!BV.hasOneUse())
6052 return SDValue();
6053
6054 // DAGCombiner visits nodes bottom-up. Check the BUILD_VECTOR operands for
6055 // any signs they may be folded by some other pattern or rule.
6056 for (SDValue Op : BV->ops()) {
6057 // Peek through bitcasts
6058 if (Op.getOpcode() == ISD::BITCAST)
6059 Op = Op.getOperand(i: 0);
6060
6061 // This may be folded into a PRMT.
6062 if (Op.getValueType() == MVT::i16 && Op.getOpcode() == ISD::TRUNCATE &&
6063 Op->getOperand(Num: 0).getValueType() == MVT::i32)
6064 return SDValue();
6065
6066 // This may be folded into cvt.bf16x2
6067 if (Op.getOpcode() == ISD::FP_ROUND)
6068 return SDValue();
6069 }
6070 Operands.append(IL: {BV.getOperand(i: 0), BV.getOperand(i: 1)});
6071 }
6072 Operands.append(in_start: N->op_end() - Back, in_end: N->op_end());
6073
6074 // Now we replace the store
6075 return DCI.DAG.getMemIntrinsicNode(Opcode, dl: SDLoc(N), VTList: N->getVTList(), Ops: Operands,
6076 MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand());
6077}
6078
6079static SDValue combineSTORE(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
6080 const NVPTXSubtarget &STI) {
6081
6082 if (DCI.isBeforeLegalize() && N->getOpcode() == ISD::STORE) {
6083 // Here is our chance to custom lower a store with a non-simple type.
6084 // Unfortunately, we can't do this in the legalizer because there is no
6085 // way to setOperationAction for an non-simple type.
6086 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
6087 if (!ST->getValue().getValueType().isSimple())
6088 return lowerSTOREVector(Op: SDValue(ST, 0), DAG&: DCI.DAG, STI);
6089 }
6090
6091 return combinePackingMovIntoStore(N, DCI, Front: 1, Back: 2);
6092}
6093
6094static SDValue combineLOAD(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
6095 const NVPTXSubtarget &STI) {
6096 if (DCI.isBeforeLegalize() && N->getOpcode() == ISD::LOAD) {
6097 // Here is our chance to custom lower a load with a non-simple type.
6098 // Unfortunately, we can't do this in the legalizer because there is no
6099 // way to setOperationAction for an non-simple type.
6100 if (!N->getValueType(ResNo: 0).isSimple())
6101 return lowerLoadVector(N, DAG&: DCI.DAG, STI);
6102 }
6103
6104 return combineUnpackingMovIntoLoad(N, DCI);
6105}
6106
6107/// PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
6108///
6109static SDValue PerformADDCombine(SDNode *N,
6110 TargetLowering::DAGCombinerInfo &DCI,
6111 CodeGenOptLevel OptLevel) {
6112 if (OptLevel == CodeGenOptLevel::None)
6113 return SDValue();
6114
6115 SDValue N0 = N->getOperand(Num: 0);
6116 SDValue N1 = N->getOperand(Num: 1);
6117
6118 // Skip non-integer, non-scalar case
6119 EVT VT = N0.getValueType();
6120 if (VT.isVector() || VT != MVT::i32)
6121 return SDValue();
6122
6123 // First try with the default operand order.
6124 if (SDValue Result = PerformADDCombineWithOperands(N, N0, N1, DCI))
6125 return Result;
6126
6127 // If that didn't work, try again with the operands commuted.
6128 return PerformADDCombineWithOperands(N, N0: N1, N1: N0, DCI);
6129}
6130
6131/// Check if a v2f32 BUILD_VECTOR provably packs values from non-adjacent
6132/// register pairs (non-coalescable).
6133static bool isNonCoalescableBuildVector(const SDValue &BV) {
6134 if (BV.getOpcode() != ISD::BUILD_VECTOR || BV.getValueType() != MVT::v2f32)
6135 return false;
6136
6137 SDValue Elt0 = BV.getOperand(i: 0);
6138 SDValue Elt1 = BV.getOperand(i: 1);
6139
6140 bool IsExt0 = Elt0.getOpcode() == ISD::EXTRACT_VECTOR_ELT;
6141 bool IsExt1 = Elt1.getOpcode() == ISD::EXTRACT_VECTOR_ELT;
6142
6143 // If neither element is an EXTRACT_VECTOR_ELT they are free-standing
6144 // scalars and the register allocator can still place them side-by-side.
6145 if (!IsExt0 && !IsExt1)
6146 return false;
6147
6148 // If exactly one element is an EXTRACT_VECTOR_ELT, the other is a scalar
6149 // that cannot generally occupy the adjacent register slot.
6150 if (IsExt0 != IsExt1)
6151 return true;
6152
6153 // At this point both sources are extracting from vectors. If they are from
6154 // different vectors, then the BUILD_VECTOR is non-coalescable.
6155 SDValue Src0 = Elt0.getOperand(i: 0);
6156 SDValue Src1 = Elt1.getOperand(i: 0);
6157 if (Src0 != Src1)
6158 return true;
6159
6160 auto *Idx0 = dyn_cast<ConstantSDNode>(Val: Elt0.getOperand(i: 1));
6161 auto *Idx1 = dyn_cast<ConstantSDNode>(Val: Elt1.getOperand(i: 1));
6162 // If both indices are dynamic they will be lowered to
6163 // loads and the vector will be spilled to local memory. The register
6164 // allocator can easily place the results in adjacent registers.
6165 if (!Idx0 && !Idx1)
6166 return false;
6167
6168 // If one index is dynamic and the other is constant, the value from the
6169 // constant load will result in an additional register to pair with the result
6170 // from the dynamic load. We consider this non-coalescable.
6171 if ((Idx0 && !Idx1) || (!Idx0 && Idx1))
6172 return true;
6173
6174 // Both are constant, adjacent pairs are coalescable
6175 return std::abs(i: Idx0->getSExtValue() - Idx1->getSExtValue()) != 1;
6176}
6177
6178/// Return true if FMUL v2f32 node \p N may be scalarized to fold each lane's
6179/// product into a scalar FMA.
6180bool NVPTXTargetLowering::mayFoldFMULIntoFMA(SDNode *N, MachineFunction &MF,
6181 CodeGenOptLevel OptLevel) const {
6182 if (N->getOpcode() != ISD::FMUL || N->getValueType(ResNo: 0) != MVT::v2f32)
6183 return false;
6184 const bool GlobalFMA = allowFMA(MF, OptLevel);
6185 if (!N->getFlags().hasAllowContract() && !GlobalFMA)
6186 return false;
6187
6188 const SDNode *FirstFAdd = nullptr;
6189 unsigned NumScalarFAdd = 0;
6190
6191 // Both lanes must feed unique FADDs
6192 for (SDNode *EE : N->users()) {
6193 if (NumScalarFAdd == 2)
6194 return false;
6195
6196 if (EE->getOpcode() != ISD::EXTRACT_VECTOR_ELT || !EE->hasOneUse() ||
6197 !isa<ConstantSDNode>(Val: EE->getOperand(Num: 1)))
6198 return false;
6199
6200 const SDNode *const FAdd = *EE->users().begin();
6201 if (FAdd->getOpcode() != ISD::FADD ||
6202 (!GlobalFMA && !FAdd->getFlags().hasAllowContract()))
6203 return false;
6204
6205 if (!FirstFAdd)
6206 FirstFAdd = FAdd;
6207 else if (FAdd == FirstFAdd)
6208 return false;
6209
6210 NumScalarFAdd++;
6211 }
6212
6213 return NumScalarFAdd == 2;
6214}
6215
6216/// Scalarize a v2f32 arithmetic node (FADD, FMUL, FSUB, FMA) when at least
6217/// one operand is a BUILD_VECTOR that repacks values from non-adjacent register
6218/// pairs. Without this combine the BUILD_VECTOR forces allocation of a
6219/// temporary 64-bit register, increasing register pressure.
6220///
6221/// Example - before:
6222/// t0: v2f32,v2f32,ch = LoadV2 ...
6223/// t1: f32 = extract_vector_elt t0, 0
6224/// t2: f32 = extract_vector_elt t0:1, 0
6225/// t3: v2f32 = BUILD_VECTOR t1, t2 ;; non-coalescable repack
6226/// t4: v2f32 = fma t_a, t3, t_c
6227///
6228/// After:
6229/// t0: v2f32,v2f32,ch = LoadV2 ...
6230/// t1: f32 = extract_vector_elt t0, 0
6231/// t2: f32 = extract_vector_elt t0:1, 0
6232/// a0: f32 = extract_vector_elt t_a, 0
6233/// a1: f32 = extract_vector_elt t_a, 1
6234/// c0: f32 = extract_vector_elt t_c, 0
6235/// c1: f32 = extract_vector_elt t_c, 1
6236/// r0: f32 = fma a0, t1, c0
6237/// r1: f32 = fma a1, t2, c1
6238/// t4: v2f32 = BUILD_VECTOR r0, r1
6239///
6240/// Also scalarizes an FMUL when all output lanes feed into scalar FADDs
6241/// to enable scalar FMA combining.
6242SDValue NVPTXTargetLowering::performScalarizeV2F32Op(
6243 SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
6244 CodeGenOptLevel OptLevel) const {
6245 EVT VT = N->getValueType(ResNo: 0);
6246 if (VT != MVT::v2f32)
6247 return SDValue();
6248
6249 if (none_of(Range: N->ops(), P: isNonCoalescableBuildVector) &&
6250 !mayFoldFMULIntoFMA(N, MF&: DCI.DAG.getMachineFunction(), OptLevel))
6251 return SDValue();
6252
6253 SelectionDAG &DAG = DCI.DAG;
6254 SDLoc DL(N);
6255 EVT EltVT = VT.getVectorElementType();
6256 unsigned Opc = N->getOpcode();
6257
6258 // For each operand, get the scalar element at the given index: if the operand
6259 // is a BUILD_VECTOR, grab the element directly; otherwise, emit an
6260 // EXTRACT_VECTOR_ELT.
6261 auto GetElement = [&](SDValue Op, unsigned Index) -> SDValue {
6262 if (Op.getOpcode() == ISD::BUILD_VECTOR)
6263 return Op.getOperand(i: Index);
6264 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Op,
6265 N2: DAG.getVectorIdxConstant(Val: Index, DL));
6266 };
6267
6268 // Build scalar operand lists for element 0 and element 1.
6269 SmallVector<SDValue, 3> Ops0, Ops1;
6270 for (const SDValue &Op : N->ops()) {
6271 Ops0.push_back(Elt: GetElement(Op, 0));
6272 Ops1.push_back(Elt: GetElement(Op, 1));
6273 }
6274
6275 SDValue Res0 = DAG.getNode(Opcode: Opc, DL, VT: EltVT, Ops: Ops0, Flags: N->getFlags());
6276 SDValue Res1 = DAG.getNode(Opcode: Opc, DL, VT: EltVT, Ops: Ops1, Flags: N->getFlags());
6277
6278 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT, N1: Res0, N2: Res1);
6279}
6280
6281/// Target-specific dag combine xforms for ISD::FADD.
6282SDValue
6283NVPTXTargetLowering::performFADDCombine(SDNode *N,
6284 TargetLowering::DAGCombinerInfo &DCI,
6285 CodeGenOptLevel OptLevel) const {
6286 if (SDValue Result = performScalarizeV2F32Op(N, DCI, OptLevel))
6287 return Result;
6288
6289 SDValue N0 = N->getOperand(Num: 0);
6290 SDValue N1 = N->getOperand(Num: 1);
6291
6292 EVT VT = N0.getValueType();
6293 if (VT.isVector() || !(VT == MVT::f32 || VT == MVT::f64))
6294 return SDValue();
6295
6296 // First try with the default operand order.
6297 if (SDValue Result = performFADDCombineWithOperands(N, N0, N1, DCI, OptLevel))
6298 return Result;
6299
6300 // If that didn't work, try again with the operands commuted.
6301 return performFADDCombineWithOperands(N, N0: N1, N1: N0, DCI, OptLevel);
6302}
6303
6304/// Get 3-input version of a 2-input min/max opcode
6305static unsigned getMinMax3Opcode(unsigned MinMax2Opcode) {
6306 switch (MinMax2Opcode) {
6307 case ISD::FMAXNUM:
6308 case ISD::FMAXIMUMNUM:
6309 return NVPTXISD::FMAXNUM3;
6310 case ISD::FMINNUM:
6311 case ISD::FMINIMUMNUM:
6312 return NVPTXISD::FMINNUM3;
6313 case ISD::FMAXIMUM:
6314 return NVPTXISD::FMAXIMUM3;
6315 case ISD::FMINIMUM:
6316 return NVPTXISD::FMINIMUM3;
6317 default:
6318 llvm_unreachable("Invalid 2-input min/max opcode");
6319 }
6320}
6321
6322/// PerformFMinMaxCombine - Combine (fmaxnum (fmaxnum a, b), c) into
6323/// (fmaxnum3 a, b, c). Also covers other llvm min/max intrinsics.
6324static SDValue PerformFMinMaxCombine(SDNode *N,
6325 TargetLowering::DAGCombinerInfo &DCI,
6326 const NVPTXSubtarget &STI) {
6327
6328 // 3-input min/max requires PTX 8.8+ and SM_100+, and only supports f32s
6329 EVT VT = N->getValueType(ResNo: 0);
6330 if (VT != MVT::f32 || !STI.hasFeature(Feature: NVPTX::PTX88) ||
6331 !STI.hasFeature(Feature: NVPTX::SM100))
6332 return SDValue();
6333
6334 SDValue Op0 = N->getOperand(Num: 0);
6335 SDValue Op1 = N->getOperand(Num: 1);
6336 unsigned MinMaxOp2 = N->getOpcode();
6337 unsigned MinMaxOp3 = getMinMax3Opcode(MinMax2Opcode: MinMaxOp2);
6338
6339 if (Op0.getOpcode() == MinMaxOp2 && Op0.hasOneUse()) {
6340 // (maxnum (maxnum a, b), c) -> (maxnum3 a, b, c)
6341 SDValue A = Op0.getOperand(i: 0);
6342 SDValue B = Op0.getOperand(i: 1);
6343 SDValue C = Op1;
6344 return DCI.DAG.getNode(Opcode: MinMaxOp3, DL: SDLoc(N), VT, N1: A, N2: B, N3: C, Flags: N->getFlags());
6345 } else if (Op1.getOpcode() == MinMaxOp2 && Op1.hasOneUse()) {
6346 // (maxnum a, (maxnum b, c)) -> (maxnum3 a, b, c)
6347 SDValue A = Op0;
6348 SDValue B = Op1.getOperand(i: 0);
6349 SDValue C = Op1.getOperand(i: 1);
6350 return DCI.DAG.getNode(Opcode: MinMaxOp3, DL: SDLoc(N), VT, N1: A, N2: B, N3: C, Flags: N->getFlags());
6351 }
6352 return SDValue();
6353}
6354
6355// sext (mul.iN nsw x, y) => mul.wide.sN x, y
6356// zext (mul.iN nuw x, y) => mul.wide.uN x, y
6357// sext (shl.iN nsw x, const) => mul.wide.sN x, (1 << const)
6358// zext (shl.iN nuw x, const) => mul.wide.uN x, (1 << const)
6359static SDValue combineSZExtToMulWide(SDNode *N,
6360 TargetLowering::DAGCombinerInfo &DCI,
6361 CodeGenOptLevel OptLevel) {
6362 assert(N->getOpcode() == ISD::SIGN_EXTEND ||
6363 N->getOpcode() == ISD::ZERO_EXTEND);
6364
6365 if (OptLevel == CodeGenOptLevel::None)
6366 return SDValue();
6367
6368 SDValue Op = N->getOperand(Num: 0);
6369 if (!Op.hasOneUse())
6370 return SDValue();
6371
6372 EVT ToVT = N->getValueType(ResNo: 0);
6373 EVT FromVT = Op.getValueType();
6374 if (!((ToVT == MVT::i32 && FromVT == MVT::i16) ||
6375 (ToVT == MVT::i64 && FromVT == MVT::i32)))
6376 return SDValue();
6377
6378 bool IsSigned = N->getOpcode() == ISD::SIGN_EXTEND;
6379 if ((IsSigned && !Op->getFlags().hasNoSignedWrap()) ||
6380 (!IsSigned && !Op->getFlags().hasNoUnsignedWrap()))
6381 return SDValue();
6382
6383 SDLoc DL(N);
6384 SDValue LHS = Op.getOperand(i: 0);
6385 SDValue RHS = Op.getOperand(i: 1);
6386 unsigned MulWideOpcode =
6387 IsSigned ? NVPTXISD::MUL_WIDE_SIGNED : NVPTXISD::MUL_WIDE_UNSIGNED;
6388 if (Op.getOpcode() == ISD::MUL) {
6389 return DCI.DAG.getNode(Opcode: MulWideOpcode, DL, VT: ToVT, N1: LHS, N2: RHS);
6390 } else if (Op.getOpcode() == ISD::SHL && isa<ConstantSDNode>(Val: RHS)) {
6391 const auto ShiftAmt = Op.getConstantOperandVal(i: 1);
6392 const auto MulVal = APInt(FromVT.getSizeInBits(), 1) << ShiftAmt;
6393
6394 // Note that the sext (shl nsw ...) case doesn't work if 1 << const
6395 // overflows to a negative value! The only valid input values in this
6396 // case are 0 and -1 (all other values yield poison because of the nsw),
6397 // and mul.wide.sN would give us the wrong sign for -1. We could use
6398 // mul.wide.uN, but since this is a weird case anyway, we might as well not
6399 // apply this transformation at all.
6400 if (IsSigned && MulVal.isNegative())
6401 return SDValue();
6402
6403 RHS = DCI.DAG.getConstant(Val: MulVal, DL, VT: FromVT);
6404 return DCI.DAG.getNode(Opcode: MulWideOpcode, DL, VT: ToVT, N1: LHS, N2: RHS);
6405 }
6406
6407 return SDValue();
6408}
6409
6410enum OperandSignedness {
6411 Signed = 0,
6412 Unsigned,
6413 Unknown
6414};
6415
6416/// IsMulWideOperandDemotable - Checks if the provided DAG node is an operand
6417/// that can be demoted to \p OptSize bits without loss of information. The
6418/// signedness of the operand, if determinable, is placed in \p S.
6419static bool IsMulWideOperandDemotable(SDValue Op,
6420 unsigned OptSize,
6421 OperandSignedness &S) {
6422 S = Unknown;
6423
6424 if (Op.getOpcode() == ISD::SIGN_EXTEND ||
6425 Op.getOpcode() == ISD::SIGN_EXTEND_INREG) {
6426 EVT OrigVT = Op.getOperand(i: 0).getValueType();
6427 if (OrigVT.getFixedSizeInBits() <= OptSize) {
6428 S = Signed;
6429 return true;
6430 }
6431 } else if (Op.getOpcode() == ISD::ZERO_EXTEND) {
6432 EVT OrigVT = Op.getOperand(i: 0).getValueType();
6433 if (OrigVT.getFixedSizeInBits() <= OptSize) {
6434 S = Unsigned;
6435 return true;
6436 }
6437 }
6438
6439 return false;
6440}
6441
6442/// AreMulWideOperandsDemotable - Checks if the given LHS and RHS operands can
6443/// be demoted to \p OptSize bits without loss of information. If the operands
6444/// contain a constant, it should appear as the RHS operand. The signedness of
6445/// the operands is placed in \p IsSigned.
6446static bool AreMulWideOperandsDemotable(SDValue LHS, SDValue RHS,
6447 unsigned OptSize,
6448 bool &IsSigned) {
6449 OperandSignedness LHSSign;
6450
6451 // The LHS operand must be a demotable op
6452 if (!IsMulWideOperandDemotable(Op: LHS, OptSize, S&: LHSSign))
6453 return false;
6454
6455 // We should have been able to determine the signedness from the LHS
6456 if (LHSSign == Unknown)
6457 return false;
6458
6459 IsSigned = (LHSSign == Signed);
6460
6461 // The RHS can be a demotable op or a constant
6462 if (ConstantSDNode *CI = dyn_cast<ConstantSDNode>(Val&: RHS)) {
6463 const APInt &Val = CI->getAPIntValue();
6464 if (LHSSign == Unsigned) {
6465 return Val.isIntN(N: OptSize);
6466 } else {
6467 return Val.isSignedIntN(N: OptSize);
6468 }
6469 } else {
6470 OperandSignedness RHSSign;
6471 if (!IsMulWideOperandDemotable(Op: RHS, OptSize, S&: RHSSign))
6472 return false;
6473
6474 return LHSSign == RHSSign;
6475 }
6476}
6477
6478/// TryMULWIDECombine - Attempt to replace a multiply of M bits with a multiply
6479/// of M/2 bits that produces an M-bit result (i.e. mul.wide). This transform
6480/// works on both multiply DAG nodes and SHL DAG nodes with a constant shift
6481/// amount.
6482static SDValue TryMULWIDECombine(SDNode *N,
6483 TargetLowering::DAGCombinerInfo &DCI) {
6484 EVT MulType = N->getValueType(ResNo: 0);
6485 if (MulType != MVT::i32 && MulType != MVT::i64) {
6486 return SDValue();
6487 }
6488
6489 SDLoc DL(N);
6490 unsigned OptSize = MulType.getSizeInBits() >> 1;
6491 SDValue LHS = N->getOperand(Num: 0);
6492 SDValue RHS = N->getOperand(Num: 1);
6493
6494 // Canonicalize the multiply so the constant (if any) is on the right
6495 if (N->getOpcode() == ISD::MUL) {
6496 if (isa<ConstantSDNode>(Val: LHS)) {
6497 std::swap(a&: LHS, b&: RHS);
6498 }
6499 }
6500
6501 // If we have a SHL, determine the actual multiply amount
6502 if (N->getOpcode() == ISD::SHL) {
6503 ConstantSDNode *ShlRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
6504 if (!ShlRHS) {
6505 return SDValue();
6506 }
6507
6508 APInt ShiftAmt = ShlRHS->getAPIntValue();
6509 unsigned BitWidth = MulType.getSizeInBits();
6510 if (ShiftAmt.sge(RHS: 0) && ShiftAmt.slt(RHS: BitWidth)) {
6511 APInt MulVal = APInt(BitWidth, 1) << ShiftAmt;
6512 RHS = DCI.DAG.getConstant(Val: MulVal, DL, VT: MulType);
6513 } else {
6514 return SDValue();
6515 }
6516 }
6517
6518 bool Signed;
6519 // Verify that our operands are demotable
6520 if (!AreMulWideOperandsDemotable(LHS, RHS, OptSize, IsSigned&: Signed)) {
6521 return SDValue();
6522 }
6523
6524 EVT DemotedVT;
6525 if (MulType == MVT::i32) {
6526 DemotedVT = MVT::i16;
6527 } else {
6528 DemotedVT = MVT::i32;
6529 }
6530
6531 // Truncate the operands to the correct size. Note that these are just for
6532 // type consistency and will (likely) be eliminated in later phases.
6533 SDValue TruncLHS =
6534 DCI.DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DemotedVT, Operand: LHS);
6535 SDValue TruncRHS =
6536 DCI.DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DemotedVT, Operand: RHS);
6537
6538 unsigned Opc;
6539 if (Signed) {
6540 Opc = NVPTXISD::MUL_WIDE_SIGNED;
6541 } else {
6542 Opc = NVPTXISD::MUL_WIDE_UNSIGNED;
6543 }
6544
6545 return DCI.DAG.getNode(Opcode: Opc, DL, VT: MulType, N1: TruncLHS, N2: TruncRHS);
6546}
6547
6548static bool isConstOne(const SDValue &Operand) {
6549 const auto *Const = dyn_cast<ConstantSDNode>(Val: Operand);
6550 return Const && Const->getZExtValue() == 1;
6551}
6552
6553static SDValue matchMADConstOnePattern(SDValue Add) {
6554 if (Add->getOpcode() != ISD::ADD)
6555 return SDValue();
6556
6557 if (isConstOne(Operand: Add->getOperand(Num: 0)))
6558 return Add->getOperand(Num: 1);
6559
6560 if (isConstOne(Operand: Add->getOperand(Num: 1)))
6561 return Add->getOperand(Num: 0);
6562
6563 return SDValue();
6564}
6565
6566static SDValue combineMADConstOne(SDValue X, SDValue Add, EVT VT, SDLoc DL,
6567 TargetLowering::DAGCombinerInfo &DCI) {
6568
6569 if (SDValue Y = matchMADConstOnePattern(Add)) {
6570 SDValue Mul = DCI.DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: X, N2: Y);
6571 return DCI.DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Mul, N2: X);
6572 }
6573
6574 return SDValue();
6575}
6576
6577static SDValue combineMulSelectConstOne(SDValue X, SDValue Select, EVT VT,
6578 SDLoc DL,
6579 TargetLowering::DAGCombinerInfo &DCI) {
6580 if (Select->getOpcode() != ISD::SELECT)
6581 return SDValue();
6582
6583 SDValue Cond = Select->getOperand(Num: 0);
6584
6585 unsigned ConstOpNo;
6586 if (isConstOne(Operand: Select->getOperand(Num: 1)))
6587 ConstOpNo = 1;
6588 else if (isConstOne(Operand: Select->getOperand(Num: 2)))
6589 ConstOpNo = 2;
6590 else
6591 return SDValue();
6592
6593 SDValue Y = Select->getOperand(Num: (ConstOpNo == 1) ? 2 : 1);
6594
6595 // Do not combine if the resulting sequence is not obviously profitable.
6596 if (!matchMADConstOnePattern(Add: Y))
6597 return SDValue();
6598
6599 SDValue NewMul = DCI.DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: X, N2: Y);
6600
6601 return DCI.DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
6602 N2: (ConstOpNo == 1) ? X : NewMul,
6603 N3: (ConstOpNo == 1) ? NewMul : X);
6604}
6605
6606static SDValue
6607PerformMULCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
6608 TargetLowering::DAGCombinerInfo &DCI) {
6609
6610 EVT VT = N0.getValueType();
6611 if (VT.isVector())
6612 return SDValue();
6613
6614 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)
6615 return SDValue();
6616
6617 SDLoc DL(N);
6618
6619 // (mul x, (add y, 1)) -> (add (mul x, y), x)
6620 if (SDValue Res = combineMADConstOne(X: N0, Add: N1, VT, DL, DCI))
6621 return Res;
6622 if (SDValue Res = combineMADConstOne(X: N1, Add: N0, VT, DL, DCI))
6623 return Res;
6624
6625 // (mul x, (select y, 1)) -> (select (mul x, y), x)
6626 if (SDValue Res = combineMulSelectConstOne(X: N0, Select: N1, VT, DL, DCI))
6627 return Res;
6628 if (SDValue Res = combineMulSelectConstOne(X: N1, Select: N0, VT, DL, DCI))
6629 return Res;
6630
6631 return SDValue();
6632}
6633
6634/// PerformMULCombine - Runs PTX-specific DAG combine patterns on MUL nodes.
6635static SDValue PerformMULCombine(SDNode *N,
6636 TargetLowering::DAGCombinerInfo &DCI,
6637 CodeGenOptLevel OptLevel) {
6638 if (OptLevel == CodeGenOptLevel::None)
6639 return SDValue();
6640
6641 if (SDValue Ret = TryMULWIDECombine(N, DCI))
6642 return Ret;
6643
6644 SDValue N0 = N->getOperand(Num: 0);
6645 SDValue N1 = N->getOperand(Num: 1);
6646 return PerformMULCombineWithOperands(N, N0, N1, DCI);
6647}
6648
6649/// Commute SHL with a bitwise logic operation when doing so exposes a common
6650/// shifted operand. For example:
6651///
6652/// Before:
6653/// N = shl (zext (LogicOp X, C)), ShiftAmount
6654/// OtherShift = shl (zext (OtherLogicOp X, OtherC)), ShiftAmount
6655///
6656/// After:
6657/// ShiftedX = shl (zext X), ShiftAmount
6658/// N = LogicOp ShiftedX, ShiftedC
6659/// OtherShift = OtherLogicOp ShiftedX, ShiftedOtherC
6660///
6661/// ShiftedC = (zext C) << ShiftAmount and ShiftedOtherC =
6662/// (zext OtherC) << ShiftAmount are folded constants. This replaces two
6663/// variable shifts with the single shared ShiftedX. Requiring another matching
6664/// shift avoids disrupting isolated address calculations where a shift may be
6665/// folded into the addressing mode.
6666static SDValue combineShiftOfLogicOp(SDNode *N,
6667 TargetLowering::DAGCombinerInfo &DCI) {
6668 using namespace SDPatternMatch;
6669
6670 struct ShiftOfLogicOp {
6671 SDNode *Shift;
6672 SDValue LogicOp;
6673 SDValue X;
6674 SDValue Constant;
6675 unsigned ExtendOpcode;
6676 };
6677
6678 // Match a logic operation, with an optional extension, inside a SHL.
6679 auto matchShiftOfLogicOp =
6680 [&](SDNode *Shift) -> std::optional<ShiftOfLogicOp> {
6681 if (Shift->getOpcode() != ISD::SHL || !Shift->getOperand(Num: 0).hasOneUse())
6682 return std::nullopt;
6683 ShiftOfLogicOp Match;
6684 Match.Shift = Shift;
6685 Match.LogicOp = Shift->getOperand(Num: 0);
6686 Match.ExtendOpcode = 0;
6687 if (ISD::isExtOpcode(Opcode: Match.LogicOp.getOpcode())) {
6688 Match.ExtendOpcode = Match.LogicOp.getOpcode();
6689 Match.LogicOp = Match.LogicOp.getOperand(i: 0);
6690 }
6691
6692 if (!sd_match(N: Match.LogicOp, P: m_OneUse(P: m_BitwiseLogic(
6693 L: m_Value(N&: Match.X),
6694 R: m_Value(N&: Match.Constant, P: m_ConstInt())))))
6695 return std::nullopt;
6696
6697 return Match;
6698 };
6699
6700 // Match N as the root shift-of-logic; bail if it does not fit the pattern.
6701 const std::optional<ShiftOfLogicOp> Root = matchShiftOfLogicOp(N);
6702 if (!Root)
6703 return SDValue();
6704
6705 // Only profitable for a constant shift amount: the per-op constant shift then
6706 // folds away instead of becoming an extra variable shift.
6707 if (!isConstOrConstSplat(N: N->getOperand(Num: 1)))
6708 return SDValue();
6709
6710 // Collect candidate shifts that share X. Reached through another user of X,
6711 // the logic result feeds the shift directly or through an optional extend.
6712 SmallVector<SDNode *, 4> CandidateShifts;
6713 for (const SDNode *CandidateLogicOp : Root->X->users()) {
6714 if (CandidateLogicOp == Root->LogicOp.getNode())
6715 continue;
6716 for (SDNode *LogicUser : CandidateLogicOp->users()) {
6717 if (ISD::isExtOpcode(Opcode: LogicUser->getOpcode())) {
6718 // shl (ext (logic X, C)): step through the extend to find the shift.
6719 for (SDNode *ExtendUser : LogicUser->users())
6720 if (ExtendUser->getOpcode() == ISD::SHL)
6721 CandidateShifts.push_back(Elt: ExtendUser);
6722 } else if (LogicUser->getOpcode() == ISD::SHL) {
6723 // shl (logic X, C): the user is already the shift.
6724 CandidateShifts.push_back(Elt: LogicUser);
6725 }
6726 }
6727 }
6728
6729 // Verify each candidate against the root's pattern: the same X, extension,
6730 // type, and shift amount.
6731 const EVT VT = N->getValueType(ResNo: 0);
6732 const SDValue ShiftAmount = N->getOperand(Num: 1);
6733 SmallVector<ShiftOfLogicOp, 4> Matches;
6734 for (SDNode *CandidateShift : CandidateShifts) {
6735 const std::optional<ShiftOfLogicOp> Candidate =
6736 matchShiftOfLogicOp(CandidateShift);
6737 if (Candidate && Candidate->X == Root->X &&
6738 Candidate->ExtendOpcode == Root->ExtendOpcode &&
6739 CandidateShift->getValueType(ResNo: 0) == VT &&
6740 CandidateShift->getOperand(Num: 1) == ShiftAmount)
6741 Matches.push_back(Elt: *Candidate);
6742 }
6743 if (Matches.empty())
6744 return SDValue();
6745
6746 // Build the shared shifted X once, then rewrite the root and every match
6747 // into a logic op over it so the shift is CSE'd.
6748 SelectionDAG &DAG = DCI.DAG;
6749 const SDValue ShiftedX =
6750 DAG.getNode(Opcode: ISD::SHL, DL: SDLoc(N), VT,
6751 N1: Root->ExtendOpcode
6752 ? DAG.getNode(Opcode: Root->ExtendOpcode, DL: SDLoc(N), VT, Operand: Root->X)
6753 : Root->X,
6754 N2: ShiftAmount);
6755
6756 // Rebuild the logic op from shared ShiftedX and a folded constant shift.
6757 auto buildCommutedLogicOp = [&](const SDValue LogicOp, SDValue C,
6758 const SDLoc &DL) {
6759 if (Root->ExtendOpcode)
6760 C = DAG.getNode(Opcode: Root->ExtendOpcode, DL, VT, Operand: C);
6761 const SDValue ShiftedC = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: C, N2: ShiftAmount);
6762 return DAG.getNode(Opcode: LogicOp.getOpcode(), DL, VT, N1: ShiftedX, N2: ShiftedC,
6763 Flags: LogicOp->getFlags());
6764 };
6765
6766 for (const ShiftOfLogicOp &Match : Matches)
6767 DCI.CombineTo(N: Match.Shift,
6768 Res: buildCommutedLogicOp(Match.LogicOp, Match.Constant,
6769 SDLoc(Match.Shift)));
6770 return buildCommutedLogicOp(Root->LogicOp, Root->Constant, SDLoc(N));
6771}
6772
6773/// PerformSHLCombine - Runs PTX-specific DAG combine patterns on SHL nodes.
6774static SDValue PerformSHLCombine(SDNode *N,
6775 TargetLowering::DAGCombinerInfo &DCI,
6776 CodeGenOptLevel OptLevel) {
6777 if (OptLevel > CodeGenOptLevel::None) {
6778 // Expose a shared shifted operand for CSE before mul.wide folding, which
6779 // would otherwise consume the shift.
6780 if (SDValue Ret = combineShiftOfLogicOp(N, DCI))
6781 return Ret;
6782
6783 // Try mul.wide combining at OptLevel > 0
6784 if (SDValue Ret = TryMULWIDECombine(N, DCI))
6785 return Ret;
6786 }
6787
6788 return SDValue();
6789}
6790
6791static SDValue PerformSETCCCombine(SDNode *N,
6792 TargetLowering::DAGCombinerInfo &DCI,
6793 const NVPTXSubtarget &STI) {
6794 EVT CCType = N->getValueType(ResNo: 0);
6795 SDValue A = N->getOperand(Num: 0);
6796 SDValue B = N->getOperand(Num: 1);
6797
6798 EVT AType = A.getValueType();
6799 if (!(CCType == MVT::v2i1 && (AType == MVT::v2f16 || AType == MVT::v2bf16)))
6800 return SDValue();
6801
6802 if (A.getValueType() == MVT::v2bf16 && !STI.hasFeature(Feature: NVPTX::SM90))
6803 return SDValue();
6804
6805 SDLoc DL(N);
6806 // setp.f16x2 returns two scalar predicates, which we need to
6807 // convert back to v2i1. The returned result will be scalarized by
6808 // the legalizer, but the comparison will remain a single vector
6809 // instruction.
6810 SDValue CCNode = DCI.DAG.getNode(
6811 Opcode: A.getValueType() == MVT::v2f16 ? NVPTXISD::SETP_F16X2
6812 : NVPTXISD::SETP_BF16X2,
6813 DL, VTList: DCI.DAG.getVTList(VT1: MVT::i1, VT2: MVT::i1), Ops: {A, B, N->getOperand(Num: 2)});
6814 return DCI.DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: CCType, N1: CCNode.getValue(R: 0),
6815 N2: CCNode.getValue(R: 1));
6816}
6817
6818static SDValue PerformEXTRACTCombine(SDNode *N,
6819 TargetLowering::DAGCombinerInfo &DCI) {
6820 SDValue Vector = peekThroughFreeze(V: N->getOperand(Num: 0));
6821 SDLoc DL(N);
6822 EVT VectorVT = Vector.getValueType();
6823 if (Vector->getOpcode() == ISD::LOAD && VectorVT.isSimple() &&
6824 IsPTXVectorType(VT: VectorVT.getSimpleVT()))
6825 return SDValue(); // Native vector loads already combine nicely w/
6826 // extract_vector_elt.
6827 // Don't mess with singletons or packed types (v2*32, v2*16, v4i8 and v8i8),
6828 // we already handle them OK.
6829 if (VectorVT.getVectorNumElements() == 1 ||
6830 NVPTX::isPackedVectorTy(VT: VectorVT) || VectorVT == MVT::v8i8)
6831 return SDValue();
6832
6833 // Don't mess with undef values as sra may be simplified to 0, not undef.
6834 if (Vector->isUndef() || ISD::allOperandsUndef(N: Vector.getNode()))
6835 return SDValue();
6836
6837 uint64_t VectorBits = VectorVT.getSizeInBits();
6838 // We only handle the types we can extract in-register.
6839 if (!(VectorBits == 16 || VectorBits == 32 || VectorBits == 64))
6840 return SDValue();
6841
6842 ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
6843 // Index == 0 is handled by generic DAG combiner.
6844 if (!Index || Index->getZExtValue() == 0)
6845 return SDValue();
6846
6847 MVT IVT = MVT::getIntegerVT(BitWidth: VectorBits);
6848 EVT EltVT = VectorVT.getVectorElementType();
6849 EVT EltIVT = EltVT.changeTypeToInteger();
6850 uint64_t EltBits = EltVT.getScalarSizeInBits();
6851
6852 SDValue Result = DCI.DAG.getNode(
6853 Opcode: ISD::TRUNCATE, DL, VT: EltIVT,
6854 Operand: DCI.DAG.getNode(
6855 Opcode: ISD::SRA, DL, VT: IVT, N1: DCI.DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IVT, Operand: Vector),
6856 N2: DCI.DAG.getConstant(Val: Index->getZExtValue() * EltBits, DL, VT: IVT)));
6857
6858 // If element has non-integer type, bitcast it back to the expected type.
6859 if (EltVT != EltIVT)
6860 Result = DCI.DAG.getNode(Opcode: ISD::BITCAST, DL, VT: EltVT, Operand: Result);
6861 // Past legalizer, we may need to extent i8 -> i16 to match the register type.
6862 if (EltVT != N->getValueType(ResNo: 0))
6863 Result = DCI.DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: N->getValueType(ResNo: 0), Operand: Result);
6864
6865 return Result;
6866}
6867
6868/// Transform patterns like:
6869/// (select (ugt shift_amt, BitWidth-1), 0, (srl/shl x, shift_amt))
6870/// (select (ult shift_amt, BitWidth), (srl/shl x, shift_amt), 0)
6871/// Into:
6872/// (NVPTXISD::SRL_CLAMP x, shift_amt) or (NVPTXISD::SHL_CLAMP x, shift_amt)
6873///
6874/// These patterns arise from code like `s >= 32 ? 0 : x >> s`. In LLVM,
6875/// over-shifting a value results in poison, but PTX shr/shl instructions clamp
6876/// the shift amount to BitWidth, making the guard redundant.
6877///
6878/// Note: We only handle SRL and SHL, not SRA, because arithmetic right shifts
6879/// can produce 0 or -1 when shift >= BitWidth.
6880/// Note: We don't handle uge or ule. These don't appear because of
6881/// canonicalization.
6882static SDValue PerformSELECTShiftCombine(SDNode *N,
6883 TargetLowering::DAGCombinerInfo &DCI) {
6884 if (!DCI.isAfterLegalizeDAG())
6885 return SDValue();
6886
6887 using namespace SDPatternMatch;
6888 unsigned BitWidth = N->getValueType(ResNo: 0).getSizeInBits();
6889 SDValue ShiftAmt, ShiftOp;
6890
6891 // Match logical shifts where the shift amount in the guard matches the shift
6892 // amount in the operation.
6893 auto LogicalShift =
6894 m_AllOf(preds: m_Value(N&: ShiftOp),
6895 preds: m_AnyOf(preds: m_Srl(L: m_Value(), R: m_TruncOrSelf(Op: m_Deferred(V&: ShiftAmt))),
6896 preds: m_Shl(L: m_Value(), R: m_TruncOrSelf(Op: m_Deferred(V&: ShiftAmt)))));
6897
6898 // shift_amt > BitWidth-1 ? 0 : shift_op
6899 bool MatchedUGT =
6900 sd_match(N, P: m_Select(Cond: m_SetCC(LHS: m_Value(N&: ShiftAmt),
6901 RHS: m_SpecificInt(V: APInt(BitWidth, BitWidth - 1)),
6902 CC: m_SpecificCondCode(CC: ISD::SETUGT)),
6903 T: m_Zero(), F: LogicalShift));
6904 // shift_amt < BitWidth ? shift_op : 0
6905 bool MatchedULT =
6906 !MatchedUGT &&
6907 sd_match(N, P: m_Select(Cond: m_SetCC(LHS: m_Value(N&: ShiftAmt),
6908 RHS: m_SpecificInt(V: APInt(BitWidth, BitWidth)),
6909 CC: m_SpecificCondCode(CC: ISD::SETULT)),
6910 T: LogicalShift, F: m_Zero()));
6911
6912 if (!MatchedUGT && !MatchedULT)
6913 return SDValue();
6914
6915 // In LLVM IR, the shift amount and the value-to-be-shifted are the same
6916 // type, whereas in PTX the shift amount is always i32. Therefore when
6917 // shifting types larger than i32, we can only do this transformation if we
6918 // know that the upper bits of the shift amount are known zero.
6919 SDValue ClampAmt = ShiftOp.getOperand(i: 1);
6920 unsigned ClampAmtBits = ClampAmt.getValueSizeInBits();
6921 if (ShiftAmt.getValueSizeInBits() > ClampAmtBits &&
6922 DCI.DAG.computeKnownBits(Op: ShiftAmt).countMaxActiveBits() > ClampAmtBits)
6923 return SDValue();
6924
6925 // Return a clamp shift operation, which has the same semantics as PTX shift.
6926 unsigned ClampOpc = ShiftOp.getOpcode() == ISD::SRL ? NVPTXISD::SRL_CLAMP
6927 : NVPTXISD::SHL_CLAMP;
6928 return DCI.DAG.getNode(Opcode: ClampOpc, DL: SDLoc(N), VT: ShiftOp.getValueType(),
6929 N1: ShiftOp.getOperand(i: 0), N2: ClampAmt);
6930}
6931
6932static SDValue PerformVSELECTCombine(SDNode *N,
6933 TargetLowering::DAGCombinerInfo &DCI) {
6934 SDValue VA = N->getOperand(Num: 1);
6935 EVT VectorVT = VA.getValueType();
6936 if (VectorVT != MVT::v4i8)
6937 return SDValue();
6938
6939 // We need to split vselect into individual per-element operations Because we
6940 // use BFE/BFI instruction for byte extraction/insertion, we do end up with
6941 // 32-bit values, so we may as well do comparison as i32 to avoid conversions
6942 // to/from i16 normally used for i8 values.
6943 SmallVector<SDValue, 4> E;
6944 SDLoc DL(N);
6945 SDValue VCond = N->getOperand(Num: 0);
6946 SDValue VB = N->getOperand(Num: 2);
6947 for (int I = 0; I < 4; ++I) {
6948 SDValue C = DCI.DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i1, N1: VCond,
6949 N2: DCI.DAG.getConstant(Val: I, DL, VT: MVT::i32));
6950 SDValue EA = DCI.DAG.getAnyExtOrTrunc(
6951 Op: DCI.DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i8, N1: VA,
6952 N2: DCI.DAG.getConstant(Val: I, DL, VT: MVT::i32)),
6953 DL, VT: MVT::i32);
6954 SDValue EB = DCI.DAG.getAnyExtOrTrunc(
6955 Op: DCI.DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i8, N1: VB,
6956 N2: DCI.DAG.getConstant(Val: I, DL, VT: MVT::i32)),
6957 DL, VT: MVT::i32);
6958 E.push_back(Elt: DCI.DAG.getAnyExtOrTrunc(
6959 Op: DCI.DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::i32, N1: C, N2: EA, N3: EB), DL, VT: MVT::i8));
6960 }
6961 return DCI.DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: MVT::v4i8, Ops: E);
6962}
6963
6964static SDValue
6965PerformBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
6966 auto VT = N->getValueType(ResNo: 0);
6967 if (!DCI.isAfterLegalizeDAG() ||
6968 // only process v2*16 types
6969 !(NVPTX::isPackedVectorTy(VT) && VT.is32BitVector() &&
6970 VT.getVectorNumElements() == 2))
6971 return SDValue();
6972
6973 auto Op0 = N->getOperand(Num: 0);
6974 auto Op1 = N->getOperand(Num: 1);
6975
6976 // Start out by assuming we want to take the lower 2 bytes of each i32
6977 // operand.
6978 uint64_t Op0Bytes = 0x10;
6979 uint64_t Op1Bytes = 0x54;
6980
6981 std::pair<SDValue *, uint64_t *> OpData[2] = {{&Op0, &Op0Bytes},
6982 {&Op1, &Op1Bytes}};
6983
6984 // Check that each operand is an i16, truncated from an i32 operand. We'll
6985 // select individual bytes from those original operands. Optionally, fold in a
6986 // shift right of that original operand.
6987 for (auto &[Op, OpBytes] : OpData) {
6988 // Eat up any bitcast
6989 if (Op->getOpcode() == ISD::BITCAST)
6990 *Op = Op->getOperand(i: 0);
6991
6992 if (!(Op->getValueType() == MVT::i16 && Op->getOpcode() == ISD::TRUNCATE &&
6993 Op->getOperand(i: 0).getValueType() == MVT::i32))
6994 return SDValue();
6995
6996 // If the truncate has multiple uses, this optimization can increase
6997 // register pressure
6998 if (!Op->hasOneUse())
6999 return SDValue();
7000
7001 *Op = Op->getOperand(i: 0);
7002
7003 // Optionally, fold in a shift-right of the original operand and let permute
7004 // pick the two higher bytes of the original value directly.
7005 if (Op->getOpcode() == ISD::SRL && isa<ConstantSDNode>(Val: Op->getOperand(i: 1))) {
7006 if (cast<ConstantSDNode>(Val: Op->getOperand(i: 1))->getZExtValue() == 16) {
7007 // Shift the PRMT byte selector to pick upper bytes from each respective
7008 // value, instead of the lower ones: 0x10 -> 0x32, 0x54 -> 0x76
7009 assert((*OpBytes == 0x10 || *OpBytes == 0x54) &&
7010 "PRMT selector values out of range");
7011 *OpBytes += 0x22;
7012 *Op = Op->getOperand(i: 0);
7013 }
7014 }
7015 }
7016
7017 SDLoc DL(N);
7018 auto &DAG = DCI.DAG;
7019
7020 auto PRMT =
7021 getPRMT(A: DAG.getBitcast(VT: MVT::i32, V: Op0), B: DAG.getBitcast(VT: MVT::i32, V: Op1),
7022 Selector: (Op1Bytes << 8) | Op0Bytes, DL, DAG);
7023 return DAG.getBitcast(VT, V: PRMT);
7024}
7025
7026static SDValue combineADDRSPACECAST(SDNode *N,
7027 TargetLowering::DAGCombinerInfo &DCI) {
7028 auto *ASCN1 = cast<AddrSpaceCastSDNode>(Val: N);
7029
7030 if (auto *ASCN2 = dyn_cast<AddrSpaceCastSDNode>(Val: ASCN1->getOperand(Num: 0))) {
7031 assert(ASCN2->getDestAddressSpace() == ASCN1->getSrcAddressSpace());
7032
7033 // Fold asc[B -> A](asc[A -> B](x)) -> x
7034 if (ASCN1->getDestAddressSpace() == ASCN2->getSrcAddressSpace())
7035 return ASCN2->getOperand(Num: 0);
7036 }
7037
7038 return SDValue();
7039}
7040
7041// Given a constant selector value and a prmt mode, return the selector value
7042// normalized to the generic prmt mode. See the PTX ISA documentation for more
7043// details:
7044// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-prmt
7045static APInt getPRMTSelector(const APInt &Selector, unsigned Mode) {
7046 assert(Selector.getBitWidth() == 32 && "PRMT must have i32 operands");
7047
7048 if (Mode == NVPTX::PTXPrmtMode::NONE)
7049 return Selector;
7050
7051 const unsigned V = Selector.trunc(width: 2).getZExtValue();
7052
7053 const auto GetSelector = [](unsigned S0, unsigned S1, unsigned S2,
7054 unsigned S3) {
7055 return APInt(32, S0 | (S1 << 4) | (S2 << 8) | (S3 << 12));
7056 };
7057
7058 switch (Mode) {
7059 case NVPTX::PTXPrmtMode::F4E:
7060 return GetSelector(V, V + 1, V + 2, V + 3);
7061 case NVPTX::PTXPrmtMode::B4E:
7062 return GetSelector(V, (V - 1) & 7, (V - 2) & 7, (V - 3) & 7);
7063 case NVPTX::PTXPrmtMode::RC8:
7064 return GetSelector(V, V, V, V);
7065 case NVPTX::PTXPrmtMode::ECL:
7066 return GetSelector(V, std::max(a: V, b: 1U), std::max(a: V, b: 2U), 3U);
7067 case NVPTX::PTXPrmtMode::ECR:
7068 return GetSelector(0, std::min(a: V, b: 1U), std::min(a: V, b: 2U), V);
7069 case NVPTX::PTXPrmtMode::RC16: {
7070 unsigned V1 = (V & 1) << 1;
7071 return GetSelector(V1, V1 + 1, V1, V1 + 1);
7072 }
7073 default:
7074 llvm_unreachable("Invalid PRMT mode");
7075 }
7076}
7077
7078static APInt computePRMT(APInt A, APInt B, APInt Selector, unsigned Mode) {
7079 assert(A.getBitWidth() == 32 && B.getBitWidth() == 32 &&
7080 Selector.getBitWidth() == 32 && "PRMT must have i32 operands");
7081 // {b, a} = {{b7, b6, b5, b4}, {b3, b2, b1, b0}}
7082 APInt BitField = B.concat(NewLSB: A);
7083 APInt SelectorVal = getPRMTSelector(Selector, Mode);
7084 APInt Result(32, 0);
7085 for (unsigned I : llvm::seq(Size: 4U)) {
7086 APInt Sel = SelectorVal.extractBits(numBits: 4, bitPosition: I * 4);
7087 unsigned Idx = Sel.getLoBits(numBits: 3).getZExtValue();
7088 unsigned Sign = Sel.getHiBits(numBits: 1).getZExtValue();
7089 APInt Byte = BitField.extractBits(numBits: 8, bitPosition: Idx * 8);
7090 if (Sign)
7091 Byte = Byte.ashr(ShiftAmt: 8);
7092 Result.insertBits(SubBits: Byte, bitPosition: I * 8);
7093 }
7094 return Result;
7095}
7096
7097static SDValue combinePRMT(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
7098 CodeGenOptLevel OptLevel) {
7099 if (OptLevel == CodeGenOptLevel::None)
7100 return SDValue();
7101
7102 // Constant fold PRMT
7103 if (isa<ConstantSDNode>(Val: N->getOperand(Num: 0)) &&
7104 isa<ConstantSDNode>(Val: N->getOperand(Num: 1)) &&
7105 isa<ConstantSDNode>(Val: N->getOperand(Num: 2)))
7106 return DCI.DAG.getConstant(Val: computePRMT(A: N->getConstantOperandAPInt(Num: 0),
7107 B: N->getConstantOperandAPInt(Num: 1),
7108 Selector: N->getConstantOperandAPInt(Num: 2),
7109 Mode: N->getConstantOperandVal(Num: 3)),
7110 DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
7111 return SDValue();
7112}
7113
7114// During call lowering we wrap the return values in a ProxyReg node which
7115// depend on the chain value produced by the completed call. This ensures that
7116// the full call is emitted in cases where libcalls are used to legalize
7117// operations. To improve the functioning of other DAG combines we pull all
7118// operations we can through one of these nodes, ensuring that the ProxyReg
7119// directly wraps a load. That is:
7120//
7121// (ProxyReg (zext (load retval0))) => (zext (ProxyReg (load retval0)))
7122//
7123static SDValue sinkProxyReg(SDValue R, SDValue Chain,
7124 TargetLowering::DAGCombinerInfo &DCI) {
7125 switch (R.getOpcode()) {
7126 case ISD::TRUNCATE:
7127 case ISD::ANY_EXTEND:
7128 case ISD::SIGN_EXTEND:
7129 case ISD::ZERO_EXTEND:
7130 case ISD::BITCAST: {
7131 if (SDValue V = sinkProxyReg(R: R.getOperand(i: 0), Chain, DCI))
7132 return DCI.DAG.getNode(Opcode: R.getOpcode(), DL: SDLoc(R), VT: R.getValueType(), Operand: V);
7133 return SDValue();
7134 }
7135 case ISD::SHL:
7136 case ISD::SRL:
7137 case ISD::SRA:
7138 case ISD::OR: {
7139 if (SDValue A = sinkProxyReg(R: R.getOperand(i: 0), Chain, DCI))
7140 if (SDValue B = sinkProxyReg(R: R.getOperand(i: 1), Chain, DCI))
7141 return DCI.DAG.getNode(Opcode: R.getOpcode(), DL: SDLoc(R), VT: R.getValueType(), N1: A, N2: B);
7142 return SDValue();
7143 }
7144 case ISD::Constant:
7145 return R;
7146 case ISD::LOAD:
7147 case NVPTXISD::LoadV2:
7148 case NVPTXISD::LoadV4: {
7149 return DCI.DAG.getNode(Opcode: NVPTXISD::ProxyReg, DL: SDLoc(R), VT: R.getValueType(),
7150 Ops: {Chain, R});
7151 }
7152 case ISD::BUILD_VECTOR: {
7153 if (DCI.isBeforeLegalize())
7154 return SDValue();
7155
7156 SmallVector<SDValue, 16> Ops;
7157 for (auto &Op : R->ops()) {
7158 SDValue V = sinkProxyReg(R: Op, Chain, DCI);
7159 if (!V)
7160 return SDValue();
7161 Ops.push_back(Elt: V);
7162 }
7163 return DCI.DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SDLoc(R), VT: R.getValueType(), Ops);
7164 }
7165 case ISD::EXTRACT_VECTOR_ELT: {
7166 if (DCI.isBeforeLegalize())
7167 return SDValue();
7168
7169 if (SDValue V = sinkProxyReg(R: R.getOperand(i: 0), Chain, DCI))
7170 return DCI.DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(R),
7171 VT: R.getValueType(), N1: V, N2: R.getOperand(i: 1));
7172 return SDValue();
7173 }
7174 default:
7175 return SDValue();
7176 }
7177}
7178
7179static unsigned getFAddWithNegOpcode(EVT VT, Intrinsic::ID IID,
7180 APFloat::roundingMode RoundingMode) {
7181 const bool IsFTZ =
7182 IID == Intrinsic::nvvm_fadd_ftz || IID == Intrinsic::nvvm_fadd_ftz_sat;
7183 const bool IsSat =
7184 IID == Intrinsic::nvvm_fadd_sat || IID == Intrinsic::nvvm_fadd_ftz_sat;
7185 switch (VT.getScalarType().getSimpleVT().SimpleTy) {
7186 case MVT::f16: {
7187 static constexpr unsigned SubRNOpcodes[2][2] = {
7188 {NVPTXISD::SUB_RN, NVPTXISD::SUB_RN_SAT},
7189 {NVPTXISD::SUB_RN_FTZ, NVPTXISD::SUB_RN_FTZ_SAT}};
7190 return SubRNOpcodes[IsFTZ][IsSat];
7191 }
7192 case MVT::bf16:
7193 return NVPTXISD::SUB_RN;
7194 case MVT::f32: {
7195 // for f32x2 inputs
7196 if (!VT.isVector() || IsSat)
7197 return 0;
7198 static constexpr unsigned SubF32x2Opcodes[4][2] = {
7199 {NVPTXISD::SUB_RZ, NVPTXISD::SUB_RZ_FTZ}, // RZ
7200 {NVPTXISD::SUB_RN, NVPTXISD::SUB_RN_FTZ}, // RN
7201 {NVPTXISD::SUB_RP, NVPTXISD::SUB_RP_FTZ}, // RP
7202 {NVPTXISD::SUB_RM, NVPTXISD::SUB_RM_FTZ}}; // RM
7203 return SubF32x2Opcodes[static_cast<unsigned>(RoundingMode)][IsFTZ];
7204 }
7205 default:
7206 return 0;
7207 }
7208}
7209
7210static SDValue combineFAddWithNeg(SDNode *N, SelectionDAG &DAG,
7211 Intrinsic::ID AddIntrinsicID,
7212 APFloat::roundingMode RoundingMode) {
7213 const EVT VT = N->getValueType(ResNo: 0);
7214 const unsigned Opc = getFAddWithNegOpcode(VT, IID: AddIntrinsicID, RoundingMode);
7215 if (!Opc)
7216 return SDValue();
7217
7218 SDValue Op1 = N->getOperand(Num: 1);
7219 SDValue Op2 = N->getOperand(Num: 2);
7220
7221 SDValue SubOp1, SubOp2;
7222
7223 if (Op1.getOpcode() == ISD::FNEG) {
7224 SubOp1 = Op2;
7225 SubOp2 = Op1.getOperand(i: 0);
7226 } else if (Op2.getOpcode() == ISD::FNEG) {
7227 SubOp1 = Op1;
7228 SubOp2 = Op2.getOperand(i: 0);
7229 } else {
7230 return SDValue();
7231 }
7232
7233 return DAG.getNode(Opcode: Opc, DL: SDLoc(N), VT, N1: SubOp1, N2: SubOp2);
7234}
7235
7236// TODO: Remove the type-legality checks here once
7237// https://github.com/llvm/llvm-project/pull/172442 lands, adding support for
7238// explicit type constraints for overloaded intrinsics in tablegen.
7239static bool isSupportedFAdd(EVT VT, const NVPTXSubtarget &STI,
7240 Intrinsic::ID IID,
7241 APFloat::roundingMode RoundingMode) {
7242 if (VT.isVector() && VT.getVectorElementCount() != ElementCount::getFixed(MinVal: 2))
7243 return false;
7244
7245 const bool IsRN = RoundingMode == APFloat::rmNearestTiesToEven;
7246 const bool IsFTZ =
7247 IID == Intrinsic::nvvm_fadd_ftz || IID == Intrinsic::nvvm_fadd_ftz_sat;
7248 const bool IsSat =
7249 IID == Intrinsic::nvvm_fadd_sat || IID == Intrinsic::nvvm_fadd_ftz_sat;
7250 switch (VT.getScalarType().getSimpleVT().SimpleTy) {
7251 case MVT::f16:
7252 return IsRN;
7253 case MVT::bf16:
7254 return IsRN && !IsSat && !IsFTZ && STI.hasNativeBF16Support(Opcode: ISD::FADD);
7255 case MVT::f32:
7256 return !VT.isVector() || (!IsSat && STI.hasF32x2Instructions());
7257 case MVT::f64:
7258 return !VT.isVector() && !IsSat && !IsFTZ;
7259 default:
7260 return false;
7261 }
7262}
7263
7264static SDValue diagnoseUnsupportedFAdd(SDNode *N, SelectionDAG &DAG,
7265 Intrinsic::ID IID,
7266 APFloat::roundingMode RoundingMode) {
7267 const EVT VT = N->getValueType(ResNo: 0);
7268 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
7269 DAG.getMachineFunction().getFunction(),
7270 Twine(Intrinsic::getBaseName(id: IID)) + " with rounding mode " +
7271 nvvm::GetRoundingModeName(RM: RoundingMode) + " and operand type " +
7272 VT.getEVTString() + " is not supported on this target",
7273 SDLoc(N).getDebugLoc()));
7274 return DAG.getPOISON(VT);
7275}
7276
7277static SDValue combineIntrinsicWOChain(SDNode *N,
7278 TargetLowering::DAGCombinerInfo &DCI,
7279 const NVPTXSubtarget &STI) {
7280 const Intrinsic::ID IID =
7281 static_cast<Intrinsic::ID>(N->getConstantOperandVal(Num: 0));
7282
7283 switch (IID) {
7284 default:
7285 break;
7286 case Intrinsic::nvvm_fadd:
7287 case Intrinsic::nvvm_fadd_ftz:
7288 case Intrinsic::nvvm_fadd_sat:
7289 case Intrinsic::nvvm_fadd_ftz_sat: {
7290 const auto RoundingMode = static_cast<APFloat::roundingMode>(
7291 N->getConstantOperandAPInt(Num: 3).getSExtValue());
7292 if (!isSupportedFAdd(VT: N->getValueType(ResNo: 0), STI, IID, RoundingMode))
7293 return diagnoseUnsupportedFAdd(N, DAG&: DCI.DAG, IID, RoundingMode);
7294 return combineFAddWithNeg(N, DAG&: DCI.DAG, AddIntrinsicID: IID, RoundingMode);
7295 }
7296 }
7297 return SDValue();
7298}
7299
7300static SDValue combineProxyReg(SDNode *N,
7301 TargetLowering::DAGCombinerInfo &DCI) {
7302
7303 SDValue Chain = N->getOperand(Num: 0);
7304 SDValue Reg = N->getOperand(Num: 1);
7305
7306 // If the ProxyReg is not wrapping a load, try to pull the operations through
7307 // the ProxyReg.
7308 if (Reg.getOpcode() != ISD::LOAD) {
7309 if (SDValue V = sinkProxyReg(R: Reg, Chain, DCI))
7310 return V;
7311 }
7312
7313 return SDValue();
7314}
7315
7316SDValue NVPTXTargetLowering::PerformDAGCombine(SDNode *N,
7317 DAGCombinerInfo &DCI) const {
7318 CodeGenOptLevel OptLevel = getTargetMachine().getOptLevel();
7319 switch (N->getOpcode()) {
7320 default:
7321 break;
7322 case ISD::ADD:
7323 return PerformADDCombine(N, DCI, OptLevel);
7324 case ISD::ADDRSPACECAST:
7325 return combineADDRSPACECAST(N, DCI);
7326 case ISD::SIGN_EXTEND:
7327 case ISD::ZERO_EXTEND:
7328 return combineSZExtToMulWide(N, DCI, OptLevel);
7329 case ISD::BUILD_VECTOR:
7330 return PerformBUILD_VECTORCombine(N, DCI);
7331 case ISD::EXTRACT_VECTOR_ELT:
7332 return PerformEXTRACTCombine(N, DCI);
7333 case ISD::FADD:
7334 return performFADDCombine(N, DCI, OptLevel);
7335 case ISD::FMA:
7336 case ISD::FMUL:
7337 case ISD::FSUB:
7338 return performScalarizeV2F32Op(N, DCI, OptLevel);
7339 case ISD::FMAXNUM:
7340 case ISD::FMINNUM:
7341 case ISD::FMAXIMUM:
7342 case ISD::FMINIMUM:
7343 case ISD::FMAXIMUMNUM:
7344 case ISD::FMINIMUMNUM:
7345 return PerformFMinMaxCombine(N, DCI, STI);
7346 case ISD::LOAD:
7347 case NVPTXISD::LoadV2:
7348 case NVPTXISD::LoadV4:
7349 return combineLOAD(N, DCI, STI);
7350 case ISD::MUL:
7351 return PerformMULCombine(N, DCI, OptLevel);
7352 case NVPTXISD::PRMT:
7353 return combinePRMT(N, DCI, OptLevel);
7354 case NVPTXISD::ProxyReg:
7355 return combineProxyReg(N, DCI);
7356 case ISD::SETCC:
7357 return PerformSETCCCombine(N, DCI, STI);
7358 case ISD::SHL:
7359 return PerformSHLCombine(N, DCI, OptLevel);
7360 case ISD::STORE:
7361 case NVPTXISD::StoreV2:
7362 case NVPTXISD::StoreV4:
7363 return combineSTORE(N, DCI, STI);
7364 case ISD::SELECT:
7365 return PerformSELECTShiftCombine(N, DCI);
7366 case ISD::VSELECT:
7367 return PerformVSELECTCombine(N, DCI);
7368 case ISD::INTRINSIC_WO_CHAIN:
7369 return combineIntrinsicWOChain(N, DCI, STI);
7370 }
7371 return SDValue();
7372}
7373
7374static void ReplaceBITCAST(SDNode *Node, SelectionDAG &DAG,
7375 SmallVectorImpl<SDValue> &Results) {
7376 // Handle bitcasting to v2i8 without hitting the default promotion
7377 // strategy which goes through stack memory.
7378 SDValue Op(Node, 0);
7379 EVT ToVT = Op->getValueType(ResNo: 0);
7380 if (ToVT != MVT::v2i8) {
7381 return;
7382 }
7383
7384 // Bitcast to i16 and unpack elements into a vector
7385 SDLoc DL(Node);
7386 SDValue AsInt = DAG.getBitcast(VT: MVT::i16, V: Op->getOperand(Num: 0));
7387 SDValue Vec0 = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i8, Operand: AsInt);
7388 SDValue Const8 = DAG.getConstant(Val: 8, DL, VT: MVT::i16);
7389 SDValue Vec1 =
7390 DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i8,
7391 Operand: DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i16, Ops: {AsInt, Const8}));
7392 Results.push_back(
7393 Elt: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: MVT::v2i8, Ops: {Vec0, Vec1}));
7394}
7395
7396static void ReplaceINTRINSIC_W_CHAIN(SDNode *N, SelectionDAG &DAG,
7397 SmallVectorImpl<SDValue> &Results) {
7398 SDValue Chain = N->getOperand(Num: 0);
7399 SDValue Intrin = N->getOperand(Num: 1);
7400 SDLoc DL(N);
7401
7402 // Get the intrinsic ID
7403 unsigned IntrinNo = Intrin.getNode()->getAsZExtVal();
7404 switch (IntrinNo) {
7405 default:
7406 return;
7407 case Intrinsic::nvvm_ldu_global_i:
7408 case Intrinsic::nvvm_ldu_global_f:
7409 case Intrinsic::nvvm_ldu_global_p: {
7410 EVT ResVT = N->getValueType(ResNo: 0);
7411
7412 if (ResVT.isVector()) {
7413 // Vector LDG/LDU
7414
7415 unsigned NumElts = ResVT.getVectorNumElements();
7416 EVT EltVT = ResVT.getVectorElementType();
7417
7418 // Since LDU/LDG are target nodes, we cannot rely on DAG type
7419 // legalization.
7420 // Therefore, we must ensure the type is legal. For i1 and i8, we set the
7421 // loaded type to i16 and propagate the "real" type as the memory type.
7422 bool NeedTrunc = false;
7423 if (EltVT.getSizeInBits() < 16) {
7424 EltVT = MVT::i16;
7425 NeedTrunc = true;
7426 }
7427
7428 unsigned Opcode = 0;
7429 SDVTList LdResVTs;
7430
7431 switch (NumElts) {
7432 default:
7433 return;
7434 case 2:
7435 Opcode = NVPTXISD::LDUV2;
7436 LdResVTs = DAG.getVTList(VT1: EltVT, VT2: EltVT, VT3: MVT::Other);
7437 break;
7438 case 4: {
7439 Opcode = NVPTXISD::LDUV4;
7440 EVT ListVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other };
7441 LdResVTs = DAG.getVTList(VTs: ListVTs);
7442 break;
7443 }
7444 }
7445
7446 SmallVector<SDValue, 8> OtherOps;
7447
7448 // Copy regular operands
7449
7450 OtherOps.push_back(Elt: Chain); // Chain
7451 // Skip operand 1 (intrinsic ID)
7452 // Others
7453 OtherOps.append(in_start: N->op_begin() + 2, in_end: N->op_end());
7454
7455 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(Val: N);
7456
7457 SDValue NewLD = DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList: LdResVTs, Ops: OtherOps,
7458 MemVT: MemSD->getMemoryVT(),
7459 MMO: MemSD->getMemOperand());
7460
7461 SmallVector<SDValue, 4> ScalarRes;
7462
7463 for (unsigned i = 0; i < NumElts; ++i) {
7464 SDValue Res = NewLD.getValue(R: i);
7465 if (NeedTrunc)
7466 Res =
7467 DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ResVT.getVectorElementType(), Operand: Res);
7468 ScalarRes.push_back(Elt: Res);
7469 }
7470
7471 SDValue LoadChain = NewLD.getValue(R: NumElts);
7472
7473 SDValue BuildVec =
7474 DAG.getBuildVector(VT: ResVT, DL, Ops: ScalarRes);
7475
7476 Results.push_back(Elt: BuildVec);
7477 Results.push_back(Elt: LoadChain);
7478 } else {
7479 // i8 LDG/LDU
7480 assert(ResVT.isSimple() && ResVT.getSimpleVT().SimpleTy == MVT::i8 &&
7481 "Custom handling of non-i8 ldu/ldg?");
7482
7483 // Just copy all operands as-is
7484 SmallVector<SDValue, 4> Ops(N->ops());
7485
7486 // Force output to i16
7487 SDVTList LdResVTs = DAG.getVTList(VT1: MVT::i16, VT2: MVT::Other);
7488
7489 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(Val: N);
7490
7491 // We make sure the memory type is i8, which will be used during isel
7492 // to select the proper instruction.
7493 SDValue NewLD =
7494 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: LdResVTs, Ops,
7495 MemVT: MVT::i8, MMO: MemSD->getMemOperand());
7496
7497 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i8,
7498 Operand: NewLD.getValue(R: 0)));
7499 Results.push_back(Elt: NewLD.getValue(R: 1));
7500 }
7501 return;
7502 }
7503
7504 case Intrinsic::nvvm_tcgen05_ld_16x64b_x1:
7505 case Intrinsic::nvvm_tcgen05_ld_16x64b_x4:
7506 case Intrinsic::nvvm_tcgen05_ld_16x64b_x8:
7507 case Intrinsic::nvvm_tcgen05_ld_16x64b_x16:
7508 case Intrinsic::nvvm_tcgen05_ld_16x64b_x32:
7509 case Intrinsic::nvvm_tcgen05_ld_16x64b_x64:
7510 case Intrinsic::nvvm_tcgen05_ld_16x64b_x128:
7511 case Intrinsic::nvvm_tcgen05_ld_32x32b_x1:
7512 case Intrinsic::nvvm_tcgen05_ld_32x32b_x4:
7513 case Intrinsic::nvvm_tcgen05_ld_32x32b_x8:
7514 case Intrinsic::nvvm_tcgen05_ld_32x32b_x16:
7515 case Intrinsic::nvvm_tcgen05_ld_32x32b_x32:
7516 case Intrinsic::nvvm_tcgen05_ld_32x32b_x64:
7517 case Intrinsic::nvvm_tcgen05_ld_32x32b_x128:
7518 case Intrinsic::nvvm_tcgen05_ld_16x128b_x2:
7519 case Intrinsic::nvvm_tcgen05_ld_16x128b_x4:
7520 case Intrinsic::nvvm_tcgen05_ld_16x128b_x8:
7521 case Intrinsic::nvvm_tcgen05_ld_16x128b_x16:
7522 case Intrinsic::nvvm_tcgen05_ld_16x128b_x32:
7523 case Intrinsic::nvvm_tcgen05_ld_16x128b_x64:
7524 case Intrinsic::nvvm_tcgen05_ld_16x256b_x1:
7525 case Intrinsic::nvvm_tcgen05_ld_16x256b_x2:
7526 case Intrinsic::nvvm_tcgen05_ld_16x256b_x4:
7527 case Intrinsic::nvvm_tcgen05_ld_16x256b_x8:
7528 case Intrinsic::nvvm_tcgen05_ld_16x256b_x16:
7529 case Intrinsic::nvvm_tcgen05_ld_16x256b_x32:
7530 if (auto Res = lowerTcgen05Ld(N, DAG)) {
7531 Results.push_back(Elt: Res->first);
7532 Results.push_back(Elt: Res->second);
7533 }
7534 return;
7535
7536 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x1:
7537 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x4:
7538 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x8:
7539 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x16:
7540 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x32:
7541 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x64:
7542 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x128:
7543 if (auto Res = lowerTcgen05Ld(N, DAG, /*HasOffset=*/true)) {
7544 Results.push_back(Elt: Res->first);
7545 Results.push_back(Elt: Res->second);
7546 }
7547 return;
7548
7549 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x8_i32:
7550 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x8_f32:
7551 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x64_i32:
7552 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x64_f32:
7553 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x4_i32:
7554 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x4_f32:
7555 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x32_i32:
7556 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x32_f32:
7557 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x16_i32:
7558 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x16_f32:
7559 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x128_i32:
7560 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x128_f32:
7561 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x8_i32:
7562 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x8_f32:
7563 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x64_i32:
7564 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x64_f32:
7565 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x4_i32:
7566 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x4_f32:
7567 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x32_i32:
7568 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x32_f32:
7569 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x16_i32:
7570 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x16_f32:
7571 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x128_i32:
7572 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x128_f32:
7573 if (auto Res = lowerTcgen05LdRed(N, DAG)) {
7574 Results.push_back(Elt: std::get<0>(t&: *Res));
7575 Results.push_back(Elt: std::get<1>(t&: *Res));
7576 Results.push_back(Elt: std::get<2>(t&: *Res));
7577 }
7578 return;
7579 }
7580}
7581
7582static void ReplaceCopyFromReg_128(SDNode *N, SelectionDAG &DAG,
7583 SmallVectorImpl<SDValue> &Results) {
7584 // Change the CopyFromReg to output 2 64-bit results instead of a 128-bit
7585 // result so that it can pass the legalization
7586 SDLoc DL(N);
7587 SDValue Chain = N->getOperand(Num: 0);
7588 SDValue Reg = N->getOperand(Num: 1);
7589 SDValue Glue = N->getOperand(Num: 2);
7590
7591 assert(Reg.getValueType() == MVT::i128 &&
7592 "Custom lowering for CopyFromReg with 128-bit reg only");
7593 SmallVector<EVT, 4> ResultsType = {MVT::i64, MVT::i64, N->getValueType(ResNo: 1),
7594 N->getValueType(ResNo: 2)};
7595 SmallVector<SDValue, 3> NewOps = {Chain, Reg, Glue};
7596
7597 SDValue NewValue = DAG.getNode(Opcode: ISD::CopyFromReg, DL, ResultTys: ResultsType, Ops: NewOps);
7598 SDValue Pair = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i128,
7599 Ops: {NewValue.getValue(R: 0), NewValue.getValue(R: 1)});
7600
7601 Results.push_back(Elt: Pair);
7602 Results.push_back(Elt: NewValue.getValue(R: 2));
7603 Results.push_back(Elt: NewValue.getValue(R: 3));
7604}
7605
7606static void replaceProxyReg(SDNode *N, SelectionDAG &DAG,
7607 const TargetLowering &TLI,
7608 SmallVectorImpl<SDValue> &Results) {
7609 SDValue Chain = N->getOperand(Num: 0);
7610 SDValue Reg = N->getOperand(Num: 1);
7611
7612 MVT VT = TLI.getRegisterType(Context&: *DAG.getContext(), VT: Reg.getValueType());
7613
7614 SDValue NewReg = DAG.getAnyExtOrTrunc(Op: Reg, DL: SDLoc(N), VT);
7615 SDValue NewProxy =
7616 DAG.getNode(Opcode: NVPTXISD::ProxyReg, DL: SDLoc(N), VT, Ops: {Chain, NewReg});
7617 SDValue Res = DAG.getAnyExtOrTrunc(Op: NewProxy, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
7618
7619 Results.push_back(Elt: Res);
7620}
7621
7622static void replaceAtomicSwap128(SDNode *N, SelectionDAG &DAG,
7623 const NVPTXSubtarget &STI,
7624 SmallVectorImpl<SDValue> &Results) {
7625 assert(N->getValueType(0) == MVT::i128 &&
7626 "Custom lowering for atomic128 only supports i128");
7627
7628 AtomicSDNode *AN = cast<AtomicSDNode>(Val: N);
7629 SDLoc dl(N);
7630
7631 if (!STI.hasAtomSwap128()) {
7632 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
7633 DAG.getMachineFunction().getFunction(),
7634 "Support for b128 atomics introduced in PTX ISA version 8.3 and "
7635 "requires target sm_90.",
7636 dl.getDebugLoc()));
7637
7638 Results.push_back(Elt: DAG.getUNDEF(VT: MVT::i128));
7639 Results.push_back(Elt: AN->getOperand(Num: 0)); // Chain
7640 return;
7641 }
7642
7643 SmallVector<SDValue, 6> Ops;
7644 Ops.push_back(Elt: AN->getOperand(Num: 0)); // Chain
7645 Ops.push_back(Elt: AN->getOperand(Num: 1)); // Ptr
7646 for (const auto &Op : AN->ops().drop_front(N: 2)) {
7647 // Low part
7648 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: dl, VT: MVT::i64, N1: Op,
7649 N2: DAG.getIntPtrConstant(Val: 0, DL: dl)));
7650 // High part
7651 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: dl, VT: MVT::i64, N1: Op,
7652 N2: DAG.getIntPtrConstant(Val: 1, DL: dl)));
7653 }
7654 unsigned Opcode = N->getOpcode() == ISD::ATOMIC_SWAP
7655 ? NVPTXISD::ATOMIC_SWAP_B128
7656 : NVPTXISD::ATOMIC_CMP_SWAP_B128;
7657 SDVTList Tys = DAG.getVTList(VT1: MVT::i64, VT2: MVT::i64, VT3: MVT::Other);
7658 SDValue Result = DAG.getMemIntrinsicNode(Opcode, dl, VTList: Tys, Ops, MemVT: MVT::i128,
7659 MMO: AN->getMemOperand());
7660 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i128,
7661 Ops: {Result.getValue(R: 0), Result.getValue(R: 1)}));
7662 Results.push_back(Elt: Result.getValue(R: 2));
7663}
7664
7665void NVPTXTargetLowering::ReplaceNodeResults(
7666 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
7667 switch (N->getOpcode()) {
7668 default:
7669 report_fatal_error(reason: "Unhandled custom legalization");
7670 case ISD::BITCAST:
7671 ReplaceBITCAST(Node: N, DAG, Results);
7672 return;
7673 case ISD::LOAD:
7674 case ISD::MLOAD:
7675 replaceLoadVector(N, DAG, Results, STI);
7676 return;
7677 case ISD::INTRINSIC_W_CHAIN:
7678 ReplaceINTRINSIC_W_CHAIN(N, DAG, Results);
7679 return;
7680 case ISD::CopyFromReg:
7681 ReplaceCopyFromReg_128(N, DAG, Results);
7682 return;
7683 case NVPTXISD::ProxyReg:
7684 replaceProxyReg(N, DAG, TLI: *this, Results);
7685 return;
7686 case ISD::ATOMIC_CMP_SWAP:
7687 case ISD::ATOMIC_SWAP:
7688 replaceAtomicSwap128(N, DAG, STI, Results);
7689 return;
7690 }
7691}
7692
7693NVPTXTargetLowering::AtomicExpansionKind
7694NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
7695 Type *Ty = AI->getValOperand()->getType();
7696
7697 // Try to lower LLVM atomicrmw fadd to PTX atomic.add. This is complicated
7698 // by the weird FTZ behavior PTX atom.add has:
7699 // - atom.add.f32 on global memory flushes denormals
7700 // - atom.add.f32 on shared memory does not flush denormals
7701 // - atom.add.f16 and atomic.add.bf16 never flush denormals
7702 //
7703 // We lower to atom.add only if the function's FTZ behavior matches that of
7704 // atom.add; otherwise, we lower to a CAS loop. But we always allow
7705 // atomic.add.bf16; even though it never flushes denormals, we never flush
7706 // bf16 denormals when doing regular arithmetic, even when FTZ is enabled.
7707 if (AI->isFloatingPointOperation() &&
7708 AI->getOperation() == AtomicRMWInst::BinOp::FAdd) {
7709 const Function *F = AI->getFunction();
7710
7711 // AllowFTZAtomics forces atom.add regardless of the FTZ mismatch.
7712 if (Ty->isFloatTy()) {
7713 const bool FTZ = F->getDenormalMode(FPType: APFloat::IEEEsingle()).Output ==
7714 DenormalMode::PreserveSign;
7715 bool UseNative = AllowFTZAtomics;
7716 switch (AI->getPointerAddressSpace()) {
7717 case llvm::ADDRESS_SPACE_GLOBAL:
7718 UseNative |= FTZ;
7719 break;
7720 case llvm::ADDRESS_SPACE_SHARED:
7721 case llvm::ADDRESS_SPACE_SHARED_CLUSTER:
7722 UseNative |= !FTZ;
7723 break;
7724 }
7725 if (UseNative)
7726 return AtomicExpansionKind::None;
7727 }
7728
7729 if (Ty->isHalfTy()) {
7730 // atom.add.f16 never flushes denormals, so it only agrees with a
7731 // function that is not in FTZ mode for f16.
7732 const bool FTZ = F->getDenormalMode(FPType: APFloat::IEEEhalf()).Output ==
7733 DenormalMode::PreserveSign;
7734 if ((!FTZ || AllowFTZAtomics) && STI.hasFeature(Feature: NVPTX::SM70) &&
7735 STI.hasFeature(Feature: NVPTX::PTX63))
7736 return AtomicExpansionKind::None;
7737 }
7738
7739 if (Ty->isBFloatTy() && STI.hasFeature(Feature: NVPTX::SM90))
7740 return AtomicExpansionKind::None;
7741
7742 if (Ty->isDoubleTy() && STI.hasAtomAddF64())
7743 return AtomicExpansionKind::None;
7744 }
7745
7746 // PTX's only atomic fp op is `add`; all other ops expand to a CAS loop.
7747 if (AI->isFloatingPointOperation())
7748 return AtomicExpansionKind::CmpXChg;
7749
7750 if (Ty->isVectorTy())
7751 return AtomicExpansionKind::CmpXChg;
7752
7753 assert(Ty->isIntegerTy() && "Ty should be integer at this point");
7754 const unsigned BitWidth = cast<IntegerType>(Val: Ty)->getBitWidth();
7755
7756 switch (AI->getOperation()) {
7757 default:
7758 return AtomicExpansionKind::CmpXChg;
7759 case AtomicRMWInst::BinOp::Xchg:
7760 if (BitWidth == 128)
7761 return AtomicExpansionKind::None;
7762 [[fallthrough]];
7763 case AtomicRMWInst::BinOp::And:
7764 case AtomicRMWInst::BinOp::Or:
7765 case AtomicRMWInst::BinOp::Xor:
7766 switch (BitWidth) {
7767 case 8:
7768 case 16:
7769 return AtomicExpansionKind::CmpXChg;
7770 case 32:
7771 return AtomicExpansionKind::None;
7772 case 64:
7773 if (STI.hasAtomBitwise64())
7774 return AtomicExpansionKind::None;
7775 return AtomicExpansionKind::CmpXChg;
7776 case 128:
7777 return AtomicExpansionKind::CmpXChg;
7778 default:
7779 llvm_unreachable("unsupported width encountered");
7780 }
7781 case AtomicRMWInst::BinOp::Add:
7782 case AtomicRMWInst::BinOp::Sub:
7783 case AtomicRMWInst::BinOp::Max:
7784 case AtomicRMWInst::BinOp::Min:
7785 case AtomicRMWInst::BinOp::UMax:
7786 case AtomicRMWInst::BinOp::UMin:
7787 switch (BitWidth) {
7788 case 8:
7789 case 16:
7790 return AtomicExpansionKind::CmpXChg;
7791 case 32:
7792 return AtomicExpansionKind::None;
7793 case 64:
7794 if (STI.hasAtomMinMax64())
7795 return AtomicExpansionKind::None;
7796 return AtomicExpansionKind::CmpXChg;
7797 case 128:
7798 return AtomicExpansionKind::CmpXChg;
7799 default:
7800 llvm_unreachable("unsupported width encountered");
7801 }
7802 case AtomicRMWInst::BinOp::UIncWrap:
7803 case AtomicRMWInst::BinOp::UDecWrap:
7804 switch (BitWidth) {
7805 case 32:
7806 return AtomicExpansionKind::None;
7807 case 8:
7808 case 16:
7809 case 64:
7810 case 128:
7811 return AtomicExpansionKind::CmpXChg;
7812 default:
7813 llvm_unreachable("unsupported width encountered");
7814 }
7815 }
7816
7817 return AtomicExpansionKind::CmpXChg;
7818}
7819
7820bool NVPTXTargetLowering::shouldInsertFencesForAtomic(
7821 const Instruction *I) const {
7822 // This function returns true iff the operation is emulated using a CAS-loop,
7823 // or if it has the memory order seq_cst (which is not natively supported in
7824 // the PTX `atom` instruction).
7825 //
7826 // atomicrmw and cmpxchg instructions not efficiently supported by PTX
7827 // are lowered to CAS emulation loops that preserve their memory order,
7828 // syncscope, and volatile semantics. For PTX, it is more efficient to use
7829 // atom.cas.relaxed.sco instructions within the loop, and fences before and
7830 // after the loop to restore order.
7831 //
7832 // Atomic instructions efficiently supported by PTX are lowered to
7833 // `atom.<op>.<sem>.<scope` instruction with their corresponding memory order
7834 // and scope. Since PTX does not support seq_cst, we emulate it by lowering to
7835 // a fence.sc followed by an atom according to the PTX atomics ABI
7836 // https://docs.nvidia.com/cuda/ptx-writers-guide-to-interoperability/atomic-abi.html
7837 if (auto *CI = dyn_cast<AtomicCmpXchgInst>(Val: I))
7838 return (cast<IntegerType>(Val: CI->getCompareOperand()->getType())
7839 ->getBitWidth() < STI.getMinCmpXchgSizeInBits()) ||
7840 CI->getMergedOrdering() == AtomicOrdering::SequentiallyConsistent;
7841 if (auto *RI = dyn_cast<AtomicRMWInst>(Val: I))
7842 return shouldExpandAtomicRMWInIR(AI: RI) == AtomicExpansionKind::CmpXChg ||
7843 RI->getOrdering() == AtomicOrdering::SequentiallyConsistent;
7844 return false;
7845}
7846
7847AtomicOrdering NVPTXTargetLowering::atomicOperationOrderAfterFenceSplit(
7848 const Instruction *I) const {
7849 // If the operation is emulated by a CAS-loop, we lower the instruction to
7850 // atom.<op>.relaxed, since AtomicExpandPass will insert fences for enforcing
7851 // the correct memory ordering around the CAS loop.
7852 //
7853 // When the operation is not emulated, but the memory order is seq_cst,
7854 // we must lower to "fence.sc.<scope>; atom.<op>.acquire.<scope>;" to conform
7855 // to the PTX atomics ABI.
7856 // https://docs.nvidia.com/cuda/ptx-writers-guide-to-interoperability/atomic-abi.html
7857 // For such cases, emitLeadingFence() will separately insert the leading
7858 // "fence.sc.<scope>;". Here, we only set the memory order to acquire.
7859 //
7860 // Otherwise, the operation is not emulated, and the memory order is not
7861 // seq_cst. In this case, the LLVM memory order is natively supported by the
7862 // PTX `atom` instruction, and we just lower to the corresponding
7863 // `atom.<op>.relaxed|acquire|release|acq_rel". For such cases, this function
7864 // will NOT be called.
7865 // prerequisite: shouldInsertFencesForAtomic() should have returned `true` for
7866 // I before its memory order was modified.
7867 if (auto *CI = dyn_cast<AtomicCmpXchgInst>(Val: I);
7868 CI && CI->getMergedOrdering() == AtomicOrdering::SequentiallyConsistent &&
7869 cast<IntegerType>(Val: CI->getCompareOperand()->getType())->getBitWidth() >=
7870 STI.getMinCmpXchgSizeInBits())
7871 return AtomicOrdering::Acquire;
7872 else if (auto *RI = dyn_cast<AtomicRMWInst>(Val: I);
7873 RI && RI->getOrdering() == AtomicOrdering::SequentiallyConsistent &&
7874 shouldExpandAtomicRMWInIR(AI: RI) == AtomicExpansionKind::None)
7875 return AtomicOrdering::Acquire;
7876
7877 return AtomicOrdering::Monotonic;
7878}
7879
7880Instruction *NVPTXTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
7881 Instruction *Inst,
7882 AtomicOrdering Ord) const {
7883 // prerequisite: shouldInsertFencesForAtomic() should have returned `true` for
7884 // `Inst` before its memory order was modified. We cannot enforce this with an
7885 // assert, because AtomicExpandPass will have modified the memory order
7886 // between the initial call to shouldInsertFencesForAtomic() and the call to
7887 // this function.
7888 if (!isa<AtomicCmpXchgInst>(Val: Inst) && !isa<AtomicRMWInst>(Val: Inst))
7889 return TargetLoweringBase::emitLeadingFence(Builder, Inst, Ord);
7890
7891 // Specialize for cmpxchg and atomicrmw
7892 auto SSID = getAtomicSyncScopeID(I: Inst);
7893 assert(SSID.has_value() && "Expected an atomic operation");
7894
7895 if (isReleaseOrStronger(AO: Ord))
7896 return Builder.CreateFence(Ordering: Ord == AtomicOrdering::SequentiallyConsistent
7897 ? AtomicOrdering::SequentiallyConsistent
7898 : AtomicOrdering::Release,
7899 SSID: SSID.value());
7900
7901 return nullptr;
7902}
7903
7904Instruction *NVPTXTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
7905 Instruction *Inst,
7906 AtomicOrdering Ord) const {
7907 // prerequisite: shouldInsertFencesForAtomic() should have returned `true` for
7908 // `Inst` before its memory order was modified. See `emitLeadingFence` for why
7909 // this cannot be enforced with an assert. Specialize for cmpxchg and
7910 // atomicrmw
7911 auto *CI = dyn_cast<AtomicCmpXchgInst>(Val: Inst);
7912 auto *RI = dyn_cast<AtomicRMWInst>(Val: Inst);
7913 if (!CI && !RI)
7914 return TargetLoweringBase::emitTrailingFence(Builder, Inst, Ord);
7915
7916 auto SSID = getAtomicSyncScopeID(I: Inst);
7917 assert(SSID.has_value() && "Expected an atomic operation");
7918
7919 bool IsEmulated =
7920 CI ? cast<IntegerType>(Val: CI->getCompareOperand()->getType())
7921 ->getBitWidth() < STI.getMinCmpXchgSizeInBits()
7922 : shouldExpandAtomicRMWInIR(AI: RI) == AtomicExpansionKind::CmpXChg;
7923
7924 if (isAcquireOrStronger(AO: Ord) && IsEmulated)
7925 return Builder.CreateFence(Ordering: AtomicOrdering::Acquire, SSID: SSID.value());
7926
7927 return nullptr;
7928}
7929
7930// Rather than default to SINT when both UINT and SINT are custom, we only
7931// change the opcode when UINT is not legal and SINT is. UINT is preferred when
7932// both are custom since unsigned CVT instructions can lead to slightly better
7933// SASS code with fewer instructions.
7934unsigned NVPTXTargetLowering::getPreferredFPToIntOpcode(unsigned Op, EVT FromVT,
7935 EVT ToVT) const {
7936 if (isOperationLegal(Op, VT: ToVT))
7937 return Op;
7938 switch (Op) {
7939 case ISD::FP_TO_UINT:
7940 if (isOperationLegal(Op: ISD::FP_TO_SINT, VT: ToVT))
7941 return ISD::FP_TO_SINT;
7942 break;
7943 case ISD::STRICT_FP_TO_UINT:
7944 if (isOperationLegal(Op: ISD::STRICT_FP_TO_SINT, VT: ToVT))
7945 return ISD::STRICT_FP_TO_SINT;
7946 break;
7947 default:
7948 break;
7949 }
7950 return Op;
7951}
7952
7953// Pin NVPTXTargetObjectFile's vtables to this file.
7954NVPTXTargetObjectFile::~NVPTXTargetObjectFile() = default;
7955
7956MCSection *NVPTXTargetObjectFile::SelectSectionForGlobal(
7957 const GlobalObject *GO, SectionKind Kind, const TargetMachine &TM) const {
7958 return getDataSection();
7959}
7960
7961static void computeKnownBitsForPRMT(const SDValue Op, KnownBits &Known,
7962 const SelectionDAG &DAG, unsigned Depth) {
7963 SDValue A = Op.getOperand(i: 0);
7964 SDValue B = Op.getOperand(i: 1);
7965 ConstantSDNode *Selector = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
7966 unsigned Mode = Op.getConstantOperandVal(i: 3);
7967
7968 if (!Selector)
7969 return;
7970
7971 KnownBits AKnown = DAG.computeKnownBits(Op: A, Depth);
7972 KnownBits BKnown = DAG.computeKnownBits(Op: B, Depth);
7973
7974 // {b, a} = {{b7, b6, b5, b4}, {b3, b2, b1, b0}}
7975 assert(AKnown.getBitWidth() == 32 && BKnown.getBitWidth() == 32 &&
7976 "PRMT must have i32 operands");
7977 assert(Known.getBitWidth() == 32 && "PRMT must have i32 result");
7978 KnownBits BitField = BKnown.concat(Lo: AKnown);
7979
7980 APInt SelectorVal = getPRMTSelector(Selector: Selector->getAPIntValue(), Mode);
7981 for (unsigned I : llvm::seq(Size: 4)) {
7982 APInt Sel = SelectorVal.extractBits(numBits: 4, bitPosition: I * 4);
7983 unsigned Idx = Sel.getLoBits(numBits: 3).getZExtValue();
7984 unsigned Sign = Sel.getHiBits(numBits: 1).getZExtValue();
7985 KnownBits Byte = BitField.extractBits(NumBits: 8, BitPosition: Idx * 8);
7986 if (Sign)
7987 Byte = KnownBits::ashr(LHS: Byte, RHS: KnownBits::makeConstant(C: APInt(8, 7)));
7988 Known.insertBits(SubBits: Byte, BitPosition: I * 8);
7989 }
7990}
7991
7992static void computeKnownBitsForLoadV(const SDValue Op, KnownBits &Known) {
7993 MemSDNode *LD = cast<MemSDNode>(Val: Op);
7994
7995 // We can't do anything without knowing the sign bit.
7996 auto ExtType = LD->getConstantOperandVal(Num: LD->getNumOperands() - 1);
7997 if (ExtType == ISD::SEXTLOAD)
7998 return;
7999
8000 // ExtLoading to vector types is weird and may not work well with known bits.
8001 auto DestVT = LD->getValueType(ResNo: 0);
8002 if (DestVT.isVector())
8003 return;
8004
8005 assert(Known.getBitWidth() == DestVT.getSizeInBits());
8006 auto ElementBitWidth = getFromTypeWidthForLoad(Mem: LD);
8007 Known.Zero.setHighBits(Known.getBitWidth() - ElementBitWidth);
8008}
8009
8010void NVPTXTargetLowering::computeKnownBitsForTargetNode(
8011 const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
8012 const SelectionDAG &DAG, unsigned Depth) const {
8013 Known.resetAll();
8014
8015 switch (Op.getOpcode()) {
8016 case NVPTXISD::PRMT:
8017 computeKnownBitsForPRMT(Op, Known, DAG, Depth);
8018 break;
8019 case NVPTXISD::LoadV2:
8020 case NVPTXISD::LoadV4:
8021 case NVPTXISD::LoadV8:
8022 computeKnownBitsForLoadV(Op, Known);
8023 break;
8024 default:
8025 break;
8026 }
8027}
8028
8029static std::pair<APInt, APInt> getPRMTDemandedBits(const APInt &SelectorVal,
8030 const APInt &DemandedBits) {
8031 APInt DemandedLHS = APInt(32, 0);
8032 APInt DemandedRHS = APInt(32, 0);
8033
8034 for (unsigned I : llvm::seq(Size: 4)) {
8035 if (DemandedBits.extractBits(numBits: 8, bitPosition: I * 8).isZero())
8036 continue;
8037
8038 APInt Sel = SelectorVal.extractBits(numBits: 4, bitPosition: I * 4);
8039 unsigned Idx = Sel.getLoBits(numBits: 3).getZExtValue();
8040 unsigned Sign = Sel.getHiBits(numBits: 1).getZExtValue();
8041
8042 APInt &Src = Idx < 4 ? DemandedLHS : DemandedRHS;
8043 unsigned ByteStart = (Idx % 4) * 8;
8044 if (Sign)
8045 Src.setBit(ByteStart + 7);
8046 else
8047 Src.setBits(loBit: ByteStart, hiBit: ByteStart + 8);
8048 }
8049
8050 return {DemandedLHS, DemandedRHS};
8051}
8052
8053// Replace undef with 0 as this is easier for other optimizations such as
8054// known bits.
8055static SDValue canonicalizePRMTInput(SDValue Op, SelectionDAG &DAG) {
8056 if (!Op)
8057 return SDValue();
8058 if (Op.isUndef())
8059 return DAG.getConstant(Val: 0, DL: SDLoc(), VT: MVT::i32);
8060 return Op;
8061}
8062
8063static SDValue simplifyDemandedBitsForPRMT(SDValue PRMT,
8064 const APInt &DemandedBits,
8065 SelectionDAG &DAG,
8066 const TargetLowering &TLI,
8067 unsigned Depth) {
8068 assert(PRMT.getOpcode() == NVPTXISD::PRMT);
8069 SDValue Op0 = PRMT.getOperand(i: 0);
8070 SDValue Op1 = PRMT.getOperand(i: 1);
8071 auto *SelectorConst = dyn_cast<ConstantSDNode>(Val: PRMT.getOperand(i: 2));
8072 if (!SelectorConst)
8073 return SDValue();
8074
8075 unsigned Mode = PRMT.getConstantOperandVal(i: 3);
8076 const APInt Selector = getPRMTSelector(Selector: SelectorConst->getAPIntValue(), Mode);
8077
8078 // Try to simplify the PRMT to one of the inputs if the used bytes are all
8079 // from the same input in the correct order.
8080 const unsigned LeadingBytes = DemandedBits.countLeadingZeros() / 8;
8081 const unsigned SelBits = (4 - LeadingBytes) * 4;
8082 if (Selector.getLoBits(numBits: SelBits) == APInt(32, 0x3210).getLoBits(numBits: SelBits))
8083 return Op0;
8084 if (Selector.getLoBits(numBits: SelBits) == APInt(32, 0x7654).getLoBits(numBits: SelBits))
8085 return Op1;
8086
8087 auto [DemandedLHS, DemandedRHS] = getPRMTDemandedBits(SelectorVal: Selector, DemandedBits);
8088
8089 // Attempt to avoid multi-use ops if we don't need anything from them.
8090 SDValue DemandedOp0 =
8091 TLI.SimplifyMultipleUseDemandedBits(Op: Op0, DemandedBits: DemandedLHS, DAG, Depth: Depth + 1);
8092 SDValue DemandedOp1 =
8093 TLI.SimplifyMultipleUseDemandedBits(Op: Op1, DemandedBits: DemandedRHS, DAG, Depth: Depth + 1);
8094
8095 DemandedOp0 = canonicalizePRMTInput(Op: DemandedOp0, DAG);
8096 DemandedOp1 = canonicalizePRMTInput(Op: DemandedOp1, DAG);
8097 if ((DemandedOp0 && DemandedOp0 != Op0) ||
8098 (DemandedOp1 && DemandedOp1 != Op1)) {
8099 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
8100 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
8101 return getPRMT(A: Op0, B: Op1, Selector: Selector.getZExtValue(), DL: SDLoc(PRMT), DAG);
8102 }
8103
8104 return SDValue();
8105}
8106
8107bool NVPTXTargetLowering::SimplifyDemandedBitsForTargetNode(
8108 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
8109 KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
8110 Known.resetAll();
8111
8112 switch (Op.getOpcode()) {
8113 case NVPTXISD::PRMT:
8114 if (SDValue Result = simplifyDemandedBitsForPRMT(PRMT: Op, DemandedBits, DAG&: TLO.DAG,
8115 TLI: *this, Depth)) {
8116 TLO.CombineTo(O: Op, N: Result);
8117 return true;
8118 }
8119 break;
8120 default:
8121 break;
8122 }
8123
8124 computeKnownBitsForTargetNode(Op, Known, DemandedElts, DAG: TLO.DAG, Depth);
8125 return false;
8126}
8127