1//===-- NVPTXISelLowering.cpp - NVPTX DAG Lowering Implementation ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that NVPTX uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "NVPTXISelLowering.h"
15#include "MCTargetDesc/NVPTXBaseInfo.h"
16#include "NVPTX.h"
17#include "NVPTXMachineFunctionInfo.h"
18#include "NVPTXSelectionDAGInfo.h"
19#include "NVPTXSubtarget.h"
20#include "NVPTXTargetMachine.h"
21#include "NVPTXTargetObjectFile.h"
22#include "NVPTXUtilities.h"
23#include "NVVMProperties.h"
24#include "llvm/ADT/APFloat.h"
25#include "llvm/ADT/APInt.h"
26#include "llvm/ADT/STLExtras.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/StringRef.h"
29#include "llvm/CodeGen/Analysis.h"
30#include "llvm/CodeGen/ISDOpcodes.h"
31#include "llvm/CodeGen/MachineFrameInfo.h"
32#include "llvm/CodeGen/MachineFunction.h"
33#include "llvm/CodeGen/MachineJumpTableInfo.h"
34#include "llvm/CodeGen/MachineMemOperand.h"
35#include "llvm/CodeGen/SDPatternMatch.h"
36#include "llvm/CodeGen/SelectionDAG.h"
37#include "llvm/CodeGen/SelectionDAGNodes.h"
38#include "llvm/CodeGen/TargetCallingConv.h"
39#include "llvm/CodeGen/TargetLowering.h"
40#include "llvm/CodeGen/ValueTypes.h"
41#include "llvm/CodeGenTypes/MachineValueType.h"
42#include "llvm/IR/Argument.h"
43#include "llvm/IR/Attributes.h"
44#include "llvm/IR/Constants.h"
45#include "llvm/IR/DataLayout.h"
46#include "llvm/IR/DerivedTypes.h"
47#include "llvm/IR/DiagnosticInfo.h"
48#include "llvm/IR/FPEnv.h"
49#include "llvm/IR/Function.h"
50#include "llvm/IR/GlobalValue.h"
51#include "llvm/IR/IRBuilder.h"
52#include "llvm/IR/Instruction.h"
53#include "llvm/IR/Instructions.h"
54#include "llvm/IR/IntrinsicsNVPTX.h"
55#include "llvm/IR/LLVMContext.h"
56#include "llvm/IR/Module.h"
57#include "llvm/IR/NVVMIntrinsicUtils.h"
58#include "llvm/IR/Type.h"
59#include "llvm/IR/Value.h"
60#include "llvm/MC/MCContext.h"
61#include "llvm/MC/MCSymbol.h"
62#include "llvm/Support/Alignment.h"
63#include "llvm/Support/AtomicOrdering.h"
64#include "llvm/Support/Casting.h"
65#include "llvm/Support/CodeGen.h"
66#include "llvm/Support/CommandLine.h"
67#include "llvm/Support/ErrorHandling.h"
68#include "llvm/Support/KnownBits.h"
69#include "llvm/Support/NVPTXAddrSpace.h"
70#include "llvm/Target/TargetMachine.h"
71#include "llvm/Target/TargetOptions.h"
72#include <algorithm>
73#include <cassert>
74#include <cmath>
75#include <cstdint>
76#include <iterator>
77#include <optional>
78#include <tuple>
79#include <utility>
80#include <vector>
81
82#define DEBUG_TYPE "nvptx-lower"
83
84using namespace llvm;
85
86static cl::opt<bool> sched4reg(
87 "nvptx-sched4reg",
88 cl::desc("NVPTX Specific: schedule for register pressue"), cl::init(Val: false));
89
90static cl::opt<unsigned> FMAContractLevelOpt(
91 "nvptx-fma-level", cl::Hidden,
92 cl::desc("NVPTX Specific: FMA contraction (0: don't do it"
93 " 1: do it 2: do it aggressively"),
94 cl::init(Val: 2));
95
96static cl::opt<NVPTX::DivPrecisionLevel> UsePrecDivF32(
97 "nvptx-prec-divf32", cl::Hidden,
98 cl::desc(
99 "NVPTX Specific: Override the precision of the lowering for f32 fdiv"),
100 cl::values(
101 clEnumValN(NVPTX::DivPrecisionLevel::Approx, "0", "Use div.approx"),
102 clEnumValN(NVPTX::DivPrecisionLevel::Full, "1", "Use div.full"),
103 clEnumValN(NVPTX::DivPrecisionLevel::IEEE754, "2",
104 "Use IEEE Compliant F32 div.rnd if available (default)"),
105 clEnumValN(NVPTX::DivPrecisionLevel::IEEE754_NoFTZ, "3",
106 "Use IEEE Compliant F32 div.rnd if available, no FTZ")),
107 cl::init(Val: NVPTX::DivPrecisionLevel::IEEE754));
108
109static cl::opt<bool> UsePrecSqrtF32(
110 "nvptx-prec-sqrtf32", cl::Hidden,
111 cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
112 cl::init(Val: true));
113
114// PTX atom.add.f32 has fixed FTZ behavior that may not match the function's
115// (see shouldExpandAtomicRMWInIR), so we'd normally fall back to a CAS loop
116// when they disagree. This option (enabled by default) allows using atom.add
117// anyway, trading correct denormal handling for the speed of the native
118// instruction.
119static cl::opt<bool> AllowFTZAtomics(
120 "nvptx-allow-ftz-atomics", cl::Hidden,
121 cl::desc("NVPTX Specific: Lower atomicrmw fadd to atom.add even when its "
122 "FTZ behavior does not match the function's denormal mode."),
123 cl::init(Val: true));
124
125/// Whereas CUDA's implementation (see libdevice) uses ex2.approx for exp2(), it
126/// does NOT use lg2.approx for log2, so this is disabled by default.
127static cl::opt<bool> UseApproxLog2F32(
128 "nvptx-approx-log2f32",
129 cl::desc("NVPTX Specific: whether to use lg2.approx for log2"),
130 cl::init(Val: false));
131
132NVPTX::DivPrecisionLevel
133NVPTXTargetLowering::getDivF32Level(const MachineFunction &MF,
134 const SDNode &N) const {
135 // If nvptx-prec-div32=N is used on the command-line, always honor it
136 if (UsePrecDivF32.getNumOccurrences() > 0)
137 return UsePrecDivF32;
138
139 const SDNodeFlags Flags = N.getFlags();
140 if (Flags.hasApproximateFuncs())
141 return NVPTX::DivPrecisionLevel::Approx;
142
143 return NVPTX::DivPrecisionLevel::IEEE754;
144}
145
146bool NVPTXTargetLowering::usePrecSqrtF32(const SDNode *N) const {
147 // If nvptx-prec-sqrtf32 is used on the command-line, always honor it
148 if (UsePrecSqrtF32.getNumOccurrences() > 0)
149 return UsePrecSqrtF32;
150
151 if (N) {
152 const SDNodeFlags Flags = N->getFlags();
153 if (Flags.hasApproximateFuncs())
154 return false;
155 }
156
157 return true;
158}
159
160bool NVPTXTargetLowering::useF32FTZ(const MachineFunction &MF) const {
161 return MF.getDenormalMode(FPType: APFloat::IEEEsingle()).Output ==
162 DenormalMode::PreserveSign;
163}
164
165static bool IsPTXVectorType(MVT VT) {
166 switch (VT.SimpleTy) {
167 default:
168 return false;
169 case MVT::v2i1:
170 case MVT::v4i1:
171 case MVT::v2i8:
172 case MVT::v4i8:
173 case MVT::v8i8: // <2 x i8x4>
174 case MVT::v16i8: // <4 x i8x4>
175 case MVT::v2i16:
176 case MVT::v4i16:
177 case MVT::v8i16: // <4 x i16x2>
178 case MVT::v2i32:
179 case MVT::v4i32:
180 case MVT::v2i64:
181 case MVT::v2f16:
182 case MVT::v4f16:
183 case MVT::v8f16: // <4 x f16x2>
184 case MVT::v2bf16:
185 case MVT::v4bf16:
186 case MVT::v8bf16: // <4 x bf16x2>
187 case MVT::v2f32:
188 case MVT::v4f32:
189 case MVT::v2f64:
190 case MVT::v4i64:
191 case MVT::v4f64:
192 case MVT::v8i32:
193 case MVT::v8f32:
194 case MVT::v16f16: // <8 x f16x2>
195 case MVT::v16bf16: // <8 x bf16x2>
196 case MVT::v16i16: // <8 x i16x2>
197 case MVT::v32i8: // <8 x i8x4>
198 return true;
199 }
200}
201
202// When legalizing vector loads/stores, this function is called, which does two
203// things:
204// 1. Determines Whether the vector is something we want to custom lower,
205// std::nullopt is returned if we do not want to custom lower it.
206// 2. If we do want to handle it, returns two parameters:
207// - unsigned int NumElts - The number of elements in the final vector
208// - EVT EltVT - The type of the elements in the final vector
209static std::optional<std::pair<unsigned int, MVT>>
210getVectorLoweringShape(EVT VectorEVT, const NVPTXSubtarget &STI,
211 unsigned AddressSpace) {
212 const bool CanLowerTo256Bit = STI.has256BitVectorLoadStore(AS: AddressSpace);
213
214 if (CanLowerTo256Bit && VectorEVT.isScalarInteger() &&
215 VectorEVT.getSizeInBits() == 256)
216 return {{4, MVT::i64}};
217
218 if (!VectorEVT.isSimple())
219 return std::nullopt;
220 const MVT VectorVT = VectorEVT.getSimpleVT();
221
222 if (!VectorVT.isVector()) {
223 if (VectorVT == MVT::i128 || VectorVT == MVT::f128)
224 return {{2, MVT::i64}};
225 return std::nullopt;
226 }
227
228 const MVT EltVT = VectorVT.getVectorElementType();
229 const unsigned NumElts = VectorVT.getVectorNumElements();
230
231 // The size of the PTX virtual register that holds a packed type.
232 unsigned PackRegSize;
233
234 // We only handle "native" vector sizes for now, e.g. <4 x double> is not
235 // legal. We can (and should) split that into 2 stores of <2 x double> here
236 // but I'm leaving that as a TODO for now.
237 switch (VectorVT.SimpleTy) {
238 default:
239 return std::nullopt;
240
241 case MVT::v4i64:
242 case MVT::v4f64:
243 // This is a "native" vector type iff the address space is global and the
244 // target supports 256-bit loads/stores
245 if (!CanLowerTo256Bit)
246 return std::nullopt;
247 [[fallthrough]];
248 case MVT::v2i8:
249 case MVT::v2i64:
250 case MVT::v2f64:
251 // This is a "native" vector type
252 return std::pair(NumElts, EltVT);
253
254 case MVT::v16f16: // <8 x f16x2>
255 case MVT::v16bf16: // <8 x bf16x2>
256 case MVT::v16i16: // <8 x i16x2>
257 case MVT::v32i8: // <8 x i8x4>
258 // This can be upsized into a "native" vector type iff the address space is
259 // global and the target supports 256-bit loads/stores.
260 if (!CanLowerTo256Bit)
261 return std::nullopt;
262 [[fallthrough]];
263 case MVT::v2i16: // <1 x i16x2>
264 case MVT::v2f16: // <1 x f16x2>
265 case MVT::v2bf16: // <1 x bf16x2>
266 case MVT::v4i8: // <1 x i8x4>
267 case MVT::v4i16: // <2 x i16x2>
268 case MVT::v4f16: // <2 x f16x2>
269 case MVT::v4bf16: // <2 x bf16x2>
270 case MVT::v8i8: // <2 x i8x4>
271 case MVT::v8f16: // <4 x f16x2>
272 case MVT::v8bf16: // <4 x bf16x2>
273 case MVT::v8i16: // <4 x i16x2>
274 case MVT::v16i8: // <4 x i8x4>
275 PackRegSize = 32;
276 break;
277
278 case MVT::v8f32: // <4 x f32x2>
279 case MVT::v8i32: // <4 x i32x2>
280 // This is a "native" vector type iff the address space is global and the
281 // target supports 256-bit loads/stores
282 if (!CanLowerTo256Bit)
283 return std::nullopt;
284 [[fallthrough]];
285 case MVT::v2f32: // <1 x f32x2>
286 case MVT::v4f32: // <2 x f32x2>
287 case MVT::v2i32: // <1 x i32x2>
288 case MVT::v4i32: // <2 x i32x2>
289 if (!STI.hasF32x2Instructions())
290 return std::pair(NumElts, EltVT);
291 PackRegSize = 64;
292 break;
293 }
294
295 // If we reach here, then we can pack 2 or more elements into a single 32-bit
296 // or 64-bit PTX register and treat the vector as a new vector containing
297 // packed elements.
298
299 // Number of elements to pack in one word.
300 const unsigned NPerReg = PackRegSize / EltVT.getSizeInBits();
301
302 return std::pair(NumElts / NPerReg, MVT::getVectorVT(VT: EltVT, NumElements: NPerReg));
303}
304
305/// ComputePTXValueVTs - For the given Type \p Ty, returns the set of primitive
306/// legal-ish MVTs that compose it. Unlike ComputeValueVTs, this will legalize
307/// the types as required by the calling convention (with special handling for
308/// i8s).
309/// NOTE: This is a band-aid for code that expects ComputeValueVTs to return the
310/// same number of types as the Ins/Outs arrays in LowerFormalArguments,
311/// LowerCall, and LowerReturn.
312static void ComputePTXValueVTs(const TargetLowering &TLI, const DataLayout &DL,
313 LLVMContext &Ctx, CallingConv::ID CallConv,
314 Type *Ty, SmallVectorImpl<EVT> &ValueVTs,
315 SmallVectorImpl<uint64_t> &Offsets,
316 uint64_t StartingOffset = 0) {
317 SmallVector<EVT, 16> TempVTs;
318 SmallVector<uint64_t, 16> TempOffsets;
319 ComputeValueVTs(TLI, DL, Ty, ValueVTs&: TempVTs, /*MemVTs=*/nullptr, FixedOffsets: &TempOffsets,
320 StartingOffset);
321
322 for (const auto [VT, Off] : zip(t&: TempVTs, u&: TempOffsets)) {
323 MVT RegisterVT = TLI.getRegisterTypeForCallingConv(Context&: Ctx, CC: CallConv, VT);
324 unsigned NumRegs = TLI.getNumRegistersForCallingConv(Context&: Ctx, CC: CallConv, VT);
325
326 // Since we actually can load/store b8, we need to ensure that we'll use
327 // the original sized type for any i8s or i8 vectors.
328 if (VT.getScalarType() == MVT::i8) {
329 if (RegisterVT == MVT::i16)
330 RegisterVT = MVT::i8;
331 else if (RegisterVT == MVT::v2i16)
332 RegisterVT = MVT::v2i8;
333 else
334 assert(RegisterVT == MVT::v4i8 &&
335 "Expected v4i8, v2i16, or i16 for i8 RegisterVT");
336 }
337
338 // TODO: This is horribly incorrect for cases where the vector elements are
339 // not a multiple of bytes (ex i1) and legal or i8. However, this problem
340 // has existed for as long as NVPTX has and no one has complained, so we'll
341 // leave it for now.
342 for (unsigned I : seq(Size: NumRegs)) {
343 ValueVTs.push_back(Elt: RegisterVT);
344 Offsets.push_back(Elt: Off + I * RegisterVT.getStoreSize());
345 }
346 }
347}
348
349// We return an EVT that can hold N VTs
350// If the VT is a vector, the resulting EVT is a flat vector with the same
351// element type as VT's element type.
352static EVT getVectorizedVT(EVT VT, unsigned N, LLVMContext &C) {
353 if (N == 1)
354 return VT;
355
356 return VT.isVector() ? EVT::getVectorVT(Context&: C, VT: VT.getScalarType(),
357 NumElements: VT.getVectorNumElements() * N)
358 : EVT::getVectorVT(Context&: C, VT, NumElements: N);
359}
360
361static SDValue getExtractVectorizedValue(SDValue V, unsigned I, EVT VT,
362 const SDLoc &dl, SelectionDAG &DAG) {
363 if (V.getValueType() == VT) {
364 assert(I == 0 && "Index must be 0 for scalar value");
365 return V;
366 }
367
368 if (!VT.isVector())
369 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT, N1: V,
370 N2: DAG.getVectorIdxConstant(Val: I, DL: dl));
371
372 return DAG.getNode(
373 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT, N1: V,
374 N2: DAG.getVectorIdxConstant(Val: I * VT.getVectorNumElements(), DL: dl));
375}
376
377template <typename T>
378static inline SDValue getBuildVectorizedValue(unsigned N, const SDLoc &dl,
379 SelectionDAG &DAG, T GetElement) {
380 if (N == 1)
381 return GetElement(0);
382
383 SmallVector<SDValue, 8> Values;
384 for (const unsigned I : llvm::seq(Size: N)) {
385 SDValue Val = GetElement(I);
386 if (Val.getValueType().isVector())
387 DAG.ExtractVectorElements(Op: Val, Args&: Values);
388 else
389 Values.push_back(Elt: Val);
390 }
391
392 EVT VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: Values[0].getValueType(),
393 NumElements: Values.size());
394 return DAG.getBuildVector(VT, DL: dl, Ops: Values);
395}
396
397/// PromoteScalarIntegerPTX
398/// Used to make sure the arguments/returns are suitable for passing
399/// and promote them to a larger size if they're not.
400///
401/// The promoted type is placed in \p PromoteVT if the function returns true.
402static EVT promoteScalarIntegerPTX(const EVT VT) {
403 if (VT.isScalarInteger()) {
404 switch (PowerOf2Ceil(A: VT.getFixedSizeInBits())) {
405 default:
406 llvm_unreachable(
407 "Promotion is not suitable for scalars of size larger than 64-bits");
408 case 1:
409 return MVT::i1;
410 case 2:
411 case 4:
412 case 8:
413 return MVT::i8;
414 case 16:
415 return MVT::i16;
416 case 32:
417 return MVT::i32;
418 case 64:
419 return MVT::i64;
420 }
421 }
422 return VT;
423}
424
425// Check whether we can merge loads/stores of some of the pieces of a
426// flattened function parameter or return value into a single vector
427// load/store.
428//
429// The flattened parameter is represented as a list of EVTs and
430// offsets, and the whole structure is aligned to ParamAlignment. This
431// function determines whether we can load/store pieces of the
432// parameter starting at index Idx using a single vectorized op of
433// size AccessSize. If so, it returns the number of param pieces
434// covered by the vector op. Otherwise, it returns 1.
435template <typename T>
436static unsigned canMergeParamLoadStoresStartingAt(
437 unsigned Idx, uint32_t AccessSize, const SmallVectorImpl<EVT> &ValueVTs,
438 const SmallVectorImpl<T> &Offsets, Align ParamAlignment) {
439
440 // Can't vectorize if param alignment is not sufficient.
441 if (ParamAlignment < AccessSize)
442 return 1;
443 // Can't vectorize if offset is not aligned.
444 if (Offsets[Idx] & (AccessSize - 1))
445 return 1;
446
447 EVT EltVT = ValueVTs[Idx];
448 unsigned EltSize = EltVT.getStoreSize();
449
450 // Element is too large to vectorize.
451 if (EltSize >= AccessSize)
452 return 1;
453
454 unsigned NumElts = AccessSize / EltSize;
455 // Can't vectorize if AccessBytes if not a multiple of EltSize.
456 if (AccessSize != EltSize * NumElts)
457 return 1;
458
459 // We don't have enough elements to vectorize.
460 if (Idx + NumElts > ValueVTs.size())
461 return 1;
462
463 // PTX ISA can only deal with 2- and 4-element vector ops.
464 if (NumElts != 4 && NumElts != 2)
465 return 1;
466
467 for (unsigned j = Idx + 1; j < Idx + NumElts; ++j) {
468 // Types do not match.
469 if (ValueVTs[j] != EltVT)
470 return 1;
471
472 // Elements are not contiguous.
473 if (Offsets[j] - Offsets[j - 1] != EltSize)
474 return 1;
475 }
476 // OK. We can vectorize ValueVTs[i..i+NumElts)
477 return NumElts;
478}
479
480// Computes whether and how we can vectorize the loads/stores of a
481// flattened function parameter or return value.
482//
483// The flattened parameter is represented as the list of ValueVTs and
484// Offsets, and is aligned to ParamAlignment bytes. We return a vector
485// of the same size as ValueVTs indicating how each piece should be
486// loaded/stored (i.e. as a scalar, or as part of a vector
487// load/store).
488template <typename T>
489static SmallVector<unsigned, 16>
490VectorizePTXValueVTs(const SmallVectorImpl<EVT> &ValueVTs,
491 const SmallVectorImpl<T> &Offsets, Align ParamAlignment,
492 bool IsVAArg = false) {
493 // Set vector size to match ValueVTs and mark all elements as
494 // scalars by default.
495
496 if (IsVAArg)
497 return SmallVector<unsigned>(ValueVTs.size(), 1);
498
499 SmallVector<unsigned, 16> VectorInfo;
500
501 const auto GetNumElts = [&](unsigned I) -> unsigned {
502 for (const unsigned AccessSize : {16, 8, 4, 2}) {
503 const unsigned NumElts = canMergeParamLoadStoresStartingAt(
504 I, AccessSize, ValueVTs, Offsets, ParamAlignment);
505 assert((NumElts == 1 || NumElts == 2 || NumElts == 4) &&
506 "Unexpected vectorization size");
507 if (NumElts != 1)
508 return NumElts;
509 }
510 return 1;
511 };
512
513 // Check what we can vectorize using 128/64/32-bit accesses.
514 for (unsigned I = 0, E = ValueVTs.size(); I != E;) {
515 const unsigned NumElts = GetNumElts(I);
516 VectorInfo.push_back(Elt: NumElts);
517 I += NumElts;
518 }
519 assert(std::accumulate(VectorInfo.begin(), VectorInfo.end(), 0u) ==
520 ValueVTs.size());
521 return VectorInfo;
522}
523
524// NVPTXTargetLowering Constructor.
525NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
526 const NVPTXSubtarget &STI)
527 : TargetLowering(TM, STI), STI(STI), GlobalUniqueCallSite(0) {
528 // always lower memset, memcpy, and memmove intrinsics to load/store
529 // instructions, rather
530 // then generating calls to memset, mempcy or memmove.
531 MaxStoresPerMemset = MaxStoresPerMemsetOptSize = (unsigned)0xFFFFFFFF;
532 MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = (unsigned) 0xFFFFFFFF;
533 MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = (unsigned) 0xFFFFFFFF;
534
535 setBooleanContents(ZeroOrNegativeOneBooleanContent);
536 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
537
538 // Jump is Expensive. Don't create extra control flow for 'and', 'or'
539 // condition branches.
540 setJumpIsExpensive(true);
541
542 // Wide divides are _very_ slow. Try to reduce the width of the divide if
543 // possible.
544 addBypassSlowDiv(SlowBitWidth: 64, FastBitWidth: 32);
545
546 // By default, use the Source scheduling
547 if (sched4reg)
548 setSchedulingPreference(Sched::RegPressure);
549 else
550 setSchedulingPreference(Sched::Source);
551
552 auto setFP16OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action,
553 LegalizeAction NoF16Action) {
554 bool IsOpSupported = STI.allowFP16Math();
555 switch (Op) {
556 // Several FP16 instructions are available on sm_80 only.
557 case ISD::FMINNUM:
558 case ISD::FMAXNUM:
559 case ISD::FMAXNUM_IEEE:
560 case ISD::FMINNUM_IEEE:
561 case ISD::FMAXIMUM:
562 case ISD::FMINIMUM:
563 case ISD::FMAXIMUMNUM:
564 case ISD::FMINIMUMNUM:
565 IsOpSupported &= STI.hasFeature(Feature: NVPTX::SM80);
566 break;
567 case ISD::FEXP2:
568 case ISD::FTANH:
569 IsOpSupported &=
570 STI.hasFeature(Feature: NVPTX::SM75) && STI.hasFeature(Feature: NVPTX::PTX70);
571 break;
572 }
573 setOperationAction(Op, VT, Action: IsOpSupported ? Action : NoF16Action);
574 };
575
576 auto setBF16OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action,
577 LegalizeAction NoBF16Action) {
578 bool IsOpSupported = STI.hasNativeBF16Support(Opcode: Op);
579 setOperationAction(
580 Op, VT, Action: IsOpSupported ? Action : NoBF16Action);
581 };
582
583 auto setI16x2OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action,
584 LegalizeAction NoI16x2Action) {
585 bool IsOpSupported = false;
586 // instructions are available on sm_90 only
587 switch (Op) {
588 case ISD::ADD:
589 case ISD::SMAX:
590 case ISD::SMIN:
591 case ISD::UMIN:
592 case ISD::UMAX:
593 IsOpSupported =
594 STI.hasFeature(Feature: NVPTX::SM90) && STI.hasFeature(Feature: NVPTX::PTX80);
595 break;
596 }
597 setOperationAction(Op, VT, Action: IsOpSupported ? Action : NoI16x2Action);
598 };
599
600 addRegisterClass(VT: MVT::i1, RC: &NVPTX::B1RegClass);
601 addRegisterClass(VT: MVT::i16, RC: &NVPTX::B16RegClass);
602 addRegisterClass(VT: MVT::v2i16, RC: &NVPTX::B32RegClass);
603 addRegisterClass(VT: MVT::v4i8, RC: &NVPTX::B32RegClass);
604 addRegisterClass(VT: MVT::i32, RC: &NVPTX::B32RegClass);
605 addRegisterClass(VT: MVT::i64, RC: &NVPTX::B64RegClass);
606 addRegisterClass(VT: MVT::f32, RC: &NVPTX::B32RegClass);
607 addRegisterClass(VT: MVT::f64, RC: &NVPTX::B64RegClass);
608 addRegisterClass(VT: MVT::f16, RC: &NVPTX::B16RegClass);
609 addRegisterClass(VT: MVT::v2f16, RC: &NVPTX::B32RegClass);
610 addRegisterClass(VT: MVT::bf16, RC: &NVPTX::B16RegClass);
611 addRegisterClass(VT: MVT::v2bf16, RC: &NVPTX::B32RegClass);
612
613 if (STI.hasF32x2Instructions()) {
614 addRegisterClass(VT: MVT::v2f32, RC: &NVPTX::B64RegClass);
615 addRegisterClass(VT: MVT::v2i32, RC: &NVPTX::B64RegClass);
616 }
617
618 // Conversion to/from FP16/FP16x2 is always legal.
619 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v2f16, Action: Custom);
620 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2f16, Action: Custom);
621 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v2f16, Action: Expand);
622 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v2f16, Action: Expand);
623
624 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: Legal);
625 if (STI.hasFeature(Feature: NVPTX::SM30))
626 setOperationAction(Op: ISD::READSTEADYCOUNTER, VT: MVT::i64, Action: Legal);
627
628 setFP16OperationAction(ISD::SETCC, MVT::f16, Legal, Promote);
629 setFP16OperationAction(ISD::SETCC, MVT::v2f16, Legal, Expand);
630
631 // Conversion to/from BFP16/BFP16x2 is always legal.
632 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v2bf16, Action: Custom);
633 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2bf16, Action: Custom);
634 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v2bf16, Action: Expand);
635 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v2bf16, Action: Expand);
636
637 setBF16OperationAction(ISD::SETCC, MVT::v2bf16, Legal, Expand);
638 setBF16OperationAction(ISD::SETCC, MVT::bf16, Legal, Promote);
639 if (getOperationAction(Op: ISD::SETCC, VT: MVT::bf16) == Promote)
640 AddPromotedToType(Opc: ISD::SETCC, OrigVT: MVT::bf16, DestVT: MVT::f32);
641
642 // Conversion to/from i16/i16x2 is always legal.
643 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v2i16, Action: Custom);
644 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2i16, Action: Custom);
645 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v2i16, Action: Expand);
646 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v2i16, Action: Expand);
647
648 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v4i8, Action: Custom);
649 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v4i8, Action: Custom);
650 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v4i8, Action: Custom);
651 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v4i8, Action: Custom);
652
653 // No support for these operations with v2f32/v2i32
654 setOperationAction(Ops: ISD::INSERT_VECTOR_ELT, VTs: {MVT::v2f32, MVT::v2i32}, Action: Expand);
655 setOperationAction(Ops: ISD::VECTOR_SHUFFLE, VTs: {MVT::v2f32, MVT::v2i32}, Action: Expand);
656
657 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v2i16, Action: Expand);
658 setOperationAction(Ops: {ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
659 VT: MVT::v2i32, Action: Expand);
660
661 // Need custom lowering in case the index is dynamic.
662 if (STI.hasF32x2Instructions())
663 setOperationAction(Ops: ISD::EXTRACT_VECTOR_ELT, VTs: {MVT::v2f32, MVT::v2i32},
664 Action: Custom);
665
666 // Custom conversions to/from v2i8.
667 setOperationAction(Op: ISD::BITCAST, VT: MVT::v2i8, Action: Custom);
668
669 // Only logical ops can be done on v4i8/v2i32 directly, others must be done
670 // elementwise.
671 setOperationAction(
672 Ops: {ISD::ABS, ISD::ADD, ISD::ADDC, ISD::ADDE,
673 ISD::BITREVERSE, ISD::CTLZ, ISD::CTPOP, ISD::CTTZ,
674 ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::FSHL, ISD::FSHR,
675 ISD::MUL, ISD::MULHS, ISD::MULHU, ISD::PARITY,
676 ISD::ROTL, ISD::ROTR, ISD::SADDO, ISD::SADDO_CARRY,
677 ISD::SADDSAT, ISD::SDIV, ISD::SDIVREM, ISD::SELECT_CC,
678 ISD::SETCC, ISD::SHL, ISD::SINT_TO_FP, ISD::SMAX,
679 ISD::SMIN, ISD::SMULO, ISD::SMUL_LOHI, ISD::SRA,
680 ISD::SREM, ISD::SRL, ISD::SSHLSAT, ISD::SSUBO,
681 ISD::SSUBO_CARRY, ISD::SSUBSAT, ISD::SUB, ISD::SUBC,
682 ISD::SUBE, ISD::UADDO, ISD::UADDO_CARRY, ISD::UADDSAT,
683 ISD::UDIV, ISD::UDIVREM, ISD::UINT_TO_FP, ISD::UMAX,
684 ISD::UMIN, ISD::UMULO, ISD::UMUL_LOHI, ISD::UREM,
685 ISD::USHLSAT, ISD::USUBO, ISD::USUBO_CARRY, ISD::VSELECT,
686 ISD::USUBSAT},
687 VTs: {MVT::v4i8, MVT::v2i32}, Action: Expand);
688
689 // Operations not directly supported by NVPTX.
690 for (MVT VT : {MVT::bf16, MVT::f16, MVT::v2bf16, MVT::v2f16, MVT::f32,
691 MVT::v2f32, MVT::f64, MVT::i1, MVT::i8, MVT::i16, MVT::v2i16,
692 MVT::v4i8, MVT::i32, MVT::v2i32, MVT::i64}) {
693 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
694 setOperationAction(Op: ISD::BR_CC, VT, Action: Expand);
695 }
696
697 setOperationAction(Ops: ISD::SDIVREM, VTs: {MVT::i32, MVT::i64}, Action: Expand);
698 setOperationAction(Ops: ISD::UDIVREM, VTs: {MVT::i32, MVT::i64}, Action: Expand);
699
700 // We don't want ops like FMINIMUM or UMAX to be lowered to SETCC+VSELECT.
701 setOperationAction(Ops: ISD::VSELECT, VTs: {MVT::v2f32, MVT::v2i32}, Action: Expand);
702
703 // Some SIGN_EXTEND_INREG can be done using cvt instruction.
704 // For others we will expand to a SHL/SRA pair.
705 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i64, Action: Legal);
706 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i32, Action: Legal);
707 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i16, Action: Legal);
708 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i8 , Action: Legal);
709 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
710 setOperationAction(Ops: ISD::SIGN_EXTEND_INREG, VTs: {MVT::v2i16, MVT::v2i32}, Action: Expand);
711
712 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i32 , Action: Custom);
713 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i32 , Action: Custom);
714 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i32 , Action: Custom);
715 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i64 , Action: Custom);
716 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i64 , Action: Custom);
717 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i64 , Action: Custom);
718
719 if (STI.hasCLMAD())
720 setOperationAction(Ops: {ISD::CLMUL, ISD::CLMULH}, VT: MVT::i64, Action: Legal);
721 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i32, Action: Legal);
722 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i64, Action: Legal);
723
724 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR},
725 VTs: {MVT::i8, MVT::i16, MVT::v2i16, MVT::i32, MVT::i64},
726 Action: Expand);
727
728 if (STI.hasHWROT32()) {
729 setOperationAction(Ops: {ISD::FSHL, ISD::FSHR}, VT: MVT::i32, Action: Legal);
730 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR, ISD::FSHL, ISD::FSHR}, VT: MVT::i64,
731 Action: Custom);
732 }
733
734 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: STI.hasBrx() ? Legal : Expand);
735 setOperationAction(Op: ISD::BRIND, VT: MVT::Other, Action: Expand);
736
737 // We want to legalize constant related memmove and memcopy
738 // intrinsics.
739 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::Other, Action: Custom);
740
741 // FP extload/truncstore is not legal in PTX. We need to expand all these.
742 for (auto FloatVTs :
743 {MVT::fp_valuetypes(), MVT::fp_fixedlen_vector_valuetypes()}) {
744 for (MVT ValVT : FloatVTs) {
745 for (MVT MemVT : FloatVTs) {
746 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT, MemVT, Action: Expand);
747 setTruncStoreAction(ValVT, MemVT, Action: Expand);
748 }
749 }
750 }
751
752 // To improve CodeGen we'll legalize any-extend loads to zext loads. This is
753 // how they'll be lowered in ISel anyway, and by doing this a little earlier
754 // we allow for more DAG combine opportunities.
755 for (auto IntVTs :
756 {MVT::integer_valuetypes(), MVT::integer_fixedlen_vector_valuetypes()})
757 for (MVT ValVT : IntVTs)
758 for (MVT MemVT : IntVTs)
759 if (isTypeLegal(VT: ValVT))
760 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT, MemVT, Action: Custom);
761
762 // PTX does not support load / store predicate registers
763 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VT: MVT::i1, Action: Custom);
764 for (MVT VT : MVT::integer_valuetypes()) {
765 setLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}, ValVT: VT, MemVT: MVT::i1,
766 Action: Promote);
767 setTruncStoreAction(ValVT: VT, MemVT: MVT::i1, Action: Expand);
768 }
769
770 // Disable generations of extload/truncstore for v2i32/v2i16/v2i8. The generic
771 // expansion for these nodes when they are unaligned is incorrect if the
772 // type is a vector.
773 //
774 // TODO: Fix the generic expansion for these nodes found in
775 // TargetLowering::expandUnalignedLoad/Store.
776 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::v2i16,
777 MemVT: MVT::v2i8, Action: Expand);
778 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::v2i32,
779 MemVTs: {MVT::v2i8, MVT::v2i16}, Action: Expand);
780 setTruncStoreAction(ValVT: MVT::v2i16, MemVT: MVT::v2i8, Action: Expand);
781 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i16, Action: Expand);
782 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i8, Action: Expand);
783
784 // Register custom handling for illegal type loads/stores. We'll try to custom
785 // lower almost all illegal types and logic in the lowering will discard cases
786 // we can't handle.
787 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VTs: {MVT::i128, MVT::i256, MVT::f128},
788 Action: Custom);
789 for (MVT VT : MVT::fixedlen_vector_valuetypes())
790 if (!isTypeLegal(VT) && VT.getStoreSizeInBits() <= 256)
791 setOperationAction(Ops: {ISD::STORE, ISD::LOAD, ISD::MSTORE, ISD::MLOAD}, VT,
792 Action: Custom);
793
794 // Custom legalization for LDU intrinsics.
795 // TODO: The logic to lower these is not very robust and we should rewrite it.
796 // Perhaps LDU should not be represented as an intrinsic at all.
797 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::i8, Action: Custom);
798 for (MVT VT : MVT::fixedlen_vector_valuetypes())
799 if (IsPTXVectorType(VT))
800 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT, Action: Custom);
801
802 setCondCodeAction(CCs: {ISD::SETNE, ISD::SETEQ, ISD::SETUGE, ISD::SETULE,
803 ISD::SETUGT, ISD::SETULT, ISD::SETGT, ISD::SETLT,
804 ISD::SETGE, ISD::SETLE},
805 VT: MVT::i1, Action: Expand);
806
807 // This is legal in NVPTX
808 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f64, Action: Legal);
809 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f32, Action: Legal);
810 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f16, Action: Legal);
811 setOperationAction(Op: ISD::ConstantFP, VT: MVT::bf16, Action: Legal);
812
813 setOperationAction(Ops: ISD::DYNAMIC_STACKALLOC, VTs: {MVT::i32, MVT::i64}, Action: Custom);
814 setOperationAction(Ops: {ISD::STACKRESTORE, ISD::STACKSAVE}, VT: MVT::Other, Action: Custom);
815
816 // TRAP can be lowered to PTX trap
817 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
818 // DEBUGTRAP can be lowered to PTX brkpt
819 setOperationAction(Op: ISD::DEBUGTRAP, VT: MVT::Other, Action: Legal);
820
821 // Support varargs.
822 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
823 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Custom);
824 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Expand);
825 setOperationAction(Op: ISD::VAEND, VT: MVT::Other, Action: Expand);
826
827 setOperationAction(Ops: {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX},
828 VTs: {MVT::i16, MVT::i32, MVT::i64}, Action: Legal);
829 // PTX abs.s is undefined for INT_MIN, so ISD::ABS (which requires
830 // abs(INT_MIN) == INT_MIN) must be expanded. ABS_MIN_POISON matches
831 // PTX abs semantics since INT_MIN input is poison/undefined.
832 setOperationAction(Ops: ISD::ABS, VTs: {MVT::i16, MVT::i32, MVT::i64}, Action: Expand);
833 setOperationAction(Ops: ISD::ABS_MIN_POISON, VTs: {MVT::i16, MVT::i32, MVT::i64},
834 Action: Legal);
835
836 setOperationAction(Ops: {ISD::CTPOP, ISD::CTLZ, ISD::CTLZ_ZERO_POISON}, VT: MVT::i16,
837 Action: Promote);
838 setOperationAction(Ops: {ISD::CTPOP, ISD::CTLZ}, VT: MVT::i32, Action: Legal);
839 setOperationAction(Ops: {ISD::CTPOP, ISD::CTLZ}, VT: MVT::i64, Action: Custom);
840
841 setI16x2OperationAction(ISD::ABS_MIN_POISON, MVT::v2i16, Legal, Custom);
842 setI16x2OperationAction(ISD::SMIN, MVT::v2i16, Legal, Custom);
843 setI16x2OperationAction(ISD::SMAX, MVT::v2i16, Legal, Custom);
844 setI16x2OperationAction(ISD::UMIN, MVT::v2i16, Legal, Custom);
845 setI16x2OperationAction(ISD::UMAX, MVT::v2i16, Legal, Custom);
846 setI16x2OperationAction(ISD::CTPOP, MVT::v2i16, Legal, Expand);
847 setI16x2OperationAction(ISD::CTLZ, MVT::v2i16, Legal, Expand);
848
849 setI16x2OperationAction(ISD::ADD, MVT::v2i16, Legal, Custom);
850 setI16x2OperationAction(ISD::SUB, MVT::v2i16, Legal, Custom);
851 setI16x2OperationAction(ISD::MUL, MVT::v2i16, Legal, Custom);
852 setI16x2OperationAction(ISD::SHL, MVT::v2i16, Legal, Custom);
853 setI16x2OperationAction(ISD::SREM, MVT::v2i16, Legal, Custom);
854 setI16x2OperationAction(ISD::UREM, MVT::v2i16, Legal, Custom);
855
856 // Other arithmetic and logic ops are unsupported.
857 setOperationAction(Ops: {ISD::SDIV, ISD::UDIV, ISD::SRA, ISD::SRL, ISD::MULHS,
858 ISD::MULHU, ISD::SMUL_LOHI, ISD::UMUL_LOHI,
859 ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::SINT_TO_FP,
860 ISD::UINT_TO_FP, ISD::SETCC},
861 VT: MVT::v2i16, Action: Expand);
862
863 // v2i32 is not supported for any arithmetic operations
864 setOperationAction(Ops: {ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX,
865 ISD::CTPOP, ISD::CTLZ, ISD::ADD, ISD::SUB, ISD::MUL,
866 ISD::SHL, ISD::SRA, ISD::SRL, ISD::OR, ISD::AND, ISD::XOR,
867 ISD::SREM, ISD::UREM},
868 VT: MVT::v2i32, Action: Expand);
869
870 setOperationAction(Op: ISD::ADDC, VT: MVT::i32, Action: Legal);
871 setOperationAction(Op: ISD::ADDE, VT: MVT::i32, Action: Legal);
872 setOperationAction(Op: ISD::SUBC, VT: MVT::i32, Action: Legal);
873 setOperationAction(Op: ISD::SUBE, VT: MVT::i32, Action: Legal);
874 if (STI.hasFeature(Feature: NVPTX::PTX43)) {
875 setOperationAction(Op: ISD::ADDC, VT: MVT::i64, Action: Legal);
876 setOperationAction(Op: ISD::ADDE, VT: MVT::i64, Action: Legal);
877 setOperationAction(Op: ISD::SUBC, VT: MVT::i64, Action: Legal);
878 setOperationAction(Op: ISD::SUBE, VT: MVT::i64, Action: Legal);
879 }
880
881 setOperationAction(Op: ISD::CTTZ, VT: MVT::i16, Action: Expand);
882 setOperationAction(Ops: ISD::CTTZ, VTs: {MVT::v2i16, MVT::v2i32}, Action: Expand);
883 setOperationAction(Op: ISD::CTTZ, VT: MVT::i32, Action: Expand);
884 setOperationAction(Op: ISD::CTTZ, VT: MVT::i64, Action: Expand);
885
886 // PTX does not directly support SELP of i1, so promote to i32 first
887 setOperationAction(Op: ISD::SELECT, VT: MVT::i1, Action: Custom);
888
889 // PTX only exposes separate mul.lo and mul.hi forms. Lower i16 and i32 to
890 // use mul.wide. i64 has no mul.wide, so it expands to MUL and MULHS/MULHU.
891 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VTs: {MVT::i16, MVT::i32},
892 Action: Custom);
893 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT: MVT::i64, Action: Expand);
894
895 // We have some custom DAG combine patterns for these nodes
896 setTargetDAGCombine({ISD::ADD,
897 ISD::AND,
898 ISD::EXTRACT_VECTOR_ELT,
899 ISD::FADD,
900 ISD::FMAXNUM,
901 ISD::FMINNUM,
902 ISD::FMAXIMUM,
903 ISD::FMINIMUM,
904 ISD::FMAXIMUMNUM,
905 ISD::FMINIMUMNUM,
906 ISD::MUL,
907 ISD::SELECT,
908 ISD::SHL,
909 ISD::SREM,
910 ISD::UREM,
911 ISD::VSELECT,
912 ISD::BUILD_VECTOR,
913 ISD::ADDRSPACECAST,
914 ISD::LOAD,
915 ISD::STORE,
916 ISD::ZERO_EXTEND,
917 ISD::SIGN_EXTEND,
918 ISD::INTRINSIC_WO_CHAIN});
919
920 // If the vector operands require register coalescing, scalarize instead
921 if (STI.hasF32x2Instructions())
922 setTargetDAGCombine({ISD::FMA, ISD::FMUL, ISD::FSUB});
923
924 // setcc for f16x2 and bf16x2 needs special handling to prevent
925 // legalizer's attempt to scalarize it due to v2i1 not being legal.
926 if (STI.allowFP16Math() || STI.hasBF16Math())
927 setTargetDAGCombine(ISD::SETCC);
928
929 // Vector reduction operations. These may be turned into shuffle or tree
930 // reductions depending on what instructions are available for each type.
931 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
932 MVT EltVT = VT.getVectorElementType();
933 if (EltVT == MVT::f32 || EltVT == MVT::f64) {
934 setOperationAction(Ops: {ISD::VECREDUCE_FMAX, ISD::VECREDUCE_FMIN,
935 ISD::VECREDUCE_FMAXIMUM, ISD::VECREDUCE_FMINIMUM},
936 VT, Action: Custom);
937 }
938 }
939
940 // Promote fp16 arithmetic if fp16 hardware isn't available or the
941 // user passed --nvptx-no-fp16-math. The flag is useful because,
942 // although sm_53+ GPUs have some sort of FP16 support in
943 // hardware, only sm_53 and sm_60 have full implementation. Others
944 // only have token amount of hardware and are likely to run faster
945 // by using fp32 units instead.
946 for (const auto &Op : {ISD::FADD, ISD::FMUL, ISD::FSUB, ISD::FMA}) {
947 setFP16OperationAction(Op, MVT::f16, Legal, Promote);
948 setFP16OperationAction(Op, MVT::v2f16, Legal, Expand);
949 setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);
950 // bf16 must be promoted to f32.
951 setBF16OperationAction(Op, MVT::bf16, Legal, Promote);
952 if (getOperationAction(Op, VT: MVT::bf16) == Promote)
953 AddPromotedToType(Opc: Op, OrigVT: MVT::bf16, DestVT: MVT::f32);
954 setOperationAction(Op, VT: MVT::v2f32,
955 Action: STI.hasF32x2Instructions() ? Legal : Expand);
956 }
957
958 // On SM80, we select add/mul/sub as fma to avoid promotion to float
959 for (const auto &Op : {ISD::FADD, ISD::FMUL, ISD::FSUB}) {
960 for (const auto &VT : {MVT::bf16, MVT::v2bf16}) {
961 if (!STI.hasNativeBF16Support(Opcode: Op) && STI.hasNativeBF16Support(Opcode: ISD::FMA)) {
962 setOperationAction(Op, VT, Action: Custom);
963 }
964 }
965 }
966
967 // f16/f16x2 neg was introduced in PTX 60, SM_53.
968 const bool IsFP16FP16x2NegAvailable = STI.hasFeature(Feature: NVPTX::SM53) &&
969 STI.hasFeature(Feature: NVPTX::PTX60) &&
970 STI.allowFP16Math();
971 for (const auto &VT : {MVT::f16, MVT::v2f16})
972 setOperationAction(Op: ISD::FNEG, VT,
973 Action: IsFP16FP16x2NegAvailable ? Legal : Expand);
974
975 setBF16OperationAction(ISD::FNEG, MVT::bf16, Legal, Expand);
976 setBF16OperationAction(ISD::FNEG, MVT::v2bf16, Legal, Expand);
977 setOperationAction(Op: ISD::FNEG, VT: MVT::v2f32, Action: Expand);
978 // (would be) Library functions.
979
980 // These map to conversion instructions for scalar FP types.
981 for (const auto &Op : {ISD::FCEIL, ISD::FFLOOR, ISD::FNEARBYINT, ISD::FRINT,
982 ISD::FROUNDEVEN, ISD::FTRUNC}) {
983 setOperationAction(Op, VT: MVT::f16, Action: Legal);
984 setOperationAction(Op, VT: MVT::f32, Action: Legal);
985 setOperationAction(Op, VT: MVT::f64, Action: Legal);
986 setOperationAction(Op, VT: MVT::v2f16, Action: Expand);
987 setOperationAction(Op, VT: MVT::v2bf16, Action: Expand);
988 setOperationAction(Op, VT: MVT::v2f32, Action: Expand);
989 setBF16OperationAction(Op, MVT::bf16, Legal, Promote);
990 if (getOperationAction(Op, VT: MVT::bf16) == Promote)
991 AddPromotedToType(Opc: Op, OrigVT: MVT::bf16, DestVT: MVT::f32);
992 }
993
994 if (!STI.hasFeature(Feature: NVPTX::SM80) || !STI.hasFeature(Feature: NVPTX::PTX71)) {
995 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f32, Action: Expand);
996 }
997 if (!STI.hasFeature(Feature: NVPTX::SM90)) {
998 for (MVT VT : {MVT::bf16, MVT::f32, MVT::f64}) {
999 setOperationAction(Op: ISD::FP_EXTEND, VT, Action: Custom);
1000 setOperationAction(Op: ISD::FP_ROUND, VT, Action: Custom);
1001 }
1002 }
1003
1004 // Expand nearest-even rounding and diagnose unsupported conversions.
1005 setOperationAction(Ops: ISD::FPTRUNC_ROUND, VTs: {MVT::f16, MVT::bf16, MVT::f32},
1006 Action: Custom);
1007
1008 // Expand v2f32 = fp_extend
1009 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v2f32, Action: Expand);
1010 // Expand v2[b]f16 = fp_round v2f32
1011 setOperationAction(Ops: ISD::FP_ROUND, VTs: {MVT::v2bf16, MVT::v2f16}, Action: Expand);
1012
1013 // sm_80 only has conversions between f32 and bf16. Custom lower all other
1014 // bf16 conversions.
1015 if (!STI.hasFeature(Feature: NVPTX::SM90)) {
1016 for (MVT VT : {MVT::i1, MVT::i16, MVT::i32, MVT::i64}) {
1017 setOperationAction(
1018 Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT},
1019 VT, Action: Custom);
1020 }
1021 setOperationAction(
1022 Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT},
1023 VT: MVT::bf16, Action: Custom);
1024 }
1025
1026 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT: MVT::i1, Action: Custom);
1027 setOperationAction(Op: ISD::FROUND, VT: MVT::f16, Action: Promote);
1028 setOperationAction(Op: ISD::FROUND, VT: MVT::v2f16, Action: Expand);
1029 setOperationAction(Op: ISD::FROUND, VT: MVT::v2bf16, Action: Expand);
1030 setOperationAction(Op: ISD::FROUND, VT: MVT::f32, Action: Custom);
1031 setOperationAction(Op: ISD::FROUND, VT: MVT::f64, Action: Custom);
1032 setOperationAction(Op: ISD::FROUND, VT: MVT::bf16, Action: Promote);
1033 AddPromotedToType(Opc: ISD::FROUND, OrigVT: MVT::bf16, DestVT: MVT::f32);
1034
1035 setOperationAction(Ops: {ISD::LROUND, ISD::LLROUND}, VTs: {MVT::f32, MVT::f64}, Action: Expand);
1036
1037 // 'Expand' implements FCOPYSIGN without calling an external library.
1038 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f16, Action: Expand);
1039 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::v2f16, Action: Expand);
1040 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::bf16, Action: Expand);
1041 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::v2bf16, Action: Expand);
1042 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f32, Action: Custom);
1043 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f64, Action: Custom);
1044
1045 // These map to corresponding instructions for f32/f64. f16 must be
1046 // promoted to f32. v2f16 is expanded to f16, which is then promoted
1047 // to f32.
1048 for (const auto &Op : {ISD::FDIV, ISD::FSQRT, ISD::FSIN, ISD::FCOS}) {
1049 setOperationAction(Op, VT: MVT::f16, Action: Promote);
1050 setOperationAction(Op, VT: MVT::f32, Action: Legal);
1051 // Only div/sqrt are legal for f64.
1052 if (Op == ISD::FDIV || Op == ISD::FSQRT) {
1053 setOperationAction(Op, VT: MVT::f64, Action: Legal);
1054 }
1055 setOperationAction(Ops: Op, VTs: {MVT::v2f16, MVT::v2bf16, MVT::v2f32}, Action: Expand);
1056 setOperationAction(Op, VT: MVT::bf16, Action: Promote);
1057 AddPromotedToType(Opc: Op, OrigVT: MVT::bf16, DestVT: MVT::f32);
1058 }
1059 // Expand remainders at the IR level using ExpandIRInsts.
1060 setOperationAction(Ops: ISD::FREM, VTs: {MVT::f16, MVT::bf16, MVT::f32, MVT::f64},
1061 Action: Expand);
1062
1063 // FTANH support:
1064 // - f32 (sm_75+, PTX 7.0+)
1065 // - f16/f16x2 (sm_75+, PTX 7.0+)
1066 // - bf16/bf16x2 (sm_90+, PTX 7.8+)
1067 // When f16/bf16 types aren't supported, they are promoted/expanded to f32.
1068 if (STI.hasFeature(Feature: NVPTX::SM75) && STI.hasFeature(Feature: NVPTX::PTX70))
1069 setOperationAction(Op: ISD::FTANH, VT: MVT::f32, Action: Legal);
1070 setOperationAction(Op: ISD::FTANH, VT: MVT::v2f32, Action: Expand);
1071
1072 // Scalar f16/bf16: promote to f32 when not natively supported.
1073 setFP16OperationAction(ISD::FTANH, MVT::f16, Legal, Promote);
1074 setBF16OperationAction(ISD::FTANH, MVT::bf16, Legal, Promote);
1075 if (getOperationAction(Op: ISD::FTANH, VT: MVT::bf16) == Promote)
1076 AddPromotedToType(Opc: ISD::FTANH, OrigVT: MVT::bf16, DestVT: MVT::f32);
1077
1078 // Vector v2f16/v2bf16: expand when not natively supported.
1079 setFP16OperationAction(ISD::FTANH, MVT::v2f16, Legal, Expand);
1080 setBF16OperationAction(ISD::FTANH, MVT::v2bf16, Legal, Expand);
1081
1082 setOperationAction(Ops: ISD::FABS, VTs: {MVT::f32, MVT::f64}, Action: Legal);
1083 setOperationAction(Op: ISD::FABS, VT: MVT::v2f32, Action: Expand);
1084 if (STI.hasFeature(Feature: NVPTX::PTX65)) {
1085 setFP16OperationAction(ISD::FABS, MVT::f16, Legal, Promote);
1086 setFP16OperationAction(ISD::FABS, MVT::v2f16, Legal, Expand);
1087 } else {
1088 setOperationAction(Op: ISD::FABS, VT: MVT::f16, Action: Promote);
1089 setOperationAction(Op: ISD::FABS, VT: MVT::v2f16, Action: Expand);
1090 }
1091 setBF16OperationAction(ISD::FABS, MVT::v2bf16, Legal, Expand);
1092 setBF16OperationAction(ISD::FABS, MVT::bf16, Legal, Promote);
1093 if (getOperationAction(Op: ISD::FABS, VT: MVT::bf16) == Promote)
1094 AddPromotedToType(Opc: ISD::FABS, OrigVT: MVT::bf16, DestVT: MVT::f32);
1095
1096 for (const auto &Op :
1097 {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM}) {
1098 setOperationAction(Op, VT: MVT::f32, Action: Legal);
1099 setOperationAction(Op, VT: MVT::f64, Action: Legal);
1100 setFP16OperationAction(Op, MVT::f16, Legal, Promote);
1101 setFP16OperationAction(Op, MVT::v2f16, Legal, Expand);
1102 setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);
1103 setBF16OperationAction(Op, MVT::bf16, Legal, Promote);
1104 if (getOperationAction(Op, VT: MVT::bf16) == Promote)
1105 AddPromotedToType(Opc: Op, OrigVT: MVT::bf16, DestVT: MVT::f32);
1106 setOperationAction(Op, VT: MVT::v2f32, Action: Expand);
1107 }
1108 bool SupportsF32MinMaxNaN = STI.hasFeature(Feature: NVPTX::SM80);
1109 for (const auto &Op : {ISD::FMINIMUM, ISD::FMAXIMUM}) {
1110 setOperationAction(Op, VT: MVT::f32, Action: SupportsF32MinMaxNaN ? Legal : Expand);
1111 setFP16OperationAction(Op, MVT::f16, Legal, Expand);
1112 setFP16OperationAction(Op, MVT::v2f16, Legal, Expand);
1113 setBF16OperationAction(Op, MVT::bf16, Legal, Expand);
1114 setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);
1115 setOperationAction(Op, VT: MVT::v2f32, Action: Expand);
1116 }
1117
1118 // Custom lowering for inline asm with 128-bit operands
1119 setOperationAction(Op: ISD::CopyToReg, VT: MVT::i128, Action: Custom);
1120 setOperationAction(Op: ISD::CopyFromReg, VT: MVT::i128, Action: Custom);
1121
1122 // FEXP2 support:
1123 // - f32
1124 // - f16/f16x2 (sm_70+, PTX 7.0+)
1125 // - bf16/bf16x2 (sm_90+, PTX 7.8+)
1126 // When f16/bf16 types aren't supported, they are promoted/expanded to f32.
1127 setOperationAction(Op: ISD::FEXP2, VT: MVT::f32, Action: Legal);
1128 setOperationAction(Op: ISD::FEXP2, VT: MVT::v2f32, Action: Expand);
1129 setFP16OperationAction(ISD::FEXP2, MVT::f16, Legal, Promote);
1130 setFP16OperationAction(ISD::FEXP2, MVT::v2f16, Legal, Expand);
1131 setBF16OperationAction(ISD::FEXP2, MVT::bf16, Legal, Promote);
1132 setBF16OperationAction(ISD::FEXP2, MVT::v2bf16, Legal, Expand);
1133
1134 // FLOG2 supports f32 only
1135 // f16/bf16 types aren't supported, but they are promoted/expanded to f32.
1136 if (UseApproxLog2F32) {
1137 setOperationAction(Op: ISD::FLOG2, VT: MVT::f32, Action: Legal);
1138 setOperationPromotedToType(Opc: ISD::FLOG2, OrigVT: MVT::f16, DestVT: MVT::f32);
1139 setOperationPromotedToType(Opc: ISD::FLOG2, OrigVT: MVT::bf16, DestVT: MVT::f32);
1140 setOperationAction(Ops: ISD::FLOG2, VTs: {MVT::v2f16, MVT::v2bf16, MVT::v2f32},
1141 Action: Expand);
1142 }
1143
1144 setOperationAction(Ops: ISD::ADDRSPACECAST, VTs: {MVT::i32, MVT::i64}, Action: Custom);
1145
1146 // atom.b128 is legal in PTX but since we don't represent i128 as a legal
1147 // type, we need to custom lower it.
1148 setOperationAction(Ops: {ISD::ATOMIC_CMP_SWAP, ISD::ATOMIC_SWAP}, VT: MVT::i128,
1149 Action: Custom);
1150
1151 // Now deduce the information based on the above mentioned
1152 // actions
1153 computeRegisterProperties(TRI: STI.getRegisterInfo());
1154
1155 // PTX support for 16-bit CAS is emulated. Only use 32+
1156 setMinCmpXchgSizeInBits(STI.getMinCmpXchgSizeInBits());
1157 setMaxAtomicSizeInBitsSupported(STI.hasAtomSwap128() ? 128 : 64);
1158 setMaxDivRemBitWidthSupported(64);
1159 setMaxLargeFPConvertBitWidthSupported(64);
1160
1161 // Custom lowering for tcgen05.ld vector operands
1162 setOperationAction(Ops: ISD::INTRINSIC_W_CHAIN,
1163 VTs: {MVT::v1i32, MVT::v2i32, MVT::v4i32, MVT::v8i32,
1164 MVT::v16i32, MVT::v32i32, MVT::v64i32, MVT::v128i32,
1165 MVT::v2f32, MVT::v4f32, MVT::v8f32, MVT::v16f32,
1166 MVT::v32f32, MVT::v64f32, MVT::v128f32},
1167 Action: Custom);
1168
1169 // Custom lowering for tcgen05.st vector operands and the st.async
1170 // i128 (.b128) operand. MVT::i8 is needed for the st.async.{sys,gpu} b8
1171 // variant.
1172 setOperationAction(Ops: ISD::INTRINSIC_VOID,
1173 VTs: {MVT::i8, MVT::v1i32, MVT::v2i32, MVT::v4i32, MVT::v8i32,
1174 MVT::v16i32, MVT::v32i32, MVT::v64i32, MVT::v128i32,
1175 MVT::i128, MVT::Other},
1176 Action: Custom);
1177
1178 // Enable custom lowering for the following:
1179 // * MVT::i128 - clusterlaunchcontrol
1180 // * MVT::i32 - prmt
1181 // * MVT::v4f32 - cvt_rs fp{4/6/8}x4 intrinsics
1182 // * MVT::Other - internal.addrspace.wrap
1183 setOperationAction(Ops: ISD::INTRINSIC_WO_CHAIN,
1184 VTs: {MVT::i32, MVT::i128, MVT::v4f32, MVT::Other}, Action: Custom);
1185
1186 // Custom lowering for bswap
1187 setOperationAction(Ops: ISD::BSWAP, VTs: {MVT::i16, MVT::i32, MVT::i64, MVT::v2i16},
1188 Action: Custom);
1189}
1190
1191TargetLoweringBase::LegalizeTypeAction
1192NVPTXTargetLowering::getPreferredVectorAction(MVT VT) const {
1193 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
1194 VT.getScalarType() == MVT::i1)
1195 return TypeSplitVector;
1196 return TargetLoweringBase::getPreferredVectorAction(VT);
1197}
1198
1199SDValue NVPTXTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,
1200 int Enabled, int &ExtraSteps,
1201 bool &UseOneConst,
1202 bool Reciprocal) const {
1203 if (!(Enabled == ReciprocalEstimate::Enabled ||
1204 (Enabled == ReciprocalEstimate::Unspecified && !usePrecSqrtF32())))
1205 return SDValue();
1206
1207 if (ExtraSteps == ReciprocalEstimate::Unspecified)
1208 ExtraSteps = 0;
1209
1210 SDLoc DL(Operand);
1211 EVT VT = Operand.getValueType();
1212 bool Ftz = useF32FTZ(MF: DAG.getMachineFunction());
1213
1214 auto MakeIntrinsicCall = [&](Intrinsic::ID IID) {
1215 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT,
1216 N1: DAG.getConstant(Val: IID, DL, VT: MVT::i32), N2: Operand);
1217 };
1218
1219 // The sqrt and rsqrt refinement processes assume we always start out with an
1220 // approximation of the rsqrt. Therefore, if we're going to do any refinement
1221 // (i.e. ExtraSteps > 0), we must return an rsqrt. But if we're *not* doing
1222 // any refinement, we must return a regular sqrt.
1223 if (Reciprocal || ExtraSteps > 0) {
1224 if (VT == MVT::f32)
1225 return MakeIntrinsicCall(Ftz ? Intrinsic::nvvm_rsqrt_approx_ftz_f
1226 : Intrinsic::nvvm_rsqrt_approx_f);
1227 else if (VT == MVT::f64)
1228 return MakeIntrinsicCall(Intrinsic::nvvm_rsqrt_approx_d);
1229 else
1230 return SDValue();
1231 } else {
1232 if (VT == MVT::f32)
1233 return MakeIntrinsicCall(Ftz ? Intrinsic::nvvm_sqrt_approx_ftz_f
1234 : Intrinsic::nvvm_sqrt_approx_f);
1235 else {
1236 // There's no sqrt.approx.f64 instruction, so we emit
1237 // reciprocal(rsqrt(x)). This is faster than
1238 // select(x == 0, 0, x * rsqrt(x)). (In fact, it's faster than plain
1239 // x * rsqrt(x).)
1240 return DAG.getNode(
1241 Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT,
1242 N1: DAG.getConstant(Val: Intrinsic::nvvm_rcp_approx_ftz_d, DL, VT: MVT::i32),
1243 N2: MakeIntrinsicCall(Intrinsic::nvvm_rsqrt_approx_d));
1244 }
1245 }
1246}
1247
1248static MachinePointerInfo refinePtrAS(SDValue &Ptr, SelectionDAG &DAG) {
1249 // Load directly from the source address space of a cast to generic.
1250 unsigned SrcAS = ADDRESS_SPACE_GENERIC;
1251 if (Ptr->getOpcode() == ISD::ADDRSPACECAST) {
1252 const auto *ASC = cast<AddrSpaceCastSDNode>(Val&: Ptr);
1253 if (ASC->getDestAddressSpace() == ADDRESS_SPACE_GENERIC) {
1254 Ptr = ASC->getOperand(Num: 0);
1255 SrcAS = ASC->getSrcAddressSpace();
1256 }
1257 }
1258
1259 // Preserve the alloca's address space through frame-index inference.
1260 if (const auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Ptr))
1261 if (const AllocaInst *AI =
1262 DAG.getMachineFunction().getFrameInfo().getObjectAllocation(
1263 ObjectIdx: FIN->getIndex()))
1264 return MachinePointerInfo(AI);
1265
1266 return MachinePointerInfo(SrcAS);
1267}
1268
1269static ISD::NodeType getExtOpcode(const ISD::ArgFlagsTy &Flags) {
1270 if (Flags.isSExt())
1271 return ISD::SIGN_EXTEND;
1272 if (Flags.isZExt())
1273 return ISD::ZERO_EXTEND;
1274 return ISD::ANY_EXTEND;
1275}
1276
1277static SDValue correctParamType(SDValue V, EVT ExpectedVT,
1278 ISD::ArgFlagsTy Flags, SelectionDAG &DAG,
1279 SDLoc dl) {
1280 const EVT ActualVT = V.getValueType();
1281 assert((ActualVT == ExpectedVT ||
1282 (ExpectedVT.isInteger() && ActualVT.isInteger())) &&
1283 "Non-integer argument type size mismatch");
1284 if (ExpectedVT.bitsGT(VT: ActualVT))
1285 return DAG.getNode(Opcode: getExtOpcode(Flags), DL: dl, VT: ExpectedVT, Operand: V);
1286 if (ExpectedVT.bitsLT(VT: ActualVT))
1287 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ExpectedVT, Operand: V);
1288
1289 return V;
1290}
1291
1292static SDValue getSymbolNode(SelectionDAG &DAG, MCSymbol *Sym, EVT T) {
1293 return DAG.getNode(Opcode: NVPTXISD::Symbol, DL: SDLoc(), VT: T, Operand: DAG.getMCSymbol(Sym, VT: T));
1294}
1295
1296static SDValue getSymbolNode(SelectionDAG &DAG, const Twine &Name, EVT T) {
1297 MCContext &Ctx = DAG.getMachineFunction().getContext();
1298 return getSymbolNode(DAG, Sym: Ctx.getOrCreateSymbol(Name), T);
1299}
1300
1301SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
1302 SmallVectorImpl<SDValue> &InVals) const {
1303
1304 if (CLI.IsVarArg &&
1305 (!STI.hasFeature(Feature: NVPTX::PTX60) || !STI.hasFeature(Feature: NVPTX::SM30)))
1306 report_fatal_error(
1307 reason: "Support for variadic functions (unsized array parameter) introduced "
1308 "in PTX ISA version 6.0 and requires target sm_30.");
1309
1310 SelectionDAG &DAG = CLI.DAG;
1311 SDLoc dl = CLI.DL;
1312 const SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
1313 SDValue Callee = CLI.Callee;
1314 ArgListTy &Args = CLI.getArgs();
1315 Type *RetTy = CLI.RetTy;
1316 const CallBase *CB = CLI.CB;
1317 const DataLayout &DL = DAG.getDataLayout();
1318 LLVMContext &Ctx = *DAG.getContext();
1319
1320 const auto GetI32 = [&](const unsigned I) {
1321 return DAG.getConstant(Val: I, DL: dl, VT: MVT::i32);
1322 };
1323
1324 const unsigned UniqueCallSite = GlobalUniqueCallSite++;
1325 const SDValue CallChain = CLI.Chain;
1326 const SDValue StartChain =
1327 DAG.getCALLSEQ_START(Chain: CallChain, InSize: UniqueCallSite, OutSize: 0, DL: dl);
1328 SDValue DeclareGlue = StartChain.getValue(R: 1);
1329
1330 SmallVector<SDValue, 16> CallPrereqs{StartChain};
1331
1332 const auto MakeDeclareScalarParam = [&](SDValue Symbol, unsigned Size) {
1333 // PTX ABI requires integral types to be at least 32 bits in size. FP16 is
1334 // loaded/stored using i16, so it's handled here as well.
1335 const unsigned SizeBits = promoteScalarArgumentSize(Size: Size * 8);
1336 SDValue Declare =
1337 DAG.getNode(Opcode: NVPTXISD::DeclareScalarParam, DL: dl, ResultTys: {MVT::Other, MVT::Glue},
1338 Ops: {StartChain, Symbol, GetI32(SizeBits), DeclareGlue});
1339 CallPrereqs.push_back(Elt: Declare);
1340 DeclareGlue = Declare.getValue(R: 1);
1341 return Declare;
1342 };
1343
1344 const auto MakeDeclareArrayParam = [&](SDValue Symbol, Align Align,
1345 unsigned Size) {
1346 SDValue Declare = DAG.getNode(
1347 Opcode: NVPTXISD::DeclareArrayParam, DL: dl, ResultTys: {MVT::Other, MVT::Glue},
1348 Ops: {StartChain, Symbol, GetI32(Align.value()), GetI32(Size), DeclareGlue});
1349 CallPrereqs.push_back(Elt: Declare);
1350 DeclareGlue = Declare.getValue(R: 1);
1351 return Declare;
1352 };
1353
1354 // Variadic arguments.
1355 //
1356 // Normally, for each argument, we declare a param scalar or a param
1357 // byte array in the .param space, and store the argument value to that
1358 // param scalar or array starting at offset 0.
1359 //
1360 // In the case of the first variadic argument, we declare a vararg byte array
1361 // with size 0. The exact size of this array isn't known at this point, so
1362 // it'll be patched later. All the variadic arguments will be stored to this
1363 // array at a certain offset (which gets tracked by 'VAOffset'). The offset is
1364 // initially set to 0, so it can be used for non-variadic arguments (which use
1365 // 0 offset) to simplify the code.
1366 //
1367 // After all vararg is processed, 'VAOffset' holds the size of the
1368 // vararg byte array.
1369 assert((CLI.IsVarArg || CLI.Args.size() <= CLI.NumFixedArgs) &&
1370 "Non-VarArg function with extra arguments");
1371
1372 const unsigned FirstVAArg = CLI.NumFixedArgs; // position of first variadic
1373 unsigned VAOffset = 0; // current offset in the param array
1374
1375 const SDValue VADeclareParam =
1376 CLI.Args.size() > FirstVAArg
1377 ? MakeDeclareArrayParam(
1378 getCallParamSymbolNode(DAG, I: FirstVAArg, T: MVT::i32),
1379 Align(STI.getMaxRequiredAlignment()), 0)
1380 : SDValue();
1381
1382 // Args.size() and Outs.size() need not match.
1383 // Outs.size() will be larger
1384 // * if there is an aggregate argument with multiple fields (each field
1385 // showing up separately in Outs)
1386 // * if there is a vector argument with more than typical vector-length
1387 // elements (generally if more than 4) where each vector element is
1388 // individually present in Outs.
1389 // So a different index should be used for indexing into Outs/OutVals.
1390 // See similar issue in LowerFormalArguments.
1391 auto AllOuts = ArrayRef(CLI.Outs);
1392 auto AllOutVals = ArrayRef(CLI.OutVals);
1393 assert(AllOuts.size() == AllOutVals.size() &&
1394 "Outs and OutVals must be the same size");
1395 // Declare the .params or .reg need to pass values
1396 // to the function
1397 for (const auto E : llvm::enumerate(First&: Args)) {
1398 const auto ArgI = E.index();
1399 const auto Arg = E.value();
1400 const auto ArgOuts =
1401 AllOuts.take_while(Pred: [&](auto O) { return O.OrigArgIndex == ArgI; });
1402 const auto ArgOutVals = AllOutVals.take_front(N: ArgOuts.size());
1403 AllOuts = AllOuts.drop_front(N: ArgOuts.size());
1404 AllOutVals = AllOutVals.drop_front(N: ArgOuts.size());
1405
1406 const bool IsVAArg = (ArgI >= FirstVAArg);
1407 const bool IsByVal = Arg.IsByVal;
1408
1409 const SDValue ParamSymbol =
1410 getCallParamSymbolNode(DAG, I: IsVAArg ? FirstVAArg : ArgI, T: MVT::i32);
1411
1412 assert((!IsByVal || Arg.IndirectType) &&
1413 "byval arg must have indirect type");
1414 Type *ETy = (IsByVal ? Arg.IndirectType : Arg.Ty);
1415
1416 const Align ArgAlign = [&]() {
1417 const unsigned ParamIdx = ArgI + AttributeList::FirstArgIndex;
1418 if (IsByVal)
1419 return getDeviceByValParamAlign(CB, ArgTy: ETy, AttrIdx: ParamIdx, DL);
1420 return getPTXParamAlign(CB, Ty: Arg.Ty, AttrIdx: ParamIdx, DL);
1421 }();
1422
1423 const unsigned TySize = DL.getTypeAllocSize(Ty: ETy);
1424 assert((!IsByVal || TySize == ArgOuts[0].Flags.getByValSize()) &&
1425 "type size mismatch");
1426
1427 const SDValue ArgDeclare = [&]() {
1428 if (IsVAArg)
1429 return VADeclareParam;
1430
1431 if (IsByVal || shouldPassAsArray(Ty: Arg.Ty))
1432 return MakeDeclareArrayParam(ParamSymbol, ArgAlign, TySize);
1433
1434 assert(ArgOuts.size() == 1 && "We must pass only one value as non-array");
1435 assert((ArgOuts[0].VT.isInteger() || ArgOuts[0].VT.isFloatingPoint()) &&
1436 "Only int and float types are supported as non-array arguments");
1437
1438 return MakeDeclareScalarParam(ParamSymbol, TySize);
1439 }();
1440
1441 if (IsByVal) {
1442 assert(ArgOutVals.size() == 1 && "We must pass only one value as byval");
1443 SDValue SrcPtr = ArgOutVals[0];
1444 const MachinePointerInfo SrcPtrInfo = refinePtrAS(Ptr&: SrcPtr, DAG);
1445 // Don't use Flags.getNonZeroByValAlign as this includes the stackalign,
1446 // which does not apply to the source pointer.
1447 const Align BaseSrcAlign = [&]() {
1448 // The align attribute on a byval argument indicates the known alignment
1449 // of the pointer passed to the function.
1450 if (CB)
1451 if (const MaybeAlign A = CB->getParamAlign(ArgNo: ArgI))
1452 return *A;
1453 // Fall back to the default alignment for the type.
1454 // TODO: This might be too aggressive but we haven't had a problem with
1455 // it yet.
1456 return getPTXParamTypeAlign(ArgTy: ETy, DL);
1457 }();
1458
1459 if (IsVAArg)
1460 VAOffset = alignTo(Size: VAOffset, A: ArgAlign);
1461
1462 SmallVector<EVT, 4> ValueVTs, MemVTs;
1463 SmallVector<TypeSize, 4> Offsets;
1464 ComputeValueVTs(TLI: *this, DL, Ty: ETy, ValueVTs, MemVTs: &MemVTs, Offsets: &Offsets);
1465
1466 unsigned J = 0;
1467 const auto VI = VectorizePTXValueVTs(ValueVTs: MemVTs, Offsets, ParamAlignment: ArgAlign, IsVAArg);
1468 for (const unsigned NumElts : VI) {
1469 EVT LoadVT = getVectorizedVT(VT: MemVTs[J], N: NumElts, C&: Ctx);
1470 Align SrcAlign = commonAlignment(A: BaseSrcAlign, Offset: Offsets[J]);
1471 SDValue SrcAddr = DAG.getObjectPtrOffset(SL: dl, Ptr: SrcPtr, Offset: Offsets[J]);
1472 SDValue SrcLoad =
1473 DAG.getLoad(VT: LoadVT, dl, Chain: CallChain, Ptr: SrcAddr,
1474 PtrInfo: SrcPtrInfo.getWithOffset(O: Offsets[J]), Alignment: SrcAlign);
1475
1476 TypeSize ParamOffset = Offsets[J].getWithIncrement(RHS: VAOffset);
1477 Align ParamAlign = commonAlignment(A: ArgAlign, Offset: ParamOffset);
1478 SDValue ParamAddr =
1479 DAG.getObjectPtrOffset(SL: dl, Ptr: ParamSymbol, Offset: ParamOffset);
1480 SDValue StoreParam = DAG.getStore(
1481 Chain: ArgDeclare, dl, Val: SrcLoad, Ptr: ParamAddr,
1482 PtrInfo: MachinePointerInfo(NVPTX::AddressSpace::DeviceParam), Alignment: ParamAlign);
1483 CallPrereqs.push_back(Elt: StoreParam);
1484
1485 J += NumElts;
1486 }
1487 if (IsVAArg)
1488 VAOffset += TySize;
1489 } else {
1490 SmallVector<EVT, 16> VTs;
1491 SmallVector<uint64_t, 16> Offsets;
1492 ComputePTXValueVTs(TLI: *this, DL, Ctx, CallConv: CLI.CallConv, Ty: Arg.Ty, ValueVTs&: VTs, Offsets,
1493 StartingOffset: VAOffset);
1494 assert(VTs.size() == Offsets.size() && "Size mismatch");
1495 assert(VTs.size() == ArgOuts.size() && "Size mismatch");
1496
1497 // PTX Interoperability Guide 3.3(A): [Integer] Values shorter
1498 // than 32-bits are sign extended or zero extended, depending on
1499 // whether they are signed or unsigned types. This case applies
1500 // only to scalar parameters and not to aggregate values.
1501 const bool ExtendIntegerParam =
1502 Arg.Ty->isIntegerTy() && DL.getTypeAllocSizeInBits(Ty: Arg.Ty) < 32;
1503
1504 const auto GetStoredValue = [&](const unsigned I) {
1505 SDValue StVal = ArgOutVals[I];
1506 assert(promoteScalarIntegerPTX(StVal.getValueType()) ==
1507 StVal.getValueType() &&
1508 "OutVal type should always be legal");
1509
1510 const EVT VTI = promoteScalarIntegerPTX(VT: VTs[I]);
1511 const EVT StoreVT =
1512 ExtendIntegerParam ? MVT::i32 : (VTI == MVT::i1 ? MVT::i8 : VTI);
1513
1514 return correctParamType(V: StVal, ExpectedVT: StoreVT, Flags: ArgOuts[I].Flags, DAG, dl);
1515 };
1516
1517 unsigned J = 0;
1518 const auto VI = VectorizePTXValueVTs(ValueVTs: VTs, Offsets, ParamAlignment: ArgAlign, IsVAArg);
1519 for (const unsigned NumElts : VI) {
1520 const EVT EltVT = promoteScalarIntegerPTX(VT: VTs[J]);
1521
1522 unsigned Offset;
1523 if (IsVAArg) {
1524 // TODO: We may need to support vector types that can be passed
1525 // as scalars in variadic arguments.
1526 assert(NumElts == 1 &&
1527 "Vectorization should be disabled for vaargs.");
1528
1529 // Align each part of the variadic argument to their type.
1530 VAOffset = alignTo(Size: VAOffset, A: DAG.getEVTAlign(MemoryVT: EltVT));
1531 Offset = VAOffset;
1532
1533 const EVT TheStoreType = ExtendIntegerParam ? MVT::i32 : EltVT;
1534 VAOffset += DL.getTypeAllocSize(Ty: TheStoreType.getTypeForEVT(Context&: Ctx));
1535 } else {
1536 assert(VAOffset == 0 && "VAOffset must be 0 for non-VA args");
1537 Offset = Offsets[J];
1538 }
1539
1540 SDValue Ptr =
1541 DAG.getObjectPtrOffset(SL: dl, Ptr: ParamSymbol, Offset: TypeSize::getFixed(ExactSize: Offset));
1542
1543 const MaybeAlign CurrentAlign = ExtendIntegerParam
1544 ? MaybeAlign(std::nullopt)
1545 : commonAlignment(A: ArgAlign, Offset);
1546
1547 SDValue Val =
1548 getBuildVectorizedValue(N: NumElts, dl, DAG, GetElement: [&](unsigned K) {
1549 return GetStoredValue(J + K);
1550 });
1551
1552 SDValue StoreParam = DAG.getStore(
1553 Chain: ArgDeclare, dl, Val, Ptr,
1554 PtrInfo: MachinePointerInfo(NVPTX::AddressSpace::DeviceParam), Alignment: CurrentAlign);
1555 CallPrereqs.push_back(Elt: StoreParam);
1556
1557 J += NumElts;
1558 }
1559 }
1560 }
1561
1562 // Handle Result
1563 if (!Ins.empty()) {
1564 const SDValue RetSymbol = getSymbolNode(DAG, Name: "retval0", T: MVT::i32);
1565 const unsigned ResultSize = DL.getTypeAllocSize(Ty: RetTy);
1566 if (shouldPassAsArray(Ty: RetTy)) {
1567 const Align RetAlign =
1568 getPTXParamAlign(CB, Ty: RetTy, AttrIdx: AttributeList::ReturnIndex, DL);
1569 MakeDeclareArrayParam(RetSymbol, RetAlign, ResultSize);
1570 } else {
1571 MakeDeclareScalarParam(RetSymbol, ResultSize);
1572 }
1573 }
1574
1575 // Set the size of the vararg param byte array if the callee is a variadic
1576 // function and the variadic part is not empty.
1577 if (VADeclareParam) {
1578 SDValue DeclareParamOps[] = {VADeclareParam.getOperand(i: 0),
1579 VADeclareParam.getOperand(i: 1),
1580 VADeclareParam.getOperand(i: 2), GetI32(VAOffset),
1581 VADeclareParam.getOperand(i: 4)};
1582 DAG.MorphNodeTo(N: VADeclareParam.getNode(), Opc: VADeclareParam.getOpcode(),
1583 VTs: VADeclareParam->getVTList(), Ops: DeclareParamOps);
1584 }
1585
1586 const auto *Func = dyn_cast<GlobalAddressSDNode>(Val: Callee.getNode());
1587 const auto *CalleeF = Func ? dyn_cast<Function>(Val: Func->getGlobal()) : nullptr;
1588
1589 // If the type of the callsite does not match that of the function, convert
1590 // the callsite to an indirect call.
1591 const bool ConvertToIndirectCall =
1592 CalleeF && CB->getFunctionType() != CalleeF->getFunctionType();
1593
1594 // Both indirect calls and libcalls have nullptr Func. In order to distinguish
1595 // between them we must rely on the call site value which is valid for
1596 // indirect calls but is always null for libcalls.
1597 const bool IsIndirectCall = (!Func && CB) || ConvertToIndirectCall;
1598
1599 if (isa<ExternalSymbolSDNode>(Val: Callee)) {
1600 Function* CalleeFunc = nullptr;
1601
1602 // Try to find the callee in the current module.
1603 Callee = DAG.getSymbolFunctionGlobalAddress(Op: Callee, TargetFunction: &CalleeFunc);
1604 assert(CalleeFunc != nullptr && "Libcall callee must be set.");
1605
1606 // Set the "libcall callee" attribute to indicate that the function
1607 // must always have a declaration.
1608 CalleeFunc->addFnAttr(Kind: "nvptx-libcall-callee", Val: "true");
1609 }
1610
1611 // In the indirect function call case, PTX requires a prototype of the form:
1612 // proto_0 : .callprototype(.param .b32 _) _ (.param .b32 _);
1613 // Where the label is to be used as the last arg of the call instruction.
1614 // We record the call site here and emit all prototypes at the
1615 // start of the function in the AsmPrinter.
1616 SDValue Proto = GetI32(0);
1617 if (IsIndirectCall) {
1618 auto *ProtoSymbol = DAG.getMachineFunction()
1619 .getInfo<NVPTXMachineFunctionInfo>()
1620 ->addCallPrototype(CB, MF&: DAG.getMachineFunction());
1621 Proto = DAG.getMCSymbol(Sym: ProtoSymbol, VT: MVT::i32);
1622 }
1623
1624 const bool IsUnknownIntrinsic =
1625 CalleeF && CalleeF->isIntrinsic() &&
1626 CalleeF->getIntrinsicID() == Intrinsic::not_intrinsic;
1627 if (IsUnknownIntrinsic) {
1628 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1629 DAG.getMachineFunction().getFunction(),
1630 "call to unknown intrinsic '" + CalleeF->getName() +
1631 "' cannot be lowered by the NVPTX backend",
1632 dl.getDebugLoc()));
1633 }
1634
1635 const unsigned NumArgs =
1636 std::min<unsigned>(a: CLI.NumFixedArgs + 1, b: Args.size());
1637 /// CALL(Chain, IsConvergent, IsIndirectCall/IsUniform, NumReturns,
1638 /// NumParams, Callee, Proto)
1639 const SDValue CallToken = DAG.getTokenFactor(DL: dl, Vals&: CallPrereqs);
1640 const SDValue Call = DAG.getNode(
1641 Opcode: NVPTXISD::CALL, DL: dl, VT: MVT::Other,
1642 Ops: {CallToken, GetI32(CLI.IsConvergent), GetI32(IsIndirectCall),
1643 GetI32(Ins.empty() ? 0 : 1), GetI32(NumArgs), Callee, Proto});
1644
1645 SmallVector<SDValue, 16> LoadChains{Call};
1646 SmallVector<SDValue, 16> ProxyRegOps;
1647 if (!Ins.empty()) {
1648 SmallVector<EVT, 16> VTs;
1649 SmallVector<uint64_t, 16> Offsets;
1650 ComputePTXValueVTs(TLI: *this, DL, Ctx, CallConv: CLI.CallConv, Ty: RetTy, ValueVTs&: VTs, Offsets);
1651 assert(VTs.size() == Ins.size() && "Bad value decomposition");
1652
1653 const Align RetAlign =
1654 getPTXParamAlign(CB, Ty: RetTy, AttrIdx: AttributeList::ReturnIndex, DL);
1655 const SDValue RetSymbol = getSymbolNode(DAG, Name: "retval0", T: MVT::i32);
1656
1657 // PTX Interoperability Guide 3.3(A): [Integer] Values shorter than
1658 // 32-bits are sign extended or zero extended, depending on whether
1659 // they are signed or unsigned types.
1660 const bool ExtendIntegerRetVal =
1661 RetTy->isIntegerTy() && DL.getTypeAllocSizeInBits(Ty: RetTy) < 32;
1662
1663 unsigned I = 0;
1664 const auto VI = VectorizePTXValueVTs(ValueVTs: VTs, Offsets, ParamAlignment: RetAlign);
1665 for (const unsigned NumElts : VI) {
1666 const MaybeAlign CurrentAlign =
1667 ExtendIntegerRetVal ? MaybeAlign(std::nullopt)
1668 : commonAlignment(A: RetAlign, Offset: Offsets[I]);
1669
1670 const EVT VTI = promoteScalarIntegerPTX(VT: VTs[I]);
1671 const EVT LoadVT =
1672 ExtendIntegerRetVal ? MVT::i32 : (VTI == MVT::i1 ? MVT::i8 : VTI);
1673 const EVT VecVT = getVectorizedVT(VT: LoadVT, N: NumElts, C&: Ctx);
1674 SDValue Ptr =
1675 DAG.getObjectPtrOffset(SL: dl, Ptr: RetSymbol, Offset: TypeSize::getFixed(ExactSize: Offsets[I]));
1676
1677 SDValue R = DAG.getLoad(
1678 VT: VecVT, dl, Chain: Call, Ptr,
1679 PtrInfo: MachinePointerInfo(NVPTX::AddressSpace::DeviceParam), Alignment: CurrentAlign);
1680
1681 LoadChains.push_back(Elt: R.getValue(R: 1));
1682 for (const unsigned J : llvm::seq(Size: NumElts))
1683 ProxyRegOps.push_back(Elt: getExtractVectorizedValue(V: R, I: J, VT: LoadVT, dl, DAG));
1684 I += NumElts;
1685 }
1686 }
1687
1688 const SDValue EndToken = DAG.getTokenFactor(DL: dl, Vals&: LoadChains);
1689 const SDValue CallEnd = DAG.getCALLSEQ_END(Chain: EndToken, Size1: UniqueCallSite,
1690 Size2: UniqueCallSite + 1, Glue: SDValue(), DL: dl);
1691
1692 // Append ProxyReg instructions to the chain to make sure that `callseq_end`
1693 // will not get lost. Otherwise, during libcalls expansion, the nodes can become
1694 // dangling.
1695 for (const auto [I, Reg] : llvm::enumerate(First&: ProxyRegOps)) {
1696 SDValue Proxy =
1697 DAG.getNode(Opcode: NVPTXISD::ProxyReg, DL: dl, VT: Reg.getValueType(), Ops: {CallEnd, Reg});
1698 SDValue Ret = correctParamType(V: Proxy, ExpectedVT: Ins[I].VT, Flags: Ins[I].Flags, DAG, dl);
1699 InVals.push_back(Elt: Ret);
1700 }
1701
1702 // set IsTailCall to false for now, until we figure out how to express
1703 // tail call optimization in PTX
1704 CLI.IsTailCall = false;
1705 return CallEnd;
1706}
1707
1708SDValue NVPTXTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
1709 SelectionDAG &DAG) const {
1710
1711 if (!STI.hasFeature(Feature: NVPTX::PTX73) || !STI.hasFeature(Feature: NVPTX::SM52)) {
1712 const Function &Fn = DAG.getMachineFunction().getFunction();
1713
1714 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1715 Fn,
1716 "Support for dynamic alloca introduced in PTX ISA version 7.3 and "
1717 "requires target sm_52.",
1718 SDLoc(Op).getDebugLoc()));
1719 auto Ops = {DAG.getConstant(Val: 0, DL: SDLoc(), VT: Op.getValueType()),
1720 Op.getOperand(i: 0)};
1721 return DAG.getMergeValues(Ops, dl: SDLoc());
1722 }
1723
1724 SDLoc DL(Op.getNode());
1725 SDValue Chain = Op.getOperand(i: 0);
1726 SDValue Size = Op.getOperand(i: 1);
1727 uint64_t Align = Op.getConstantOperandVal(i: 2);
1728
1729 // The alignment on a ISD::DYNAMIC_STACKALLOC node may be 0 to indicate that
1730 // the default stack alignment should be used.
1731 if (Align == 0)
1732 Align = DAG.getSubtarget().getFrameLowering()->getStackAlign().value();
1733
1734 // The size for ptx alloca instruction is 64-bit for m64 and 32-bit for m32.
1735 const MVT LocalVT = getPointerTy(DL: DAG.getDataLayout(), AS: ADDRESS_SPACE_LOCAL);
1736
1737 SDValue Alloc =
1738 DAG.getNode(Opcode: NVPTXISD::DYNAMIC_STACKALLOC, DL, ResultTys: {LocalVT, MVT::Other},
1739 Ops: {Chain, DAG.getZExtOrTrunc(Op: Size, DL, VT: LocalVT),
1740 DAG.getTargetConstant(Val: Align, DL, VT: MVT::i32)});
1741
1742 // NVPTXLowerAlloca puts allocas in the local address space, so a local
1743 // pointer is requested here; escapes are explicit addrspacecasts in the IR.
1744 assert(Op.getValueType() == LocalVT && "Unexpected alloca pointer size");
1745
1746 return DAG.getMergeValues(Ops: {Alloc, SDValue(Alloc.getNode(), 1)}, dl: DL);
1747}
1748
1749SDValue NVPTXTargetLowering::LowerSTACKRESTORE(SDValue Op,
1750 SelectionDAG &DAG) const {
1751 SDLoc DL(Op.getNode());
1752 if (!STI.hasFeature(Feature: NVPTX::PTX73) || !STI.hasFeature(Feature: NVPTX::SM52)) {
1753 const Function &Fn = DAG.getMachineFunction().getFunction();
1754
1755 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1756 Fn,
1757 "Support for stackrestore requires PTX ISA version >= 7.3 and target "
1758 ">= sm_52.",
1759 DL.getDebugLoc()));
1760 return Op.getOperand(i: 0);
1761 }
1762
1763 const MVT LocalVT = getPointerTy(DL: DAG.getDataLayout(), AS: ADDRESS_SPACE_LOCAL);
1764 SDValue Chain = Op.getOperand(i: 0);
1765 SDValue Ptr = Op.getOperand(i: 1);
1766 SDValue ASC = DAG.getAddrSpaceCast(dl: DL, VT: LocalVT, Ptr, SrcAS: ADDRESS_SPACE_GENERIC,
1767 DestAS: ADDRESS_SPACE_LOCAL);
1768 return DAG.getNode(Opcode: NVPTXISD::STACKRESTORE, DL, VT: MVT::Other, Ops: {Chain, ASC});
1769}
1770
1771SDValue NVPTXTargetLowering::LowerSTACKSAVE(SDValue Op,
1772 SelectionDAG &DAG) const {
1773 SDLoc DL(Op.getNode());
1774 if (!STI.hasFeature(Feature: NVPTX::PTX73) || !STI.hasFeature(Feature: NVPTX::SM52)) {
1775 const Function &Fn = DAG.getMachineFunction().getFunction();
1776
1777 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1778 Fn,
1779 "Support for stacksave requires PTX ISA version >= 7.3 and target >= "
1780 "sm_52.",
1781 DL.getDebugLoc()));
1782 auto Ops = {DAG.getConstant(Val: 0, DL, VT: Op.getValueType()), Op.getOperand(i: 0)};
1783 return DAG.getMergeValues(Ops, dl: DL);
1784 }
1785
1786 const MVT LocalVT = getPointerTy(DL: DAG.getDataLayout(), AS: ADDRESS_SPACE_LOCAL);
1787 SDValue Chain = Op.getOperand(i: 0);
1788 SDValue SS =
1789 DAG.getNode(Opcode: NVPTXISD::STACKSAVE, DL, ResultTys: {LocalVT, MVT::Other}, Ops: Chain);
1790 SDValue ASC = DAG.getAddrSpaceCast(
1791 dl: DL, VT: Op.getValueType(), Ptr: SS, SrcAS: ADDRESS_SPACE_LOCAL, DestAS: ADDRESS_SPACE_GENERIC);
1792 return DAG.getMergeValues(Ops: {ASC, SDValue(SS.getNode(), 1)}, dl: DL);
1793}
1794
1795// By default CONCAT_VECTORS is lowered by ExpandVectorBuildThroughStack()
1796// (see LegalizeDAG.cpp). This is slow and uses local memory.
1797// We use extract/insert/build vector just as what LegalizeOp() does in llvm 2.5
1798SDValue
1799NVPTXTargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const {
1800 SDNode *Node = Op.getNode();
1801 SDLoc dl(Node);
1802 SmallVector<SDValue, 8> Ops;
1803 unsigned NumOperands = Node->getNumOperands();
1804 for (unsigned i = 0; i < NumOperands; ++i) {
1805 SDValue SubOp = Node->getOperand(Num: i);
1806 EVT VVT = SubOp.getNode()->getValueType(ResNo: 0);
1807 EVT EltVT = VVT.getVectorElementType();
1808 unsigned NumSubElem = VVT.getVectorNumElements();
1809 for (unsigned j = 0; j < NumSubElem; ++j) {
1810 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: SubOp,
1811 N2: DAG.getIntPtrConstant(Val: j, DL: dl)));
1812 }
1813 }
1814 return DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL: dl, Ops);
1815}
1816
1817static SDValue getPRMT(SDValue A, SDValue B, SDValue Selector, SDLoc DL,
1818 SelectionDAG &DAG,
1819 unsigned Mode = NVPTX::PTXPrmtMode::NONE) {
1820 assert(A.getValueType() == MVT::i32 && B.getValueType() == MVT::i32 &&
1821 Selector.getValueType() == MVT::i32 && "PRMT must have i32 operands");
1822 return DAG.getNode(Opcode: NVPTXISD::PRMT, DL, VT: MVT::i32,
1823 Ops: {A, B, Selector, DAG.getConstant(Val: Mode, DL, VT: MVT::i32)});
1824}
1825
1826static SDValue getPRMT(SDValue A, SDValue B, uint64_t Selector, SDLoc DL,
1827 SelectionDAG &DAG,
1828 unsigned Mode = NVPTX::PTXPrmtMode::NONE) {
1829 return getPRMT(A, B, Selector: DAG.getConstant(Val: Selector, DL, VT: MVT::i32), DL, DAG, Mode);
1830}
1831
1832/// Reduces the elements using the scalar operations provided. The operations
1833/// are sorted descending in number of inputs they take. The flags on the
1834/// original reduction operation will be propagated to each scalar operation.
1835/// Nearby elements are grouped in tree reduction, unlike the shuffle reduction
1836/// used in ExpandReductions and SelectionDAG.
1837static SDValue buildTreeReduction(
1838 const SmallVector<SDValue> &Elements, EVT EltTy,
1839 ArrayRef<std::pair<unsigned /*NodeType*/, unsigned /*NumInputs*/>> Ops,
1840 const SDLoc &DL, const SDNodeFlags Flags, SelectionDAG &DAG) {
1841 // Build the reduction tree at each level, starting with all the elements.
1842 SmallVector<SDValue> Level = Elements;
1843
1844 unsigned OpIdx = 0;
1845 while (Level.size() > 1) {
1846 // Try to reduce this level using the current operator.
1847 const auto [Op, NumInputs] = Ops[OpIdx];
1848
1849 // Build the next level by partially reducing all elements.
1850 SmallVector<SDValue> ReducedLevel;
1851 unsigned I = 0, E = Level.size();
1852 for (; I + NumInputs <= E; I += NumInputs) {
1853 // Reduce elements in groups of [NumInputs], as much as possible.
1854 ReducedLevel.push_back(Elt: DAG.getNode(
1855 Opcode: Op, DL, VT: EltTy, Ops: ArrayRef<SDValue>(Level).slice(N: I, M: NumInputs), Flags));
1856 }
1857
1858 if (I < E) {
1859 // Handle leftover elements.
1860
1861 if (ReducedLevel.empty()) {
1862 // We didn't reduce anything at this level. We need to pick a smaller
1863 // operator.
1864 ++OpIdx;
1865 assert(OpIdx < Ops.size() && "no smaller operators for reduction");
1866 continue;
1867 }
1868
1869 // We reduced some things but there's still more left, meaning the
1870 // operator's number of inputs doesn't evenly divide this level size. Move
1871 // these elements to the next level.
1872 for (; I < E; ++I)
1873 ReducedLevel.push_back(Elt: Level[I]);
1874 }
1875
1876 // Process the next level.
1877 Level = ReducedLevel;
1878 }
1879
1880 return *Level.begin();
1881}
1882
1883// Get scalar reduction opcode
1884static ISD::NodeType getScalarOpcodeForReduction(unsigned ReductionOpcode) {
1885 switch (ReductionOpcode) {
1886 case ISD::VECREDUCE_FMAX:
1887 return ISD::FMAXNUM;
1888 case ISD::VECREDUCE_FMIN:
1889 return ISD::FMINNUM;
1890 case ISD::VECREDUCE_FMAXIMUM:
1891 return ISD::FMAXIMUM;
1892 case ISD::VECREDUCE_FMINIMUM:
1893 return ISD::FMINIMUM;
1894 default:
1895 llvm_unreachable("unhandled reduction opcode");
1896 }
1897}
1898
1899/// Get 3-input scalar reduction opcode
1900static std::optional<unsigned>
1901getScalar3OpcodeForReduction(unsigned ReductionOpcode) {
1902 switch (ReductionOpcode) {
1903 case ISD::VECREDUCE_FMAX:
1904 return NVPTXISD::FMAXNUM3;
1905 case ISD::VECREDUCE_FMIN:
1906 return NVPTXISD::FMINNUM3;
1907 case ISD::VECREDUCE_FMAXIMUM:
1908 return NVPTXISD::FMAXIMUM3;
1909 case ISD::VECREDUCE_FMINIMUM:
1910 return NVPTXISD::FMINIMUM3;
1911 default:
1912 return std::nullopt;
1913 }
1914}
1915
1916/// Lower reductions to either a sequence of operations or a tree if
1917/// reassociations are allowed. This method will use larger operations like
1918/// max3/min3 when the target supports them.
1919SDValue NVPTXTargetLowering::LowerVECREDUCE(SDValue Op,
1920 SelectionDAG &DAG) const {
1921 SDLoc DL(Op);
1922 const SDNodeFlags Flags = Op->getFlags();
1923 SDValue Vector = Op.getOperand(i: 0);
1924
1925 const unsigned Opcode = Op->getOpcode();
1926 const EVT EltTy = Vector.getValueType().getVectorElementType();
1927
1928 // Whether we can use 3-input min/max when expanding the reduction.
1929 const bool CanUseMinMax3 =
1930 EltTy == MVT::f32 && STI.hasFeature(Feature: NVPTX::SM100) &&
1931 STI.hasFeature(Feature: NVPTX::PTX88) &&
1932 (Opcode == ISD::VECREDUCE_FMAX || Opcode == ISD::VECREDUCE_FMIN ||
1933 Opcode == ISD::VECREDUCE_FMAXIMUM || Opcode == ISD::VECREDUCE_FMINIMUM);
1934
1935 // A list of SDNode opcodes with equivalent semantics, sorted descending by
1936 // number of inputs they take.
1937 SmallVector<std::pair<unsigned /*Op*/, unsigned /*NumIn*/>, 2> ScalarOps;
1938
1939 if (auto Opcode3Elem = getScalar3OpcodeForReduction(ReductionOpcode: Opcode);
1940 CanUseMinMax3 && Opcode3Elem)
1941 ScalarOps.push_back(Elt: {*Opcode3Elem, 3});
1942 ScalarOps.push_back(Elt: {getScalarOpcodeForReduction(ReductionOpcode: Opcode), 2});
1943
1944 SmallVector<SDValue> Elements;
1945 DAG.ExtractVectorElements(Op: Vector, Args&: Elements);
1946
1947 return buildTreeReduction(Elements, EltTy, Ops: ScalarOps, DL, Flags, DAG);
1948}
1949
1950SDValue NVPTXTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
1951 // Handle bitcasting from v2i8 without hitting the default promotion
1952 // strategy which goes through stack memory.
1953 EVT FromVT = Op->getOperand(Num: 0)->getValueType(ResNo: 0);
1954 if (FromVT != MVT::v2i8) {
1955 return Op;
1956 }
1957
1958 // Pack vector elements into i16 and bitcast to final type
1959 SDLoc DL(Op);
1960 SDValue Vec0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i8,
1961 N1: Op->getOperand(Num: 0), N2: DAG.getIntPtrConstant(Val: 0, DL));
1962 SDValue Vec1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i8,
1963 N1: Op->getOperand(Num: 0), N2: DAG.getIntPtrConstant(Val: 1, DL));
1964 SDValue Extend0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i16, Operand: Vec0);
1965 SDValue Extend1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i16, Operand: Vec1);
1966 SDValue Const8 = DAG.getConstant(Val: 8, DL, VT: MVT::i16);
1967 SDValue AsInt = DAG.getNode(
1968 Opcode: ISD::OR, DL, VT: MVT::i16,
1969 Ops: {Extend0, DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i16, Ops: {Extend1, Const8})});
1970 EVT ToVT = Op->getValueType(ResNo: 0);
1971 return DAG.getBitcast(VT: ToVT, V: AsInt);
1972}
1973
1974// We can init constant f16x2/v2i16/v4i8 with a single .b32 move. Normally it
1975// would get lowered as two constant loads and vector-packing move.
1976// Instead we want just a constant move:
1977// mov.b32 %r2, 0x40003C00
1978SDValue NVPTXTargetLowering::LowerBUILD_VECTOR(SDValue Op,
1979 SelectionDAG &DAG) const {
1980 EVT VT = Op->getValueType(ResNo: 0);
1981 if (!(NVPTX::isPackedVectorTy(VT) && VT.is32BitVector()))
1982 return Op;
1983 SDLoc DL(Op);
1984
1985 if (!llvm::all_of(Range: Op->ops(), P: [](SDValue Operand) {
1986 return Operand->isUndef() || isa<ConstantSDNode>(Val: Operand) ||
1987 isa<ConstantFPSDNode>(Val: Operand);
1988 })) {
1989 if (VT != MVT::v4i8)
1990 return Op;
1991 // Lower non-const v4i8 vector as byte-wise constructed i32, which allows us
1992 // to optimize calculation of constant parts.
1993 auto GetPRMT = [&](const SDValue Left, const SDValue Right, bool Cast,
1994 uint64_t SelectionValue) -> SDValue {
1995 SDValue L = Left;
1996 SDValue R = Right;
1997 if (Cast) {
1998 L = DAG.getAnyExtOrTrunc(Op: L, DL, VT: MVT::i32);
1999 R = DAG.getAnyExtOrTrunc(Op: R, DL, VT: MVT::i32);
2000 }
2001 return getPRMT(A: L, B: R, Selector: SelectionValue, DL, DAG);
2002 };
2003 auto PRMT__10 = GetPRMT(Op->getOperand(Num: 0), Op->getOperand(Num: 1), true, 0x3340);
2004 auto PRMT__32 = GetPRMT(Op->getOperand(Num: 2), Op->getOperand(Num: 3), true, 0x3340);
2005 auto PRMT3210 = GetPRMT(PRMT__10, PRMT__32, false, 0x5410);
2006 return DAG.getBitcast(VT, V: PRMT3210);
2007 }
2008
2009 // Get value or the Nth operand as an APInt(32). Undef values treated as 0.
2010 auto GetOperand = [](SDValue Op, int N) -> APInt {
2011 const SDValue &Operand = Op->getOperand(Num: N);
2012 EVT VT = Op->getValueType(ResNo: 0);
2013 if (Operand->isUndef())
2014 return APInt(32, 0);
2015 APInt Value;
2016 if (VT == MVT::v2f16 || VT == MVT::v2bf16)
2017 Value = cast<ConstantFPSDNode>(Val: Operand)->getValueAPF().bitcastToAPInt();
2018 else if (VT == MVT::v2i16 || VT == MVT::v4i8)
2019 Value = Operand->getAsAPIntVal();
2020 else
2021 llvm_unreachable("Unsupported type");
2022 // i8 values are carried around as i16, so we need to zero out upper bits,
2023 // so they do not get in the way of combining individual byte values
2024 if (VT == MVT::v4i8)
2025 Value = Value.trunc(width: 8);
2026 return Value.zext(width: 32);
2027 };
2028
2029 // Construct a 32-bit constant by shifting into place smaller values
2030 // (elements of the vector type VT).
2031 // For example, if VT has 2 elements, then N == 2:
2032 // ShiftAmount = 32 / N = 16
2033 // Value |= Op0 (b16) << 0
2034 // Value |= Op1 (b16) << 16
2035 // If N == 4:
2036 // ShiftAmount = 32 / N = 8
2037 // Value |= Op0 (b8) << 0
2038 // Value |= Op1 (b8) << 8
2039 // Value |= Op2 (b8) << 16
2040 // Value |= Op3 (b8) << 24
2041 // ...etc
2042 APInt Value(32, 0);
2043 const unsigned NumElements = VT.getVectorNumElements();
2044 assert(32 % NumElements == 0 && "must evenly divide bit length");
2045 const unsigned ShiftAmount = 32 / NumElements;
2046 for (unsigned ElementNo : seq(Size: NumElements))
2047 Value |= GetOperand(Op, ElementNo).shl(shiftAmt: ElementNo * ShiftAmount);
2048 SDValue Const = DAG.getConstant(Val: Value, DL, VT: MVT::i32);
2049 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Op->getValueType(ResNo: 0), Operand: Const);
2050}
2051
2052SDValue NVPTXTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
2053 SelectionDAG &DAG) const {
2054 SDValue Index = Op->getOperand(Num: 1);
2055 SDValue Vector = Op->getOperand(Num: 0);
2056 SDLoc DL(Op);
2057 EVT VectorVT = Vector.getValueType();
2058
2059 if (VectorVT == MVT::v4i8) {
2060 SDValue Selector = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32,
2061 N1: DAG.getZExtOrTrunc(Op: Index, DL, VT: MVT::i32),
2062 N2: DAG.getConstant(Val: 0x7770, DL, VT: MVT::i32));
2063 SDValue PRMT = getPRMT(A: DAG.getBitcast(VT: MVT::i32, V: Vector),
2064 B: DAG.getConstant(Val: 0, DL, VT: MVT::i32), Selector, DL, DAG);
2065 SDValue Ext = DAG.getAnyExtOrTrunc(Op: PRMT, DL, VT: Op->getValueType(ResNo: 0));
2066 SDNodeFlags Flags;
2067 Flags.setNoSignedWrap(Ext.getScalarValueSizeInBits() > 8);
2068 Flags.setNoUnsignedWrap(Ext.getScalarValueSizeInBits() >= 8);
2069 Ext->setFlags(Flags);
2070 return Ext;
2071 }
2072
2073 // Constant index will be matched by tablegen.
2074 if (isa<ConstantSDNode>(Val: Index.getNode()))
2075 return Op;
2076
2077 // Extract individual elements and select one of them.
2078 assert(NVPTX::isPackedVectorTy(VectorVT) &&
2079 VectorVT.getVectorNumElements() == 2 && "Unexpected vector type.");
2080 EVT EltVT = VectorVT.getVectorElementType();
2081
2082 SDLoc dl(Op.getNode());
2083 SDValue E0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Vector,
2084 N2: DAG.getIntPtrConstant(Val: 0, DL: dl));
2085 SDValue E1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Vector,
2086 N2: DAG.getIntPtrConstant(Val: 1, DL: dl));
2087 return DAG.getSelectCC(DL: dl, LHS: Index, RHS: DAG.getIntPtrConstant(Val: 0, DL: dl), True: E0, False: E1,
2088 Cond: ISD::CondCode::SETEQ);
2089}
2090
2091SDValue NVPTXTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
2092 SelectionDAG &DAG) const {
2093 SDValue Vector = Op->getOperand(Num: 0);
2094 EVT VectorVT = Vector.getValueType();
2095
2096 if (VectorVT != MVT::v4i8)
2097 return Op;
2098 SDLoc DL(Op);
2099 SDValue Value = Op->getOperand(Num: 1);
2100 if (Value->isUndef())
2101 return Vector;
2102
2103 SDValue Index = Op->getOperand(Num: 2);
2104
2105 SDValue BFI =
2106 DAG.getNode(Opcode: NVPTXISD::BFI, DL, VT: MVT::i32,
2107 Ops: {DAG.getZExtOrTrunc(Op: Value, DL, VT: MVT::i32),
2108 DAG.getBitcast(VT: MVT::i32, V: Vector),
2109 DAG.getNode(Opcode: ISD::MUL, DL, VT: MVT::i32,
2110 N1: DAG.getZExtOrTrunc(Op: Index, DL, VT: MVT::i32),
2111 N2: DAG.getConstant(Val: 8, DL, VT: MVT::i32)),
2112 DAG.getConstant(Val: 8, DL, VT: MVT::i32)});
2113 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Op->getValueType(ResNo: 0), Operand: BFI);
2114}
2115
2116SDValue NVPTXTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
2117 SelectionDAG &DAG) const {
2118 SDValue V1 = Op.getOperand(i: 0);
2119 EVT VectorVT = V1.getValueType();
2120 if (VectorVT != MVT::v4i8 || Op.getValueType() != MVT::v4i8)
2121 return Op;
2122
2123 // Lower shuffle to PRMT instruction.
2124 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: Op.getNode());
2125 SDValue V2 = Op.getOperand(i: 1);
2126 uint32_t Selector = 0;
2127 for (auto I : llvm::enumerate(First: SVN->getMask())) {
2128 if (I.value() != -1) // -1 is a placeholder for undef.
2129 Selector |= (I.value() << (I.index() * 4));
2130 }
2131
2132 SDLoc DL(Op);
2133 SDValue PRMT = getPRMT(A: DAG.getBitcast(VT: MVT::i32, V: V1),
2134 B: DAG.getBitcast(VT: MVT::i32, V: V2), Selector, DL, DAG);
2135 return DAG.getBitcast(VT: Op.getValueType(), V: PRMT);
2136}
2137/// LowerShiftRightParts - Lower SRL_PARTS, SRA_PARTS, which
2138/// 1) returns two i32 values and take a 2 x i32 value to shift plus a shift
2139/// amount, or
2140/// 2) returns two i64 values and take a 2 x i64 value to shift plus a shift
2141/// amount.
2142SDValue NVPTXTargetLowering::LowerShiftRightParts(SDValue Op,
2143 SelectionDAG &DAG) const {
2144 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
2145 assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS);
2146
2147 EVT VT = Op.getValueType();
2148 unsigned VTBits = VT.getSizeInBits();
2149 SDLoc dl(Op);
2150 SDValue ShOpLo = Op.getOperand(i: 0);
2151 SDValue ShOpHi = Op.getOperand(i: 1);
2152 SDValue ShAmt = Op.getOperand(i: 2);
2153 unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;
2154
2155 if (VTBits == 32 && STI.hasFeature(Feature: NVPTX::SM35)) {
2156 // For 32bit and sm35, we can use the funnel shift 'shf' instruction.
2157 // {dHi, dLo} = {aHi, aLo} >> Amt
2158 // dHi = aHi >> Amt
2159 // dLo = shf.r.clamp aLo, aHi, Amt
2160
2161 SDValue Hi = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: ShOpHi, N2: ShAmt);
2162 SDValue Lo =
2163 DAG.getNode(Opcode: NVPTXISD::FSHR_CLAMP, DL: dl, VT, N1: ShOpHi, N2: ShOpLo, N3: ShAmt);
2164
2165 SDValue Ops[2] = { Lo, Hi };
2166 return DAG.getMergeValues(Ops, dl);
2167 } else {
2168 // {dHi, dLo} = {aHi, aLo} >> Amt
2169 // - if (Amt>=size) then
2170 // dLo = aHi >> (Amt-size)
2171 // dHi = aHi >> Amt (this is either all 0 or all 1)
2172 // else
2173 // dLo = (aLo >>logic Amt) | (aHi << (size-Amt))
2174 // dHi = aHi >> Amt
2175
2176 SDValue RevShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
2177 N1: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32),
2178 N2: ShAmt);
2179 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: ShOpLo, N2: ShAmt);
2180 SDValue ExtraShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: ShAmt,
2181 N2: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32));
2182 SDValue Tmp2 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpHi, N2: RevShAmt);
2183 SDValue FalseVal = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp1, N2: Tmp2);
2184 SDValue TrueVal = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: ShOpHi, N2: ExtraShAmt);
2185
2186 SDValue Cmp = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: ShAmt,
2187 RHS: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32),
2188 Cond: ISD::SETGE);
2189 SDValue Hi = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: ShOpHi, N2: ShAmt);
2190 SDValue Lo = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cmp, N2: TrueVal, N3: FalseVal);
2191
2192 SDValue Ops[2] = { Lo, Hi };
2193 return DAG.getMergeValues(Ops, dl);
2194 }
2195}
2196
2197/// LowerShiftLeftParts - Lower SHL_PARTS, which
2198/// 1) returns two i32 values and take a 2 x i32 value to shift plus a shift
2199/// amount, or
2200/// 2) returns two i64 values and take a 2 x i64 value to shift plus a shift
2201/// amount.
2202SDValue NVPTXTargetLowering::LowerShiftLeftParts(SDValue Op,
2203 SelectionDAG &DAG) const {
2204 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
2205 assert(Op.getOpcode() == ISD::SHL_PARTS);
2206
2207 EVT VT = Op.getValueType();
2208 unsigned VTBits = VT.getSizeInBits();
2209 SDLoc dl(Op);
2210 SDValue ShOpLo = Op.getOperand(i: 0);
2211 SDValue ShOpHi = Op.getOperand(i: 1);
2212 SDValue ShAmt = Op.getOperand(i: 2);
2213
2214 if (VTBits == 32 && STI.hasFeature(Feature: NVPTX::SM35)) {
2215 // For 32bit and sm35, we can use the funnel shift 'shf' instruction.
2216 // {dHi, dLo} = {aHi, aLo} << Amt
2217 // dHi = shf.l.clamp aLo, aHi, Amt
2218 // dLo = aLo << Amt
2219
2220 SDValue Hi =
2221 DAG.getNode(Opcode: NVPTXISD::FSHL_CLAMP, DL: dl, VT, N1: ShOpHi, N2: ShOpLo, N3: ShAmt);
2222 SDValue Lo = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpLo, N2: ShAmt);
2223
2224 SDValue Ops[2] = { Lo, Hi };
2225 return DAG.getMergeValues(Ops, dl);
2226 } else {
2227 // {dHi, dLo} = {aHi, aLo} << Amt
2228 // - if (Amt>=size) then
2229 // dLo = aLo << Amt (all 0)
2230 // dLo = aLo << (Amt-size)
2231 // else
2232 // dLo = aLo << Amt
2233 // dHi = (aHi << Amt) | (aLo >> (size-Amt))
2234
2235 SDValue RevShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
2236 N1: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32),
2237 N2: ShAmt);
2238 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpHi, N2: ShAmt);
2239 SDValue ExtraShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: ShAmt,
2240 N2: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32));
2241 SDValue Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: ShOpLo, N2: RevShAmt);
2242 SDValue FalseVal = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp1, N2: Tmp2);
2243 SDValue TrueVal = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpLo, N2: ExtraShAmt);
2244
2245 SDValue Cmp = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: ShAmt,
2246 RHS: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32),
2247 Cond: ISD::SETGE);
2248 SDValue Lo = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpLo, N2: ShAmt);
2249 SDValue Hi = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cmp, N2: TrueVal, N3: FalseVal);
2250
2251 SDValue Ops[2] = { Lo, Hi };
2252 return DAG.getMergeValues(Ops, dl);
2253 }
2254}
2255
2256/// If the types match, convert the generic copysign to the NVPTXISD version,
2257/// otherwise bail ensuring that mismatched cases are properly expaned.
2258SDValue NVPTXTargetLowering::LowerFCOPYSIGN(SDValue Op,
2259 SelectionDAG &DAG) const {
2260 EVT VT = Op.getValueType();
2261 SDLoc DL(Op);
2262
2263 SDValue In1 = Op.getOperand(i: 0);
2264 SDValue In2 = Op.getOperand(i: 1);
2265 EVT SrcVT = In2.getValueType();
2266
2267 if (!SrcVT.bitsEq(VT))
2268 return SDValue();
2269
2270 return DAG.getNode(Opcode: NVPTXISD::FCOPYSIGN, DL, VT, N1: In1, N2: In2);
2271}
2272
2273SDValue NVPTXTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const {
2274 EVT VT = Op.getValueType();
2275
2276 if (VT == MVT::f32)
2277 return LowerFROUND32(Op, DAG);
2278
2279 if (VT == MVT::f64)
2280 return LowerFROUND64(Op, DAG);
2281
2282 llvm_unreachable("unhandled type");
2283}
2284
2285// This is the the rounding method used in CUDA libdevice in C like code:
2286// float roundf(float A)
2287// {
2288// float RoundedA = (float) (int) ( A > 0 ? (A + 0.5f) : (A - 0.5f));
2289// RoundedA = abs(A) > 0x1.0p23 ? A : RoundedA;
2290// return abs(A) < 0.5 ? (float)(int)A : RoundedA;
2291// }
2292SDValue NVPTXTargetLowering::LowerFROUND32(SDValue Op,
2293 SelectionDAG &DAG) const {
2294 SDLoc SL(Op);
2295 SDValue A = Op.getOperand(i: 0);
2296 EVT VT = Op.getValueType();
2297
2298 SDValue AbsA = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT, Operand: A);
2299
2300 // RoundedA = (float) (int) ( A > 0 ? (A + 0.5f) : (A - 0.5f))
2301 SDValue Bitcast = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: A);
2302 const unsigned SignBitMask = 0x80000000;
2303 SDValue Sign = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: Bitcast,
2304 N2: DAG.getConstant(Val: SignBitMask, DL: SL, VT: MVT::i32));
2305 const unsigned PointFiveInBits = 0x3F000000;
2306 SDValue PointFiveWithSignRaw =
2307 DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: Sign,
2308 N2: DAG.getConstant(Val: PointFiveInBits, DL: SL, VT: MVT::i32));
2309 SDValue PointFiveWithSign =
2310 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: PointFiveWithSignRaw);
2311 SDValue AdjustedA = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: A, N2: PointFiveWithSign);
2312 SDValue RoundedA = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: AdjustedA);
2313
2314 // RoundedA = abs(A) > 0x1.0p23 ? A : RoundedA;
2315 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Ctx&: *DAG.getContext(), VT);
2316 SDValue IsLarge =
2317 DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: AbsA, RHS: DAG.getConstantFP(Val: pow(x: 2.0, y: 23.0), DL: SL, VT),
2318 Cond: ISD::SETOGT);
2319 RoundedA = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsLarge, N2: A, N3: RoundedA);
2320
2321 // return abs(A) < 0.5 ? (float)(int)A : RoundedA;
2322 SDValue IsSmall =DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: AbsA,
2323 RHS: DAG.getConstantFP(Val: 0.5, DL: SL, VT), Cond: ISD::SETOLT);
2324 SDValue RoundedAForSmallA = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: A);
2325 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsSmall, N2: RoundedAForSmallA, N3: RoundedA);
2326}
2327
2328// The implementation of round(double) is similar to that of round(float) in
2329// that they both separate the value range into three regions and use a method
2330// specific to the region to round the values. However, round(double) first
2331// calculates the round of the absolute value and then adds the sign back while
2332// round(float) directly rounds the value with sign.
2333SDValue NVPTXTargetLowering::LowerFROUND64(SDValue Op,
2334 SelectionDAG &DAG) const {
2335 SDLoc SL(Op);
2336 SDValue A = Op.getOperand(i: 0);
2337 EVT VT = Op.getValueType();
2338
2339 SDValue AbsA = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT, Operand: A);
2340
2341 // double RoundedA = (double) (int) (abs(A) + 0.5f);
2342 SDValue AdjustedA = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: AbsA,
2343 N2: DAG.getConstantFP(Val: 0.5, DL: SL, VT));
2344 SDValue RoundedA = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: AdjustedA);
2345
2346 // RoundedA = abs(A) < 0.5 ? (double)0 : RoundedA;
2347 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Ctx&: *DAG.getContext(), VT);
2348 SDValue IsSmall =DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: AbsA,
2349 RHS: DAG.getConstantFP(Val: 0.5, DL: SL, VT), Cond: ISD::SETOLT);
2350 RoundedA = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsSmall,
2351 N2: DAG.getConstantFP(Val: 0, DL: SL, VT),
2352 N3: RoundedA);
2353
2354 // Add sign to rounded_A
2355 RoundedA = DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT, N1: RoundedA, N2: A);
2356 DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: A);
2357
2358 // RoundedA = abs(A) > 0x1.0p52 ? A : RoundedA;
2359 SDValue IsLarge =
2360 DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: AbsA, RHS: DAG.getConstantFP(Val: pow(x: 2.0, y: 52.0), DL: SL, VT),
2361 Cond: ISD::SETOGT);
2362 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsLarge, N2: A, N3: RoundedA);
2363}
2364
2365static SDValue PromoteBinOpToF32(SDNode *N, SelectionDAG &DAG) {
2366 EVT VT = N->getValueType(ResNo: 0);
2367 EVT NVT = MVT::f32;
2368 if (VT.isVector()) {
2369 NVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NVT, EC: VT.getVectorElementCount());
2370 }
2371 SDLoc DL(N);
2372 SDValue Tmp0 = DAG.getFPExtendOrRound(Op: N->getOperand(Num: 0), DL, VT: NVT);
2373 SDValue Tmp1 = DAG.getFPExtendOrRound(Op: N->getOperand(Num: 1), DL, VT: NVT);
2374 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL, VT: NVT, N1: Tmp0, N2: Tmp1, Flags: N->getFlags());
2375 return DAG.getFPExtendOrRound(Op: Res, DL, VT);
2376}
2377
2378SDValue NVPTXTargetLowering::PromoteBinOpIfF32FTZ(SDValue Op,
2379 SelectionDAG &DAG) const {
2380 if (useF32FTZ(MF: DAG.getMachineFunction())) {
2381 return PromoteBinOpToF32(N: Op.getNode(), DAG);
2382 }
2383 return Op;
2384}
2385
2386SDValue NVPTXTargetLowering::LowerINT_TO_FP(SDValue Op,
2387 SelectionDAG &DAG) const {
2388 assert(!STI.hasFeature(NVPTX::SM90));
2389
2390 if (Op.getValueType() == MVT::bf16) {
2391 SDLoc Loc(Op);
2392 return DAG.getNode(
2393 Opcode: ISD::FP_ROUND, DL: Loc, VT: MVT::bf16,
2394 N1: DAG.getNode(Opcode: Op.getOpcode(), DL: Loc, VT: MVT::f32, Operand: Op.getOperand(i: 0)),
2395 N2: DAG.getIntPtrConstant(Val: 0, DL: Loc, /*isTarget=*/true));
2396 }
2397
2398 // Everything else is considered legal.
2399 return Op;
2400}
2401
2402SDValue NVPTXTargetLowering::LowerFP_TO_INT(SDValue Op,
2403 SelectionDAG &DAG) const {
2404 assert(!STI.hasFeature(NVPTX::SM90));
2405
2406 if (Op.getOperand(i: 0).getValueType() == MVT::bf16) {
2407 SDLoc Loc(Op);
2408 return DAG.getNode(
2409 Opcode: Op.getOpcode(), DL: Loc, VT: Op.getValueType(),
2410 Operand: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: Loc, VT: MVT::f32, Operand: Op.getOperand(i: 0)));
2411 }
2412
2413 // Everything else is considered legal.
2414 return Op;
2415}
2416
2417SDValue NVPTXTargetLowering::LowerFP_ROUND(SDValue Op,
2418 SelectionDAG &DAG) const {
2419 EVT NarrowVT = Op.getValueType();
2420 SDValue Wide = Op.getOperand(i: 0);
2421 EVT WideVT = Wide.getValueType();
2422 if (NarrowVT.getScalarType() == MVT::bf16) {
2423 const TargetLowering *TLI = STI.getTargetLowering();
2424 if (!STI.hasFeature(Feature: NVPTX::SM80)) {
2425 return TLI->expandFP_ROUND(Node: Op.getNode(), DAG);
2426 }
2427 if (!STI.hasFeature(Feature: NVPTX::SM90)) {
2428 // sm_80 was the first architecture to support f32 -> bf16.
2429 if (WideVT.getScalarType() == MVT::f32) {
2430 return Op;
2431 }
2432 if (WideVT.getScalarType() == MVT::f64) {
2433 SDLoc Loc(Op);
2434 // Round-inexact-to-odd f64 to f32, then do the final rounding using
2435 // the hardware f32 -> bf16 instruction.
2436 SDValue rod = TLI->expandRoundInexactToOdd(
2437 ResultVT: WideVT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::f32), Op: Wide, DL: Loc,
2438 DAG);
2439 return DAG.getFPExtendOrRound(Op: rod, DL: Loc, VT: NarrowVT);
2440 }
2441 return TLI->expandFP_ROUND(Node: Op.getNode(), DAG);
2442 }
2443 }
2444
2445 // Everything else is considered legal.
2446 return Op;
2447}
2448
2449static SDValue lowerMUL_LOHI(SDValue Op, SelectionDAG &DAG) {
2450 const SDLoc DL(Op);
2451 const EVT VT = Op.getValueType();
2452 const unsigned Opc = Op.getOpcode() == ISD::SMUL_LOHI
2453 ? NVPTXISD::MUL_WIDE_SIGNED
2454 : NVPTXISD::MUL_WIDE_UNSIGNED;
2455
2456 SDValue Wide =
2457 DAG.getNode(Opcode: Opc, DL, VT: VT.widenIntegerElementType(Context&: *DAG.getContext()),
2458 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1));
2459 auto [Lo, Hi] = DAG.SplitScalar(N: Wide, DL, LoVT: VT, HiVT: VT);
2460 return DAG.getMergeValues(Ops: {Lo, Hi}, dl: DL);
2461}
2462
2463static SDValue lowerFPTRUNC_ROUND(SDValue Op, SelectionDAG &DAG,
2464 const NVPTXSubtarget &STI) {
2465 EVT SrcVT = Op.getOperand(i: 0).getValueType();
2466 EVT DstVT = Op.getValueType();
2467 auto RM = static_cast<RoundingMode>(Op.getConstantOperandVal(i: 1));
2468 if (RM == RoundingMode::NearestTiesToEven) {
2469 // Reuse the native selection and fallback expansion for ordinary fptrunc.
2470 SDLoc DL(Op);
2471 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: DstVT, N1: Op.getOperand(i: 0),
2472 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true),
2473 Flags: Op.getNode()->getFlags());
2474 }
2475
2476 bool RoundToInfinity =
2477 RM == RoundingMode::TowardNegative || RM == RoundingMode::TowardPositive;
2478
2479 bool Supported = RM == RoundingMode::TowardZero || RoundToInfinity;
2480 if (DstVT == MVT::bf16) {
2481 if (SrcVT == MVT::f64 || RoundToInfinity)
2482 Supported &= STI.hasFeature(Feature: NVPTX::SM90);
2483 else
2484 Supported &= STI.hasFeature(Feature: NVPTX::SM80);
2485 }
2486
2487 if (Supported)
2488 return Op;
2489
2490 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
2491 DAG.getMachineFunction().getFunction(),
2492 "unsupported conversion or rounding mode for llvm.fptrunc.round",
2493 SDLoc(Op).getDebugLoc()));
2494 return DAG.getPOISON(VT: DstVT);
2495}
2496
2497SDValue NVPTXTargetLowering::LowerFP_EXTEND(SDValue Op,
2498 SelectionDAG &DAG) const {
2499 SDValue Narrow = Op.getOperand(i: 0);
2500 EVT NarrowVT = Narrow.getValueType();
2501 EVT WideVT = Op.getValueType();
2502 if (NarrowVT.getScalarType() == MVT::bf16) {
2503 if (WideVT.getScalarType() == MVT::f32 &&
2504 (!STI.hasFeature(Feature: NVPTX::SM80) || !STI.hasFeature(Feature: NVPTX::PTX71))) {
2505 SDLoc Loc(Op);
2506 return DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: Loc, VT: WideVT, Operand: Narrow);
2507 }
2508 if (WideVT.getScalarType() == MVT::f64 && !STI.hasFeature(Feature: NVPTX::SM90)) {
2509 EVT F32 = NarrowVT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::f32);
2510 SDLoc Loc(Op);
2511 if (STI.hasFeature(Feature: NVPTX::SM80) && STI.hasFeature(Feature: NVPTX::PTX71)) {
2512 Op = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: Loc, VT: F32, Operand: Narrow);
2513 } else {
2514 Op = DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: Loc, VT: F32, Operand: Narrow);
2515 }
2516 return DAG.getNode(Opcode: ISD::FP_EXTEND, DL: Loc, VT: WideVT, Operand: Op);
2517 }
2518 }
2519
2520 // Everything else is considered legal.
2521 return Op;
2522}
2523
2524static SDValue LowerVectorArith(SDValue Op, SelectionDAG &DAG) {
2525 SDLoc DL(Op);
2526 if (Op.getValueType() != MVT::v2i16)
2527 return Op;
2528 EVT EltVT = Op.getValueType().getVectorElementType();
2529 SmallVector<SDValue> VecElements;
2530 for (int I = 0, E = Op.getValueType().getVectorNumElements(); I < E; I++) {
2531 SmallVector<SDValue> ScalarArgs;
2532 llvm::transform(Range: Op->ops(), d_first: std::back_inserter(x&: ScalarArgs),
2533 F: [&](const SDUse &O) {
2534 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT,
2535 N1: O.get(), N2: DAG.getIntPtrConstant(Val: I, DL));
2536 });
2537 VecElements.push_back(Elt: DAG.getNode(Opcode: Op.getOpcode(), DL, VT: EltVT, Ops: ScalarArgs));
2538 }
2539 SDValue V =
2540 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: Op.getValueType(), Ops: VecElements);
2541 return V;
2542}
2543
2544static SDValue lowerTcgen05St(SDValue Op, SelectionDAG &DAG,
2545 bool hasOffset = false) {
2546 // skip lowering if the vector operand is already legalized
2547 if (!Op->getOperand(Num: hasOffset ? 4 : 3).getValueType().isVector())
2548 return Op;
2549
2550 SDNode *N = Op.getNode();
2551 SDLoc DL(N);
2552 SmallVector<SDValue, 32> Ops;
2553
2554 // split the vector argument
2555 for (size_t I = 0; I < N->getNumOperands(); I++) {
2556 SDValue Val = N->getOperand(Num: I);
2557 EVT ValVT = Val.getValueType();
2558 if (ValVT.isVector()) {
2559 EVT EltVT = ValVT.getVectorElementType();
2560 for (unsigned J = 0, NElts = ValVT.getVectorNumElements(); J < NElts; J++)
2561 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Val,
2562 N2: DAG.getIntPtrConstant(Val: J, DL)));
2563 } else
2564 Ops.push_back(Elt: Val);
2565 }
2566
2567 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(Val: N);
2568 SDValue Tcgen05StNode =
2569 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl: DL, VTList: N->getVTList(), Ops,
2570 MemVT: MemSD->getMemoryVT(), MMO: MemSD->getMemOperand());
2571
2572 return Tcgen05StNode;
2573}
2574
2575static SDValue lowerBSWAP(SDValue Op, SelectionDAG &DAG) {
2576 SDLoc DL(Op);
2577 SDValue Src = Op.getOperand(i: 0);
2578 EVT VT = Op.getValueType();
2579
2580 switch (VT.getSimpleVT().SimpleTy) {
2581 case MVT::i16: {
2582 SDValue Extended = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Src);
2583 SDValue Swapped =
2584 getPRMT(A: Extended, B: DAG.getConstant(Val: 0, DL, VT: MVT::i32), Selector: 0x7701, DL, DAG);
2585 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: Swapped);
2586 }
2587 case MVT::i32: {
2588 return getPRMT(A: Src, B: DAG.getConstant(Val: 0, DL, VT: MVT::i32), Selector: 0x0123, DL, DAG);
2589 }
2590 case MVT::v2i16: {
2591 SDValue Converted = DAG.getBitcast(VT: MVT::i32, V: Src);
2592 SDValue Swapped =
2593 getPRMT(A: Converted, B: DAG.getConstant(Val: 0, DL, VT: MVT::i32), Selector: 0x2301, DL, DAG);
2594 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i16, Operand: Swapped);
2595 }
2596 case MVT::i64: {
2597 SDValue UnpackSrc =
2598 DAG.getNode(Opcode: NVPTXISD::UNPACK_VECTOR, DL, ResultTys: {MVT::i32, MVT::i32}, Ops: Src);
2599 SDValue SwappedLow =
2600 getPRMT(A: UnpackSrc.getValue(R: 0), B: DAG.getConstant(Val: 0, DL, VT: MVT::i32), Selector: 0x0123,
2601 DL, DAG);
2602 SDValue SwappedHigh =
2603 getPRMT(A: UnpackSrc.getValue(R: 1), B: DAG.getConstant(Val: 0, DL, VT: MVT::i32), Selector: 0x0123,
2604 DL, DAG);
2605 return DAG.getNode(Opcode: NVPTXISD::BUILD_VECTOR, DL, VT: MVT::i64,
2606 Ops: {SwappedHigh, SwappedLow});
2607 }
2608 default:
2609 llvm_unreachable("unsupported type for bswap");
2610 }
2611}
2612
2613static SDValue lowerStAsyncWithMbarrier(SDValue Op, SelectionDAG &DAG) {
2614 const Function &Fn = DAG.getMachineFunction().getFunction();
2615 SDNode *N = Op.getNode();
2616 SDLoc DL(N);
2617 Intrinsic::ID IntrinsicID = N->getConstantOperandVal(Num: 1);
2618 SDValue DestAddr = N->getOperand(Num: 2);
2619 SDValue Value = N->getOperand(Num: 3);
2620 SDValue MbarAddr = N->getOperand(Num: 4);
2621
2622 MVT ValueVT = Value.getSimpleValueType();
2623
2624 if (ValueVT == MVT::i32 || ValueVT == MVT::i64)
2625 return Op;
2626
2627 if (ValueVT == MVT::i128) {
2628 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: Value);
2629 SDValue ValueLo = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i64, N1: Cast,
2630 N2: DAG.getIntPtrConstant(Val: 0, DL));
2631 SDValue ValueHi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i64, N1: Cast,
2632 N2: DAG.getIntPtrConstant(Val: 1, DL));
2633 SDValue Ops[] = {N->getOperand(Num: 0), DestAddr, ValueLo, ValueHi, MbarAddr};
2634 return DAG.getNode(Opcode: NVPTXISD::ST_ASYNC_MBARRIER_B128, DL, VT: MVT::Other, Ops);
2635 }
2636
2637 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
2638 Fn,
2639 Twine("unsupported argument type ") + llvm::EVT(ValueVT).getEVTString() +
2640 " for " + llvm::Intrinsic::getName(id: IntrinsicID) + " intrinsic",
2641 DiagnosticLocation(DL.getDebugLoc())));
2642 return Op.getOperand(i: 0); // Return only the chain
2643}
2644
2645static SDValue lowerStAsyncRelease(SDValue Op, SelectionDAG &DAG) {
2646 const Function &Fn = DAG.getMachineFunction().getFunction();
2647 SDNode *N = Op.getNode();
2648 SDLoc DL(N);
2649 Intrinsic::ID IntrinsicID = N->getConstantOperandVal(Num: 1);
2650 SDValue DestAddr = N->getOperand(Num: 2);
2651 SDValue Value = N->getOperand(Num: 3);
2652
2653 MVT ValueVT = Value.getSimpleValueType();
2654
2655 if (ValueVT == MVT::i16 || ValueVT == MVT::i32 || ValueVT == MVT::i64)
2656 return Op;
2657
2658 if (ValueVT == MVT::i8) {
2659 unsigned OpCode;
2660 switch (IntrinsicID) {
2661 case Intrinsic::nvvm_st_async_sys:
2662 OpCode = NVPTXISD::ST_ASYNC_SYS_B8;
2663 break;
2664 case Intrinsic::nvvm_st_async_gpu:
2665 OpCode = NVPTXISD::ST_ASYNC_GPU_B8;
2666 break;
2667 case Intrinsic::nvvm_st_async_mmio_sys:
2668 OpCode = NVPTXISD::ST_ASYNC_MMIO_SYS_B8;
2669 break;
2670 default:
2671 llvm_unreachable("unexpected intrinsic ID for st.async.release");
2672 }
2673
2674 Value = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i16, Operand: Value);
2675
2676 // The `.mmio` variant has no multimem form and therefore no `isMultimem`
2677 // operand.
2678 if (IntrinsicID == Intrinsic::nvvm_st_async_mmio_sys) {
2679 SDValue Ops[] = {N->getOperand(Num: 0), DestAddr, Value};
2680 return DAG.getNode(Opcode: OpCode, DL, VT: MVT::Other, Ops);
2681 }
2682
2683 SDValue IsMultimem =
2684 DAG.getTargetConstant(Val: N->getConstantOperandVal(Num: 4), DL, VT: MVT::i1);
2685 SDValue Ops[] = {N->getOperand(Num: 0), DestAddr, Value, IsMultimem};
2686 return DAG.getNode(Opcode: OpCode, DL, VT: MVT::Other, Ops);
2687 }
2688
2689 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
2690 Fn,
2691 Twine("unsupported argument type ") + llvm::EVT(ValueVT).getEVTString() +
2692 " for " + llvm::Intrinsic::getName(id: IntrinsicID) + " intrinsic",
2693 DiagnosticLocation(DL.getDebugLoc())));
2694 return Op.getOperand(i: 0); // Return only the chain
2695}
2696
2697static unsigned getTcgen05MMADisableOutputLane(unsigned IID) {
2698 switch (IID) {
2699 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg1:
2700 return NVPTXISD::TCGEN05_MMA_SHARED_DISABLE_OUTPUT_LANE_CG1;
2701 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg2:
2702 return NVPTXISD::TCGEN05_MMA_SHARED_DISABLE_OUTPUT_LANE_CG2;
2703 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg1:
2704 return NVPTXISD::TCGEN05_MMA_SHARED_SCALE_D_DISABLE_OUTPUT_LANE_CG1;
2705 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg2:
2706 return NVPTXISD::TCGEN05_MMA_SHARED_SCALE_D_DISABLE_OUTPUT_LANE_CG2;
2707 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1:
2708 return NVPTXISD::TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG1;
2709 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2:
2710 return NVPTXISD::TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG2;
2711 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1:
2712 return NVPTXISD::TCGEN05_MMA_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG1;
2713 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2:
2714 return NVPTXISD::TCGEN05_MMA_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG2;
2715 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1_ashift:
2716 return NVPTXISD::TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG1_ASHIFT;
2717 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2_ashift:
2718 return NVPTXISD::TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG2_ASHIFT;
2719 case Intrinsic::
2720 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1_ashift:
2721 return NVPTXISD::TCGEN05_MMA_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG1_ASHIFT;
2722 case Intrinsic::
2723 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2_ashift:
2724 return NVPTXISD::TCGEN05_MMA_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG2_ASHIFT;
2725 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg1:
2726 return NVPTXISD::TCGEN05_MMA_SP_SHARED_DISABLE_OUTPUT_LANE_CG1;
2727 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg2:
2728 return NVPTXISD::TCGEN05_MMA_SP_SHARED_DISABLE_OUTPUT_LANE_CG2;
2729 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg1:
2730 return NVPTXISD::TCGEN05_MMA_SP_SHARED_SCALE_D_DISABLE_OUTPUT_LANE_CG1;
2731 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg2:
2732 return NVPTXISD::TCGEN05_MMA_SP_SHARED_SCALE_D_DISABLE_OUTPUT_LANE_CG2;
2733 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1:
2734 return NVPTXISD::TCGEN05_MMA_SP_TENSOR_DISABLE_OUTPUT_LANE_CG1;
2735 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2:
2736 return NVPTXISD::TCGEN05_MMA_SP_TENSOR_DISABLE_OUTPUT_LANE_CG2;
2737 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1_ashift:
2738 return NVPTXISD::TCGEN05_MMA_SP_TENSOR_DISABLE_OUTPUT_LANE_CG1_ASHIFT;
2739 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2_ashift:
2740 return NVPTXISD::TCGEN05_MMA_SP_TENSOR_DISABLE_OUTPUT_LANE_CG2_ASHIFT;
2741 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1:
2742 return NVPTXISD::TCGEN05_MMA_SP_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG1;
2743 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2:
2744 return NVPTXISD::TCGEN05_MMA_SP_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG2;
2745 case Intrinsic::
2746 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1_ashift:
2747 return NVPTXISD::
2748 TCGEN05_MMA_SP_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG1_ASHIFT;
2749 case Intrinsic::
2750 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2_ashift:
2751 return NVPTXISD::
2752 TCGEN05_MMA_SP_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG2_ASHIFT;
2753 case Intrinsic::
2754 nvvm_tcgen05_mma_shared_f8f6f4_disable_output_lane_cg1_decompress_b:
2755 return NVPTXISD::TCGEN05_MMA_SHARED_DISABLE_OUTPUT_LANE_CG1_DECOMPRESS_B;
2756 case Intrinsic::
2757 nvvm_tcgen05_mma_shared_f8f6f4_disable_output_lane_cg2_decompress_b:
2758 return NVPTXISD::TCGEN05_MMA_SHARED_DISABLE_OUTPUT_LANE_CG2_DECOMPRESS_B;
2759 case Intrinsic::
2760 nvvm_tcgen05_mma_tensor_f8f6f4_disable_output_lane_cg1_decompress_b:
2761 return NVPTXISD::TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG1_DECOMPRESS_B;
2762 case Intrinsic::
2763 nvvm_tcgen05_mma_tensor_f8f6f4_disable_output_lane_cg2_decompress_b:
2764 return NVPTXISD::TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG2_DECOMPRESS_B;
2765 };
2766 llvm_unreachable("unhandled tcgen05.mma.disable_output_lane intrinsic");
2767}
2768
2769static SDValue LowerTcgen05MMADisableOutputLane(SDValue Op, SelectionDAG &DAG) {
2770 SDNode *N = Op.getNode();
2771 SDLoc DL(N);
2772 unsigned IID = cast<ConstantSDNode>(Val: N->getOperand(Num: 1))->getZExtValue();
2773
2774 SmallVector<SDValue, 16> Ops;
2775 // split the vector argument
2776 for (size_t I = 0; I < N->getNumOperands(); I++) {
2777 if (I == 1)
2778 continue; // skip IID
2779 SDValue Val = N->getOperand(Num: I);
2780 EVT ValVT = Val.getValueType();
2781 if (ValVT.isVector()) {
2782 EVT EltVT = ValVT.getVectorElementType();
2783 for (unsigned J = 0, NElts = ValVT.getVectorNumElements(); J < NElts; J++)
2784 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Val,
2785 N2: DAG.getIntPtrConstant(Val: J, DL)));
2786 } else
2787 Ops.push_back(Elt: Val);
2788 }
2789
2790 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(Val: N);
2791 SDValue Tcgen05MMANode = DAG.getMemIntrinsicNode(
2792 Opcode: getTcgen05MMADisableOutputLane(IID), dl: DL, VTList: N->getVTList(), Ops,
2793 MemVT: MemSD->getMemoryVT(), MMO: MemSD->getMemOperand());
2794
2795 return Tcgen05MMANode;
2796}
2797
2798// Lower vector return type of tcgen05.ld intrinsics
2799static std::optional<std::pair<SDValue, SDValue>>
2800lowerTcgen05Ld(SDNode *N, SelectionDAG &DAG, bool HasOffset = false) {
2801 SDLoc DL(N);
2802 EVT ResVT = N->getValueType(ResNo: 0);
2803 if (!ResVT.isVector())
2804 return {}; // already legalized.
2805
2806 const unsigned NumElts = ResVT.getVectorNumElements();
2807
2808 // Create the return type of the instructions
2809 SmallVector<EVT, 5> ListVTs;
2810 for (unsigned i = 0; i < NumElts; ++i)
2811 ListVTs.push_back(Elt: MVT::i32);
2812
2813 ListVTs.push_back(Elt: N->getValueType(ResNo: 1)); // Chain
2814
2815 SDVTList ResVTs = DAG.getVTList(VTs: ListVTs);
2816
2817 SmallVector<SDValue, 8> Ops{N->getOperand(Num: 0), N->getOperand(Num: 1),
2818 N->getOperand(Num: 2)};
2819
2820 if (HasOffset) {
2821 Ops.push_back(Elt: N->getOperand(Num: 3)); // offset
2822 Ops.push_back(Elt: N->getOperand(Num: 4)); // Pack flag
2823 } else
2824 Ops.push_back(Elt: N->getOperand(Num: 3)); // Pack flag
2825
2826 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(Val: N);
2827 SDValue NewNode =
2828 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: ResVTs, Ops,
2829 MemVT: MemSD->getMemoryVT(), MMO: MemSD->getMemOperand());
2830
2831 // split the vector result
2832 SmallVector<SDValue, 4> ScalarRes;
2833 for (unsigned i = 0; i < NumElts; ++i) {
2834 SDValue Res = NewNode.getValue(R: i);
2835 ScalarRes.push_back(Elt: Res);
2836 }
2837
2838 SDValue Chain = NewNode.getValue(R: NumElts);
2839 SDValue BuildVector = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: ResVT, Ops: ScalarRes);
2840 return {{BuildVector, Chain}};
2841}
2842
2843static SDValue reportInvalidTensormapReplaceUsage(SDValue Op, SelectionDAG &DAG,
2844 unsigned Val) {
2845 SDNode *N = Op.getNode();
2846 SDLoc DL(N);
2847
2848 const Function &Fn = DAG.getMachineFunction().getFunction();
2849
2850 unsigned AS = 0;
2851 if (auto *MemN = dyn_cast<MemIntrinsicSDNode>(Val: N))
2852 AS = MemN->getAddressSpace();
2853 Type *PtrTy = PointerType::get(C&: *DAG.getContext(), AddressSpace: AS);
2854 Module *M = DAG.getMachineFunction().getFunction().getParent();
2855
2856 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
2857 Fn,
2858 "Intrinsic " +
2859 Intrinsic::getName(Id: N->getConstantOperandVal(Num: 1), OverloadTys: {PtrTy}, M) +
2860 " with value " + Twine(Val) +
2861 " is not supported on the given target.",
2862 DL.getDebugLoc()));
2863 return Op.getOperand(i: 0);
2864}
2865
2866static SDValue lowerTensormapReplaceElemtype(SDValue Op, SelectionDAG &DAG) {
2867 SDNode *N = Op.getNode();
2868 SDLoc DL(N);
2869
2870 // immediate argument representing elemtype
2871 unsigned Val = N->getConstantOperandVal(Num: 3);
2872
2873 if (!DAG.getSubtarget<NVPTXSubtarget>().hasTensormapReplaceElemtypeSupport(
2874 ElemType: Val))
2875 return reportInvalidTensormapReplaceUsage(Op, DAG, Val);
2876
2877 return Op;
2878}
2879
2880static SDValue lowerTensormapReplaceSwizzleMode(SDValue Op, SelectionDAG &DAG) {
2881 SDNode *N = Op.getNode();
2882 SDLoc DL(N);
2883
2884 // immediate argument representing swizzle mode
2885 unsigned Val = N->getConstantOperandVal(Num: 3);
2886
2887 if (!DAG.getSubtarget<NVPTXSubtarget>().hasTensormapReplaceSwizzleModeSupport(
2888 SwizzleMode: Val))
2889 return reportInvalidTensormapReplaceUsage(Op, DAG, Val);
2890
2891 return Op;
2892}
2893
2894static SDValue lowerIntrinsicVoid(SDValue Op, SelectionDAG &DAG) {
2895 SDNode *N = Op.getNode();
2896 SDValue Intrin = N->getOperand(Num: 1);
2897
2898 // Get the intrinsic ID
2899 unsigned IntrinNo = cast<ConstantSDNode>(Val: Intrin.getNode())->getZExtValue();
2900 switch (IntrinNo) {
2901 default:
2902 break;
2903 case Intrinsic::nvvm_st_async:
2904 return lowerStAsyncWithMbarrier(Op, DAG);
2905 case Intrinsic::nvvm_st_async_sys:
2906 case Intrinsic::nvvm_st_async_gpu:
2907 case Intrinsic::nvvm_st_async_mmio_sys:
2908 return lowerStAsyncRelease(Op, DAG);
2909
2910 case Intrinsic::nvvm_tcgen05_st_16x64b_x1:
2911 case Intrinsic::nvvm_tcgen05_st_16x64b_x2:
2912 case Intrinsic::nvvm_tcgen05_st_16x64b_x4:
2913 case Intrinsic::nvvm_tcgen05_st_16x64b_x8:
2914 case Intrinsic::nvvm_tcgen05_st_16x64b_x16:
2915 case Intrinsic::nvvm_tcgen05_st_16x64b_x32:
2916 case Intrinsic::nvvm_tcgen05_st_16x64b_x128:
2917 case Intrinsic::nvvm_tcgen05_st_16x128b_x1:
2918 case Intrinsic::nvvm_tcgen05_st_16x128b_x2:
2919 case Intrinsic::nvvm_tcgen05_st_16x128b_x4:
2920 case Intrinsic::nvvm_tcgen05_st_16x128b_x8:
2921 case Intrinsic::nvvm_tcgen05_st_16x128b_x16:
2922 case Intrinsic::nvvm_tcgen05_st_16x128b_x32:
2923 case Intrinsic::nvvm_tcgen05_st_16x128b_x64:
2924 case Intrinsic::nvvm_tcgen05_st_16x256b_x1:
2925 case Intrinsic::nvvm_tcgen05_st_16x256b_x2:
2926 case Intrinsic::nvvm_tcgen05_st_16x256b_x4:
2927 case Intrinsic::nvvm_tcgen05_st_16x256b_x8:
2928 case Intrinsic::nvvm_tcgen05_st_16x256b_x16:
2929 case Intrinsic::nvvm_tcgen05_st_16x256b_x32:
2930 case Intrinsic::nvvm_tcgen05_st_32x32b_x1:
2931 case Intrinsic::nvvm_tcgen05_st_32x32b_x2:
2932 case Intrinsic::nvvm_tcgen05_st_32x32b_x4:
2933 case Intrinsic::nvvm_tcgen05_st_32x32b_x8:
2934 case Intrinsic::nvvm_tcgen05_st_32x32b_x16:
2935 case Intrinsic::nvvm_tcgen05_st_32x32b_x32:
2936 case Intrinsic::nvvm_tcgen05_st_16x64b_x64:
2937 case Intrinsic::nvvm_tcgen05_st_32x32b_x64:
2938 case Intrinsic::nvvm_tcgen05_st_32x32b_x128:
2939 return lowerTcgen05St(Op, DAG);
2940 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x1:
2941 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x2:
2942 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x4:
2943 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x8:
2944 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x16:
2945 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x32:
2946 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x64:
2947 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x128:
2948 return lowerTcgen05St(Op, DAG, /* hasOffset */ true);
2949 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg1:
2950 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg2:
2951 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg1:
2952 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg2:
2953 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg1:
2954 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg2:
2955 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg1:
2956 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg2:
2957 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1:
2958 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2:
2959 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1:
2960 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2:
2961 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1:
2962 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2:
2963 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1:
2964 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2:
2965 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1_ashift:
2966 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2_ashift:
2967 case Intrinsic::
2968 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1_ashift:
2969 case Intrinsic::
2970 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2_ashift:
2971 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1_ashift:
2972 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2_ashift:
2973 case Intrinsic::
2974 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1_ashift:
2975 case Intrinsic::
2976 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2_ashift:
2977 case Intrinsic::
2978 nvvm_tcgen05_mma_shared_f8f6f4_disable_output_lane_cg1_decompress_b:
2979 case Intrinsic::
2980 nvvm_tcgen05_mma_shared_f8f6f4_disable_output_lane_cg2_decompress_b:
2981 case Intrinsic::
2982 nvvm_tcgen05_mma_tensor_f8f6f4_disable_output_lane_cg1_decompress_b:
2983 case Intrinsic::
2984 nvvm_tcgen05_mma_tensor_f8f6f4_disable_output_lane_cg2_decompress_b:
2985 return LowerTcgen05MMADisableOutputLane(Op, DAG);
2986 case Intrinsic::nvvm_tensormap_replace_elemtype:
2987 return lowerTensormapReplaceElemtype(Op, DAG);
2988 case Intrinsic::nvvm_tensormap_replace_swizzle_mode:
2989 return lowerTensormapReplaceSwizzleMode(Op, DAG);
2990 }
2991 return Op;
2992}
2993
2994static SDValue LowerClusterLaunchControlQueryCancel(SDValue Op,
2995 SelectionDAG &DAG) {
2996
2997 SDNode *N = Op.getNode();
2998 if (N->getOperand(Num: 1).getValueType() != MVT::i128) {
2999 // return, if the operand is already lowered
3000 return SDValue();
3001 }
3002
3003 unsigned IID =
3004 cast<ConstantSDNode>(Val: N->getOperand(Num: 0).getNode())->getZExtValue();
3005 auto Opcode = [&]() {
3006 switch (IID) {
3007 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_is_canceled:
3008 return NVPTXISD::CLUSTERLAUNCHCONTROL_QUERY_CANCEL_IS_CANCELED;
3009 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_x:
3010 return NVPTXISD::CLUSTERLAUNCHCONTROL_QUERY_CANCEL_GET_FIRST_CTAID_X;
3011 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_y:
3012 return NVPTXISD::CLUSTERLAUNCHCONTROL_QUERY_CANCEL_GET_FIRST_CTAID_Y;
3013 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_z:
3014 return NVPTXISD::CLUSTERLAUNCHCONTROL_QUERY_CANCEL_GET_FIRST_CTAID_Z;
3015 default:
3016 llvm_unreachable("unsupported/unhandled intrinsic");
3017 }
3018 }();
3019
3020 SDLoc DL(N);
3021 SDValue TryCancelResponse = N->getOperand(Num: 1);
3022 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: TryCancelResponse);
3023 SDValue TryCancelResponse0 =
3024 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i64, N1: Cast,
3025 N2: DAG.getIntPtrConstant(Val: 0, DL));
3026 SDValue TryCancelResponse1 =
3027 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i64, N1: Cast,
3028 N2: DAG.getIntPtrConstant(Val: 1, DL));
3029
3030 return DAG.getNode(Opcode, DL, VTList: N->getVTList(),
3031 Ops: {TryCancelResponse0, TryCancelResponse1});
3032}
3033
3034static SDValue lowerCvtRSIntrinsics(SDValue Op, SelectionDAG &DAG) {
3035 SDNode *N = Op.getNode();
3036 SDLoc DL(N);
3037 SDValue F32Vec = N->getOperand(Num: 1);
3038 SDValue RBits = N->getOperand(Num: 2);
3039
3040 unsigned IntrinsicID = N->getConstantOperandVal(Num: 0);
3041
3042 // Extract the 4 float elements from the vector
3043 SmallVector<SDValue, 6> Ops;
3044 for (unsigned i = 0; i < 4; ++i)
3045 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: F32Vec,
3046 N2: DAG.getIntPtrConstant(Val: i, DL)));
3047
3048 using NVPTX::PTXCvtMode::CvtMode;
3049
3050 auto [OpCode, RetTy, CvtModeFlag] =
3051 [&]() -> std::tuple<unsigned, MVT::SimpleValueType, uint32_t> {
3052 switch (IntrinsicID) {
3053 case Intrinsic::nvvm_f32x4_to_e4m3x4_rs_relu_satfinite:
3054 return {NVPTXISD::CVT_E4M3X4_F32X4_RS_SF, MVT::v4i8,
3055 CvtMode::RS | CvtMode::RELU_FLAG};
3056 case Intrinsic::nvvm_f32x4_to_e4m3x4_rs_satfinite:
3057 return {NVPTXISD::CVT_E4M3X4_F32X4_RS_SF, MVT::v4i8, CvtMode::RS};
3058 case Intrinsic::nvvm_f32x4_to_e5m2x4_rs_relu_satfinite:
3059 return {NVPTXISD::CVT_E5M2X4_F32X4_RS_SF, MVT::v4i8,
3060 CvtMode::RS | CvtMode::RELU_FLAG};
3061 case Intrinsic::nvvm_f32x4_to_e5m2x4_rs_satfinite:
3062 return {NVPTXISD::CVT_E5M2X4_F32X4_RS_SF, MVT::v4i8, CvtMode::RS};
3063 case Intrinsic::nvvm_f32x4_to_e2m3x4_rs_relu_satfinite:
3064 return {NVPTXISD::CVT_E2M3X4_F32X4_RS_SF, MVT::v4i8,
3065 CvtMode::RS | CvtMode::RELU_FLAG};
3066 case Intrinsic::nvvm_f32x4_to_e2m3x4_rs_satfinite:
3067 return {NVPTXISD::CVT_E2M3X4_F32X4_RS_SF, MVT::v4i8, CvtMode::RS};
3068 case Intrinsic::nvvm_f32x4_to_e3m2x4_rs_relu_satfinite:
3069 return {NVPTXISD::CVT_E3M2X4_F32X4_RS_SF, MVT::v4i8,
3070 CvtMode::RS | CvtMode::RELU_FLAG};
3071 case Intrinsic::nvvm_f32x4_to_e3m2x4_rs_satfinite:
3072 return {NVPTXISD::CVT_E3M2X4_F32X4_RS_SF, MVT::v4i8, CvtMode::RS};
3073 case Intrinsic::nvvm_f32x4_to_e2m1x4_rs_relu_satfinite:
3074 return {NVPTXISD::CVT_E2M1X4_F32X4_RS_SF, MVT::i16,
3075 CvtMode::RS | CvtMode::RELU_FLAG};
3076 case Intrinsic::nvvm_f32x4_to_e2m1x4_rs_satfinite:
3077 return {NVPTXISD::CVT_E2M1X4_F32X4_RS_SF, MVT::i16, CvtMode::RS};
3078 default:
3079 llvm_unreachable("unsupported/unhandled intrinsic");
3080 }
3081 }();
3082
3083 Ops.push_back(Elt: RBits);
3084 Ops.push_back(Elt: DAG.getConstant(Val: CvtModeFlag, DL, VT: MVT::i32));
3085
3086 return DAG.getNode(Opcode: OpCode, DL, VT: RetTy, Ops);
3087}
3088
3089static SDValue lowerPrmtIntrinsic(SDValue Op, SelectionDAG &DAG) {
3090 const unsigned Mode = [&]() {
3091 switch (Op->getConstantOperandVal(Num: 0)) {
3092 case Intrinsic::nvvm_prmt:
3093 return NVPTX::PTXPrmtMode::NONE;
3094 case Intrinsic::nvvm_prmt_b4e:
3095 return NVPTX::PTXPrmtMode::B4E;
3096 case Intrinsic::nvvm_prmt_ecl:
3097 return NVPTX::PTXPrmtMode::ECL;
3098 case Intrinsic::nvvm_prmt_ecr:
3099 return NVPTX::PTXPrmtMode::ECR;
3100 case Intrinsic::nvvm_prmt_f4e:
3101 return NVPTX::PTXPrmtMode::F4E;
3102 case Intrinsic::nvvm_prmt_rc16:
3103 return NVPTX::PTXPrmtMode::RC16;
3104 case Intrinsic::nvvm_prmt_rc8:
3105 return NVPTX::PTXPrmtMode::RC8;
3106 default:
3107 llvm_unreachable("unsupported/unhandled intrinsic");
3108 }
3109 }();
3110 SDLoc DL(Op);
3111 SDValue A = Op->getOperand(Num: 1);
3112 SDValue B = Op.getNumOperands() == 4 ? Op.getOperand(i: 2)
3113 : DAG.getConstant(Val: 0, DL, VT: MVT::i32);
3114 SDValue Selector = (Op->op_end() - 1)->get();
3115 return getPRMT(A, B, Selector, DL, DAG, Mode);
3116}
3117
3118#define TCGEN05_LD_RED_INTR(SHAPE, NUM, TYPE) \
3119 Intrinsic::nvvm_tcgen05_ld_red_##SHAPE##_x##NUM##_##TYPE
3120
3121#define TCGEN05_LD_RED_INST(SHAPE, NUM, TYPE) \
3122 NVPTXISD::TCGEN05_LD_RED_##SHAPE##_X##NUM##_##TYPE
3123
3124static unsigned getTcgen05LdRedID(Intrinsic::ID IID) {
3125 switch (IID) {
3126 case TCGEN05_LD_RED_INTR(32x32b, 2, f32):
3127 return TCGEN05_LD_RED_INST(32x32b, 2, F32);
3128 case TCGEN05_LD_RED_INTR(32x32b, 4, f32):
3129 return TCGEN05_LD_RED_INST(32x32b, 4, F32);
3130 case TCGEN05_LD_RED_INTR(32x32b, 8, f32):
3131 return TCGEN05_LD_RED_INST(32x32b, 8, F32);
3132 case TCGEN05_LD_RED_INTR(32x32b, 16, f32):
3133 return TCGEN05_LD_RED_INST(32x32b, 16, F32);
3134 case TCGEN05_LD_RED_INTR(32x32b, 32, f32):
3135 return TCGEN05_LD_RED_INST(32x32b, 32, F32);
3136 case TCGEN05_LD_RED_INTR(32x32b, 64, f32):
3137 return TCGEN05_LD_RED_INST(32x32b, 64, F32);
3138 case TCGEN05_LD_RED_INTR(32x32b, 128, f32):
3139 return TCGEN05_LD_RED_INST(32x32b, 128, F32);
3140 case TCGEN05_LD_RED_INTR(16x32bx2, 2, f32):
3141 return TCGEN05_LD_RED_INST(16x32bx2, 2, F32);
3142 case TCGEN05_LD_RED_INTR(16x32bx2, 4, f32):
3143 return TCGEN05_LD_RED_INST(16x32bx2, 4, F32);
3144 case TCGEN05_LD_RED_INTR(16x32bx2, 8, f32):
3145 return TCGEN05_LD_RED_INST(16x32bx2, 8, F32);
3146 case TCGEN05_LD_RED_INTR(16x32bx2, 16, f32):
3147 return TCGEN05_LD_RED_INST(16x32bx2, 16, F32);
3148 case TCGEN05_LD_RED_INTR(16x32bx2, 32, f32):
3149 return TCGEN05_LD_RED_INST(16x32bx2, 32, F32);
3150 case TCGEN05_LD_RED_INTR(16x32bx2, 64, f32):
3151 return TCGEN05_LD_RED_INST(16x32bx2, 64, F32);
3152 case TCGEN05_LD_RED_INTR(16x32bx2, 128, f32):
3153 return TCGEN05_LD_RED_INST(16x32bx2, 128, F32);
3154 case TCGEN05_LD_RED_INTR(32x32b, 2, i32):
3155 return TCGEN05_LD_RED_INST(32x32b, 2, I32);
3156 case TCGEN05_LD_RED_INTR(32x32b, 4, i32):
3157 return TCGEN05_LD_RED_INST(32x32b, 4, I32);
3158 case TCGEN05_LD_RED_INTR(32x32b, 8, i32):
3159 return TCGEN05_LD_RED_INST(32x32b, 8, I32);
3160 case TCGEN05_LD_RED_INTR(32x32b, 16, i32):
3161 return TCGEN05_LD_RED_INST(32x32b, 16, I32);
3162 case TCGEN05_LD_RED_INTR(32x32b, 32, i32):
3163 return TCGEN05_LD_RED_INST(32x32b, 32, I32);
3164 case TCGEN05_LD_RED_INTR(32x32b, 64, i32):
3165 return TCGEN05_LD_RED_INST(32x32b, 64, I32);
3166 case TCGEN05_LD_RED_INTR(32x32b, 128, i32):
3167 return TCGEN05_LD_RED_INST(32x32b, 128, I32);
3168 case TCGEN05_LD_RED_INTR(16x32bx2, 2, i32):
3169 return TCGEN05_LD_RED_INST(16x32bx2, 2, I32);
3170 case TCGEN05_LD_RED_INTR(16x32bx2, 4, i32):
3171 return TCGEN05_LD_RED_INST(16x32bx2, 4, I32);
3172 case TCGEN05_LD_RED_INTR(16x32bx2, 8, i32):
3173 return TCGEN05_LD_RED_INST(16x32bx2, 8, I32);
3174 case TCGEN05_LD_RED_INTR(16x32bx2, 16, i32):
3175 return TCGEN05_LD_RED_INST(16x32bx2, 16, I32);
3176 case TCGEN05_LD_RED_INTR(16x32bx2, 32, i32):
3177 return TCGEN05_LD_RED_INST(16x32bx2, 32, I32);
3178 case TCGEN05_LD_RED_INTR(16x32bx2, 64, i32):
3179 return TCGEN05_LD_RED_INST(16x32bx2, 64, I32);
3180 case TCGEN05_LD_RED_INTR(16x32bx2, 128, i32):
3181 return TCGEN05_LD_RED_INST(16x32bx2, 128, I32);
3182 default:
3183 llvm_unreachable("Invalid tcgen05.ld.red intrinsic ID");
3184 }
3185}
3186
3187// Lower vector return type of tcgen05.ld intrinsics
3188static std::optional<std::tuple<SDValue, SDValue, SDValue>>
3189lowerTcgen05LdRed(SDNode *N, SelectionDAG &DAG) {
3190 SDLoc DL(N);
3191 EVT ResVT = N->getValueType(ResNo: 0);
3192 if (!ResVT.isVector())
3193 return {}; // already legalized.
3194
3195 const unsigned NumElts = ResVT.getVectorNumElements();
3196
3197 // Create the return type of the instructions
3198 // +1 represents the reduction value
3199 SmallVector<EVT, 132> ListVTs{
3200 NumElts + 1,
3201 ResVT.getVectorElementType().isFloatingPoint() ? MVT::f32 : MVT::i32};
3202
3203 ListVTs.push_back(Elt: MVT::Other); // Chain
3204
3205 SDVTList ResVTs = DAG.getVTList(VTs: ListVTs);
3206
3207 // Prepare the Operands
3208 SmallVector<SDValue, 8> Ops{N->getOperand(Num: 0)}; // Chain
3209
3210 // skip IID at index 1
3211 for (unsigned i = 2; i < N->getNumOperands(); i++)
3212 Ops.push_back(Elt: N->getOperand(Num: i));
3213
3214 unsigned IID = cast<ConstantSDNode>(Val: N->getOperand(Num: 1))->getZExtValue();
3215 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(Val: N);
3216 SDValue NewNode =
3217 DAG.getMemIntrinsicNode(Opcode: getTcgen05LdRedID(IID), dl: DL, VTList: ResVTs, Ops,
3218 MemVT: MemSD->getMemoryVT(), MMO: MemSD->getMemOperand());
3219
3220 // Split vector result
3221 SmallVector<SDValue, 132> ScalarRes;
3222 for (unsigned i = 0; i < NumElts; ++i) {
3223 SDValue Res = NewNode.getValue(R: i);
3224 ScalarRes.push_back(Elt: Res);
3225 }
3226
3227 SDValue BuildVector = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: ResVT, Ops: ScalarRes);
3228 SDValue RedResult = NewNode.getValue(R: NumElts);
3229 SDValue Chain = NewNode.getValue(R: NumElts + 1);
3230 return {{BuildVector, RedResult, Chain}};
3231}
3232
3233static SDValue lowerIntrinsicWChain(SDValue Op, SelectionDAG &DAG) {
3234 switch (Op->getConstantOperandVal(Num: 1)) {
3235 default:
3236 return Op;
3237
3238 // These tcgen05 intrinsics return a v2i32, which is legal, so we have to
3239 // lower them through LowerOperation() instead of ReplaceNodeResults().
3240 case Intrinsic::nvvm_tcgen05_ld_16x64b_x2:
3241 case Intrinsic::nvvm_tcgen05_ld_16x128b_x1:
3242 case Intrinsic::nvvm_tcgen05_ld_32x32b_x2:
3243 if (auto Res = lowerTcgen05Ld(N: Op.getNode(), DAG))
3244 return DAG.getMergeValues(Ops: {Res->first, Res->second}, dl: SDLoc(Op));
3245 return SDValue();
3246
3247 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x2:
3248 if (auto Res = lowerTcgen05Ld(N: Op.getNode(), DAG, /*HasOffset=*/true))
3249 return DAG.getMergeValues(Ops: {Res->first, Res->second}, dl: SDLoc(Op));
3250 return SDValue();
3251
3252 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x2_f32:
3253 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x2_i32:
3254 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x2_f32:
3255 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x2_i32:
3256 if (auto Res = lowerTcgen05LdRed(N: Op.getNode(), DAG))
3257 return DAG.getMergeValues(
3258 Ops: {std::get<0>(t&: *Res), std::get<1>(t&: *Res), std::get<2>(t&: *Res)}, dl: SDLoc(Op));
3259 return SDValue();
3260 }
3261}
3262
3263static unsigned getSPNumRegs(EVT VT) {
3264 return divideCeil(Numerator: VT.getSizeInBits(), Denominator: 32);
3265}
3266
3267static EVT getSPRegType(EVT VT) {
3268 assert(VT.isVector() && "Expected a vector type");
3269 EVT EltVT = VT.getVectorElementType();
3270 if (EltVT == MVT::i8)
3271 return MVT::v4i8;
3272 if (EltVT == MVT::i16)
3273 return MVT::v2i16;
3274 return EltVT;
3275}
3276
3277static void appendSplitSPOperands(ArrayRef<SDUse> Input,
3278 SmallVectorImpl<SDValue> &Output,
3279 SelectionDAG &DAG) {
3280 for (SDValue Operand : Input) {
3281 EVT VT = Operand.getValueType();
3282 SDLoc DL(Operand);
3283 if (!VT.isVector()) {
3284 Output.push_back(Elt: Operand);
3285 continue;
3286 }
3287
3288 // Pack partial .b32 operands at their natural width before extending them
3289 // to avoid redundant cvt/prmt sequences.
3290 if (VT.getSizeInBits() < 32) {
3291 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
3292 Output.push_back(
3293 Elt: DAG.getAnyExtOrTrunc(Op: DAG.getBitcast(VT: IntVT, V: Operand), DL, VT: MVT::i32));
3294 continue;
3295 }
3296
3297 EVT RegVT = getSPRegType(VT);
3298 if (!RegVT.isVector()) {
3299 DAG.ExtractVectorElements(Op: Operand, Args&: Output);
3300 continue;
3301 }
3302
3303 unsigned RegElts = RegVT.getVectorNumElements();
3304 for (unsigned I : llvm::seq(Size: getSPNumRegs(VT)))
3305 Output.push_back(Elt: DAG.getBitcast(
3306 VT: MVT::i32, V: DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: RegVT, N1: Operand,
3307 N2: DAG.getVectorIdxConstant(Val: I * RegElts, DL))));
3308 }
3309}
3310
3311static SDValue joinSPRegs(EVT VT, ArrayRef<SDValue> Regs, const SDLoc &DL,
3312 SelectionDAG &DAG) {
3313 assert(Regs.size() == getSPNumRegs(VT) && VT.getSizeInBits() % 32 == 0 &&
3314 "Incorrect number of SP vector registers");
3315 if (!VT.isVector())
3316 return Regs.front();
3317
3318 EVT RegVT = getSPRegType(VT);
3319 SmallVector<SDValue> Parts;
3320 for (SDValue Reg : Regs)
3321 Parts.push_back(Elt: DAG.getBitcast(VT: RegVT, V: Reg));
3322
3323 if (!RegVT.isVector())
3324 return DAG.getBuildVector(VT, DL, Ops: Parts);
3325 if (Parts.size() == 1)
3326 return Parts.front();
3327 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, Ops: Parts);
3328}
3329
3330static unsigned getSPCompressNodeOpcode(unsigned NumResults) {
3331#define SPCOMPRESS_NODE_CASE(NumResults) \
3332 case NumResults: \
3333 return NVPTXISD::SPCOMPRESS_R##NumResults
3334
3335 switch (NumResults) {
3336 SPCOMPRESS_NODE_CASE(2);
3337 SPCOMPRESS_NODE_CASE(3);
3338 SPCOMPRESS_NODE_CASE(5);
3339 SPCOMPRESS_NODE_CASE(6);
3340 SPCOMPRESS_NODE_CASE(9);
3341 SPCOMPRESS_NODE_CASE(10);
3342 SPCOMPRESS_NODE_CASE(12);
3343 SPCOMPRESS_NODE_CASE(18);
3344 SPCOMPRESS_NODE_CASE(20);
3345 SPCOMPRESS_NODE_CASE(24);
3346 SPCOMPRESS_NODE_CASE(36);
3347 SPCOMPRESS_NODE_CASE(40);
3348 SPCOMPRESS_NODE_CASE(48);
3349 SPCOMPRESS_NODE_CASE(72);
3350 SPCOMPRESS_NODE_CASE(80);
3351 SPCOMPRESS_NODE_CASE(96);
3352 default:
3353 llvm_unreachable("Invalid spcompress result count");
3354 }
3355
3356#undef SPCOMPRESS_NODE_CASE
3357}
3358
3359static unsigned getSPDecompressNodeOpcode(unsigned NumResults) {
3360#define SPDECOMPRESS_NODE_CASE(NumResults) \
3361 case NumResults: \
3362 return NVPTXISD::SPDECOMPRESS_R##NumResults
3363
3364 switch (NumResults) {
3365 SPDECOMPRESS_NODE_CASE(1);
3366 SPDECOMPRESS_NODE_CASE(2);
3367 SPDECOMPRESS_NODE_CASE(4);
3368 SPDECOMPRESS_NODE_CASE(8);
3369 SPDECOMPRESS_NODE_CASE(16);
3370 SPDECOMPRESS_NODE_CASE(32);
3371 SPDECOMPRESS_NODE_CASE(64);
3372 SPDECOMPRESS_NODE_CASE(128);
3373 default:
3374 llvm_unreachable("Invalid spdecompress result count");
3375 }
3376
3377#undef SPDECOMPRESS_NODE_CASE
3378}
3379
3380static SDValue lowerSPIntrinsic(SDValue Op, SelectionDAG &DAG) {
3381 SDNode *N = Op.getNode();
3382 SDLoc DL(N);
3383
3384 auto IID = static_cast<Intrinsic::ID>(N->getConstantOperandVal(Num: 0));
3385 bool IsCompress = IID == Intrinsic::nvvm_spcompress;
3386 assert((IsCompress || IID == Intrinsic::nvvm_spdecompress) &&
3387 "Unexpected SP intrinsic");
3388
3389 EVT DataVT =
3390 IsCompress ? N->getOperand(Num: 1).getValueType() : N->getValueType(ResNo: 0);
3391 EVT CDataVT =
3392 IsCompress ? N->getValueType(ResNo: 1) : N->getOperand(Num: 2).getValueType();
3393
3394 unsigned ElemSize = DataVT.getScalarSizeInBits();
3395 unsigned IdxSize = N->getConstantOperandVal(Num: 3);
3396 unsigned NumTgt = N->getConstantOperandVal(Num: 4);
3397 unsigned NumSrc =
3398 NumTgt * CDataVT.getVectorNumElements() / DataVT.getVectorNumElements();
3399 unsigned RepeatFactor = IsCompress ? getSPNumRegs(VT: DataVT) / 2
3400 : DataVT.getVectorNumElements() / NumTgt;
3401
3402 SmallVector<SDValue, 16> Ops;
3403 for (unsigned Qualifier : {ElemSize, IdxSize, RepeatFactor, NumSrc, NumTgt})
3404 Ops.push_back(Elt: DAG.getTargetConstant(Val: Qualifier, DL, VT: MVT::i32));
3405 appendSplitSPOperands(Input: N->ops().slice(N: 1, M: 2), Output&: Ops, DAG);
3406
3407 SmallVector<EVT, 16> ResultVTs;
3408 for (const EVT VT : N->values())
3409 ResultVTs.append(NumInputs: getSPNumRegs(VT), Elt: MVT::i32);
3410
3411 unsigned Opcode = IsCompress ? getSPCompressNodeOpcode(NumResults: ResultVTs.size())
3412 : getSPDecompressNodeOpcode(NumResults: ResultVTs.size());
3413 SDValue Lowered = DAG.getNode(Opcode, DL, ResultTys: ResultVTs, Ops);
3414
3415 SmallVector<SDValue, 2> Retvals;
3416 unsigned Reg = 0;
3417 for (const EVT VT : N->values()) {
3418 SmallVector<SDValue> Regs;
3419 for (unsigned R : llvm::seq(Size: getSPNumRegs(VT)))
3420 Regs.push_back(Elt: Lowered.getValue(R: Reg + R));
3421 Reg += Regs.size();
3422 Retvals.push_back(Elt: joinSPRegs(VT, Regs, DL, DAG));
3423 }
3424
3425 return DAG.getMergeValues(Ops: Retvals, dl: DL);
3426}
3427
3428static SDValue lowerIntrinsicWOChain(SDValue Op, SelectionDAG &DAG) {
3429 switch (Op->getConstantOperandVal(Num: 0)) {
3430 default:
3431 return Op;
3432 case Intrinsic::nvvm_prmt:
3433 case Intrinsic::nvvm_prmt_b4e:
3434 case Intrinsic::nvvm_prmt_ecl:
3435 case Intrinsic::nvvm_prmt_ecr:
3436 case Intrinsic::nvvm_prmt_f4e:
3437 case Intrinsic::nvvm_prmt_rc16:
3438 case Intrinsic::nvvm_prmt_rc8:
3439 return lowerPrmtIntrinsic(Op, DAG);
3440 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_is_canceled:
3441 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_x:
3442 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_y:
3443 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_z:
3444 return LowerClusterLaunchControlQueryCancel(Op, DAG);
3445 case Intrinsic::nvvm_f32x4_to_e4m3x4_rs_satfinite:
3446 case Intrinsic::nvvm_f32x4_to_e4m3x4_rs_relu_satfinite:
3447 case Intrinsic::nvvm_f32x4_to_e5m2x4_rs_satfinite:
3448 case Intrinsic::nvvm_f32x4_to_e5m2x4_rs_relu_satfinite:
3449 case Intrinsic::nvvm_f32x4_to_e2m3x4_rs_satfinite:
3450 case Intrinsic::nvvm_f32x4_to_e2m3x4_rs_relu_satfinite:
3451 case Intrinsic::nvvm_f32x4_to_e3m2x4_rs_satfinite:
3452 case Intrinsic::nvvm_f32x4_to_e3m2x4_rs_relu_satfinite:
3453 case Intrinsic::nvvm_f32x4_to_e2m1x4_rs_satfinite:
3454 case Intrinsic::nvvm_f32x4_to_e2m1x4_rs_relu_satfinite:
3455 return lowerCvtRSIntrinsics(Op, DAG);
3456
3457 case Intrinsic::nvvm_spcompress:
3458 case Intrinsic::nvvm_spdecompress:
3459 return lowerSPIntrinsic(Op, DAG);
3460 }
3461}
3462
3463// In PTX 64-bit CTLZ and CTPOP are supported, but they return a 32-bit value.
3464// Lower these into a node returning the correct type which is zero-extended
3465// back to the correct size.
3466static SDValue lowerCTLZCTPOP(SDValue Op, SelectionDAG &DAG) {
3467 SDValue V = Op->getOperand(Num: 0);
3468 assert(V.getValueType() == MVT::i64 &&
3469 "Unexpected CTLZ/CTPOP type to legalize");
3470
3471 SDLoc DL(Op);
3472 SDValue CT = DAG.getNode(Opcode: Op->getOpcode(), DL, VT: MVT::i32, Operand: V);
3473 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: CT, Flags: SDNodeFlags::NonNeg);
3474}
3475
3476static SDValue expandFSH64(SDValue A, SDValue B, SDValue ShiftAmount, SDLoc DL,
3477 unsigned Opcode, SelectionDAG &DAG) {
3478 assert(A.getValueType() == MVT::i64 && B.getValueType() == MVT::i64);
3479
3480 const auto *AmtConst = dyn_cast<ConstantSDNode>(Val&: ShiftAmount);
3481 if (!AmtConst)
3482 return SDValue();
3483 const auto Amt = AmtConst->getZExtValue() & 63;
3484
3485 SDValue UnpackA =
3486 DAG.getNode(Opcode: NVPTXISD::UNPACK_VECTOR, DL, ResultTys: {MVT::i32, MVT::i32}, Ops: A);
3487 SDValue UnpackB =
3488 DAG.getNode(Opcode: NVPTXISD::UNPACK_VECTOR, DL, ResultTys: {MVT::i32, MVT::i32}, Ops: B);
3489
3490 // Arch is Little endiain: 0 = low bits, 1 = high bits
3491 SDValue ALo = UnpackA.getValue(R: 0);
3492 SDValue AHi = UnpackA.getValue(R: 1);
3493 SDValue BLo = UnpackB.getValue(R: 0);
3494 SDValue BHi = UnpackB.getValue(R: 1);
3495
3496 // The bitfeild consists of { AHi : ALo : BHi : BLo }
3497 //
3498 // * FSHL, Amt < 32 - The window will contain { AHi : ALo : BHi }
3499 // * FSHL, Amt >= 32 - The window will contain { ALo : BHi : BLo }
3500 // * FSHR, Amt < 32 - The window will contain { ALo : BHi : BLo }
3501 // * FSHR, Amt >= 32 - The window will contain { AHi : ALo : BHi }
3502 //
3503 // Note that Amt = 0 and Amt = 32 are special cases where 32-bit funnel shifts
3504 // are not needed at all. Amt = 0 is a no-op producing either A or B depending
3505 // on the direction. Amt = 32 can be implemented by a packing and unpacking
3506 // move to select and arrange the 32bit values. For simplicity, these cases
3507 // are not handled here explicitly and instead we rely on DAGCombiner to
3508 // remove the no-op funnel shifts we insert.
3509 auto [High, Mid, Low] = ((Opcode == ISD::FSHL) == (Amt < 32))
3510 ? std::make_tuple(args&: AHi, args&: ALo, args&: BHi)
3511 : std::make_tuple(args&: ALo, args&: BHi, args&: BLo);
3512
3513 SDValue NewAmt = DAG.getConstant(Val: Amt & 31, DL, VT: MVT::i32);
3514 SDValue RHi = DAG.getNode(Opcode, DL, VT: MVT::i32, Ops: {High, Mid, NewAmt});
3515 SDValue RLo = DAG.getNode(Opcode, DL, VT: MVT::i32, Ops: {Mid, Low, NewAmt});
3516
3517 return DAG.getNode(Opcode: NVPTXISD::BUILD_VECTOR, DL, VT: MVT::i64, Ops: {RLo, RHi});
3518}
3519
3520static SDValue lowerFSH(SDValue Op, SelectionDAG &DAG) {
3521 return expandFSH64(A: Op->getOperand(Num: 0), B: Op->getOperand(Num: 1), ShiftAmount: Op->getOperand(Num: 2),
3522 DL: SDLoc(Op), Opcode: Op->getOpcode(), DAG);
3523}
3524
3525static SDValue lowerROT(SDValue Op, SelectionDAG &DAG) {
3526 unsigned Opcode = Op->getOpcode() == ISD::ROTL ? ISD::FSHL : ISD::FSHR;
3527 return expandFSH64(A: Op->getOperand(Num: 0), B: Op->getOperand(Num: 0), ShiftAmount: Op->getOperand(Num: 1),
3528 DL: SDLoc(Op), Opcode, DAG);
3529}
3530
3531static SDValue lowerSELECT(SDValue Op, SelectionDAG &DAG) {
3532 assert(Op.getValueType() == MVT::i1 && "Custom lowering enabled only for i1");
3533
3534 SDValue Cond = Op->getOperand(Num: 0);
3535 SDValue TrueVal = Op->getOperand(Num: 1);
3536 SDValue FalseVal = Op->getOperand(Num: 2);
3537 SDLoc DL(Op);
3538
3539 // If both operands are truncated, we push the select through the truncates.
3540 if (TrueVal.getOpcode() == ISD::TRUNCATE &&
3541 FalseVal.getOpcode() == ISD::TRUNCATE) {
3542 TrueVal = TrueVal.getOperand(i: 0);
3543 FalseVal = FalseVal.getOperand(i: 0);
3544
3545 EVT VT = TrueVal.getSimpleValueType().bitsLE(VT: FalseVal.getSimpleValueType())
3546 ? TrueVal.getValueType()
3547 : FalseVal.getValueType();
3548 TrueVal = DAG.getAnyExtOrTrunc(Op: TrueVal, DL, VT);
3549 FalseVal = DAG.getAnyExtOrTrunc(Op: FalseVal, DL, VT);
3550 SDValue Select = DAG.getSelect(DL, VT, Cond, LHS: TrueVal, RHS: FalseVal);
3551 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Select);
3552 }
3553
3554 // Otherwise, expand the select into a series of logical operations. These
3555 // often can be folded into other operations either by us or ptxas.
3556 TrueVal = DAG.getFreeze(V: TrueVal);
3557 FalseVal = DAG.getFreeze(V: FalseVal);
3558 SDValue And1 = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i1, N1: Cond, N2: TrueVal);
3559 SDValue NotCond = DAG.getNOT(DL, Val: Cond, VT: MVT::i1);
3560 SDValue And2 = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i1, N1: NotCond, N2: FalseVal);
3561 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i1, N1: And1, N2: And2);
3562 return Or;
3563}
3564
3565static SDValue lowerMSTORE(SDValue Op, SelectionDAG &DAG) {
3566 SDNode *N = Op.getNode();
3567
3568 SDValue Chain = N->getOperand(Num: 0);
3569 SDValue Val = N->getOperand(Num: 1);
3570 SDValue BasePtr = N->getOperand(Num: 2);
3571 SDValue Offset = N->getOperand(Num: 3);
3572 SDValue Mask = N->getOperand(Num: 4);
3573
3574 SDLoc DL(N);
3575 EVT ValVT = Val.getValueType();
3576 MemSDNode *MemSD = cast<MemSDNode>(Val: N);
3577 assert(ValVT.isVector() && "Masked vector store must have vector type");
3578 assert(MemSD->getAlign() >= DAG.getEVTAlign(ValVT) &&
3579 "Unexpected alignment for masked store");
3580
3581 unsigned Opcode = 0;
3582 switch (ValVT.getSimpleVT().SimpleTy) {
3583 default:
3584 llvm_unreachable("Unexpected masked vector store type");
3585 case MVT::v4i64:
3586 case MVT::v4f64: {
3587 Opcode = NVPTXISD::StoreV4;
3588 break;
3589 }
3590 case MVT::v8i32:
3591 case MVT::v8f32: {
3592 Opcode = NVPTXISD::StoreV8;
3593 break;
3594 }
3595 }
3596
3597 SmallVector<SDValue, 8> Ops;
3598
3599 // Construct the new SDNode. First operand is the chain.
3600 Ops.push_back(Elt: Chain);
3601
3602 // The next N operands are the values to store. Encode the mask into the
3603 // values using the sentinel register 0 to represent a masked-off element.
3604 assert(Mask.getValueType().isVector() &&
3605 Mask.getValueType().getVectorElementType() == MVT::i1 &&
3606 "Mask must be a vector of i1");
3607 assert(Mask.getOpcode() == ISD::BUILD_VECTOR &&
3608 "Mask expected to be a BUILD_VECTOR");
3609 assert(Mask.getValueType().getVectorNumElements() ==
3610 ValVT.getVectorNumElements() &&
3611 "Mask size must be the same as the vector size");
3612 for (auto [I, Op] : enumerate(First: Mask->ops())) {
3613 // Mask elements must be constants.
3614 if (Op.getNode()->getAsZExtVal() == 0) {
3615 // Append a sentinel register 0 to the Ops vector to represent a masked
3616 // off element, this will be handled in tablegen
3617 Ops.push_back(Elt: DAG.getRegister(Reg: MCRegister::NoRegister,
3618 VT: ValVT.getVectorElementType()));
3619 } else {
3620 // Extract the element from the vector to store
3621 SDValue ExtVal =
3622 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ValVT.getVectorElementType(),
3623 N1: Val, N2: DAG.getIntPtrConstant(Val: I, DL));
3624 Ops.push_back(Elt: ExtVal);
3625 }
3626 }
3627
3628 // Next, the pointer operand.
3629 Ops.push_back(Elt: BasePtr);
3630
3631 // Finally, the offset operand. We expect this to always be undef, and it will
3632 // be ignored in lowering, but to mirror the handling of the other vector
3633 // store instructions we include it in the new SDNode.
3634 assert(Offset.isUndef() && "Offset operand expected to be undef or poison");
3635 Ops.push_back(Elt: Offset);
3636
3637 SDValue NewSt =
3638 DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops,
3639 MemVT: MemSD->getMemoryVT(), MMO: MemSD->getMemOperand());
3640
3641 return NewSt;
3642}
3643
3644SDValue
3645NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
3646 switch (Op.getOpcode()) {
3647 case ISD::RETURNADDR:
3648 return SDValue();
3649 case ISD::FRAMEADDR:
3650 return SDValue();
3651 case ISD::ADDRSPACECAST:
3652 return LowerADDRSPACECAST(Op, DAG);
3653 case ISD::INTRINSIC_W_CHAIN:
3654 return lowerIntrinsicWChain(Op, DAG);
3655 case ISD::INTRINSIC_WO_CHAIN:
3656 return lowerIntrinsicWOChain(Op, DAG);
3657 case ISD::INTRINSIC_VOID:
3658 return lowerIntrinsicVoid(Op, DAG);
3659 case ISD::BUILD_VECTOR:
3660 return LowerBUILD_VECTOR(Op, DAG);
3661 case ISD::BITCAST:
3662 return LowerBITCAST(Op, DAG);
3663 case ISD::EXTRACT_SUBVECTOR:
3664 return Op;
3665 case ISD::EXTRACT_VECTOR_ELT:
3666 return LowerEXTRACT_VECTOR_ELT(Op, DAG);
3667 case ISD::INSERT_VECTOR_ELT:
3668 return LowerINSERT_VECTOR_ELT(Op, DAG);
3669 case ISD::VECTOR_SHUFFLE:
3670 return LowerVECTOR_SHUFFLE(Op, DAG);
3671 case ISD::CONCAT_VECTORS:
3672 return LowerCONCAT_VECTORS(Op, DAG);
3673 case ISD::VECREDUCE_FMAX:
3674 case ISD::VECREDUCE_FMIN:
3675 case ISD::VECREDUCE_FMAXIMUM:
3676 case ISD::VECREDUCE_FMINIMUM:
3677 return LowerVECREDUCE(Op, DAG);
3678 case ISD::STORE:
3679 return LowerSTORE(Op, DAG);
3680 case ISD::MSTORE: {
3681 assert(STI.has256BitVectorLoadStore(
3682 cast<MemSDNode>(Op.getNode())->getAddressSpace()) &&
3683 "Masked store vector not supported on subtarget.");
3684 return lowerMSTORE(Op, DAG);
3685 }
3686 case ISD::LOAD:
3687 return LowerLOAD(Op, DAG);
3688 case ISD::MLOAD:
3689 return LowerMLOAD(Op, DAG);
3690 case ISD::SHL_PARTS:
3691 return LowerShiftLeftParts(Op, DAG);
3692 case ISD::SRA_PARTS:
3693 case ISD::SRL_PARTS:
3694 return LowerShiftRightParts(Op, DAG);
3695 case ISD::SELECT:
3696 return lowerSELECT(Op, DAG);
3697 case ISD::SMUL_LOHI:
3698 case ISD::UMUL_LOHI:
3699 return lowerMUL_LOHI(Op, DAG);
3700 case ISD::FROUND:
3701 return LowerFROUND(Op, DAG);
3702 case ISD::FCOPYSIGN:
3703 return LowerFCOPYSIGN(Op, DAG);
3704 case ISD::SINT_TO_FP:
3705 case ISD::UINT_TO_FP:
3706 return LowerINT_TO_FP(Op, DAG);
3707 case ISD::FP_TO_SINT:
3708 case ISD::FP_TO_UINT:
3709 // fptosi/fptoui to i1 truncate toward zero, so the only defined results
3710 // are {0,-1} (signed) and {0,1} (unsigned); every other input results in
3711 // poison. Thus we can simply lower to `x <= -1.0` or `x >= 1.0`.
3712 if (Op.getValueType() == MVT::i1) {
3713 SDLoc DL(Op);
3714 SDValue X = Op.getOperand(i: 0);
3715 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT;
3716 return DAG.getSetCC(
3717 DL, VT: MVT::i1, LHS: X,
3718 RHS: DAG.getConstantFP(Val: IsSigned ? -1.0 : 1.0, DL, VT: X.getValueType()),
3719 Cond: IsSigned ? ISD::SETOLE : ISD::SETOGE);
3720 }
3721 return LowerFP_TO_INT(Op, DAG);
3722 case ISD::FP_ROUND:
3723 return LowerFP_ROUND(Op, DAG);
3724 case ISD::FPTRUNC_ROUND:
3725 return lowerFPTRUNC_ROUND(Op, DAG, STI);
3726 case ISD::FP_EXTEND:
3727 return LowerFP_EXTEND(Op, DAG);
3728 case ISD::VAARG:
3729 return LowerVAARG(Op, DAG);
3730 case ISD::VASTART:
3731 return LowerVASTART(Op, DAG);
3732 case ISD::FSHL:
3733 case ISD::FSHR:
3734 return lowerFSH(Op, DAG);
3735 case ISD::ROTL:
3736 case ISD::ROTR:
3737 return lowerROT(Op, DAG);
3738 case ISD::ABS:
3739 case ISD::ABS_MIN_POISON:
3740 case ISD::SMIN:
3741 case ISD::SMAX:
3742 case ISD::UMIN:
3743 case ISD::UMAX:
3744 case ISD::ADD:
3745 case ISD::SUB:
3746 case ISD::MUL:
3747 case ISD::SHL:
3748 case ISD::SREM:
3749 case ISD::UREM:
3750 return LowerVectorArith(Op, DAG);
3751 case ISD::DYNAMIC_STACKALLOC:
3752 return LowerDYNAMIC_STACKALLOC(Op, DAG);
3753 case ISD::STACKRESTORE:
3754 return LowerSTACKRESTORE(Op, DAG);
3755 case ISD::STACKSAVE:
3756 return LowerSTACKSAVE(Op, DAG);
3757 case ISD::CopyToReg:
3758 return LowerCopyToReg_128(Op, DAG);
3759 case ISD::FADD:
3760 case ISD::FSUB:
3761 case ISD::FMUL:
3762 // Used only for bf16 on SM80, where we select fma for non-ftz operation
3763 return PromoteBinOpIfF32FTZ(Op, DAG);
3764 case ISD::CTPOP:
3765 case ISD::CTLZ:
3766 return lowerCTLZCTPOP(Op, DAG);
3767 case ISD::BSWAP:
3768 return lowerBSWAP(Op, DAG);
3769 default:
3770 llvm_unreachable("Custom lowering not defined for operation");
3771 }
3772}
3773
3774// This will prevent AsmPrinter from trying to print the jump tables itself.
3775unsigned NVPTXTargetLowering::getJumpTableEncoding() const {
3776 return MachineJumpTableInfo::EK_Inline;
3777}
3778
3779SDValue NVPTXTargetLowering::LowerADDRSPACECAST(SDValue Op,
3780 SelectionDAG &DAG) const {
3781 AddrSpaceCastSDNode *N = cast<AddrSpaceCastSDNode>(Val: Op.getNode());
3782 unsigned SrcAS = N->getSrcAddressSpace();
3783 unsigned DestAS = N->getDestAddressSpace();
3784 if (SrcAS != llvm::ADDRESS_SPACE_GENERIC &&
3785 DestAS != llvm::ADDRESS_SPACE_GENERIC) {
3786 // Shared and SharedCluster can be converted to each other through generic
3787 // space
3788 if ((SrcAS == llvm::ADDRESS_SPACE_SHARED &&
3789 DestAS == llvm::ADDRESS_SPACE_SHARED_CLUSTER) ||
3790 (SrcAS == llvm::ADDRESS_SPACE_SHARED_CLUSTER &&
3791 DestAS == llvm::ADDRESS_SPACE_SHARED)) {
3792 SDLoc DL(Op.getNode());
3793 const MVT GenerictVT =
3794 getPointerTy(DL: DAG.getDataLayout(), AS: ADDRESS_SPACE_GENERIC);
3795 SDValue GenericConversion = DAG.getAddrSpaceCast(
3796 dl: DL, VT: GenerictVT, Ptr: Op.getOperand(i: 0), SrcAS, DestAS: ADDRESS_SPACE_GENERIC);
3797 SDValue SharedClusterConversion =
3798 DAG.getAddrSpaceCast(dl: DL, VT: Op.getValueType(), Ptr: GenericConversion,
3799 SrcAS: ADDRESS_SPACE_GENERIC, DestAS);
3800 return SharedClusterConversion;
3801 }
3802
3803 return DAG.getUNDEF(VT: Op.getValueType());
3804 }
3805
3806 return Op;
3807}
3808
3809// This function is almost a copy of SelectionDAG::expandVAArg().
3810// The only diff is that this one produces loads from local address space.
3811SDValue NVPTXTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
3812 const TargetLowering *TLI = STI.getTargetLowering();
3813 SDLoc DL(Op);
3814
3815 SDNode *Node = Op.getNode();
3816 const Value *V = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 2))->getValue();
3817 EVT VT = Node->getValueType(ResNo: 0);
3818 auto *Ty = VT.getTypeForEVT(Context&: *DAG.getContext());
3819 SDValue Tmp1 = Node->getOperand(Num: 0);
3820 SDValue Tmp2 = Node->getOperand(Num: 1);
3821 const MaybeAlign MA(Node->getConstantOperandVal(Num: 3));
3822
3823 SDValue VAListLoad = DAG.getLoad(VT: TLI->getPointerTy(DL: DAG.getDataLayout()), dl: DL,
3824 Chain: Tmp1, Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
3825 SDValue VAList = VAListLoad;
3826
3827 if (MA && *MA > TLI->getMinStackArgumentAlignment()) {
3828 VAList = DAG.getNode(
3829 Opcode: ISD::ADD, DL, VT: VAList.getValueType(), N1: VAList,
3830 N2: DAG.getConstant(Val: MA->value() - 1, DL, VT: VAList.getValueType()));
3831
3832 VAList = DAG.getNode(Opcode: ISD::AND, DL, VT: VAList.getValueType(), N1: VAList,
3833 N2: DAG.getSignedConstant(Val: -(int64_t)MA->value(), DL,
3834 VT: VAList.getValueType()));
3835 }
3836
3837 // Increment the pointer, VAList, to the next vaarg
3838 Tmp1 = DAG.getNode(Opcode: ISD::ADD, DL, VT: VAList.getValueType(), N1: VAList,
3839 N2: DAG.getConstant(Val: DAG.getDataLayout().getTypeAllocSize(Ty),
3840 DL, VT: VAList.getValueType()));
3841
3842 // Store the incremented VAList to the legalized pointer
3843 Tmp1 = DAG.getStore(Chain: VAListLoad.getValue(R: 1), dl: DL, Val: Tmp1, Ptr: Tmp2,
3844 PtrInfo: MachinePointerInfo(V));
3845
3846 const Value *SrcV = Constant::getNullValue(
3847 Ty: PointerType::get(C&: *DAG.getContext(), AddressSpace: ADDRESS_SPACE_LOCAL));
3848
3849 // Load the actual argument out of the pointer VAList
3850 return DAG.getLoad(VT, dl: DL, Chain: Tmp1, Ptr: VAList, PtrInfo: MachinePointerInfo(SrcV));
3851}
3852
3853SDValue NVPTXTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
3854 const TargetLowering *TLI = STI.getTargetLowering();
3855 SDLoc DL(Op);
3856 EVT PtrVT = TLI->getPointerTy(DL: DAG.getDataLayout());
3857
3858 // Store the address of unsized array <function>_vararg[] in the ap object.
3859 SDValue VAReg = getParamSymbolNode(DAG, /* vararg */ I: -1, T: PtrVT);
3860
3861 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
3862 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL, Val: VAReg, Ptr: Op.getOperand(i: 1),
3863 PtrInfo: MachinePointerInfo(SV));
3864}
3865
3866static std::pair<MemSDNode *, uint32_t>
3867convertMLOADToLoadWithUsedBytesMask(MemSDNode *N, SelectionDAG &DAG,
3868 const NVPTXSubtarget &STI) {
3869 SDValue Chain = N->getOperand(Num: 0);
3870 SDValue BasePtr = N->getOperand(Num: 1);
3871 SDValue Mask = N->getOperand(Num: 3);
3872 [[maybe_unused]] SDValue Passthru = N->getOperand(Num: 4);
3873
3874 SDLoc DL(N);
3875 EVT ResVT = N->getValueType(ResNo: 0);
3876 assert(ResVT.isVector() && "Masked vector load must have vector type");
3877 // While we only expect poison passthru vectors as an input to the backend,
3878 // when the legalization framework splits a poison vector in half, it creates
3879 // two undef vectors, so we can technically expect those too.
3880 assert((Passthru.getOpcode() == ISD::POISON ||
3881 Passthru.getOpcode() == ISD::UNDEF) &&
3882 "Passthru operand expected to be poison or undef");
3883
3884 // Extract the mask and convert it to a uint32_t representing the used bytes
3885 // of the entire vector load
3886 uint32_t UsedBytesMask = 0;
3887 uint32_t ElementSizeInBits = ResVT.getVectorElementType().getSizeInBits();
3888 assert(ElementSizeInBits % 8 == 0 && "Unexpected element size");
3889 uint32_t ElementSizeInBytes = ElementSizeInBits / 8;
3890 uint32_t ElementMask = (1u << ElementSizeInBytes) - 1u;
3891
3892 for (SDValue Op : reverse(C: Mask->ops())) {
3893 // We technically only want to do this shift for every
3894 // iteration *but* the first, but in the first iteration UsedBytesMask is 0,
3895 // so this shift is a no-op.
3896 UsedBytesMask <<= ElementSizeInBytes;
3897
3898 // Mask elements must be constants.
3899 if (Op->getAsZExtVal() != 0)
3900 UsedBytesMask |= ElementMask;
3901 }
3902
3903 assert(UsedBytesMask != 0 && UsedBytesMask != UINT32_MAX &&
3904 "Unexpected masked load with elements masked all on or all off");
3905
3906 // Create a new load sd node to be handled normally by ReplaceLoadVector.
3907 MemSDNode *NewLD = cast<MemSDNode>(
3908 Val: DAG.getLoad(VT: ResVT, dl: DL, Chain, Ptr: BasePtr, MMO: N->getMemOperand()).getNode());
3909
3910 // If our subtarget does not support the used bytes mask pragma, "drop" the
3911 // mask by setting it to UINT32_MAX
3912 if (!STI.hasUsedBytesMaskPragma())
3913 UsedBytesMask = UINT32_MAX;
3914
3915 return {NewLD, UsedBytesMask};
3916}
3917
3918/// replaceLoadVector - Convert vector loads into multi-output scalar loads.
3919static std::optional<std::pair<SDValue, SDValue>>
3920replaceLoadVector(SDNode *N, SelectionDAG &DAG, const NVPTXSubtarget &STI) {
3921 MemSDNode *LD = cast<MemSDNode>(Val: N);
3922 const EVT ResVT = LD->getValueType(ResNo: 0);
3923 const EVT MemVT = LD->getMemoryVT();
3924
3925 // If we're doing sign/zero extension as part of the load, avoid lowering to
3926 // a LoadV node. TODO: consider relaxing this restriction.
3927 if (ResVT != MemVT)
3928 return std::nullopt;
3929
3930 const auto NumEltsAndEltVT =
3931 getVectorLoweringShape(VectorEVT: ResVT, STI, AddressSpace: LD->getAddressSpace());
3932 if (!NumEltsAndEltVT)
3933 return std::nullopt;
3934 const auto [NumElts, EltVT] = NumEltsAndEltVT.value();
3935
3936 Align Alignment = LD->getAlign();
3937 const auto &TD = DAG.getDataLayout();
3938 Align PrefAlign = TD.getPrefTypeAlign(Ty: MemVT.getTypeForEVT(Context&: *DAG.getContext()));
3939 if (Alignment < PrefAlign) {
3940 // This load is not sufficiently aligned, so bail out and let this vector
3941 // load be scalarized. Note that we may still be able to emit smaller
3942 // vector loads. For example, if we are loading a <4 x float> with an
3943 // alignment of 8, this check will fail but the legalizer will try again
3944 // with 2 x <2 x float>, which will succeed with an alignment of 8.
3945 return std::nullopt;
3946 }
3947
3948 // If we have a masked load, convert it to a normal load now
3949 std::optional<uint32_t> UsedBytesMask = std::nullopt;
3950 if (LD->getOpcode() == ISD::MLOAD)
3951 std::tie(args&: LD, args&: UsedBytesMask) =
3952 convertMLOADToLoadWithUsedBytesMask(N: LD, DAG, STI);
3953
3954 // Since LoadV2 is a target node, we cannot rely on DAG type legalization.
3955 // Therefore, we must ensure the type is legal. For i1 and i8, we set the
3956 // loaded type to i16 and propagate the "real" type as the memory type.
3957 const MVT LoadEltVT = (EltVT.getSizeInBits() < 16) ? MVT::i16 : EltVT;
3958
3959 unsigned Opcode;
3960 switch (NumElts) {
3961 default:
3962 return std::nullopt;
3963 case 2:
3964 Opcode = NVPTXISD::LoadV2;
3965 break;
3966 case 4:
3967 Opcode = NVPTXISD::LoadV4;
3968 break;
3969 case 8:
3970 Opcode = NVPTXISD::LoadV8;
3971 break;
3972 }
3973 auto ListVTs = SmallVector<EVT, 9>(NumElts, LoadEltVT);
3974 ListVTs.push_back(Elt: MVT::Other);
3975 SDVTList LdResVTs = DAG.getVTList(VTs: ListVTs);
3976
3977 SDLoc DL(LD);
3978
3979 // Copy regular operands
3980 SmallVector<SDValue, 8> OtherOps(LD->ops());
3981
3982 OtherOps.push_back(
3983 Elt: DAG.getConstant(Val: UsedBytesMask.value_or(UINT32_MAX), DL, VT: MVT::i32));
3984
3985 // The select routine does not have access to the LoadSDNode instance, so
3986 // pass along the extension information
3987 OtherOps.push_back(
3988 Elt: DAG.getIntPtrConstant(Val: cast<LoadSDNode>(Val: LD)->getExtensionType(), DL));
3989
3990 SDValue NewLD = DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList: LdResVTs, Ops: OtherOps, MemVT,
3991 MMO: LD->getMemOperand());
3992
3993 SmallVector<SDValue> ScalarRes;
3994 if (EltVT.isVector()) {
3995 assert(EVT(EltVT.getVectorElementType()) == ResVT.getVectorElementType());
3996 assert(NumElts * EltVT.getVectorNumElements() ==
3997 ResVT.getVectorNumElements());
3998 // Generate EXTRACT_VECTOR_ELTs to split v2[i,f,bf]16/v4i8 subvectors back
3999 // into individual elements.
4000 for (const unsigned I : llvm::seq(Size: NumElts)) {
4001 SDValue SubVector = NewLD.getValue(R: I);
4002 DAG.ExtractVectorElements(Op: SubVector, Args&: ScalarRes);
4003 }
4004 } else {
4005 for (const unsigned I : llvm::seq(Size: NumElts)) {
4006 SDValue Res = NewLD.getValue(R: I);
4007 if (LoadEltVT != EltVT)
4008 Res = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Res);
4009 ScalarRes.push_back(Elt: Res);
4010 }
4011 }
4012
4013 SDValue LoadChain = NewLD.getValue(R: NumElts);
4014
4015 const MVT BuildVecVT =
4016 MVT::getVectorVT(VT: EltVT.getScalarType(), NumElements: ScalarRes.size());
4017 SDValue BuildVec = DAG.getBuildVector(VT: BuildVecVT, DL, Ops: ScalarRes);
4018 SDValue LoadValue = DAG.getBitcast(VT: ResVT, V: BuildVec);
4019
4020 return {{LoadValue, LoadChain}};
4021}
4022
4023static void replaceLoadVector(SDNode *N, SelectionDAG &DAG,
4024 SmallVectorImpl<SDValue> &Results,
4025 const NVPTXSubtarget &STI) {
4026 if (auto Res = replaceLoadVector(N, DAG, STI))
4027 Results.append(IL: {Res->first, Res->second});
4028}
4029
4030static SDValue lowerLoadVector(SDNode *N, SelectionDAG &DAG,
4031 const NVPTXSubtarget &STI) {
4032 if (auto Res = replaceLoadVector(N, DAG, STI))
4033 return DAG.getMergeValues(Ops: {Res->first, Res->second}, dl: SDLoc(N));
4034 return SDValue();
4035}
4036
4037// v = ld i1* addr
4038// =>
4039// v1 = ld i8* addr (-> i16)
4040// v = trunc i16 to i1
4041static SDValue lowerLOADi1(LoadSDNode *LD, SelectionDAG &DAG) {
4042 SDLoc dl(LD);
4043 assert(LD->getExtensionType() == ISD::NON_EXTLOAD);
4044 assert(LD->getValueType(0) == MVT::i1 && "Custom lowering for i1 load only");
4045 SDValue newLD = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: MVT::i16, Chain: LD->getChain(),
4046 Ptr: LD->getBasePtr(), PtrInfo: LD->getPointerInfo(),
4047 MemVT: MVT::i8, Alignment: LD->getAlign(),
4048 MMOFlags: LD->getMemOperand()->getFlags());
4049 SDValue result = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i1, Operand: newLD);
4050 // The legalizer (the caller) is expecting two values from the legalized
4051 // load, so we build a MergeValues node for it. See ExpandUnalignedLoad()
4052 // in LegalizeDAG.cpp which also uses MergeValues.
4053 return DAG.getMergeValues(Ops: {result, newLD.getValue(R: 1)}, dl);
4054}
4055
4056SDValue NVPTXTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
4057 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
4058
4059 if (Op.getValueType() == MVT::i1)
4060 return lowerLOADi1(LD, DAG);
4061
4062 // To improve CodeGen we'll legalize any-extend loads to zext loads. This is
4063 // how they'll be lowered in ISel anyway, and by doing this a little earlier
4064 // we allow for more DAG combine opportunities.
4065 if (LD->getExtensionType() == ISD::EXTLOAD) {
4066 assert(LD->getValueType(0).isInteger() && LD->getMemoryVT().isInteger() &&
4067 "Unexpected fpext-load");
4068 return DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl: SDLoc(Op), VT: Op.getValueType(),
4069 Chain: LD->getChain(), Ptr: LD->getBasePtr(), MemVT: LD->getMemoryVT(),
4070 MMO: LD->getMemOperand());
4071 }
4072
4073 llvm_unreachable("Unexpected custom lowering for load");
4074}
4075
4076SDValue NVPTXTargetLowering::LowerMLOAD(SDValue Op, SelectionDAG &DAG) const {
4077 // v2f16/v2bf16/v2i16/v4i8 are legal, so we can't rely on legalizer to handle
4078 // masked loads of these types and have to handle them here.
4079 // v2f32 also needs to be handled here if the subtarget has f32x2
4080 // instructions, making it legal.
4081 //
4082 // Note: misaligned masked loads should never reach this point
4083 // because the override of isLegalMaskedLoad in NVPTXTargetTransformInfo.cpp
4084 // will validate alignment. Therefore, we do not need to special case handle
4085 // them here.
4086 EVT VT = Op.getValueType();
4087 if (NVPTX::isPackedVectorTy(VT)) {
4088 auto Result = convertMLOADToLoadWithUsedBytesMask(
4089 N: cast<MemSDNode>(Val: Op.getNode()), DAG, STI);
4090 MemSDNode *LD = std::get<0>(in&: Result);
4091 uint32_t UsedBytesMask = std::get<1>(in&: Result);
4092
4093 SDLoc DL(LD);
4094
4095 // Copy regular operands
4096 SmallVector<SDValue, 8> OtherOps(LD->ops());
4097
4098 OtherOps.push_back(Elt: DAG.getConstant(Val: UsedBytesMask, DL, VT: MVT::i32));
4099
4100 // We currently are not lowering extending loads, but pass the extension
4101 // type anyway as later handling expects it.
4102 OtherOps.push_back(
4103 Elt: DAG.getIntPtrConstant(Val: cast<LoadSDNode>(Val: LD)->getExtensionType(), DL));
4104 SDValue NewLD =
4105 DAG.getMemIntrinsicNode(Opcode: NVPTXISD::MLoad, dl: DL, VTList: LD->getVTList(), Ops: OtherOps,
4106 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
4107 return NewLD;
4108 }
4109 return SDValue();
4110}
4111
4112static SDValue lowerSTOREVector(SDValue Op, SelectionDAG &DAG,
4113 const NVPTXSubtarget &STI) {
4114 MemSDNode *N = cast<MemSDNode>(Val: Op.getNode());
4115 SDValue Val = N->getOperand(Num: 1);
4116 SDLoc DL(N);
4117 const EVT ValVT = Val.getValueType();
4118 const EVT MemVT = N->getMemoryVT();
4119
4120 // If we're truncating as part of the store, avoid lowering to a StoreV node.
4121 // TODO: consider relaxing this restriction.
4122 if (ValVT != MemVT)
4123 return SDValue();
4124
4125 const auto NumEltsAndEltVT =
4126 getVectorLoweringShape(VectorEVT: ValVT, STI, AddressSpace: N->getAddressSpace());
4127 if (!NumEltsAndEltVT)
4128 return SDValue();
4129 const auto [NumElts, EltVT] = NumEltsAndEltVT.value();
4130
4131 const DataLayout &TD = DAG.getDataLayout();
4132
4133 Align Alignment = N->getAlign();
4134 Align PrefAlign = TD.getPrefTypeAlign(Ty: ValVT.getTypeForEVT(Context&: *DAG.getContext()));
4135 if (Alignment < PrefAlign) {
4136 // This store is not sufficiently aligned, so bail out and let this vector
4137 // store be scalarized. Note that we may still be able to emit smaller
4138 // vector stores. For example, if we are storing a <4 x float> with an
4139 // alignment of 8, this check will fail but the legalizer will try again
4140 // with 2 x <2 x float>, which will succeed with an alignment of 8.
4141 return SDValue();
4142 }
4143
4144 unsigned Opcode;
4145 switch (NumElts) {
4146 default:
4147 return SDValue();
4148 case 2:
4149 Opcode = NVPTXISD::StoreV2;
4150 break;
4151 case 4:
4152 Opcode = NVPTXISD::StoreV4;
4153 break;
4154 case 8:
4155 Opcode = NVPTXISD::StoreV8;
4156 break;
4157 }
4158
4159 SmallVector<SDValue, 8> Ops;
4160
4161 // First is the chain
4162 Ops.push_back(Elt: N->getOperand(Num: 0));
4163
4164 // Then the split values
4165 if (EltVT.isVector()) {
4166 assert(EVT(EltVT.getVectorElementType()) == ValVT.getVectorElementType());
4167 assert(NumElts * EltVT.getVectorNumElements() ==
4168 ValVT.getVectorNumElements());
4169 // Combine individual elements into v2[i,f,bf]16/v4i8 subvectors to be
4170 // stored as b32s
4171 const unsigned NumEltsPerSubVector = EltVT.getVectorNumElements();
4172 for (const unsigned I : llvm::seq(Size: NumElts)) {
4173 SmallVector<SDValue, 4> SubVectorElts;
4174 DAG.ExtractVectorElements(Op: Val, Args&: SubVectorElts, Start: I * NumEltsPerSubVector,
4175 Count: NumEltsPerSubVector);
4176 Ops.push_back(Elt: DAG.getBuildVector(VT: EltVT, DL, Ops: SubVectorElts));
4177 }
4178 } else {
4179 SDValue V = DAG.getBitcast(VT: MVT::getVectorVT(VT: EltVT, NumElements: NumElts), V: Val);
4180 for (const unsigned I : llvm::seq(Size: NumElts)) {
4181 SDValue ExtVal = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: V,
4182 N2: DAG.getIntPtrConstant(Val: I, DL));
4183
4184 // Since StoreV2 is a target node, we cannot rely on DAG type
4185 // legalization. Therefore, we must ensure the type is legal. For i1 and
4186 // i8, we set the stored type to i16 and propagate the "real" type as the
4187 // memory type.
4188 if (EltVT.getSizeInBits() < 16)
4189 ExtVal = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i16, Operand: ExtVal);
4190 Ops.push_back(Elt: ExtVal);
4191 }
4192 }
4193
4194 // Then any remaining arguments
4195 Ops.append(in_start: N->op_begin() + 2, in_end: N->op_end());
4196
4197 SDValue NewSt =
4198 DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops,
4199 MemVT: N->getMemoryVT(), MMO: N->getMemOperand());
4200
4201 // return DCI.CombineTo(N, NewSt, true);
4202 return NewSt;
4203}
4204
4205SDValue NVPTXTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
4206 StoreSDNode *Store = cast<StoreSDNode>(Val&: Op);
4207 EVT VT = Store->getMemoryVT();
4208
4209 // Leave truncating stores to i1 to the generic legalizer.
4210 if (VT == MVT::i1 && !Store->isTruncatingStore())
4211 return LowerSTOREi1(Op, DAG);
4212
4213 // Lower store of any other vector type, including v2f32 as we want to break
4214 // it apart since this is not a widely-supported type.
4215 return lowerSTOREVector(Op, DAG, STI);
4216}
4217
4218// st i1 v, addr
4219// =>
4220// v1 = zxt v to i16
4221// st.u8 i16, addr
4222SDValue NVPTXTargetLowering::LowerSTOREi1(SDValue Op, SelectionDAG &DAG) const {
4223 SDNode *Node = Op.getNode();
4224 SDLoc dl(Node);
4225 StoreSDNode *ST = cast<StoreSDNode>(Val: Node);
4226 SDValue Tmp1 = ST->getChain();
4227 SDValue Tmp2 = ST->getBasePtr();
4228 SDValue Tmp3 = ST->getValue();
4229 assert(Tmp3.getValueType() == MVT::i1 && "Custom lowering for i1 store only");
4230 Tmp3 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::i16, Operand: Tmp3);
4231 SDValue Result =
4232 DAG.getTruncStore(Chain: Tmp1, dl, Val: Tmp3, Ptr: Tmp2, PtrInfo: ST->getPointerInfo(), SVT: MVT::i8,
4233 Alignment: ST->getAlign(), MMOFlags: ST->getMemOperand()->getFlags());
4234 return Result;
4235}
4236
4237SDValue NVPTXTargetLowering::LowerCopyToReg_128(SDValue Op,
4238 SelectionDAG &DAG) const {
4239 // Change the CopyToReg to take in two 64-bit operands instead of a 128-bit
4240 // operand so that it can pass the legalization.
4241
4242 assert(Op.getOperand(1).getValueType() == MVT::i128 &&
4243 "Custom lowering for 128-bit CopyToReg only");
4244
4245 SDNode *Node = Op.getNode();
4246 SDLoc DL(Node);
4247
4248 SDValue Cast = DAG.getBitcast(VT: MVT::v2i64, V: Op->getOperand(Num: 2));
4249 SDValue Lo = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i64, N1: Cast,
4250 N2: DAG.getIntPtrConstant(Val: 0, DL));
4251 SDValue Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i64, N1: Cast,
4252 N2: DAG.getIntPtrConstant(Val: 1, DL));
4253
4254 SmallVector<SDValue, 5> NewOps(Op->getNumOperands() + 1);
4255 SmallVector<EVT, 3> ResultsType(Node->values());
4256
4257 NewOps[0] = Op->getOperand(Num: 0); // Chain
4258 NewOps[1] = Op->getOperand(Num: 1); // Dst Reg
4259 NewOps[2] = Lo; // Lower 64-bit
4260 NewOps[3] = Hi; // Higher 64-bit
4261 if (Op.getNumOperands() == 4)
4262 NewOps[4] = Op->getOperand(Num: 3); // Glue if exists
4263
4264 return DAG.getNode(Opcode: ISD::CopyToReg, DL, ResultTys: ResultsType, Ops: NewOps);
4265}
4266
4267unsigned NVPTXTargetLowering::getNumRegisters(
4268 LLVMContext &Context, EVT VT,
4269 std::optional<MVT> RegisterVT = std::nullopt) const {
4270 if (VT == MVT::i128 && RegisterVT == MVT::i128)
4271 return 1;
4272 return TargetLoweringBase::getNumRegisters(Context, VT, RegisterVT);
4273}
4274
4275bool NVPTXTargetLowering::splitValueIntoRegisterParts(
4276 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
4277 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
4278 if (Val.getValueType() == MVT::i128 && NumParts == 1) {
4279 Parts[0] = Val;
4280 return true;
4281 }
4282 return false;
4283}
4284
4285SDValue NVPTXTargetLowering::getParamSymbolNode(SelectionDAG &DAG, int I,
4286 EVT T) const {
4287 const MachineFunction &MF = DAG.getMachineFunction();
4288 return getSymbolNode(
4289 DAG, Sym: getParamSymbol(Ctx&: MF.getContext(), F: &MF.getFunction(), Idx: I), T);
4290}
4291
4292SDValue NVPTXTargetLowering::getCallParamSymbolNode(SelectionDAG &DAG, int I,
4293 EVT T) const {
4294 return getSymbolNode(DAG, Name: "param" + Twine(I), T);
4295}
4296
4297SDValue NVPTXTargetLowering::LowerFormalArguments(
4298 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4299 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
4300 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4301 const DataLayout &DL = DAG.getDataLayout();
4302 LLVMContext &Ctx = *DAG.getContext();
4303
4304 const Function &F = DAG.getMachineFunction().getFunction();
4305 const bool IsKernel = isKernelFunction(F);
4306
4307 const MVT PtrVT = getPointerTy(DL, AS: IsKernel ? ADDRESS_SPACE_ENTRY_PARAM
4308 : ADDRESS_SPACE_LOCAL);
4309
4310 SDValue Root = DAG.getRoot();
4311 SmallVector<SDValue, 16> OutChains;
4312
4313 // argTypes.size() (or theArgs.size()) and Ins.size() need not match.
4314 // Ins.size() will be larger
4315 // * if there is an aggregate argument with multiple fields (each field
4316 // showing up separately in Ins)
4317 // * if there is a vector argument with more than typical vector-length
4318 // elements (generally if more than 4) where each vector element is
4319 // individually present in Ins.
4320 // So a different index should be used for indexing into Ins.
4321 // See similar issue in LowerCall.
4322
4323 auto AllIns = ArrayRef(Ins);
4324 const auto NonEmptyArgs = make_filter_range(
4325 Range: F.args(), Pred: [](const Argument &A) { return !A.getType()->isEmptyTy(); });
4326 for (const auto &[ParamI, Arg] : enumerate(First: NonEmptyArgs)) {
4327 const unsigned ArgNo = Arg.getArgNo();
4328 const auto ArgIns =
4329 AllIns.take_while(Pred: [&](auto I) { return I.OrigArgIndex == ArgNo; });
4330 AllIns = AllIns.drop_front(N: ArgIns.size());
4331
4332 Type *Ty = Arg.getType();
4333 assert(!ArgIns.empty() &&
4334 "Non-empty argument produced no parameter values");
4335
4336 if (Arg.use_empty()) {
4337 // argument is dead
4338 for (const auto &In : ArgIns) {
4339 assert(!In.Used && "Arg.use_empty() is true but Arg is used?");
4340 InVals.push_back(Elt: DAG.getUNDEF(VT: In.VT));
4341 }
4342 continue;
4343 }
4344
4345 SDValue ArgSymbol = getParamSymbolNode(DAG, I: ParamI, T: PtrVT);
4346
4347 // In the following cases, assign a node order of "i+1"
4348 // to newly created nodes. The SDNodes for params have to
4349 // appear in the same order as their order of appearance
4350 // in the original function. "i+1" holds that order.
4351 if (Arg.hasByValAttr()) {
4352 // Param has ByVal attribute
4353 // Return MoveParam(param symbol).
4354 // Ideally, the param symbol can be returned directly,
4355 // but when SDNode builder decides to use it in a CopyToReg(),
4356 // machine instruction fails because TargetExternalSymbol
4357 // (not lowered) is target dependent, and CopyToReg assumes
4358 // the source is lowered.
4359 assert(ArgIns.size() == 1 && "ByVal argument must be a pointer");
4360 const auto &ByvalIn = ArgIns[0];
4361 assert(getValueType(DL, Ty) == ByvalIn.VT &&
4362 "Ins type did not match function type");
4363
4364 SDValue P;
4365 if (IsKernel) {
4366 assert(Ty->getPointerAddressSpace() == ADDRESS_SPACE_ENTRY_PARAM &&
4367 "Kernel ByVal argument must be lowered to the param address "
4368 "space by NVPTXLowerArgs");
4369 P = ArgSymbol;
4370 P.getNode()->setIROrder(Arg.getArgNo() + 1);
4371 } else {
4372 P = DAG.getNode(Opcode: NVPTXISD::MoveParam, DL: dl, VT: ArgSymbol.getValueType(),
4373 Operand: ArgSymbol);
4374 P.getNode()->setIROrder(Arg.getArgNo() + 1);
4375 P = DAG.getAddrSpaceCast(dl, VT: ByvalIn.VT, Ptr: P, SrcAS: ADDRESS_SPACE_LOCAL,
4376 DestAS: ADDRESS_SPACE_GENERIC);
4377 }
4378 InVals.push_back(Elt: P);
4379 } else {
4380 SmallVector<EVT, 16> VTs;
4381 SmallVector<uint64_t, 16> Offsets;
4382 ComputePTXValueVTs(TLI: *this, DL, Ctx, CallConv, Ty, ValueVTs&: VTs, Offsets);
4383 assert(VTs.size() == ArgIns.size() && "Size mismatch");
4384 assert(VTs.size() == Offsets.size() && "Size mismatch");
4385
4386 const Align ArgAlign = getPTXParamAlign(
4387 F: &F, Ty, AttrIdx: Arg.getArgNo() + AttributeList::FirstArgIndex, DL);
4388
4389 unsigned I = 0;
4390 const auto VI = VectorizePTXValueVTs(ValueVTs: VTs, Offsets, ParamAlignment: ArgAlign);
4391 for (const unsigned NumElts : VI) {
4392 // i1 is loaded/stored as i8
4393 const EVT LoadVT = VTs[I] == MVT::i1 ? MVT::i8 : VTs[I];
4394 const EVT VecVT = getVectorizedVT(VT: LoadVT, N: NumElts, C&: Ctx);
4395
4396 SDValue VecAddr = DAG.getObjectPtrOffset(
4397 SL: dl, Ptr: ArgSymbol, Offset: TypeSize::getFixed(ExactSize: Offsets[I]));
4398
4399 const Align PartAlign = commonAlignment(A: ArgAlign, Offset: Offsets[I]);
4400 const unsigned AS = IsKernel ? NVPTX::AddressSpace::EntryParam
4401 : NVPTX::AddressSpace::DeviceParam;
4402 SDValue P = DAG.getLoad(VT: VecVT, dl, Chain: Root, Ptr: VecAddr,
4403 PtrInfo: MachinePointerInfo(AS), Alignment: PartAlign,
4404 MMOFlags: MachineMemOperand::MODereferenceable |
4405 MachineMemOperand::MOInvariant);
4406 P.getNode()->setIROrder(Arg.getArgNo() + 1);
4407 for (const unsigned J : llvm::seq(Size: NumElts)) {
4408 SDValue Elt = getExtractVectorizedValue(V: P, I: J, VT: LoadVT, dl, DAG);
4409
4410 Elt = correctParamType(V: Elt, ExpectedVT: ArgIns[I + J].VT, Flags: ArgIns[I + J].Flags,
4411 DAG, dl);
4412 InVals.push_back(Elt);
4413 }
4414 I += NumElts;
4415 }
4416 }
4417 }
4418
4419 if (!OutChains.empty())
4420 DAG.setRoot(DAG.getTokenFactor(DL: dl, Vals&: OutChains));
4421
4422 return Chain;
4423}
4424
4425SDValue
4426NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
4427 bool isVarArg,
4428 const SmallVectorImpl<ISD::OutputArg> &Outs,
4429 const SmallVectorImpl<SDValue> &OutVals,
4430 const SDLoc &dl, SelectionDAG &DAG) const {
4431 const Function &F = DAG.getMachineFunction().getFunction();
4432 Type *RetTy = F.getReturnType();
4433
4434 if (RetTy->isVoidTy()) {
4435 assert(OutVals.empty() && Outs.empty() && "Return value expected for void");
4436 return DAG.getNode(Opcode: NVPTXISD::RET_GLUE, DL: dl, VT: MVT::Other, Operand: Chain);
4437 }
4438
4439 const DataLayout &DL = DAG.getDataLayout();
4440 LLVMContext &Ctx = *DAG.getContext();
4441
4442 const SDValue RetSymbol = getSymbolNode(DAG, Name: "func_retval0", T: MVT::i32);
4443 const auto RetAlign =
4444 getPTXParamAlign(F: &F, Ty: RetTy, AttrIdx: AttributeList::ReturnIndex, DL);
4445
4446 // PTX Interoperability Guide 3.3(A): [Integer] Values shorter than
4447 // 32-bits are sign extended or zero extended, depending on whether
4448 // they are signed or unsigned types.
4449 const bool ExtendIntegerRetVal =
4450 RetTy->isIntegerTy() && DL.getTypeAllocSizeInBits(Ty: RetTy) < 32;
4451
4452 SmallVector<EVT, 16> VTs;
4453 SmallVector<uint64_t, 16> Offsets;
4454 ComputePTXValueVTs(TLI: *this, DL, Ctx, CallConv, Ty: RetTy, ValueVTs&: VTs, Offsets);
4455 assert(VTs.size() == OutVals.size() && "Bad return value decomposition");
4456
4457 const auto GetRetVal = [&](unsigned I) -> SDValue {
4458 SDValue RetVal = OutVals[I];
4459 assert(promoteScalarIntegerPTX(RetVal.getValueType()) ==
4460 RetVal.getValueType() &&
4461 "OutVal type should always be legal");
4462
4463 const EVT VTI = promoteScalarIntegerPTX(VT: VTs[I]);
4464 const EVT StoreVT =
4465 ExtendIntegerRetVal ? MVT::i32 : (VTI == MVT::i1 ? MVT::i8 : VTI);
4466 return correctParamType(V: RetVal, ExpectedVT: StoreVT, Flags: Outs[I].Flags, DAG, dl);
4467 };
4468
4469 unsigned I = 0;
4470 const auto VI = VectorizePTXValueVTs(ValueVTs: VTs, Offsets, ParamAlignment: RetAlign);
4471 for (const unsigned NumElts : VI) {
4472 const MaybeAlign CurrentAlign = ExtendIntegerRetVal
4473 ? MaybeAlign(std::nullopt)
4474 : commonAlignment(A: RetAlign, Offset: Offsets[I]);
4475
4476 SDValue Val = getBuildVectorizedValue(
4477 N: NumElts, dl, DAG, GetElement: [&](unsigned K) { return GetRetVal(I + K); });
4478
4479 SDValue Ptr =
4480 DAG.getObjectPtrOffset(SL: dl, Ptr: RetSymbol, Offset: TypeSize::getFixed(ExactSize: Offsets[I]));
4481
4482 Chain = DAG.getStore(Chain, dl, Val, Ptr,
4483 PtrInfo: MachinePointerInfo(NVPTX::AddressSpace::DeviceParam),
4484 Alignment: CurrentAlign);
4485
4486 I += NumElts;
4487 }
4488
4489 return DAG.getNode(Opcode: NVPTXISD::RET_GLUE, DL: dl, VT: MVT::Other, Operand: Chain);
4490}
4491
4492void NVPTXTargetLowering::LowerAsmOperandForConstraint(
4493 SDValue Op, StringRef Constraint, std::vector<SDValue> &Ops,
4494 SelectionDAG &DAG) const {
4495 if (Constraint.size() > 1)
4496 return;
4497 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
4498}
4499
4500// llvm.ptx.memcpy.const and llvm.ptx.memmove.const need to be modeled as
4501// TgtMemIntrinsic
4502// because we need the information that is only available in the "Value" type
4503// of destination
4504// pointer. In particular, the address space information.
4505void NVPTXTargetLowering::getTgtMemIntrinsic(
4506 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
4507 MachineFunction &MF, unsigned Intrinsic) const {
4508 IntrinsicInfo Info;
4509 switch (Intrinsic) {
4510 default:
4511 return;
4512 case Intrinsic::nvvm_match_all_sync_i32p:
4513 case Intrinsic::nvvm_match_all_sync_i64p:
4514 Info.opc = ISD::INTRINSIC_W_CHAIN;
4515 // memVT is bogus. These intrinsics have IntrInaccessibleMemOnly attribute
4516 // in order to model data exchange with other threads, but perform no real
4517 // memory accesses.
4518 Info.memVT = MVT::i1;
4519
4520 // Our result depends on both our and other thread's arguments.
4521 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
4522 Infos.push_back(Elt: Info);
4523 return;
4524 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_col:
4525 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_row:
4526 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_col_stride:
4527 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_row_stride:
4528 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_col:
4529 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_row:
4530 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_col_stride:
4531 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_row_stride:
4532 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_col:
4533 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_row:
4534 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_col_stride:
4535 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_row_stride:
4536 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_col:
4537 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_row:
4538 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_col_stride:
4539 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_row_stride:
4540 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_col:
4541 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_row:
4542 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_col_stride:
4543 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_row_stride:
4544 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_col:
4545 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_row:
4546 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_col_stride:
4547 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_row_stride: {
4548 Info.opc = ISD::INTRINSIC_W_CHAIN;
4549 Info.memVT = MVT::v8f16;
4550 Info.ptrVal = I.getArgOperand(i: 0);
4551 Info.offset = 0;
4552 Info.flags = MachineMemOperand::MOLoad;
4553 Info.align = Align(16);
4554 Infos.push_back(Elt: Info);
4555 return;
4556 }
4557 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_col:
4558 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_col_stride:
4559 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_col_stride:
4560 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_col:
4561 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_row:
4562 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_row_stride:
4563 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_row_stride:
4564 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_row:
4565 case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_col:
4566 case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_col_stride:
4567 case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_row:
4568 case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_row_stride:
4569 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_col:
4570 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_col_stride:
4571 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_col_stride:
4572 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_col:
4573 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_row:
4574 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_row_stride:
4575 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_row_stride:
4576 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_row:
4577 case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_col:
4578 case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_col_stride:
4579 case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_row:
4580 case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_row_stride: {
4581 Info.opc = ISD::INTRINSIC_W_CHAIN;
4582 Info.memVT = MVT::v2i32;
4583 Info.ptrVal = I.getArgOperand(i: 0);
4584 Info.offset = 0;
4585 Info.flags = MachineMemOperand::MOLoad;
4586 Info.align = Align(8);
4587 Infos.push_back(Elt: Info);
4588 return;
4589 }
4590
4591 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_col:
4592 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_col_stride:
4593 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_col_stride:
4594 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_col:
4595 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_row:
4596 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_row_stride:
4597 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_row_stride:
4598 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_row:
4599 case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_col:
4600 case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_col_stride:
4601 case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_row:
4602 case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_row_stride:
4603 case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_col:
4604 case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_col_stride:
4605 case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_row:
4606 case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_row_stride:
4607
4608 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_col:
4609 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_col_stride:
4610 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_col_stride:
4611 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_col:
4612 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_row:
4613 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_row_stride:
4614 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_row_stride:
4615 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_row:
4616 case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_col:
4617 case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_col_stride:
4618 case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_row:
4619 case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_row_stride:
4620 case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_col:
4621 case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_col_stride:
4622 case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_row:
4623 case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_row_stride:
4624 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x4_b16:
4625 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x4_trans_b16:
4626 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x2_trans_b8:
4627 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x2_trans_b8x16_b4x16_p64:
4628 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x2_trans_b8x16_b6x16_p32:
4629 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x4_b8x16_b4x16_p64:
4630 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x4_b8x16_b6x16_p32:
4631 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x4_s8_s4: {
4632 Info.opc = ISD::INTRINSIC_W_CHAIN;
4633 Info.memVT = MVT::v4i32;
4634 Info.ptrVal = I.getArgOperand(i: 0);
4635 Info.offset = 0;
4636 Info.flags = MachineMemOperand::MOLoad;
4637 Info.align = Align(16);
4638 Infos.push_back(Elt: Info);
4639 return;
4640 }
4641
4642 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_col:
4643 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_col_stride:
4644 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_col_stride:
4645 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_col:
4646 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_row:
4647 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_row_stride:
4648 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_row_stride:
4649 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_row:
4650
4651 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_col:
4652 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_col_stride:
4653 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_col_stride:
4654 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_col:
4655 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_row:
4656 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_row_stride:
4657 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_row_stride:
4658 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_row:
4659 case Intrinsic::nvvm_wmma_m8n8k128_load_a_b1_row:
4660 case Intrinsic::nvvm_wmma_m8n8k128_load_a_b1_row_stride:
4661 case Intrinsic::nvvm_wmma_m8n8k128_load_b_b1_col:
4662 case Intrinsic::nvvm_wmma_m8n8k128_load_b_b1_col_stride:
4663 case Intrinsic::nvvm_wmma_m8n8k32_load_a_s4_row:
4664 case Intrinsic::nvvm_wmma_m8n8k32_load_a_s4_row_stride:
4665 case Intrinsic::nvvm_wmma_m8n8k32_load_a_u4_row_stride:
4666 case Intrinsic::nvvm_wmma_m8n8k32_load_a_u4_row:
4667 case Intrinsic::nvvm_wmma_m8n8k32_load_b_s4_col:
4668 case Intrinsic::nvvm_wmma_m8n8k32_load_b_s4_col_stride:
4669 case Intrinsic::nvvm_wmma_m8n8k32_load_b_u4_col_stride:
4670 case Intrinsic::nvvm_wmma_m8n8k32_load_b_u4_col:
4671 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x1_b16:
4672 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x1_trans_b16:
4673 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x1_b8x16_b4x16_p64:
4674 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x1_b8x16_b6x16_p32:
4675 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x1_s8_s4: {
4676 Info.opc = ISD::INTRINSIC_W_CHAIN;
4677 Info.memVT = MVT::i32;
4678 Info.ptrVal = I.getArgOperand(i: 0);
4679 Info.offset = 0;
4680 Info.flags = MachineMemOperand::MOLoad;
4681 Info.align = Align(4);
4682 Infos.push_back(Elt: Info);
4683 return;
4684 }
4685
4686 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_col:
4687 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_row:
4688 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_col_stride:
4689 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_row_stride:
4690 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_col:
4691 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_row:
4692 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_col_stride:
4693 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_row_stride:
4694 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_col:
4695 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_row:
4696 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_col_stride:
4697 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_row_stride: {
4698 Info.opc = ISD::INTRINSIC_W_CHAIN;
4699 Info.memVT = MVT::v4f16;
4700 Info.ptrVal = I.getArgOperand(i: 0);
4701 Info.offset = 0;
4702 Info.flags = MachineMemOperand::MOLoad;
4703 Info.align = Align(16);
4704 Infos.push_back(Elt: Info);
4705 return;
4706 }
4707
4708 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_col:
4709 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_row:
4710 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_col_stride:
4711 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_row_stride:
4712 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_col:
4713 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_row:
4714 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_col_stride:
4715 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_row_stride:
4716 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_col:
4717 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_row:
4718 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_col_stride:
4719 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_row_stride:
4720 case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_col:
4721 case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_row:
4722 case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_col_stride:
4723 case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_row_stride: {
4724 Info.opc = ISD::INTRINSIC_W_CHAIN;
4725 Info.memVT = MVT::v8f32;
4726 Info.ptrVal = I.getArgOperand(i: 0);
4727 Info.offset = 0;
4728 Info.flags = MachineMemOperand::MOLoad;
4729 Info.align = Align(16);
4730 Infos.push_back(Elt: Info);
4731 return;
4732 }
4733
4734 case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_col:
4735 case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_col_stride:
4736 case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_row:
4737 case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_row_stride:
4738
4739 case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_col:
4740 case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_col_stride:
4741 case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_row:
4742 case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_row_stride:
4743
4744 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_col:
4745 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_col_stride:
4746 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_row:
4747 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_row_stride:
4748 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_col:
4749 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_col_stride:
4750 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_row:
4751 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_row_stride:
4752 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_col:
4753 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_col_stride:
4754 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_row:
4755 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_row_stride: {
4756 Info.opc = ISD::INTRINSIC_W_CHAIN;
4757 Info.memVT = MVT::v8i32;
4758 Info.ptrVal = I.getArgOperand(i: 0);
4759 Info.offset = 0;
4760 Info.flags = MachineMemOperand::MOLoad;
4761 Info.align = Align(16);
4762 Infos.push_back(Elt: Info);
4763 return;
4764 }
4765
4766 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_col:
4767 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_col_stride:
4768 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_row:
4769 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_row_stride:
4770 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_col:
4771 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_col_stride:
4772 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_row:
4773 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_row_stride:
4774 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x2_b16:
4775 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x2_trans_b16:
4776 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x1_trans_b8:
4777 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x1_trans_b8x16_b4x16_p64:
4778 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x1_trans_b8x16_b6x16_p32:
4779 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x2_b8x16_b4x16_p64:
4780 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x2_b8x16_b6x16_p32:
4781 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x2_s8_s4: {
4782 Info.opc = ISD::INTRINSIC_W_CHAIN;
4783 Info.memVT = MVT::v2i32;
4784 Info.ptrVal = I.getArgOperand(i: 0);
4785 Info.offset = 0;
4786 Info.flags = MachineMemOperand::MOLoad;
4787 Info.align = Align(8);
4788 Infos.push_back(Elt: Info);
4789 return;
4790 }
4791
4792 case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_col:
4793 case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_col_stride:
4794 case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_row:
4795 case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_row_stride:
4796
4797 case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_col:
4798 case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_col_stride:
4799 case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_row:
4800 case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_row_stride: {
4801 Info.opc = ISD::INTRINSIC_W_CHAIN;
4802 Info.memVT = MVT::f64;
4803 Info.ptrVal = I.getArgOperand(i: 0);
4804 Info.offset = 0;
4805 Info.flags = MachineMemOperand::MOLoad;
4806 Info.align = Align(8);
4807 Infos.push_back(Elt: Info);
4808 return;
4809 }
4810
4811 case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_col:
4812 case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_col_stride:
4813 case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_row:
4814 case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_row_stride: {
4815 Info.opc = ISD::INTRINSIC_W_CHAIN;
4816 Info.memVT = MVT::v2f64;
4817 Info.ptrVal = I.getArgOperand(i: 0);
4818 Info.offset = 0;
4819 Info.flags = MachineMemOperand::MOLoad;
4820 Info.align = Align(16);
4821 Infos.push_back(Elt: Info);
4822 return;
4823 }
4824
4825 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_col:
4826 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_row:
4827 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_col_stride:
4828 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_row_stride:
4829 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_col:
4830 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_row:
4831 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_col_stride:
4832 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_row_stride:
4833 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_col:
4834 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_row:
4835 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_col_stride:
4836 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_row_stride: {
4837 Info.opc = ISD::INTRINSIC_VOID;
4838 Info.memVT = MVT::v4f16;
4839 Info.ptrVal = I.getArgOperand(i: 0);
4840 Info.offset = 0;
4841 Info.flags = MachineMemOperand::MOStore;
4842 Info.align = Align(16);
4843 Infos.push_back(Elt: Info);
4844 return;
4845 }
4846
4847 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_col:
4848 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_row:
4849 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_col_stride:
4850 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_row_stride:
4851 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_col:
4852 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_row:
4853 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_col_stride:
4854 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_row_stride:
4855 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_col:
4856 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_row:
4857 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_col_stride:
4858 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_row_stride:
4859 case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_col:
4860 case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_row:
4861 case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_col_stride:
4862 case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_row_stride: {
4863 Info.opc = ISD::INTRINSIC_VOID;
4864 Info.memVT = MVT::v8f32;
4865 Info.ptrVal = I.getArgOperand(i: 0);
4866 Info.offset = 0;
4867 Info.flags = MachineMemOperand::MOStore;
4868 Info.align = Align(16);
4869 Infos.push_back(Elt: Info);
4870 return;
4871 }
4872
4873 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_col:
4874 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_col_stride:
4875 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_row:
4876 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_row_stride:
4877 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_col:
4878 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_col_stride:
4879 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_row:
4880 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_row_stride:
4881 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_col:
4882 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_col_stride:
4883 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_row:
4884 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_row_stride: {
4885 Info.opc = ISD::INTRINSIC_VOID;
4886 Info.memVT = MVT::v8i32;
4887 Info.ptrVal = I.getArgOperand(i: 0);
4888 Info.offset = 0;
4889 Info.flags = MachineMemOperand::MOStore;
4890 Info.align = Align(16);
4891 Infos.push_back(Elt: Info);
4892 return;
4893 }
4894
4895 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_col:
4896 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_col_stride:
4897 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_row:
4898 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_row_stride:
4899 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_col:
4900 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_col_stride:
4901 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_row:
4902 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_row_stride:
4903 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x2_b16:
4904 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x2_trans_b16:
4905 case Intrinsic::nvvm_stmatrix_sync_aligned_m16n8_x2_trans_b8: {
4906 Info.opc = ISD::INTRINSIC_VOID;
4907 Info.memVT = MVT::v2i32;
4908 Info.ptrVal = I.getArgOperand(i: 0);
4909 Info.offset = 0;
4910 Info.flags = MachineMemOperand::MOStore;
4911 Info.align = Align(8);
4912 Infos.push_back(Elt: Info);
4913 return;
4914 }
4915
4916 case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_col:
4917 case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_col_stride:
4918 case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_row:
4919 case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_row_stride: {
4920 Info.opc = ISD::INTRINSIC_VOID;
4921 Info.memVT = MVT::v2f64;
4922 Info.ptrVal = I.getArgOperand(i: 0);
4923 Info.offset = 0;
4924 Info.flags = MachineMemOperand::MOStore;
4925 Info.align = Align(16);
4926 Infos.push_back(Elt: Info);
4927 return;
4928 }
4929
4930 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x1_b16:
4931 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x1_trans_b16:
4932 case Intrinsic::nvvm_stmatrix_sync_aligned_m16n8_x1_trans_b8: {
4933 Info.opc = ISD::INTRINSIC_VOID;
4934 Info.memVT = MVT::i32;
4935 Info.ptrVal = I.getArgOperand(i: 0);
4936 Info.offset = 0;
4937 Info.flags = MachineMemOperand::MOStore;
4938 Info.align = Align(4);
4939 Infos.push_back(Elt: Info);
4940 return;
4941 }
4942
4943 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x4_b16:
4944 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x4_trans_b16:
4945 case Intrinsic::nvvm_stmatrix_sync_aligned_m16n8_x4_trans_b8: {
4946 Info.opc = ISD::INTRINSIC_VOID;
4947 Info.memVT = MVT::v4i32;
4948 Info.ptrVal = I.getArgOperand(i: 0);
4949 Info.offset = 0;
4950 Info.flags = MachineMemOperand::MOStore;
4951 Info.align = Align(16);
4952 Infos.push_back(Elt: Info);
4953 return;
4954 }
4955
4956 case Intrinsic::nvvm_st_bulk: {
4957 Value *Dst = I.getArgOperand(i: 0);
4958 Value *Val = I.getArgOperand(i: 2);
4959 Info.opc = ISD::INTRINSIC_VOID;
4960 Info.memVT = MVT::getVT(Ty: Val->getType());
4961 Info.ptrVal = Dst;
4962 Info.offset = 0;
4963 Info.flags = MachineMemOperand::MOStore;
4964 Info.align.reset();
4965 Info.size = MemoryLocation::UnknownSize;
4966 Infos.push_back(Elt: Info);
4967 return;
4968 }
4969
4970 case Intrinsic::nvvm_prefetch_tensormap: {
4971 auto &DL = I.getDataLayout();
4972 Info.opc = ISD::INTRINSIC_VOID;
4973 Info.memVT = getPointerTy(DL);
4974 Info.ptrVal = I.getArgOperand(i: 0);
4975 Info.offset = 0;
4976 Info.flags =
4977 MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable;
4978 Info.align.reset();
4979 Infos.push_back(Elt: Info);
4980 return;
4981 }
4982
4983 case Intrinsic::nvvm_prefetch_L1_32B_valid_addr: {
4984 Info.opc = ISD::INTRINSIC_VOID;
4985 Info.memVT = MVT::i8;
4986 Info.ptrVal = I.getArgOperand(i: 0);
4987 Info.offset = 0;
4988 Info.flags = MachineMemOperand::MOLoad;
4989 Info.align = Align(1);
4990 Infos.push_back(Elt: Info);
4991 return;
4992 }
4993
4994 case Intrinsic::nvvm_mbarrier_init: {
4995 Info.opc = ISD::INTRINSIC_VOID;
4996 Info.memVT = MVT::i64;
4997 Info.ptrVal = I.getArgOperand(i: 0);
4998 Info.offset = 0;
4999 Info.flags = MachineMemOperand::MOStore;
5000 Info.align = Align(8);
5001 Infos.push_back(Elt: Info);
5002 return;
5003 }
5004
5005 case Intrinsic::nvvm_mbarrier_check_layout: {
5006 Info.opc = ISD::INTRINSIC_W_CHAIN;
5007 Info.memVT = MVT::i64;
5008 Info.ptrVal = I.getArgOperand(i: 0);
5009 Info.offset = 0;
5010 Info.flags = MachineMemOperand::MOLoad;
5011 Info.align = Align(8);
5012 Infos.push_back(Elt: Info);
5013 return;
5014 }
5015
5016 case Intrinsic::nvvm_tensormap_replace_global_address:
5017 case Intrinsic::nvvm_tensormap_replace_global_stride: {
5018 Info.opc = ISD::INTRINSIC_VOID;
5019 Info.memVT = MVT::i64;
5020 Info.ptrVal = I.getArgOperand(i: 0);
5021 Info.offset = 0;
5022 Info.flags = MachineMemOperand::MOStore;
5023 Info.align.reset();
5024 Infos.push_back(Elt: Info);
5025 return;
5026 }
5027
5028 case Intrinsic::nvvm_tensormap_replace_rank:
5029 case Intrinsic::nvvm_tensormap_replace_box_dim:
5030 case Intrinsic::nvvm_tensormap_replace_global_dim:
5031 case Intrinsic::nvvm_tensormap_replace_element_stride:
5032 case Intrinsic::nvvm_tensormap_replace_elemtype:
5033 case Intrinsic::nvvm_tensormap_replace_interleave_layout:
5034 case Intrinsic::nvvm_tensormap_replace_swizzle_mode:
5035 case Intrinsic::nvvm_tensormap_replace_swizzle_atomicity:
5036 case Intrinsic::nvvm_tensormap_replace_fill_mode: {
5037 Info.opc = ISD::INTRINSIC_VOID;
5038 Info.memVT = MVT::i32;
5039 Info.ptrVal = I.getArgOperand(i: 0);
5040 Info.offset = 0;
5041 Info.flags = MachineMemOperand::MOStore;
5042 Info.align.reset();
5043 Infos.push_back(Elt: Info);
5044 return;
5045 }
5046
5047 case Intrinsic::nvvm_ldu_global_i:
5048 case Intrinsic::nvvm_ldu_global_f:
5049 case Intrinsic::nvvm_ldu_global_p: {
5050 Info.opc = ISD::INTRINSIC_W_CHAIN;
5051 Info.memVT = getValueType(DL: I.getDataLayout(), Ty: I.getType());
5052 Info.ptrVal = I.getArgOperand(i: 0);
5053 Info.offset = 0;
5054 Info.flags = MachineMemOperand::MOLoad;
5055 Info.align = cast<ConstantInt>(Val: I.getArgOperand(i: 1))->getMaybeAlignValue();
5056
5057 Infos.push_back(Elt: Info);
5058 return;
5059 }
5060 case Intrinsic::nvvm_tex_1d_v4f32_s32:
5061 case Intrinsic::nvvm_tex_1d_v4f32_f32:
5062 case Intrinsic::nvvm_tex_1d_level_v4f32_f32:
5063 case Intrinsic::nvvm_tex_1d_grad_v4f32_f32:
5064 case Intrinsic::nvvm_tex_1d_array_v4f32_s32:
5065 case Intrinsic::nvvm_tex_1d_array_v4f32_f32:
5066 case Intrinsic::nvvm_tex_1d_array_level_v4f32_f32:
5067 case Intrinsic::nvvm_tex_1d_array_grad_v4f32_f32:
5068 case Intrinsic::nvvm_tex_2d_v4f32_s32:
5069 case Intrinsic::nvvm_tex_2d_v4f32_f32:
5070 case Intrinsic::nvvm_tex_2d_level_v4f32_f32:
5071 case Intrinsic::nvvm_tex_2d_grad_v4f32_f32:
5072 case Intrinsic::nvvm_tex_2d_array_v4f32_s32:
5073 case Intrinsic::nvvm_tex_2d_array_v4f32_f32:
5074 case Intrinsic::nvvm_tex_2d_array_level_v4f32_f32:
5075 case Intrinsic::nvvm_tex_2d_array_grad_v4f32_f32:
5076 case Intrinsic::nvvm_tex_3d_v4f32_s32:
5077 case Intrinsic::nvvm_tex_3d_v4f32_f32:
5078 case Intrinsic::nvvm_tex_3d_level_v4f32_f32:
5079 case Intrinsic::nvvm_tex_3d_grad_v4f32_f32:
5080 case Intrinsic::nvvm_tex_cube_v4f32_f32:
5081 case Intrinsic::nvvm_tex_cube_level_v4f32_f32:
5082 case Intrinsic::nvvm_tex_cube_array_v4f32_f32:
5083 case Intrinsic::nvvm_tex_cube_array_level_v4f32_f32:
5084 case Intrinsic::nvvm_tld4_r_2d_v4f32_f32:
5085 case Intrinsic::nvvm_tld4_g_2d_v4f32_f32:
5086 case Intrinsic::nvvm_tld4_b_2d_v4f32_f32:
5087 case Intrinsic::nvvm_tld4_a_2d_v4f32_f32:
5088 case Intrinsic::nvvm_tex_unified_1d_v4f32_s32:
5089 case Intrinsic::nvvm_tex_unified_1d_v4f32_f32:
5090 case Intrinsic::nvvm_tex_unified_1d_level_v4f32_f32:
5091 case Intrinsic::nvvm_tex_unified_1d_grad_v4f32_f32:
5092 case Intrinsic::nvvm_tex_unified_1d_array_v4f32_s32:
5093 case Intrinsic::nvvm_tex_unified_1d_array_v4f32_f32:
5094 case Intrinsic::nvvm_tex_unified_1d_array_level_v4f32_f32:
5095 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4f32_f32:
5096 case Intrinsic::nvvm_tex_unified_2d_v4f32_s32:
5097 case Intrinsic::nvvm_tex_unified_2d_v4f32_f32:
5098 case Intrinsic::nvvm_tex_unified_2d_level_v4f32_f32:
5099 case Intrinsic::nvvm_tex_unified_2d_grad_v4f32_f32:
5100 case Intrinsic::nvvm_tex_unified_2d_array_v4f32_s32:
5101 case Intrinsic::nvvm_tex_unified_2d_array_v4f32_f32:
5102 case Intrinsic::nvvm_tex_unified_2d_array_level_v4f32_f32:
5103 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4f32_f32:
5104 case Intrinsic::nvvm_tex_unified_3d_v4f32_s32:
5105 case Intrinsic::nvvm_tex_unified_3d_v4f32_f32:
5106 case Intrinsic::nvvm_tex_unified_3d_level_v4f32_f32:
5107 case Intrinsic::nvvm_tex_unified_3d_grad_v4f32_f32:
5108 case Intrinsic::nvvm_tex_unified_cube_v4f32_f32:
5109 case Intrinsic::nvvm_tex_unified_cube_level_v4f32_f32:
5110 case Intrinsic::nvvm_tex_unified_cube_array_v4f32_f32:
5111 case Intrinsic::nvvm_tex_unified_cube_array_level_v4f32_f32:
5112 case Intrinsic::nvvm_tex_unified_cube_grad_v4f32_f32:
5113 case Intrinsic::nvvm_tex_unified_cube_array_grad_v4f32_f32:
5114 case Intrinsic::nvvm_tld4_unified_r_2d_v4f32_f32:
5115 case Intrinsic::nvvm_tld4_unified_g_2d_v4f32_f32:
5116 case Intrinsic::nvvm_tld4_unified_b_2d_v4f32_f32:
5117 case Intrinsic::nvvm_tld4_unified_a_2d_v4f32_f32:
5118 Info.opc = ISD::INTRINSIC_W_CHAIN;
5119 Info.memVT = MVT::v4f32;
5120 Info.ptrVal = nullptr;
5121 Info.offset = 0;
5122 Info.flags = MachineMemOperand::MOLoad;
5123 Info.align = Align(16);
5124 Infos.push_back(Elt: Info);
5125 return;
5126
5127 case Intrinsic::nvvm_tex_1d_v4s32_s32:
5128 case Intrinsic::nvvm_tex_1d_v4s32_f32:
5129 case Intrinsic::nvvm_tex_1d_level_v4s32_f32:
5130 case Intrinsic::nvvm_tex_1d_grad_v4s32_f32:
5131 case Intrinsic::nvvm_tex_1d_array_v4s32_s32:
5132 case Intrinsic::nvvm_tex_1d_array_v4s32_f32:
5133 case Intrinsic::nvvm_tex_1d_array_level_v4s32_f32:
5134 case Intrinsic::nvvm_tex_1d_array_grad_v4s32_f32:
5135 case Intrinsic::nvvm_tex_2d_v4s32_s32:
5136 case Intrinsic::nvvm_tex_2d_v4s32_f32:
5137 case Intrinsic::nvvm_tex_2d_level_v4s32_f32:
5138 case Intrinsic::nvvm_tex_2d_grad_v4s32_f32:
5139 case Intrinsic::nvvm_tex_2d_array_v4s32_s32:
5140 case Intrinsic::nvvm_tex_2d_array_v4s32_f32:
5141 case Intrinsic::nvvm_tex_2d_array_level_v4s32_f32:
5142 case Intrinsic::nvvm_tex_2d_array_grad_v4s32_f32:
5143 case Intrinsic::nvvm_tex_3d_v4s32_s32:
5144 case Intrinsic::nvvm_tex_3d_v4s32_f32:
5145 case Intrinsic::nvvm_tex_3d_level_v4s32_f32:
5146 case Intrinsic::nvvm_tex_3d_grad_v4s32_f32:
5147 case Intrinsic::nvvm_tex_cube_v4s32_f32:
5148 case Intrinsic::nvvm_tex_cube_level_v4s32_f32:
5149 case Intrinsic::nvvm_tex_cube_array_v4s32_f32:
5150 case Intrinsic::nvvm_tex_cube_array_level_v4s32_f32:
5151 case Intrinsic::nvvm_tex_cube_v4u32_f32:
5152 case Intrinsic::nvvm_tex_cube_level_v4u32_f32:
5153 case Intrinsic::nvvm_tex_cube_array_v4u32_f32:
5154 case Intrinsic::nvvm_tex_cube_array_level_v4u32_f32:
5155 case Intrinsic::nvvm_tex_1d_v4u32_s32:
5156 case Intrinsic::nvvm_tex_1d_v4u32_f32:
5157 case Intrinsic::nvvm_tex_1d_level_v4u32_f32:
5158 case Intrinsic::nvvm_tex_1d_grad_v4u32_f32:
5159 case Intrinsic::nvvm_tex_1d_array_v4u32_s32:
5160 case Intrinsic::nvvm_tex_1d_array_v4u32_f32:
5161 case Intrinsic::nvvm_tex_1d_array_level_v4u32_f32:
5162 case Intrinsic::nvvm_tex_1d_array_grad_v4u32_f32:
5163 case Intrinsic::nvvm_tex_2d_v4u32_s32:
5164 case Intrinsic::nvvm_tex_2d_v4u32_f32:
5165 case Intrinsic::nvvm_tex_2d_level_v4u32_f32:
5166 case Intrinsic::nvvm_tex_2d_grad_v4u32_f32:
5167 case Intrinsic::nvvm_tex_2d_array_v4u32_s32:
5168 case Intrinsic::nvvm_tex_2d_array_v4u32_f32:
5169 case Intrinsic::nvvm_tex_2d_array_level_v4u32_f32:
5170 case Intrinsic::nvvm_tex_2d_array_grad_v4u32_f32:
5171 case Intrinsic::nvvm_tex_3d_v4u32_s32:
5172 case Intrinsic::nvvm_tex_3d_v4u32_f32:
5173 case Intrinsic::nvvm_tex_3d_level_v4u32_f32:
5174 case Intrinsic::nvvm_tex_3d_grad_v4u32_f32:
5175 case Intrinsic::nvvm_tld4_r_2d_v4s32_f32:
5176 case Intrinsic::nvvm_tld4_g_2d_v4s32_f32:
5177 case Intrinsic::nvvm_tld4_b_2d_v4s32_f32:
5178 case Intrinsic::nvvm_tld4_a_2d_v4s32_f32:
5179 case Intrinsic::nvvm_tld4_r_2d_v4u32_f32:
5180 case Intrinsic::nvvm_tld4_g_2d_v4u32_f32:
5181 case Intrinsic::nvvm_tld4_b_2d_v4u32_f32:
5182 case Intrinsic::nvvm_tld4_a_2d_v4u32_f32:
5183 case Intrinsic::nvvm_tex_unified_1d_v4s32_s32:
5184 case Intrinsic::nvvm_tex_unified_1d_v4s32_f32:
5185 case Intrinsic::nvvm_tex_unified_1d_level_v4s32_f32:
5186 case Intrinsic::nvvm_tex_unified_1d_grad_v4s32_f32:
5187 case Intrinsic::nvvm_tex_unified_1d_array_v4s32_s32:
5188 case Intrinsic::nvvm_tex_unified_1d_array_v4s32_f32:
5189 case Intrinsic::nvvm_tex_unified_1d_array_level_v4s32_f32:
5190 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4s32_f32:
5191 case Intrinsic::nvvm_tex_unified_2d_v4s32_s32:
5192 case Intrinsic::nvvm_tex_unified_2d_v4s32_f32:
5193 case Intrinsic::nvvm_tex_unified_2d_level_v4s32_f32:
5194 case Intrinsic::nvvm_tex_unified_2d_grad_v4s32_f32:
5195 case Intrinsic::nvvm_tex_unified_2d_array_v4s32_s32:
5196 case Intrinsic::nvvm_tex_unified_2d_array_v4s32_f32:
5197 case Intrinsic::nvvm_tex_unified_2d_array_level_v4s32_f32:
5198 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4s32_f32:
5199 case Intrinsic::nvvm_tex_unified_3d_v4s32_s32:
5200 case Intrinsic::nvvm_tex_unified_3d_v4s32_f32:
5201 case Intrinsic::nvvm_tex_unified_3d_level_v4s32_f32:
5202 case Intrinsic::nvvm_tex_unified_3d_grad_v4s32_f32:
5203 case Intrinsic::nvvm_tex_unified_1d_v4u32_s32:
5204 case Intrinsic::nvvm_tex_unified_1d_v4u32_f32:
5205 case Intrinsic::nvvm_tex_unified_1d_level_v4u32_f32:
5206 case Intrinsic::nvvm_tex_unified_1d_grad_v4u32_f32:
5207 case Intrinsic::nvvm_tex_unified_1d_array_v4u32_s32:
5208 case Intrinsic::nvvm_tex_unified_1d_array_v4u32_f32:
5209 case Intrinsic::nvvm_tex_unified_1d_array_level_v4u32_f32:
5210 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4u32_f32:
5211 case Intrinsic::nvvm_tex_unified_2d_v4u32_s32:
5212 case Intrinsic::nvvm_tex_unified_2d_v4u32_f32:
5213 case Intrinsic::nvvm_tex_unified_2d_level_v4u32_f32:
5214 case Intrinsic::nvvm_tex_unified_2d_grad_v4u32_f32:
5215 case Intrinsic::nvvm_tex_unified_2d_array_v4u32_s32:
5216 case Intrinsic::nvvm_tex_unified_2d_array_v4u32_f32:
5217 case Intrinsic::nvvm_tex_unified_2d_array_level_v4u32_f32:
5218 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4u32_f32:
5219 case Intrinsic::nvvm_tex_unified_3d_v4u32_s32:
5220 case Intrinsic::nvvm_tex_unified_3d_v4u32_f32:
5221 case Intrinsic::nvvm_tex_unified_3d_level_v4u32_f32:
5222 case Intrinsic::nvvm_tex_unified_3d_grad_v4u32_f32:
5223 case Intrinsic::nvvm_tex_unified_cube_v4s32_f32:
5224 case Intrinsic::nvvm_tex_unified_cube_level_v4s32_f32:
5225 case Intrinsic::nvvm_tex_unified_cube_array_v4s32_f32:
5226 case Intrinsic::nvvm_tex_unified_cube_array_level_v4s32_f32:
5227 case Intrinsic::nvvm_tex_unified_cube_v4u32_f32:
5228 case Intrinsic::nvvm_tex_unified_cube_level_v4u32_f32:
5229 case Intrinsic::nvvm_tex_unified_cube_array_v4u32_f32:
5230 case Intrinsic::nvvm_tex_unified_cube_array_level_v4u32_f32:
5231 case Intrinsic::nvvm_tex_unified_cube_grad_v4s32_f32:
5232 case Intrinsic::nvvm_tex_unified_cube_grad_v4u32_f32:
5233 case Intrinsic::nvvm_tex_unified_cube_array_grad_v4s32_f32:
5234 case Intrinsic::nvvm_tex_unified_cube_array_grad_v4u32_f32:
5235 case Intrinsic::nvvm_tld4_unified_r_2d_v4s32_f32:
5236 case Intrinsic::nvvm_tld4_unified_g_2d_v4s32_f32:
5237 case Intrinsic::nvvm_tld4_unified_b_2d_v4s32_f32:
5238 case Intrinsic::nvvm_tld4_unified_a_2d_v4s32_f32:
5239 case Intrinsic::nvvm_tld4_unified_r_2d_v4u32_f32:
5240 case Intrinsic::nvvm_tld4_unified_g_2d_v4u32_f32:
5241 case Intrinsic::nvvm_tld4_unified_b_2d_v4u32_f32:
5242 case Intrinsic::nvvm_tld4_unified_a_2d_v4u32_f32:
5243 Info.opc = ISD::INTRINSIC_W_CHAIN;
5244 Info.memVT = MVT::v4i32;
5245 Info.ptrVal = nullptr;
5246 Info.offset = 0;
5247 Info.flags = MachineMemOperand::MOLoad;
5248 Info.align = Align(16);
5249 Infos.push_back(Elt: Info);
5250 return;
5251
5252 case Intrinsic::nvvm_suld_1d_i8_clamp:
5253 case Intrinsic::nvvm_suld_1d_v2i8_clamp:
5254 case Intrinsic::nvvm_suld_1d_v4i8_clamp:
5255 case Intrinsic::nvvm_suld_1d_array_i8_clamp:
5256 case Intrinsic::nvvm_suld_1d_array_v2i8_clamp:
5257 case Intrinsic::nvvm_suld_1d_array_v4i8_clamp:
5258 case Intrinsic::nvvm_suld_2d_i8_clamp:
5259 case Intrinsic::nvvm_suld_2d_v2i8_clamp:
5260 case Intrinsic::nvvm_suld_2d_v4i8_clamp:
5261 case Intrinsic::nvvm_suld_2d_array_i8_clamp:
5262 case Intrinsic::nvvm_suld_2d_array_v2i8_clamp:
5263 case Intrinsic::nvvm_suld_2d_array_v4i8_clamp:
5264 case Intrinsic::nvvm_suld_3d_i8_clamp:
5265 case Intrinsic::nvvm_suld_3d_v2i8_clamp:
5266 case Intrinsic::nvvm_suld_3d_v4i8_clamp:
5267 case Intrinsic::nvvm_suld_1d_i8_trap:
5268 case Intrinsic::nvvm_suld_1d_v2i8_trap:
5269 case Intrinsic::nvvm_suld_1d_v4i8_trap:
5270 case Intrinsic::nvvm_suld_1d_array_i8_trap:
5271 case Intrinsic::nvvm_suld_1d_array_v2i8_trap:
5272 case Intrinsic::nvvm_suld_1d_array_v4i8_trap:
5273 case Intrinsic::nvvm_suld_2d_i8_trap:
5274 case Intrinsic::nvvm_suld_2d_v2i8_trap:
5275 case Intrinsic::nvvm_suld_2d_v4i8_trap:
5276 case Intrinsic::nvvm_suld_2d_array_i8_trap:
5277 case Intrinsic::nvvm_suld_2d_array_v2i8_trap:
5278 case Intrinsic::nvvm_suld_2d_array_v4i8_trap:
5279 case Intrinsic::nvvm_suld_3d_i8_trap:
5280 case Intrinsic::nvvm_suld_3d_v2i8_trap:
5281 case Intrinsic::nvvm_suld_3d_v4i8_trap:
5282 case Intrinsic::nvvm_suld_1d_i8_zero:
5283 case Intrinsic::nvvm_suld_1d_v2i8_zero:
5284 case Intrinsic::nvvm_suld_1d_v4i8_zero:
5285 case Intrinsic::nvvm_suld_1d_array_i8_zero:
5286 case Intrinsic::nvvm_suld_1d_array_v2i8_zero:
5287 case Intrinsic::nvvm_suld_1d_array_v4i8_zero:
5288 case Intrinsic::nvvm_suld_2d_i8_zero:
5289 case Intrinsic::nvvm_suld_2d_v2i8_zero:
5290 case Intrinsic::nvvm_suld_2d_v4i8_zero:
5291 case Intrinsic::nvvm_suld_2d_array_i8_zero:
5292 case Intrinsic::nvvm_suld_2d_array_v2i8_zero:
5293 case Intrinsic::nvvm_suld_2d_array_v4i8_zero:
5294 case Intrinsic::nvvm_suld_3d_i8_zero:
5295 case Intrinsic::nvvm_suld_3d_v2i8_zero:
5296 case Intrinsic::nvvm_suld_3d_v4i8_zero:
5297 Info.opc = ISD::INTRINSIC_W_CHAIN;
5298 Info.memVT = MVT::i8;
5299 Info.ptrVal = nullptr;
5300 Info.offset = 0;
5301 Info.flags = MachineMemOperand::MOLoad;
5302 Info.align = Align(16);
5303 Infos.push_back(Elt: Info);
5304 return;
5305
5306 case Intrinsic::nvvm_suld_1d_i16_clamp:
5307 case Intrinsic::nvvm_suld_1d_v2i16_clamp:
5308 case Intrinsic::nvvm_suld_1d_v4i16_clamp:
5309 case Intrinsic::nvvm_suld_1d_array_i16_clamp:
5310 case Intrinsic::nvvm_suld_1d_array_v2i16_clamp:
5311 case Intrinsic::nvvm_suld_1d_array_v4i16_clamp:
5312 case Intrinsic::nvvm_suld_2d_i16_clamp:
5313 case Intrinsic::nvvm_suld_2d_v2i16_clamp:
5314 case Intrinsic::nvvm_suld_2d_v4i16_clamp:
5315 case Intrinsic::nvvm_suld_2d_array_i16_clamp:
5316 case Intrinsic::nvvm_suld_2d_array_v2i16_clamp:
5317 case Intrinsic::nvvm_suld_2d_array_v4i16_clamp:
5318 case Intrinsic::nvvm_suld_3d_i16_clamp:
5319 case Intrinsic::nvvm_suld_3d_v2i16_clamp:
5320 case Intrinsic::nvvm_suld_3d_v4i16_clamp:
5321 case Intrinsic::nvvm_suld_1d_i16_trap:
5322 case Intrinsic::nvvm_suld_1d_v2i16_trap:
5323 case Intrinsic::nvvm_suld_1d_v4i16_trap:
5324 case Intrinsic::nvvm_suld_1d_array_i16_trap:
5325 case Intrinsic::nvvm_suld_1d_array_v2i16_trap:
5326 case Intrinsic::nvvm_suld_1d_array_v4i16_trap:
5327 case Intrinsic::nvvm_suld_2d_i16_trap:
5328 case Intrinsic::nvvm_suld_2d_v2i16_trap:
5329 case Intrinsic::nvvm_suld_2d_v4i16_trap:
5330 case Intrinsic::nvvm_suld_2d_array_i16_trap:
5331 case Intrinsic::nvvm_suld_2d_array_v2i16_trap:
5332 case Intrinsic::nvvm_suld_2d_array_v4i16_trap:
5333 case Intrinsic::nvvm_suld_3d_i16_trap:
5334 case Intrinsic::nvvm_suld_3d_v2i16_trap:
5335 case Intrinsic::nvvm_suld_3d_v4i16_trap:
5336 case Intrinsic::nvvm_suld_1d_i16_zero:
5337 case Intrinsic::nvvm_suld_1d_v2i16_zero:
5338 case Intrinsic::nvvm_suld_1d_v4i16_zero:
5339 case Intrinsic::nvvm_suld_1d_array_i16_zero:
5340 case Intrinsic::nvvm_suld_1d_array_v2i16_zero:
5341 case Intrinsic::nvvm_suld_1d_array_v4i16_zero:
5342 case Intrinsic::nvvm_suld_2d_i16_zero:
5343 case Intrinsic::nvvm_suld_2d_v2i16_zero:
5344 case Intrinsic::nvvm_suld_2d_v4i16_zero:
5345 case Intrinsic::nvvm_suld_2d_array_i16_zero:
5346 case Intrinsic::nvvm_suld_2d_array_v2i16_zero:
5347 case Intrinsic::nvvm_suld_2d_array_v4i16_zero:
5348 case Intrinsic::nvvm_suld_3d_i16_zero:
5349 case Intrinsic::nvvm_suld_3d_v2i16_zero:
5350 case Intrinsic::nvvm_suld_3d_v4i16_zero:
5351 Info.opc = ISD::INTRINSIC_W_CHAIN;
5352 Info.memVT = MVT::i16;
5353 Info.ptrVal = nullptr;
5354 Info.offset = 0;
5355 Info.flags = MachineMemOperand::MOLoad;
5356 Info.align = Align(16);
5357 Infos.push_back(Elt: Info);
5358 return;
5359
5360 case Intrinsic::nvvm_suld_1d_i32_clamp:
5361 case Intrinsic::nvvm_suld_1d_v2i32_clamp:
5362 case Intrinsic::nvvm_suld_1d_v4i32_clamp:
5363 case Intrinsic::nvvm_suld_1d_array_i32_clamp:
5364 case Intrinsic::nvvm_suld_1d_array_v2i32_clamp:
5365 case Intrinsic::nvvm_suld_1d_array_v4i32_clamp:
5366 case Intrinsic::nvvm_suld_2d_i32_clamp:
5367 case Intrinsic::nvvm_suld_2d_v2i32_clamp:
5368 case Intrinsic::nvvm_suld_2d_v4i32_clamp:
5369 case Intrinsic::nvvm_suld_2d_array_i32_clamp:
5370 case Intrinsic::nvvm_suld_2d_array_v2i32_clamp:
5371 case Intrinsic::nvvm_suld_2d_array_v4i32_clamp:
5372 case Intrinsic::nvvm_suld_3d_i32_clamp:
5373 case Intrinsic::nvvm_suld_3d_v2i32_clamp:
5374 case Intrinsic::nvvm_suld_3d_v4i32_clamp:
5375 case Intrinsic::nvvm_suld_1d_i32_trap:
5376 case Intrinsic::nvvm_suld_1d_v2i32_trap:
5377 case Intrinsic::nvvm_suld_1d_v4i32_trap:
5378 case Intrinsic::nvvm_suld_1d_array_i32_trap:
5379 case Intrinsic::nvvm_suld_1d_array_v2i32_trap:
5380 case Intrinsic::nvvm_suld_1d_array_v4i32_trap:
5381 case Intrinsic::nvvm_suld_2d_i32_trap:
5382 case Intrinsic::nvvm_suld_2d_v2i32_trap:
5383 case Intrinsic::nvvm_suld_2d_v4i32_trap:
5384 case Intrinsic::nvvm_suld_2d_array_i32_trap:
5385 case Intrinsic::nvvm_suld_2d_array_v2i32_trap:
5386 case Intrinsic::nvvm_suld_2d_array_v4i32_trap:
5387 case Intrinsic::nvvm_suld_3d_i32_trap:
5388 case Intrinsic::nvvm_suld_3d_v2i32_trap:
5389 case Intrinsic::nvvm_suld_3d_v4i32_trap:
5390 case Intrinsic::nvvm_suld_1d_i32_zero:
5391 case Intrinsic::nvvm_suld_1d_v2i32_zero:
5392 case Intrinsic::nvvm_suld_1d_v4i32_zero:
5393 case Intrinsic::nvvm_suld_1d_array_i32_zero:
5394 case Intrinsic::nvvm_suld_1d_array_v2i32_zero:
5395 case Intrinsic::nvvm_suld_1d_array_v4i32_zero:
5396 case Intrinsic::nvvm_suld_2d_i32_zero:
5397 case Intrinsic::nvvm_suld_2d_v2i32_zero:
5398 case Intrinsic::nvvm_suld_2d_v4i32_zero:
5399 case Intrinsic::nvvm_suld_2d_array_i32_zero:
5400 case Intrinsic::nvvm_suld_2d_array_v2i32_zero:
5401 case Intrinsic::nvvm_suld_2d_array_v4i32_zero:
5402 case Intrinsic::nvvm_suld_3d_i32_zero:
5403 case Intrinsic::nvvm_suld_3d_v2i32_zero:
5404 case Intrinsic::nvvm_suld_3d_v4i32_zero:
5405 Info.opc = ISD::INTRINSIC_W_CHAIN;
5406 Info.memVT = MVT::i32;
5407 Info.ptrVal = nullptr;
5408 Info.offset = 0;
5409 Info.flags = MachineMemOperand::MOLoad;
5410 Info.align = Align(16);
5411 Infos.push_back(Elt: Info);
5412 return;
5413
5414 case Intrinsic::nvvm_suld_1d_i64_clamp:
5415 case Intrinsic::nvvm_suld_1d_v2i64_clamp:
5416 case Intrinsic::nvvm_suld_1d_array_i64_clamp:
5417 case Intrinsic::nvvm_suld_1d_array_v2i64_clamp:
5418 case Intrinsic::nvvm_suld_2d_i64_clamp:
5419 case Intrinsic::nvvm_suld_2d_v2i64_clamp:
5420 case Intrinsic::nvvm_suld_2d_array_i64_clamp:
5421 case Intrinsic::nvvm_suld_2d_array_v2i64_clamp:
5422 case Intrinsic::nvvm_suld_3d_i64_clamp:
5423 case Intrinsic::nvvm_suld_3d_v2i64_clamp:
5424 case Intrinsic::nvvm_suld_1d_i64_trap:
5425 case Intrinsic::nvvm_suld_1d_v2i64_trap:
5426 case Intrinsic::nvvm_suld_1d_array_i64_trap:
5427 case Intrinsic::nvvm_suld_1d_array_v2i64_trap:
5428 case Intrinsic::nvvm_suld_2d_i64_trap:
5429 case Intrinsic::nvvm_suld_2d_v2i64_trap:
5430 case Intrinsic::nvvm_suld_2d_array_i64_trap:
5431 case Intrinsic::nvvm_suld_2d_array_v2i64_trap:
5432 case Intrinsic::nvvm_suld_3d_i64_trap:
5433 case Intrinsic::nvvm_suld_3d_v2i64_trap:
5434 case Intrinsic::nvvm_suld_1d_i64_zero:
5435 case Intrinsic::nvvm_suld_1d_v2i64_zero:
5436 case Intrinsic::nvvm_suld_1d_array_i64_zero:
5437 case Intrinsic::nvvm_suld_1d_array_v2i64_zero:
5438 case Intrinsic::nvvm_suld_2d_i64_zero:
5439 case Intrinsic::nvvm_suld_2d_v2i64_zero:
5440 case Intrinsic::nvvm_suld_2d_array_i64_zero:
5441 case Intrinsic::nvvm_suld_2d_array_v2i64_zero:
5442 case Intrinsic::nvvm_suld_3d_i64_zero:
5443 case Intrinsic::nvvm_suld_3d_v2i64_zero:
5444 Info.opc = ISD::INTRINSIC_W_CHAIN;
5445 Info.memVT = MVT::i64;
5446 Info.ptrVal = nullptr;
5447 Info.offset = 0;
5448 Info.flags = MachineMemOperand::MOLoad;
5449 Info.align = Align(16);
5450 Infos.push_back(Elt: Info);
5451 return;
5452
5453 case Intrinsic::nvvm_tcgen05_ld_16x64b_x1:
5454 case Intrinsic::nvvm_tcgen05_ld_32x32b_x1:
5455 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x1: {
5456 Info.opc = ISD::INTRINSIC_W_CHAIN;
5457 Info.memVT = MVT::v1i32;
5458 Info.ptrVal = I.getArgOperand(i: 0);
5459 Info.offset = 0;
5460 Info.flags = MachineMemOperand::MOLoad;
5461 Info.align.reset();
5462 Infos.push_back(Elt: Info);
5463 return;
5464 }
5465
5466 case Intrinsic::nvvm_tcgen05_ld_16x64b_x2:
5467 case Intrinsic::nvvm_tcgen05_ld_16x128b_x1:
5468 case Intrinsic::nvvm_tcgen05_ld_32x32b_x2:
5469 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x2:
5470 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x2_i32:
5471 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x2_i32: {
5472 Info.opc = ISD::INTRINSIC_W_CHAIN;
5473 Info.memVT = MVT::v2i32;
5474 Info.ptrVal = I.getArgOperand(i: 0);
5475 Info.offset = 0;
5476 Info.flags = MachineMemOperand::MOLoad;
5477 Info.align.reset();
5478 Infos.push_back(Elt: Info);
5479 return;
5480 }
5481
5482 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x2_f32:
5483 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x2_f32: {
5484 Info.opc = ISD::INTRINSIC_W_CHAIN;
5485 Info.memVT = MVT::v2f32;
5486 Info.ptrVal = I.getArgOperand(i: 0);
5487 Info.offset = 0;
5488 Info.flags = MachineMemOperand::MOLoad;
5489 Info.align.reset();
5490 Infos.push_back(Elt: Info);
5491 return;
5492 }
5493
5494 case Intrinsic::nvvm_tcgen05_ld_16x64b_x4:
5495 case Intrinsic::nvvm_tcgen05_ld_16x128b_x2:
5496 case Intrinsic::nvvm_tcgen05_ld_32x32b_x4:
5497 case Intrinsic::nvvm_tcgen05_ld_16x256b_x1:
5498 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x4:
5499 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x4_i32:
5500 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x4_i32: {
5501 Info.opc = ISD::INTRINSIC_W_CHAIN;
5502 Info.memVT = MVT::v4i32;
5503 Info.ptrVal = I.getArgOperand(i: 0);
5504 Info.offset = 0;
5505 Info.flags = MachineMemOperand::MOLoad;
5506 Info.align.reset();
5507 Infos.push_back(Elt: Info);
5508 return;
5509 }
5510
5511 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x4_f32:
5512 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x4_f32: {
5513 Info.opc = ISD::INTRINSIC_W_CHAIN;
5514 Info.memVT = MVT::v4f32;
5515 Info.ptrVal = I.getArgOperand(i: 0);
5516 Info.offset = 0;
5517 Info.flags = MachineMemOperand::MOLoad;
5518 Info.align.reset();
5519 Infos.push_back(Elt: Info);
5520 return;
5521 }
5522
5523 case Intrinsic::nvvm_tcgen05_ld_16x64b_x8:
5524 case Intrinsic::nvvm_tcgen05_ld_16x128b_x4:
5525 case Intrinsic::nvvm_tcgen05_ld_16x256b_x2:
5526 case Intrinsic::nvvm_tcgen05_ld_32x32b_x8:
5527 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x8:
5528 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x8_i32:
5529 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x8_i32: {
5530 Info.opc = ISD::INTRINSIC_W_CHAIN;
5531 Info.memVT = MVT::v8i32;
5532 Info.ptrVal = I.getArgOperand(i: 0);
5533 Info.offset = 0;
5534 Info.flags = MachineMemOperand::MOLoad;
5535 Info.align.reset();
5536 Infos.push_back(Elt: Info);
5537 return;
5538 }
5539
5540 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x8_f32:
5541 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x8_f32: {
5542 Info.opc = ISD::INTRINSIC_W_CHAIN;
5543 Info.memVT = MVT::v8f32;
5544 Info.ptrVal = I.getArgOperand(i: 0);
5545 Info.offset = 0;
5546 Info.flags = MachineMemOperand::MOLoad;
5547 Info.align.reset();
5548 Infos.push_back(Elt: Info);
5549 return;
5550 }
5551
5552 case Intrinsic::nvvm_tcgen05_ld_16x64b_x16:
5553 case Intrinsic::nvvm_tcgen05_ld_16x128b_x8:
5554 case Intrinsic::nvvm_tcgen05_ld_16x256b_x4:
5555 case Intrinsic::nvvm_tcgen05_ld_32x32b_x16:
5556 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x16:
5557 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x16_i32:
5558 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x16_i32: {
5559 Info.opc = ISD::INTRINSIC_W_CHAIN;
5560 Info.memVT = MVT::v16i32;
5561 Info.ptrVal = I.getArgOperand(i: 0);
5562 Info.offset = 0;
5563 Info.flags = MachineMemOperand::MOLoad;
5564 Info.align.reset();
5565 Infos.push_back(Elt: Info);
5566 return;
5567 }
5568
5569 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x16_f32:
5570 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x16_f32: {
5571 Info.opc = ISD::INTRINSIC_W_CHAIN;
5572 Info.memVT = MVT::v16f32;
5573 Info.ptrVal = I.getArgOperand(i: 0);
5574 Info.offset = 0;
5575 Info.flags = MachineMemOperand::MOLoad;
5576 Info.align.reset();
5577 Infos.push_back(Elt: Info);
5578 return;
5579 }
5580
5581 case Intrinsic::nvvm_tcgen05_ld_16x64b_x32:
5582 case Intrinsic::nvvm_tcgen05_ld_16x128b_x16:
5583 case Intrinsic::nvvm_tcgen05_ld_16x256b_x8:
5584 case Intrinsic::nvvm_tcgen05_ld_32x32b_x32:
5585 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x32:
5586 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x32_i32:
5587 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x32_i32: {
5588 Info.opc = ISD::INTRINSIC_W_CHAIN;
5589 Info.memVT = MVT::v32i32;
5590 Info.ptrVal = I.getArgOperand(i: 0);
5591 Info.offset = 0;
5592 Info.flags = MachineMemOperand::MOLoad;
5593 Info.align.reset();
5594 Infos.push_back(Elt: Info);
5595 return;
5596 }
5597
5598 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x32_f32:
5599 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x32_f32: {
5600 Info.opc = ISD::INTRINSIC_W_CHAIN;
5601 Info.memVT = MVT::v32f32;
5602 Info.ptrVal = I.getArgOperand(i: 0);
5603 Info.offset = 0;
5604 Info.flags = MachineMemOperand::MOLoad;
5605 Info.align.reset();
5606 Infos.push_back(Elt: Info);
5607 return;
5608 }
5609
5610 case Intrinsic::nvvm_tcgen05_ld_16x64b_x64:
5611 case Intrinsic::nvvm_tcgen05_ld_16x128b_x32:
5612 case Intrinsic::nvvm_tcgen05_ld_16x256b_x16:
5613 case Intrinsic::nvvm_tcgen05_ld_32x32b_x64:
5614 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x64:
5615 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x64_i32:
5616 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x64_i32: {
5617 Info.opc = ISD::INTRINSIC_W_CHAIN;
5618 Info.memVT = MVT::v64i32;
5619 Info.ptrVal = I.getArgOperand(i: 0);
5620 Info.offset = 0;
5621 Info.flags = MachineMemOperand::MOLoad;
5622 Info.align.reset();
5623 Infos.push_back(Elt: Info);
5624 return;
5625 }
5626
5627 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x64_f32:
5628 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x64_f32: {
5629 Info.opc = ISD::INTRINSIC_W_CHAIN;
5630 Info.memVT = MVT::v64f32;
5631 Info.ptrVal = I.getArgOperand(i: 0);
5632 Info.offset = 0;
5633 Info.flags = MachineMemOperand::MOLoad;
5634 Info.align.reset();
5635 Infos.push_back(Elt: Info);
5636 return;
5637 }
5638
5639 case Intrinsic::nvvm_tcgen05_ld_16x64b_x128:
5640 case Intrinsic::nvvm_tcgen05_ld_16x128b_x64:
5641 case Intrinsic::nvvm_tcgen05_ld_16x256b_x32:
5642 case Intrinsic::nvvm_tcgen05_ld_32x32b_x128:
5643 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x128:
5644 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x128_i32:
5645 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x128_i32: {
5646 Info.opc = ISD::INTRINSIC_W_CHAIN;
5647 Info.memVT = MVT::v128i32;
5648 Info.ptrVal = I.getArgOperand(i: 0);
5649 Info.offset = 0;
5650 Info.flags = MachineMemOperand::MOLoad;
5651 Info.align.reset();
5652 Infos.push_back(Elt: Info);
5653 return;
5654 }
5655
5656 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x128_f32:
5657 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x128_f32: {
5658 Info.opc = ISD::INTRINSIC_W_CHAIN;
5659 Info.memVT = MVT::v128f32;
5660 Info.ptrVal = I.getArgOperand(i: 0);
5661 Info.offset = 0;
5662 Info.flags = MachineMemOperand::MOLoad;
5663 Info.align.reset();
5664 Infos.push_back(Elt: Info);
5665 return;
5666 }
5667
5668 case Intrinsic::nvvm_tcgen05_st_16x64b_x1:
5669 case Intrinsic::nvvm_tcgen05_st_32x32b_x1:
5670 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x1: {
5671 Info.opc = ISD::INTRINSIC_VOID;
5672 Info.memVT = MVT::v1i32;
5673 Info.ptrVal = I.getArgOperand(i: 0);
5674 Info.offset = 0;
5675 Info.flags = MachineMemOperand::MOStore;
5676 Info.align.reset();
5677 Infos.push_back(Elt: Info);
5678 return;
5679 }
5680
5681 case Intrinsic::nvvm_tcgen05_st_16x64b_x2:
5682 case Intrinsic::nvvm_tcgen05_st_16x128b_x1:
5683 case Intrinsic::nvvm_tcgen05_st_32x32b_x2:
5684 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x2: {
5685 Info.opc = ISD::INTRINSIC_VOID;
5686 Info.memVT = MVT::v2i32;
5687 Info.ptrVal = I.getArgOperand(i: 0);
5688 Info.offset = 0;
5689 Info.flags = MachineMemOperand::MOStore;
5690 Info.align.reset();
5691 Infos.push_back(Elt: Info);
5692 return;
5693 }
5694
5695 case Intrinsic::nvvm_tcgen05_st_16x64b_x4:
5696 case Intrinsic::nvvm_tcgen05_st_16x128b_x2:
5697 case Intrinsic::nvvm_tcgen05_st_16x256b_x1:
5698 case Intrinsic::nvvm_tcgen05_st_32x32b_x4:
5699 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x4: {
5700 Info.opc = ISD::INTRINSIC_VOID;
5701 Info.memVT = MVT::v4i32;
5702 Info.ptrVal = I.getArgOperand(i: 0);
5703 Info.offset = 0;
5704 Info.flags = MachineMemOperand::MOStore;
5705 Info.align.reset();
5706 Infos.push_back(Elt: Info);
5707 return;
5708 }
5709
5710 case Intrinsic::nvvm_tcgen05_st_16x64b_x8:
5711 case Intrinsic::nvvm_tcgen05_st_16x128b_x4:
5712 case Intrinsic::nvvm_tcgen05_st_16x256b_x2:
5713 case Intrinsic::nvvm_tcgen05_st_32x32b_x8:
5714 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x8: {
5715 Info.opc = ISD::INTRINSIC_VOID;
5716 Info.memVT = MVT::v8i32;
5717 Info.ptrVal = I.getArgOperand(i: 0);
5718 Info.offset = 0;
5719 Info.flags = MachineMemOperand::MOStore;
5720 Info.align.reset();
5721 Infos.push_back(Elt: Info);
5722 return;
5723 }
5724
5725 case Intrinsic::nvvm_tcgen05_st_16x64b_x16:
5726 case Intrinsic::nvvm_tcgen05_st_16x128b_x8:
5727 case Intrinsic::nvvm_tcgen05_st_16x256b_x4:
5728 case Intrinsic::nvvm_tcgen05_st_32x32b_x16:
5729 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x16: {
5730 Info.opc = ISD::INTRINSIC_VOID;
5731 Info.memVT = MVT::v16i32;
5732 Info.ptrVal = I.getArgOperand(i: 0);
5733 Info.offset = 0;
5734 Info.flags = MachineMemOperand::MOStore;
5735 Info.align.reset();
5736 Infos.push_back(Elt: Info);
5737 return;
5738 }
5739
5740 case Intrinsic::nvvm_tcgen05_st_16x64b_x32:
5741 case Intrinsic::nvvm_tcgen05_st_16x128b_x16:
5742 case Intrinsic::nvvm_tcgen05_st_16x256b_x8:
5743 case Intrinsic::nvvm_tcgen05_st_32x32b_x32:
5744 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x32: {
5745 Info.opc = ISD::INTRINSIC_VOID;
5746 Info.memVT = MVT::v32i32;
5747 Info.ptrVal = I.getArgOperand(i: 0);
5748 Info.offset = 0;
5749 Info.flags = MachineMemOperand::MOStore;
5750 Info.align.reset();
5751 Infos.push_back(Elt: Info);
5752 return;
5753 }
5754
5755 case Intrinsic::nvvm_tcgen05_st_16x64b_x64:
5756 case Intrinsic::nvvm_tcgen05_st_16x128b_x32:
5757 case Intrinsic::nvvm_tcgen05_st_16x256b_x16:
5758 case Intrinsic::nvvm_tcgen05_st_32x32b_x64:
5759 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x64: {
5760 Info.opc = ISD::INTRINSIC_VOID;
5761 Info.memVT = MVT::v64i32;
5762 Info.ptrVal = I.getArgOperand(i: 0);
5763 Info.offset = 0;
5764 Info.flags = MachineMemOperand::MOStore;
5765 Info.align.reset();
5766 Infos.push_back(Elt: Info);
5767 return;
5768 }
5769
5770 case Intrinsic::nvvm_tcgen05_st_16x64b_x128:
5771 case Intrinsic::nvvm_tcgen05_st_16x128b_x64:
5772 case Intrinsic::nvvm_tcgen05_st_16x256b_x32:
5773 case Intrinsic::nvvm_tcgen05_st_32x32b_x128:
5774 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x128: {
5775 Info.opc = ISD::INTRINSIC_VOID;
5776 Info.memVT = MVT::v128i32;
5777 Info.ptrVal = I.getArgOperand(i: 0);
5778 Info.offset = 0;
5779 Info.flags = MachineMemOperand::MOStore;
5780 Info.align.reset();
5781 Infos.push_back(Elt: Info);
5782 return;
5783 }
5784 case Intrinsic::
5785 nvvm_tcgen05_mma_shared_f8f6f4_disable_output_lane_cg1_decompress_b:
5786 case Intrinsic::
5787 nvvm_tcgen05_mma_tensor_f8f6f4_disable_output_lane_cg1_decompress_b:
5788 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg1:
5789 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg1:
5790 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg1:
5791 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg1:
5792 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1:
5793 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1:
5794 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1_ashift:
5795 case Intrinsic::
5796 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1_ashift:
5797 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1:
5798 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1:
5799 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1_ashift:
5800 case Intrinsic::
5801 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1_ashift: {
5802 // We are reading and writing back to TMem
5803 Info.opc = ISD::INTRINSIC_VOID;
5804 Info.memVT = MVT::v4i32;
5805 Info.ptrVal = I.getArgOperand(i: 0);
5806 Info.offset = 0;
5807 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
5808 Info.align = Align(16);
5809 Infos.push_back(Elt: Info);
5810 return;
5811 }
5812
5813 case Intrinsic::
5814 nvvm_tcgen05_mma_shared_f8f6f4_disable_output_lane_cg2_decompress_b:
5815 case Intrinsic::
5816 nvvm_tcgen05_mma_tensor_f8f6f4_disable_output_lane_cg2_decompress_b:
5817 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg2:
5818 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg2:
5819 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg2:
5820 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg2:
5821 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2:
5822 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2:
5823 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2:
5824 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2:
5825 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2_ashift:
5826 case Intrinsic::
5827 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2_ashift:
5828 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2_ashift:
5829 case Intrinsic::
5830 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2_ashift: {
5831 // We are reading and writing back to TMem
5832 Info.opc = ISD::INTRINSIC_VOID;
5833 Info.memVT = MVT::v8i32;
5834 Info.ptrVal = I.getArgOperand(i: 0);
5835 Info.offset = 0;
5836 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
5837 Info.align = Align(16);
5838 Infos.push_back(Elt: Info);
5839 return;
5840 }
5841 case Intrinsic::nvvm_tcgen05_alloc_cg1:
5842 case Intrinsic::nvvm_tcgen05_alloc_cg2:
5843 Info.opc = ISD::INTRINSIC_VOID;
5844 Info.memVT = MVT::i32;
5845 Info.ptrVal = I.getArgOperand(i: 0);
5846 Info.offset = 0;
5847 Info.flags = MachineMemOperand::MOStore;
5848 Info.align = Align(4);
5849 Infos.push_back(Elt: Info);
5850 return;
5851 }
5852}
5853
5854// Helper for getting a function parameter symbol. Its name is composed from
5855// the function name and the parameter index. Negative index corresponds to the
5856// special parameter (unsized array) used for passing variable arguments.
5857MCSymbol *NVPTXTargetLowering::getParamSymbol(MCContext &Ctx, const Function *F,
5858 int Idx) const {
5859 const StringRef FuncName = getTargetMachine().getSymbol(GV: F)->getName();
5860 if (Idx < 0)
5861 return Ctx.getOrCreateSymbol(Name: FuncName + "_vararg");
5862 return Ctx.getOrCreateSymbol(Name: FuncName + "_param_" + Twine(Idx));
5863}
5864
5865/// isLegalAddressingMode - Return true if the addressing mode represented
5866/// by AM is legal for this target, for a load/store of the specified type.
5867/// Used to guide target specific optimizations, like loop strength reduction
5868/// (LoopStrengthReduce.cpp) and memory optimization for address mode
5869/// (CodeGenPrepare.cpp)
5870bool NVPTXTargetLowering::isLegalAddressingMode(const DataLayout &DL,
5871 const AddrMode &AM, Type *Ty,
5872 unsigned AS, Instruction *I) const {
5873 // AddrMode - This represents an addressing mode of:
5874 // BaseGV + BaseOffs + BaseReg + Scale*ScaleReg
5875 //
5876 // The legal address modes are
5877 // - [avar]
5878 // - [areg]
5879 // - [areg+immoff]
5880 // - [immAddr]
5881
5882 // immoff must fit in a signed 32-bit int
5883 if (!APInt(64, AM.BaseOffs).isSignedIntN(N: 32))
5884 return false;
5885
5886 if (AM.BaseGV)
5887 return !AM.BaseOffs && !AM.HasBaseReg && !AM.Scale;
5888
5889 switch (AM.Scale) {
5890 case 0: // "r", "r+i" or "i" is allowed
5891 break;
5892 case 1:
5893 if (AM.HasBaseReg) // "r+r+i" or "r+r" is not allowed.
5894 return false;
5895 // Otherwise we have r+i.
5896 break;
5897 default:
5898 // No scale > 1 is allowed
5899 return false;
5900 }
5901 return true;
5902}
5903
5904//===----------------------------------------------------------------------===//
5905// NVPTX Inline Assembly Support
5906//===----------------------------------------------------------------------===//
5907
5908/// getConstraintType - Given a constraint letter, return the type of
5909/// constraint it is for this target.
5910NVPTXTargetLowering::ConstraintType
5911NVPTXTargetLowering::getConstraintType(StringRef Constraint) const {
5912 if (Constraint.size() == 1) {
5913 switch (Constraint[0]) {
5914 default:
5915 break;
5916 case 'b':
5917 case 'r':
5918 case 'h':
5919 case 'c':
5920 case 'l':
5921 case 'f':
5922 case 'd':
5923 case 'q':
5924 case '0':
5925 case 'N':
5926 return C_RegisterClass;
5927 }
5928 }
5929 return TargetLowering::getConstraintType(Constraint);
5930}
5931
5932std::pair<unsigned, const TargetRegisterClass *>
5933NVPTXTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
5934 StringRef Constraint,
5935 MVT VT) const {
5936 if (Constraint.size() == 1) {
5937 switch (Constraint[0]) {
5938 case 'b':
5939 return std::make_pair(x: 0U, y: &NVPTX::B1RegClass);
5940 case 'c':
5941 case 'h':
5942 return std::make_pair(x: 0U, y: &NVPTX::B16RegClass);
5943 case 'r':
5944 case 'f':
5945 return std::make_pair(x: 0U, y: &NVPTX::B32RegClass);
5946 case 'l':
5947 case 'N':
5948 case 'd':
5949 return std::make_pair(x: 0U, y: &NVPTX::B64RegClass);
5950 case 'q': {
5951 if (!STI.hasFeature(Feature: NVPTX::SM70))
5952 report_fatal_error(reason: "Inline asm with 128 bit operands is only "
5953 "supported for sm_70 and higher!");
5954 return std::make_pair(x: 0U, y: &NVPTX::B128RegClass);
5955 }
5956 }
5957 }
5958 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
5959}
5960
5961//===----------------------------------------------------------------------===//
5962// NVPTX DAG Combining
5963//===----------------------------------------------------------------------===//
5964
5965bool NVPTXTargetLowering::allowFMA(MachineFunction &MF,
5966 CodeGenOptLevel OptLevel) const {
5967 // Always honor command-line argument
5968 if (FMAContractLevelOpt.getNumOccurrences() > 0)
5969 return FMAContractLevelOpt > 0;
5970
5971 // Do not contract if we're not optimizing the code.
5972 if (OptLevel == CodeGenOptLevel::None)
5973 return false;
5974
5975 return false;
5976}
5977
5978static bool isConstZero(const SDValue &Operand) {
5979 const auto *Const = dyn_cast<ConstantSDNode>(Val: Operand);
5980 return Const && Const->getZExtValue() == 0;
5981}
5982
5983/// PerformADDCombineWithOperands - Try DAG combinations for an ADD with
5984/// operands N0 and N1. This is a helper for PerformADDCombine that is
5985/// called with the default operands, and if that fails, with commuted
5986/// operands.
5987static SDValue
5988PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
5989 TargetLowering::DAGCombinerInfo &DCI) {
5990 EVT VT = N0.getValueType();
5991
5992 // Since integer multiply-add costs the same as integer multiply
5993 // but is more costly than integer add, do the fusion only when
5994 // the mul is only used in the add.
5995 // TODO: this may not be true for later architectures, consider relaxing this
5996 if (!N0.getNode()->hasOneUse())
5997 return SDValue();
5998
5999 // fold (add (select cond, 0, (mul a, b)), c)
6000 // -> (select cond, c, (add (mul a, b), c))
6001 //
6002 if (N0.getOpcode() == ISD::SELECT) {
6003 unsigned ZeroOpNum;
6004 if (isConstZero(Operand: N0->getOperand(Num: 1)))
6005 ZeroOpNum = 1;
6006 else if (isConstZero(Operand: N0->getOperand(Num: 2)))
6007 ZeroOpNum = 2;
6008 else
6009 return SDValue();
6010
6011 SDValue M = N0->getOperand(Num: (ZeroOpNum == 1) ? 2 : 1);
6012 if (M->getOpcode() != ISD::MUL || !M.getNode()->hasOneUse())
6013 return SDValue();
6014
6015 SDLoc DL(N);
6016 SDValue Mul =
6017 DCI.DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: M->getOperand(Num: 0), N2: M->getOperand(Num: 1));
6018 SDValue MAD = DCI.DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Mul, N2: N1);
6019 return DCI.DAG.getSelect(DL: SDLoc(N), VT, Cond: N0->getOperand(Num: 0),
6020 LHS: ((ZeroOpNum == 1) ? N1 : MAD),
6021 RHS: ((ZeroOpNum == 1) ? MAD : N1));
6022 }
6023
6024 return SDValue();
6025}
6026
6027SDValue NVPTXTargetLowering::performFADDCombineWithOperands(
6028 SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI,
6029 CodeGenOptLevel OptLevel) const {
6030 EVT VT = N0.getValueType();
6031 if (N0.getOpcode() == ISD::FMUL) {
6032 if (!(allowFMA(MF&: DCI.DAG.getMachineFunction(), OptLevel) ||
6033 (N->getFlags().hasAllowContract() &&
6034 N0->getFlags().hasAllowContract())))
6035 return SDValue();
6036
6037 // For floating point:
6038 // Do the fusion only when the mul has less than 5 uses and all
6039 // are add.
6040 // The heuristic is that if a use is not an add, then that use
6041 // cannot be fused into fma, therefore mul is still needed anyway.
6042 // If there are more than 4 uses, even if they are all add, fusing
6043 // them will increase register pressue.
6044 //
6045 int numUses = 0;
6046 int nonAddCount = 0;
6047 for (const SDNode *User : N0.getNode()->users()) {
6048 numUses++;
6049 if (User->getOpcode() != ISD::FADD)
6050 ++nonAddCount;
6051 if (numUses >= 5)
6052 return SDValue();
6053 }
6054 if (nonAddCount) {
6055 int orderNo = N->getIROrder();
6056 int orderNo2 = N0.getNode()->getIROrder();
6057 // simple heuristics here for considering potential register
6058 // pressure, the logics here is that the differnce are used
6059 // to measure the distance between def and use, the longer distance
6060 // more likely cause register pressure.
6061 if (orderNo - orderNo2 < 500)
6062 return SDValue();
6063
6064 // Now, check if at least one of the FMUL's operands is live beyond the
6065 // node N, which guarantees that the FMA will not increase register
6066 // pressure at node N.
6067 bool opIsLive = false;
6068 const SDNode *left = N0.getOperand(i: 0).getNode();
6069 const SDNode *right = N0.getOperand(i: 1).getNode();
6070
6071 if (isa<ConstantSDNode>(Val: left) || isa<ConstantSDNode>(Val: right))
6072 opIsLive = true;
6073
6074 if (!opIsLive)
6075 for (const SDNode *User : left->users()) {
6076 int orderNo3 = User->getIROrder();
6077 if (orderNo3 > orderNo) {
6078 opIsLive = true;
6079 break;
6080 }
6081 }
6082
6083 if (!opIsLive)
6084 for (const SDNode *User : right->users()) {
6085 int orderNo3 = User->getIROrder();
6086 if (orderNo3 > orderNo) {
6087 opIsLive = true;
6088 break;
6089 }
6090 }
6091
6092 if (!opIsLive)
6093 return SDValue();
6094 }
6095
6096 return DCI.DAG.getNode(Opcode: ISD::FMA, DL: SDLoc(N), VT, N1: N0.getOperand(i: 0),
6097 N2: N0.getOperand(i: 1), N3: N1);
6098 }
6099
6100 return SDValue();
6101}
6102
6103/// Fold unpacking movs into a load by increasing the number of return values.
6104///
6105/// ex:
6106/// L: v2f16,ch = load <p>
6107/// a: f16 = extractelt L:0, 0
6108/// b: f16 = extractelt L:0, 1
6109/// use(a, b)
6110///
6111/// ...is turned into...
6112///
6113/// L: f16,f16,ch = LoadV2 <p>
6114/// use(L:0, L:1)
6115static SDValue
6116combineUnpackingMovIntoLoad(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
6117 // Don't run this optimization before the legalizer
6118 if (!DCI.isAfterLegalizeDAG())
6119 return SDValue();
6120
6121 EVT ElementVT = N->getValueType(ResNo: 0);
6122 // Avoid non-packed types and v4i8
6123 if (!NVPTX::isPackedVectorTy(VT: ElementVT) || ElementVT == MVT::v4i8)
6124 return SDValue();
6125
6126 // Check whether all outputs are either used by an extractelt or are
6127 // glue/chain nodes
6128 if (!all_of(Range: N->uses(), P: [&](SDUse &U) {
6129 // Skip glue, chain nodes
6130 if (U.getValueType() == MVT::Glue || U.getValueType() == MVT::Other)
6131 return true;
6132 if (U.getUser()->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
6133 if (N->getOpcode() != ISD::LOAD)
6134 return true;
6135 // Since this is an ISD::LOAD, check all extractelts are used. If
6136 // any are not used, we don't want to defeat another optimization that
6137 // will narrow the load.
6138 //
6139 // For example:
6140 //
6141 // L: v2f16,ch = load <p>
6142 // e0: f16 = extractelt L:0, 0
6143 // e1: f16 = extractelt L:0, 1 <-- unused
6144 // store e0
6145 //
6146 // Can be optimized by DAGCombiner to:
6147 //
6148 // L: f16,ch = load <p>
6149 // store L:0
6150 return !U.getUser()->use_empty();
6151 }
6152
6153 // Otherwise, this use prevents us from splitting a value.
6154 return false;
6155 }))
6156 return SDValue();
6157
6158 auto *LD = cast<MemSDNode>(Val: N);
6159 SDLoc DL(LD);
6160
6161 // the new opcode after we double the number of operands
6162 unsigned Opcode;
6163 SmallVector<SDValue> Operands(LD->ops());
6164 unsigned OldNumOutputs; // non-glue, non-chain outputs
6165 switch (LD->getOpcode()) {
6166 case ISD::LOAD:
6167 OldNumOutputs = 1;
6168 // Any packed type is legal, so the legalizer will not have lowered
6169 // ISD::LOAD -> NVPTXISD::Load (unless it's under-aligned). We have to do it
6170 // here.
6171 Opcode = NVPTXISD::LoadV2;
6172 // append a "full" used bytes mask operand right before the extension type
6173 // operand, signifying that all bytes are used.
6174 Operands.push_back(Elt: DCI.DAG.getConstant(UINT32_MAX, DL, VT: MVT::i32));
6175 Operands.push_back(Elt: DCI.DAG.getIntPtrConstant(
6176 Val: cast<LoadSDNode>(Val: LD)->getExtensionType(), DL));
6177 break;
6178 case NVPTXISD::LoadV2:
6179 OldNumOutputs = 2;
6180 Opcode = NVPTXISD::LoadV4;
6181 break;
6182 case NVPTXISD::LoadV4:
6183 // V8 is only supported for f32/i32. Don't forget, we're not changing the
6184 // load size here. This is already a 256-bit load.
6185 if (ElementVT != MVT::v2f32 && ElementVT != MVT::v2i32)
6186 return SDValue();
6187 OldNumOutputs = 4;
6188 Opcode = NVPTXISD::LoadV8;
6189 break;
6190 case NVPTXISD::LoadV8:
6191 // PTX doesn't support the next doubling of outputs
6192 return SDValue();
6193 }
6194
6195 // the non-glue, non-chain outputs in the new load
6196 const unsigned NewNumOutputs = OldNumOutputs * 2;
6197 SmallVector<EVT> NewVTs(NewNumOutputs, ElementVT.getVectorElementType());
6198 // add remaining chain and glue values
6199 NewVTs.append(in_start: LD->value_begin() + OldNumOutputs, in_end: LD->value_end());
6200
6201 // Create the new load
6202 SDValue NewLoad = DCI.DAG.getMemIntrinsicNode(
6203 Opcode, dl: DL, VTList: DCI.DAG.getVTList(VTs: NewVTs), Ops: Operands, MemVT: LD->getMemoryVT(),
6204 MMO: LD->getMemOperand());
6205
6206 // Now we use a combination of BUILD_VECTORs and a MERGE_VALUES node to keep
6207 // the outputs the same. These nodes will be optimized away in later
6208 // DAGCombiner iterations.
6209 SmallVector<SDValue> Results;
6210 for (unsigned I : seq(Size: OldNumOutputs))
6211 Results.push_back(Elt: DCI.DAG.getBuildVector(
6212 VT: ElementVT, DL, Ops: {NewLoad.getValue(R: I * 2), NewLoad.getValue(R: I * 2 + 1)}));
6213 // Add remaining chain and glue nodes
6214 for (unsigned I : seq(Size: NewLoad->getNumValues() - NewNumOutputs))
6215 Results.push_back(Elt: NewLoad.getValue(R: NewNumOutputs + I));
6216
6217 return DCI.DAG.getMergeValues(Ops: Results, dl: DL);
6218}
6219
6220/// Fold packing movs into a store.
6221///
6222/// ex:
6223/// v1: v2f16 = BUILD_VECTOR a:f16, b:f16
6224/// v2: v2f16 = BUILD_VECTOR c:f16, d:f16
6225/// StoreV2 v1, v2
6226///
6227/// ...is turned into...
6228///
6229/// StoreV4 a, b, c, d
6230static SDValue combinePackingMovIntoStore(SDNode *N,
6231 TargetLowering::DAGCombinerInfo &DCI,
6232 unsigned Front, unsigned Back) {
6233 // We want to run this as late as possible since other optimizations may
6234 // eliminate the BUILD_VECTORs.
6235 if (!DCI.isAfterLegalizeDAG())
6236 return SDValue();
6237
6238 // Get the type of the operands being stored.
6239 EVT ElementVT = N->getOperand(Num: Front).getValueType();
6240
6241 // Avoid non-packed types and v4i8
6242 if (!NVPTX::isPackedVectorTy(VT: ElementVT) || ElementVT == MVT::v4i8)
6243 return SDValue();
6244
6245 auto *ST = cast<MemSDNode>(Val: N);
6246
6247 // The new opcode after we double the number of operands.
6248 unsigned Opcode;
6249 switch (N->getOpcode()) {
6250 case ISD::STORE:
6251 // Any packed type is legal, so the legalizer will not have lowered
6252 // ISD::STORE -> NVPTXISD::Store (unless it's under-aligned). We have to do
6253 // it here.
6254 Opcode = NVPTXISD::StoreV2;
6255 break;
6256 case NVPTXISD::StoreV2:
6257 Opcode = NVPTXISD::StoreV4;
6258 break;
6259 case NVPTXISD::StoreV4:
6260 // V8 is only supported for f32/i32. Don't forget, we're not changing the
6261 // store size here. This is already a 256-bit store.
6262 if (ElementVT != MVT::v2f32 && ElementVT != MVT::v2i32)
6263 return SDValue();
6264 Opcode = NVPTXISD::StoreV8;
6265 break;
6266 case NVPTXISD::StoreV8:
6267 // PTX doesn't support the next doubling of operands
6268 return SDValue();
6269 default:
6270 llvm_unreachable("Unhandled store opcode");
6271 }
6272
6273 // Scan the operands and if they're all BUILD_VECTORs, we'll have gathered
6274 // their elements.
6275 SmallVector<SDValue, 4> Operands(N->ops().take_front(N: Front));
6276 for (SDValue BV : N->ops().drop_front(N: Front).drop_back(N: Back)) {
6277 if (BV.getOpcode() != ISD::BUILD_VECTOR)
6278 return SDValue();
6279
6280 // If the operand has multiple uses, this optimization can increase register
6281 // pressure.
6282 if (!BV.hasOneUse())
6283 return SDValue();
6284
6285 // DAGCombiner visits nodes bottom-up. Check the BUILD_VECTOR operands for
6286 // any signs they may be folded by some other pattern or rule.
6287 for (SDValue Op : BV->ops()) {
6288 // Peek through bitcasts
6289 if (Op.getOpcode() == ISD::BITCAST)
6290 Op = Op.getOperand(i: 0);
6291
6292 // This may be folded into a PRMT.
6293 if (Op.getValueType() == MVT::i16 && Op.getOpcode() == ISD::TRUNCATE &&
6294 Op->getOperand(Num: 0).getValueType() == MVT::i32)
6295 return SDValue();
6296
6297 // This may be folded into cvt.bf16x2
6298 if (Op.getOpcode() == ISD::FP_ROUND)
6299 return SDValue();
6300 }
6301 Operands.append(IL: {BV.getOperand(i: 0), BV.getOperand(i: 1)});
6302 }
6303 Operands.append(in_start: N->op_end() - Back, in_end: N->op_end());
6304
6305 // Now we replace the store
6306 return DCI.DAG.getMemIntrinsicNode(Opcode, dl: SDLoc(N), VTList: N->getVTList(), Ops: Operands,
6307 MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand());
6308}
6309
6310static SDValue combineSTORE(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
6311 const NVPTXSubtarget &STI) {
6312
6313 if (DCI.isBeforeLegalize() && N->getOpcode() == ISD::STORE) {
6314 // Here is our chance to custom lower a store with a non-simple type.
6315 // Unfortunately, we can't do this in the legalizer because there is no
6316 // way to setOperationAction for an non-simple type.
6317 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
6318 if (!ST->getValue().getValueType().isSimple())
6319 return lowerSTOREVector(Op: SDValue(ST, 0), DAG&: DCI.DAG, STI);
6320 }
6321
6322 return combinePackingMovIntoStore(N, DCI, Front: 1, Back: 2);
6323}
6324
6325static SDValue combineLOAD(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
6326 const NVPTXSubtarget &STI) {
6327 if (DCI.isBeforeLegalize() && N->getOpcode() == ISD::LOAD) {
6328 // Here is our chance to custom lower a load with a non-simple type.
6329 // Unfortunately, we can't do this in the legalizer because there is no
6330 // way to setOperationAction for an non-simple type.
6331 if (!N->getValueType(ResNo: 0).isSimple())
6332 return lowerLoadVector(N, DAG&: DCI.DAG, STI);
6333 }
6334
6335 return combineUnpackingMovIntoLoad(N, DCI);
6336}
6337
6338/// PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
6339///
6340static SDValue PerformADDCombine(SDNode *N,
6341 TargetLowering::DAGCombinerInfo &DCI,
6342 CodeGenOptLevel OptLevel) {
6343 if (OptLevel == CodeGenOptLevel::None)
6344 return SDValue();
6345
6346 SDValue N0 = N->getOperand(Num: 0);
6347 SDValue N1 = N->getOperand(Num: 1);
6348
6349 // Skip non-integer, non-scalar case
6350 EVT VT = N0.getValueType();
6351 if (VT.isVector() || VT != MVT::i32)
6352 return SDValue();
6353
6354 // First try with the default operand order.
6355 if (SDValue Result = PerformADDCombineWithOperands(N, N0, N1, DCI))
6356 return Result;
6357
6358 // If that didn't work, try again with the operands commuted.
6359 return PerformADDCombineWithOperands(N, N0: N1, N1: N0, DCI);
6360}
6361
6362/// Check if a v2f32 BUILD_VECTOR provably packs values from non-adjacent
6363/// register pairs (non-coalescable).
6364static bool isNonCoalescableBuildVector(const SDValue &BV) {
6365 if (BV.getOpcode() != ISD::BUILD_VECTOR || BV.getValueType() != MVT::v2f32)
6366 return false;
6367
6368 SDValue Elt0 = BV.getOperand(i: 0);
6369 SDValue Elt1 = BV.getOperand(i: 1);
6370
6371 bool IsExt0 = Elt0.getOpcode() == ISD::EXTRACT_VECTOR_ELT;
6372 bool IsExt1 = Elt1.getOpcode() == ISD::EXTRACT_VECTOR_ELT;
6373
6374 // If neither element is an EXTRACT_VECTOR_ELT they are free-standing
6375 // scalars and the register allocator can still place them side-by-side.
6376 if (!IsExt0 && !IsExt1)
6377 return false;
6378
6379 // If exactly one element is an EXTRACT_VECTOR_ELT, the other is a scalar
6380 // that cannot generally occupy the adjacent register slot.
6381 if (IsExt0 != IsExt1)
6382 return true;
6383
6384 // At this point both sources are extracting from vectors. If they are from
6385 // different vectors, then the BUILD_VECTOR is non-coalescable.
6386 SDValue Src0 = Elt0.getOperand(i: 0);
6387 SDValue Src1 = Elt1.getOperand(i: 0);
6388 if (Src0 != Src1)
6389 return true;
6390
6391 auto *Idx0 = dyn_cast<ConstantSDNode>(Val: Elt0.getOperand(i: 1));
6392 auto *Idx1 = dyn_cast<ConstantSDNode>(Val: Elt1.getOperand(i: 1));
6393 // If both indices are dynamic they will be lowered to
6394 // loads and the vector will be spilled to local memory. The register
6395 // allocator can easily place the results in adjacent registers.
6396 if (!Idx0 && !Idx1)
6397 return false;
6398
6399 // If one index is dynamic and the other is constant, the value from the
6400 // constant load will result in an additional register to pair with the result
6401 // from the dynamic load. We consider this non-coalescable.
6402 if ((Idx0 && !Idx1) || (!Idx0 && Idx1))
6403 return true;
6404
6405 // Both are constant, adjacent pairs are coalescable
6406 return std::abs(i: Idx0->getSExtValue() - Idx1->getSExtValue()) != 1;
6407}
6408
6409/// Return true if FMUL v2f32 node \p N may be scalarized to fold each lane's
6410/// product into a scalar FMA.
6411bool NVPTXTargetLowering::mayFoldFMULIntoFMA(SDNode *N, MachineFunction &MF,
6412 CodeGenOptLevel OptLevel) const {
6413 if (N->getOpcode() != ISD::FMUL || N->getValueType(ResNo: 0) != MVT::v2f32)
6414 return false;
6415 const bool GlobalFMA = allowFMA(MF, OptLevel);
6416 if (!N->getFlags().hasAllowContract() && !GlobalFMA)
6417 return false;
6418
6419 const SDNode *FirstFAdd = nullptr;
6420 unsigned NumScalarFAdd = 0;
6421
6422 // Both lanes must feed unique FADDs
6423 for (SDNode *EE : N->users()) {
6424 if (NumScalarFAdd == 2)
6425 return false;
6426
6427 if (EE->getOpcode() != ISD::EXTRACT_VECTOR_ELT || !EE->hasOneUse() ||
6428 !isa<ConstantSDNode>(Val: EE->getOperand(Num: 1)))
6429 return false;
6430
6431 const SDNode *const FAdd = *EE->users().begin();
6432 if (FAdd->getOpcode() != ISD::FADD ||
6433 (!GlobalFMA && !FAdd->getFlags().hasAllowContract()))
6434 return false;
6435
6436 if (!FirstFAdd)
6437 FirstFAdd = FAdd;
6438 else if (FAdd == FirstFAdd)
6439 return false;
6440
6441 NumScalarFAdd++;
6442 }
6443
6444 return NumScalarFAdd == 2;
6445}
6446
6447/// Scalarize a v2f32 arithmetic node (FADD, FMUL, FSUB, FMA) when at least
6448/// one operand is a BUILD_VECTOR that repacks values from non-adjacent register
6449/// pairs. Without this combine the BUILD_VECTOR forces allocation of a
6450/// temporary 64-bit register, increasing register pressure.
6451///
6452/// Example - before:
6453/// t0: v2f32,v2f32,ch = LoadV2 ...
6454/// t1: f32 = extract_vector_elt t0, 0
6455/// t2: f32 = extract_vector_elt t0:1, 0
6456/// t3: v2f32 = BUILD_VECTOR t1, t2 ;; non-coalescable repack
6457/// t4: v2f32 = fma t_a, t3, t_c
6458///
6459/// After:
6460/// t0: v2f32,v2f32,ch = LoadV2 ...
6461/// t1: f32 = extract_vector_elt t0, 0
6462/// t2: f32 = extract_vector_elt t0:1, 0
6463/// a0: f32 = extract_vector_elt t_a, 0
6464/// a1: f32 = extract_vector_elt t_a, 1
6465/// c0: f32 = extract_vector_elt t_c, 0
6466/// c1: f32 = extract_vector_elt t_c, 1
6467/// r0: f32 = fma a0, t1, c0
6468/// r1: f32 = fma a1, t2, c1
6469/// t4: v2f32 = BUILD_VECTOR r0, r1
6470///
6471/// Also scalarizes an FMUL when all output lanes feed into scalar FADDs
6472/// to enable scalar FMA combining.
6473SDValue NVPTXTargetLowering::performScalarizeV2F32Op(
6474 SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
6475 CodeGenOptLevel OptLevel) const {
6476 EVT VT = N->getValueType(ResNo: 0);
6477 if (VT != MVT::v2f32)
6478 return SDValue();
6479
6480 if (none_of(Range: N->ops(), P: isNonCoalescableBuildVector) &&
6481 !mayFoldFMULIntoFMA(N, MF&: DCI.DAG.getMachineFunction(), OptLevel))
6482 return SDValue();
6483
6484 SelectionDAG &DAG = DCI.DAG;
6485 SDLoc DL(N);
6486 EVT EltVT = VT.getVectorElementType();
6487 unsigned Opc = N->getOpcode();
6488
6489 // For each operand, get the scalar element at the given index: if the operand
6490 // is a BUILD_VECTOR, grab the element directly; otherwise, emit an
6491 // EXTRACT_VECTOR_ELT.
6492 auto GetElement = [&](SDValue Op, unsigned Index) -> SDValue {
6493 if (Op.getOpcode() == ISD::BUILD_VECTOR)
6494 return Op.getOperand(i: Index);
6495 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Op,
6496 N2: DAG.getVectorIdxConstant(Val: Index, DL));
6497 };
6498
6499 // Build scalar operand lists for element 0 and element 1.
6500 SmallVector<SDValue, 3> Ops0, Ops1;
6501 for (const SDValue &Op : N->ops()) {
6502 Ops0.push_back(Elt: GetElement(Op, 0));
6503 Ops1.push_back(Elt: GetElement(Op, 1));
6504 }
6505
6506 SDValue Res0 = DAG.getNode(Opcode: Opc, DL, VT: EltVT, Ops: Ops0, Flags: N->getFlags());
6507 SDValue Res1 = DAG.getNode(Opcode: Opc, DL, VT: EltVT, Ops: Ops1, Flags: N->getFlags());
6508
6509 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT, N1: Res0, N2: Res1);
6510}
6511
6512/// Target-specific dag combine xforms for ISD::FADD.
6513SDValue
6514NVPTXTargetLowering::performFADDCombine(SDNode *N,
6515 TargetLowering::DAGCombinerInfo &DCI,
6516 CodeGenOptLevel OptLevel) const {
6517 if (SDValue Result = performScalarizeV2F32Op(N, DCI, OptLevel))
6518 return Result;
6519
6520 SDValue N0 = N->getOperand(Num: 0);
6521 SDValue N1 = N->getOperand(Num: 1);
6522
6523 EVT VT = N0.getValueType();
6524 if (VT.isVector() || !(VT == MVT::f32 || VT == MVT::f64))
6525 return SDValue();
6526
6527 // First try with the default operand order.
6528 if (SDValue Result = performFADDCombineWithOperands(N, N0, N1, DCI, OptLevel))
6529 return Result;
6530
6531 // If that didn't work, try again with the operands commuted.
6532 return performFADDCombineWithOperands(N, N0: N1, N1: N0, DCI, OptLevel);
6533}
6534
6535/// Get 3-input version of a 2-input min/max opcode
6536static unsigned getMinMax3Opcode(unsigned MinMax2Opcode) {
6537 switch (MinMax2Opcode) {
6538 case ISD::FMAXNUM:
6539 case ISD::FMAXIMUMNUM:
6540 return NVPTXISD::FMAXNUM3;
6541 case ISD::FMINNUM:
6542 case ISD::FMINIMUMNUM:
6543 return NVPTXISD::FMINNUM3;
6544 case ISD::FMAXIMUM:
6545 return NVPTXISD::FMAXIMUM3;
6546 case ISD::FMINIMUM:
6547 return NVPTXISD::FMINIMUM3;
6548 default:
6549 llvm_unreachable("Invalid 2-input min/max opcode");
6550 }
6551}
6552
6553/// PerformFMinMaxCombine - Combine (fmaxnum (fmaxnum a, b), c) into
6554/// (fmaxnum3 a, b, c). Also covers other llvm min/max intrinsics.
6555static SDValue PerformFMinMaxCombine(SDNode *N,
6556 TargetLowering::DAGCombinerInfo &DCI,
6557 const NVPTXSubtarget &STI) {
6558
6559 // 3-input min/max requires PTX 8.8+ and SM_100+, and only supports f32s
6560 EVT VT = N->getValueType(ResNo: 0);
6561 if (VT != MVT::f32 || !STI.hasFeature(Feature: NVPTX::PTX88) ||
6562 !STI.hasFeature(Feature: NVPTX::SM100))
6563 return SDValue();
6564
6565 SDValue Op0 = N->getOperand(Num: 0);
6566 SDValue Op1 = N->getOperand(Num: 1);
6567 unsigned MinMaxOp2 = N->getOpcode();
6568 unsigned MinMaxOp3 = getMinMax3Opcode(MinMax2Opcode: MinMaxOp2);
6569
6570 if (Op0.getOpcode() == MinMaxOp2 && Op0.hasOneUse()) {
6571 // (maxnum (maxnum a, b), c) -> (maxnum3 a, b, c)
6572 SDValue A = Op0.getOperand(i: 0);
6573 SDValue B = Op0.getOperand(i: 1);
6574 SDValue C = Op1;
6575 return DCI.DAG.getNode(Opcode: MinMaxOp3, DL: SDLoc(N), VT, N1: A, N2: B, N3: C, Flags: N->getFlags());
6576 } else if (Op1.getOpcode() == MinMaxOp2 && Op1.hasOneUse()) {
6577 // (maxnum a, (maxnum b, c)) -> (maxnum3 a, b, c)
6578 SDValue A = Op0;
6579 SDValue B = Op1.getOperand(i: 0);
6580 SDValue C = Op1.getOperand(i: 1);
6581 return DCI.DAG.getNode(Opcode: MinMaxOp3, DL: SDLoc(N), VT, N1: A, N2: B, N3: C, Flags: N->getFlags());
6582 }
6583 return SDValue();
6584}
6585
6586// sext (mul.iN nsw x, y) => mul.wide.sN x, y
6587// zext (mul.iN nuw x, y) => mul.wide.uN x, y
6588// sext (shl.iN nsw x, const) => mul.wide.sN x, (1 << const)
6589// zext (shl.iN nuw x, const) => mul.wide.uN x, (1 << const)
6590static SDValue combineSZExtToMulWide(SDNode *N,
6591 TargetLowering::DAGCombinerInfo &DCI,
6592 CodeGenOptLevel OptLevel) {
6593 assert(N->getOpcode() == ISD::SIGN_EXTEND ||
6594 N->getOpcode() == ISD::ZERO_EXTEND);
6595
6596 if (OptLevel == CodeGenOptLevel::None)
6597 return SDValue();
6598
6599 SDValue Op = N->getOperand(Num: 0);
6600 if (!Op.hasOneUse())
6601 return SDValue();
6602
6603 EVT ToVT = N->getValueType(ResNo: 0);
6604 EVT FromVT = Op.getValueType();
6605 if (!((ToVT == MVT::i32 && FromVT == MVT::i16) ||
6606 (ToVT == MVT::i64 && FromVT == MVT::i32)))
6607 return SDValue();
6608
6609 bool IsSigned = N->getOpcode() == ISD::SIGN_EXTEND;
6610 if ((IsSigned && !Op->getFlags().hasNoSignedWrap()) ||
6611 (!IsSigned && !Op->getFlags().hasNoUnsignedWrap()))
6612 return SDValue();
6613
6614 SDLoc DL(N);
6615 SDValue LHS = Op.getOperand(i: 0);
6616 SDValue RHS = Op.getOperand(i: 1);
6617 unsigned MulWideOpcode =
6618 IsSigned ? NVPTXISD::MUL_WIDE_SIGNED : NVPTXISD::MUL_WIDE_UNSIGNED;
6619 if (Op.getOpcode() == ISD::MUL) {
6620 return DCI.DAG.getNode(Opcode: MulWideOpcode, DL, VT: ToVT, N1: LHS, N2: RHS);
6621 } else if (Op.getOpcode() == ISD::SHL && isa<ConstantSDNode>(Val: RHS)) {
6622 const auto ShiftAmt = Op.getConstantOperandVal(i: 1);
6623 const auto MulVal = APInt(FromVT.getSizeInBits(), 1) << ShiftAmt;
6624
6625 // Note that the sext (shl nsw ...) case doesn't work if 1 << const
6626 // overflows to a negative value! The only valid input values in this
6627 // case are 0 and -1 (all other values yield poison because of the nsw),
6628 // and mul.wide.sN would give us the wrong sign for -1. We could use
6629 // mul.wide.uN, but since this is a weird case anyway, we might as well not
6630 // apply this transformation at all.
6631 if (IsSigned && MulVal.isNegative())
6632 return SDValue();
6633
6634 RHS = DCI.DAG.getConstant(Val: MulVal, DL, VT: FromVT);
6635 return DCI.DAG.getNode(Opcode: MulWideOpcode, DL, VT: ToVT, N1: LHS, N2: RHS);
6636 }
6637
6638 return SDValue();
6639}
6640
6641/// Check whether Op is explicitly extended from at most \p OptSize bits,
6642/// setting \p IsSigned to the signedness of the extension.
6643static bool IsExtendedFromLte(SDValue Op, unsigned OptSize, bool &IsSigned) {
6644 if (Op.getOpcode() == ISD::SIGN_EXTEND ||
6645 Op.getOpcode() == ISD::SIGN_EXTEND_INREG) {
6646 EVT OrigVT = Op.getOperand(i: 0).getValueType();
6647 if (OrigVT.getFixedSizeInBits() <= OptSize) {
6648 IsSigned = true;
6649 return true;
6650 }
6651 } else if (Op.getOpcode() == ISD::ZERO_EXTEND) {
6652 EVT OrigVT = Op.getOperand(i: 0).getValueType();
6653 if (OrigVT.getFixedSizeInBits() <= OptSize) {
6654 IsSigned = false;
6655 return true;
6656 }
6657 }
6658
6659 return false;
6660}
6661
6662/// AreMulWideOperandsDemotable - Checks if the given LHS and RHS operands can
6663/// be demoted to \p OptSize bits without loss of information using the same
6664/// signedness. The signedness of the operands is placed in \p IsSigned.
6665static bool AreMulWideOperandsDemotable(SDValue LHS, SDValue RHS,
6666 unsigned OptSize,
6667 const SelectionDAG &DAG,
6668 bool &IsSigned) {
6669 // The native register size is 32 bits, so extending i16 to i32 is often
6670 // cheap, whereas truncating i64 to i32 is free. Use known bits to narrow
6671 // i64 products, and require explicit extensions for i32 products.
6672 if (LHS.getValueType() != MVT::i64) {
6673 if (!IsExtendedFromLte(Op: LHS, OptSize, IsSigned))
6674 return false;
6675
6676 if (auto *CI = dyn_cast<ConstantSDNode>(Val&: RHS)) {
6677 const APInt &Val = CI->getAPIntValue();
6678 return IsSigned ? Val.isSignedIntN(N: OptSize) : Val.isIntN(N: OptSize);
6679 }
6680
6681 bool RHSSigned;
6682 return IsExtendedFromLte(Op: RHS, OptSize, IsSigned&: RHSSigned) && IsSigned == RHSSigned;
6683 }
6684
6685 // Extensions may have been folded into masks, shifts, or other operations.
6686 // Check the values' widths rather than requiring explicit extension nodes.
6687 if (DAG.computeKnownBits(Op: LHS).countMaxActiveBits() <= OptSize &&
6688 DAG.computeKnownBits(Op: RHS).countMaxActiveBits() <= OptSize) {
6689 IsSigned = false;
6690 return true;
6691 }
6692 if (DAG.ComputeMaxSignificantBits(Op: LHS) <= OptSize &&
6693 DAG.ComputeMaxSignificantBits(Op: RHS) <= OptSize) {
6694 IsSigned = true;
6695 return true;
6696 }
6697 return false;
6698}
6699
6700/// TryMULWIDECombine - Attempt to replace a multiply of M bits with a multiply
6701/// of M/2 bits that produces an M-bit result (i.e. mul.wide). This transform
6702/// works on both multiply DAG nodes and SHL DAG nodes with a constant shift
6703/// amount.
6704static SDValue TryMULWIDECombine(SDNode *N,
6705 TargetLowering::DAGCombinerInfo &DCI) {
6706 EVT MulType = N->getValueType(ResNo: 0);
6707 if (MulType != MVT::i32 && MulType != MVT::i64) {
6708 return SDValue();
6709 }
6710
6711 SDLoc DL(N);
6712 unsigned OptSize = MulType.getSizeInBits() >> 1;
6713 SDValue LHS = N->getOperand(Num: 0);
6714 SDValue RHS = N->getOperand(Num: 1);
6715
6716 // Canonicalize the multiply so the constant (if any) is on the right
6717 if (N->getOpcode() == ISD::MUL) {
6718 if (isa<ConstantSDNode>(Val: LHS)) {
6719 std::swap(a&: LHS, b&: RHS);
6720 }
6721 }
6722
6723 bool Signed;
6724 // If we have a SHL, determine the actual multiply amount.
6725 if (N->getOpcode() == ISD::SHL) {
6726 // A plain shift is already one instruction. Only fold it when doing so
6727 // can also remove an explicit extension.
6728 if (!IsExtendedFromLte(Op: LHS, OptSize, IsSigned&: Signed))
6729 return SDValue();
6730
6731 ConstantSDNode *ShlRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
6732 if (!ShlRHS) {
6733 return SDValue();
6734 }
6735
6736 APInt ShiftAmt = ShlRHS->getAPIntValue();
6737 unsigned BitWidth = MulType.getSizeInBits();
6738 unsigned Limit = Signed ? OptSize - 1 : OptSize;
6739 if (ShiftAmt.sge(RHS: 0) && ShiftAmt.slt(RHS: Limit)) {
6740 APInt MulVal = APInt(BitWidth, 1) << ShiftAmt;
6741 RHS = DCI.DAG.getConstant(Val: MulVal, DL, VT: MulType);
6742 } else {
6743 return SDValue();
6744 }
6745 } else if (!AreMulWideOperandsDemotable(LHS, RHS, OptSize, DAG: DCI.DAG, IsSigned&: Signed)) {
6746 return SDValue();
6747 }
6748
6749 if (N->getOpcode() == ISD::MUL) {
6750 // Keep matching low/high products together so ptxas can share their
6751 // partial products. Widening only the low product can duplicate work and
6752 // increase register pressure, even when its operands fit the narrow type.
6753 for (unsigned Opc : {ISD::MULHU, ISD::MULHS}) {
6754 if (SDNode *Hi = DCI.DAG.getNodeIfExists(Opcode: Opc, VTList: N->getVTList(), Ops: {LHS, RHS}))
6755 if (!Hi->use_empty())
6756 return SDValue();
6757 if (SDNode *Hi = DCI.DAG.getNodeIfExists(Opcode: Opc, VTList: N->getVTList(), Ops: {RHS, LHS}))
6758 if (!Hi->use_empty())
6759 return SDValue();
6760 }
6761 }
6762
6763 EVT DemotedVT;
6764 if (MulType == MVT::i32) {
6765 DemotedVT = MVT::i16;
6766 } else {
6767 DemotedVT = MVT::i32;
6768 }
6769
6770 // Truncate the operands to the correct size. Note that these are just for
6771 // type consistency and will (likely) be eliminated in later phases.
6772 SDValue TruncLHS =
6773 DCI.DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DemotedVT, Operand: LHS);
6774 SDValue TruncRHS =
6775 DCI.DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DemotedVT, Operand: RHS);
6776
6777 unsigned Opc;
6778 if (Signed) {
6779 Opc = NVPTXISD::MUL_WIDE_SIGNED;
6780 } else {
6781 Opc = NVPTXISD::MUL_WIDE_UNSIGNED;
6782 }
6783
6784 return DCI.DAG.getNode(Opcode: Opc, DL, VT: MulType, N1: TruncLHS, N2: TruncRHS);
6785}
6786
6787static bool isConstOne(const SDValue &Operand) {
6788 const auto *Const = dyn_cast<ConstantSDNode>(Val: Operand);
6789 return Const && Const->getZExtValue() == 1;
6790}
6791
6792static SDValue matchMADConstOnePattern(SDValue Add) {
6793 if (Add->getOpcode() != ISD::ADD)
6794 return SDValue();
6795
6796 if (isConstOne(Operand: Add->getOperand(Num: 0)))
6797 return Add->getOperand(Num: 1);
6798
6799 if (isConstOne(Operand: Add->getOperand(Num: 1)))
6800 return Add->getOperand(Num: 0);
6801
6802 return SDValue();
6803}
6804
6805static SDValue combineMADConstOne(SDValue X, SDValue Add, EVT VT, SDLoc DL,
6806 TargetLowering::DAGCombinerInfo &DCI) {
6807
6808 if (SDValue Y = matchMADConstOnePattern(Add)) {
6809 SDValue Mul = DCI.DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: X, N2: Y);
6810 return DCI.DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Mul, N2: X);
6811 }
6812
6813 return SDValue();
6814}
6815
6816static SDValue combineMulSelectConstOne(SDValue X, SDValue Select, EVT VT,
6817 SDLoc DL,
6818 TargetLowering::DAGCombinerInfo &DCI) {
6819 if (Select->getOpcode() != ISD::SELECT)
6820 return SDValue();
6821
6822 SDValue Cond = Select->getOperand(Num: 0);
6823
6824 unsigned ConstOpNo;
6825 if (isConstOne(Operand: Select->getOperand(Num: 1)))
6826 ConstOpNo = 1;
6827 else if (isConstOne(Operand: Select->getOperand(Num: 2)))
6828 ConstOpNo = 2;
6829 else
6830 return SDValue();
6831
6832 SDValue Y = Select->getOperand(Num: (ConstOpNo == 1) ? 2 : 1);
6833
6834 // Do not combine if the resulting sequence is not obviously profitable.
6835 if (!matchMADConstOnePattern(Add: Y))
6836 return SDValue();
6837
6838 SDValue NewMul = DCI.DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: X, N2: Y);
6839
6840 return DCI.DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
6841 N2: (ConstOpNo == 1) ? X : NewMul,
6842 N3: (ConstOpNo == 1) ? NewMul : X);
6843}
6844
6845static SDValue
6846PerformMULCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
6847 TargetLowering::DAGCombinerInfo &DCI) {
6848
6849 EVT VT = N0.getValueType();
6850 if (VT.isVector())
6851 return SDValue();
6852
6853 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)
6854 return SDValue();
6855
6856 SDLoc DL(N);
6857
6858 // (mul x, (add y, 1)) -> (add (mul x, y), x)
6859 if (SDValue Res = combineMADConstOne(X: N0, Add: N1, VT, DL, DCI))
6860 return Res;
6861 if (SDValue Res = combineMADConstOne(X: N1, Add: N0, VT, DL, DCI))
6862 return Res;
6863
6864 // (mul x, (select y, 1)) -> (select (mul x, y), x)
6865 if (SDValue Res = combineMulSelectConstOne(X: N0, Select: N1, VT, DL, DCI))
6866 return Res;
6867 if (SDValue Res = combineMulSelectConstOne(X: N1, Select: N0, VT, DL, DCI))
6868 return Res;
6869
6870 return SDValue();
6871}
6872
6873/// PerformMULCombine - Runs PTX-specific DAG combine patterns on MUL nodes.
6874static SDValue PerformMULCombine(SDNode *N,
6875 TargetLowering::DAGCombinerInfo &DCI,
6876 CodeGenOptLevel OptLevel) {
6877 if (OptLevel == CodeGenOptLevel::None)
6878 return SDValue();
6879
6880 if (SDValue Ret = TryMULWIDECombine(N, DCI))
6881 return Ret;
6882
6883 SDValue N0 = N->getOperand(Num: 0);
6884 SDValue N1 = N->getOperand(Num: 1);
6885 return PerformMULCombineWithOperands(N, N0, N1, DCI);
6886}
6887
6888/// Commute SHL with a bitwise logic operation when doing so exposes a common
6889/// shifted operand. For example:
6890///
6891/// Before:
6892/// N = shl (zext (LogicOp X, C)), ShiftAmount
6893/// OtherShift = shl (zext (OtherLogicOp X, OtherC)), ShiftAmount
6894///
6895/// After:
6896/// ShiftedX = shl (zext X), ShiftAmount
6897/// N = LogicOp ShiftedX, ShiftedC
6898/// OtherShift = OtherLogicOp ShiftedX, ShiftedOtherC
6899///
6900/// ShiftedC = (zext C) << ShiftAmount and ShiftedOtherC =
6901/// (zext OtherC) << ShiftAmount are folded constants. This replaces two
6902/// variable shifts with the single shared ShiftedX. Requiring another matching
6903/// shift avoids disrupting isolated address calculations where a shift may be
6904/// folded into the addressing mode.
6905static SDValue combineShiftOfLogicOp(SDNode *N,
6906 TargetLowering::DAGCombinerInfo &DCI) {
6907 using namespace SDPatternMatch;
6908
6909 struct ShiftOfLogicOp {
6910 SDNode *Shift;
6911 SDValue LogicOp;
6912 SDValue X;
6913 SDValue Constant;
6914 unsigned ExtendOpcode;
6915 };
6916
6917 // Match a logic operation, with an optional extension, inside a SHL.
6918 auto matchShiftOfLogicOp =
6919 [&](SDNode *Shift) -> std::optional<ShiftOfLogicOp> {
6920 if (Shift->getOpcode() != ISD::SHL || !Shift->getOperand(Num: 0).hasOneUse())
6921 return std::nullopt;
6922 ShiftOfLogicOp Match;
6923 Match.Shift = Shift;
6924 Match.LogicOp = Shift->getOperand(Num: 0);
6925 Match.ExtendOpcode = 0;
6926 if (ISD::isExtOpcode(Opcode: Match.LogicOp.getOpcode())) {
6927 Match.ExtendOpcode = Match.LogicOp.getOpcode();
6928 Match.LogicOp = Match.LogicOp.getOperand(i: 0);
6929 }
6930
6931 if (!sd_match(N: Match.LogicOp, P: m_OneUse(P: m_BitwiseLogic(
6932 L: m_Value(N&: Match.X),
6933 R: m_Value(N&: Match.Constant, P: m_ConstInt())))))
6934 return std::nullopt;
6935
6936 return Match;
6937 };
6938
6939 // Match N as the root shift-of-logic; bail if it does not fit the pattern.
6940 const std::optional<ShiftOfLogicOp> Root = matchShiftOfLogicOp(N);
6941 if (!Root)
6942 return SDValue();
6943
6944 // Only profitable for a constant shift amount: the per-op constant shift then
6945 // folds away instead of becoming an extra variable shift.
6946 if (!isConstOrConstSplat(N: N->getOperand(Num: 1)))
6947 return SDValue();
6948
6949 // Collect candidate shifts that share X. Reached through another user of X,
6950 // the logic result feeds the shift directly or through an optional extend.
6951 SmallVector<SDNode *, 4> CandidateShifts;
6952 for (const SDNode *CandidateLogicOp : Root->X->users()) {
6953 if (CandidateLogicOp == Root->LogicOp.getNode())
6954 continue;
6955 for (SDNode *LogicUser : CandidateLogicOp->users()) {
6956 if (ISD::isExtOpcode(Opcode: LogicUser->getOpcode())) {
6957 // shl (ext (logic X, C)): step through the extend to find the shift.
6958 for (SDNode *ExtendUser : LogicUser->users())
6959 if (ExtendUser->getOpcode() == ISD::SHL)
6960 CandidateShifts.push_back(Elt: ExtendUser);
6961 } else if (LogicUser->getOpcode() == ISD::SHL) {
6962 // shl (logic X, C): the user is already the shift.
6963 CandidateShifts.push_back(Elt: LogicUser);
6964 }
6965 }
6966 }
6967
6968 // Verify each candidate against the root's pattern: the same X, extension,
6969 // type, and shift amount.
6970 const EVT VT = N->getValueType(ResNo: 0);
6971 const SDValue ShiftAmount = N->getOperand(Num: 1);
6972 SmallVector<ShiftOfLogicOp, 4> Matches;
6973 for (SDNode *CandidateShift : CandidateShifts) {
6974 const std::optional<ShiftOfLogicOp> Candidate =
6975 matchShiftOfLogicOp(CandidateShift);
6976 if (Candidate && Candidate->X == Root->X &&
6977 Candidate->ExtendOpcode == Root->ExtendOpcode &&
6978 CandidateShift->getValueType(ResNo: 0) == VT &&
6979 CandidateShift->getOperand(Num: 1) == ShiftAmount)
6980 Matches.push_back(Elt: *Candidate);
6981 }
6982 if (Matches.empty())
6983 return SDValue();
6984
6985 // Build the shared shifted X once, then rewrite the root and every match
6986 // into a logic op over it so the shift is CSE'd.
6987 SelectionDAG &DAG = DCI.DAG;
6988 const SDValue ShiftedX =
6989 DAG.getNode(Opcode: ISD::SHL, DL: SDLoc(N), VT,
6990 N1: Root->ExtendOpcode
6991 ? DAG.getNode(Opcode: Root->ExtendOpcode, DL: SDLoc(N), VT, Operand: Root->X)
6992 : Root->X,
6993 N2: ShiftAmount);
6994
6995 // Rebuild the logic op from shared ShiftedX and a folded constant shift.
6996 auto buildCommutedLogicOp = [&](const SDValue LogicOp, SDValue C,
6997 const SDLoc &DL) {
6998 if (Root->ExtendOpcode)
6999 C = DAG.getNode(Opcode: Root->ExtendOpcode, DL, VT, Operand: C);
7000 const SDValue ShiftedC = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: C, N2: ShiftAmount);
7001 return DAG.getNode(Opcode: LogicOp.getOpcode(), DL, VT, N1: ShiftedX, N2: ShiftedC,
7002 Flags: LogicOp->getFlags());
7003 };
7004
7005 for (const ShiftOfLogicOp &Match : Matches)
7006 DCI.CombineTo(N: Match.Shift,
7007 Res: buildCommutedLogicOp(Match.LogicOp, Match.Constant,
7008 SDLoc(Match.Shift)));
7009 return buildCommutedLogicOp(Root->LogicOp, Root->Constant, SDLoc(N));
7010}
7011
7012/// PerformSHLCombine - Runs PTX-specific DAG combine patterns on SHL nodes.
7013static SDValue PerformSHLCombine(SDNode *N,
7014 TargetLowering::DAGCombinerInfo &DCI,
7015 CodeGenOptLevel OptLevel) {
7016 if (OptLevel > CodeGenOptLevel::None) {
7017 // Expose a shared shifted operand for CSE before mul.wide folding, which
7018 // would otherwise consume the shift.
7019 if (SDValue Ret = combineShiftOfLogicOp(N, DCI))
7020 return Ret;
7021
7022 // Try mul.wide combining at OptLevel > 0
7023 if (SDValue Ret = TryMULWIDECombine(N, DCI))
7024 return Ret;
7025 }
7026
7027 return SDValue();
7028}
7029
7030static SDValue PerformSETCCCombine(SDNode *N,
7031 TargetLowering::DAGCombinerInfo &DCI,
7032 const NVPTXSubtarget &STI) {
7033 EVT CCType = N->getValueType(ResNo: 0);
7034 SDValue A = N->getOperand(Num: 0);
7035 SDValue B = N->getOperand(Num: 1);
7036
7037 EVT AType = A.getValueType();
7038 if (!(CCType == MVT::v2i1 && (AType == MVT::v2f16 || AType == MVT::v2bf16)))
7039 return SDValue();
7040
7041 if (A.getValueType() == MVT::v2bf16 && !STI.hasFeature(Feature: NVPTX::SM90))
7042 return SDValue();
7043
7044 SDLoc DL(N);
7045 // setp.f16x2 returns two scalar predicates, which we need to
7046 // convert back to v2i1. The returned result will be scalarized by
7047 // the legalizer, but the comparison will remain a single vector
7048 // instruction.
7049 SDValue CCNode = DCI.DAG.getNode(
7050 Opcode: A.getValueType() == MVT::v2f16 ? NVPTXISD::SETP_F16X2
7051 : NVPTXISD::SETP_BF16X2,
7052 DL, VTList: DCI.DAG.getVTList(VT1: MVT::i1, VT2: MVT::i1), Ops: {A, B, N->getOperand(Num: 2)});
7053 return DCI.DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: CCType, N1: CCNode.getValue(R: 0),
7054 N2: CCNode.getValue(R: 1));
7055}
7056
7057static SDValue PerformEXTRACTCombine(SDNode *N,
7058 TargetLowering::DAGCombinerInfo &DCI) {
7059 SDValue Vector = peekThroughFreeze(V: N->getOperand(Num: 0));
7060 SDLoc DL(N);
7061 EVT VectorVT = Vector.getValueType();
7062 if (Vector->getOpcode() == ISD::LOAD && VectorVT.isSimple() &&
7063 IsPTXVectorType(VT: VectorVT.getSimpleVT()))
7064 return SDValue(); // Native vector loads already combine nicely w/
7065 // extract_vector_elt.
7066 // Don't mess with singletons or packed types (v2*32, v2*16, v4i8 and v8i8),
7067 // we already handle them OK.
7068 if (VectorVT.getVectorNumElements() == 1 ||
7069 NVPTX::isPackedVectorTy(VT: VectorVT) || VectorVT == MVT::v8i8)
7070 return SDValue();
7071
7072 // Don't mess with undef values as sra may be simplified to 0, not undef.
7073 if (Vector->isUndef() || ISD::allOperandsUndef(N: Vector.getNode()))
7074 return SDValue();
7075
7076 uint64_t VectorBits = VectorVT.getSizeInBits();
7077 // We only handle the types we can extract in-register.
7078 if (!(VectorBits == 16 || VectorBits == 32 || VectorBits == 64))
7079 return SDValue();
7080
7081 ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
7082 // Index == 0 is handled by generic DAG combiner.
7083 if (!Index || Index->getZExtValue() == 0)
7084 return SDValue();
7085
7086 MVT IVT = MVT::getIntegerVT(BitWidth: VectorBits);
7087 EVT EltVT = VectorVT.getVectorElementType();
7088 EVT EltIVT = EltVT.changeTypeToInteger();
7089 uint64_t EltBits = EltVT.getScalarSizeInBits();
7090
7091 SDValue Result = DCI.DAG.getNode(
7092 Opcode: ISD::TRUNCATE, DL, VT: EltIVT,
7093 Operand: DCI.DAG.getNode(
7094 Opcode: ISD::SRA, DL, VT: IVT, N1: DCI.DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IVT, Operand: Vector),
7095 N2: DCI.DAG.getConstant(Val: Index->getZExtValue() * EltBits, DL, VT: IVT)));
7096
7097 // If element has non-integer type, bitcast it back to the expected type.
7098 if (EltVT != EltIVT)
7099 Result = DCI.DAG.getNode(Opcode: ISD::BITCAST, DL, VT: EltVT, Operand: Result);
7100 // Past legalizer, we may need to extent i8 -> i16 to match the register type.
7101 if (EltVT != N->getValueType(ResNo: 0))
7102 Result = DCI.DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: N->getValueType(ResNo: 0), Operand: Result);
7103
7104 return Result;
7105}
7106
7107/// Transform patterns like:
7108/// (select (ugt shift_amt, BitWidth-1), 0, (srl/shl x, shift_amt))
7109/// (select (ult shift_amt, BitWidth), (srl/shl x, shift_amt), 0)
7110/// Into:
7111/// (NVPTXISD::SRL_CLAMP x, shift_amt) or (NVPTXISD::SHL_CLAMP x, shift_amt)
7112///
7113/// These patterns arise from code like `s >= 32 ? 0 : x >> s`. In LLVM,
7114/// over-shifting a value results in poison, but PTX shr/shl instructions clamp
7115/// the shift amount to BitWidth, making the guard redundant.
7116///
7117/// Note: We only handle SRL and SHL, not SRA, because arithmetic right shifts
7118/// can produce 0 or -1 when shift >= BitWidth.
7119/// Note: We don't handle uge or ule. These don't appear because of
7120/// canonicalization.
7121static SDValue PerformSELECTShiftCombine(SDNode *N,
7122 TargetLowering::DAGCombinerInfo &DCI) {
7123 if (!DCI.isAfterLegalizeDAG())
7124 return SDValue();
7125
7126 using namespace SDPatternMatch;
7127 unsigned BitWidth = N->getValueType(ResNo: 0).getSizeInBits();
7128 SDValue ShiftAmt, ShiftOp;
7129
7130 // Match logical shifts where the shift amount in the guard matches the shift
7131 // amount in the operation.
7132 auto LogicalShift = m_Value(
7133 N&: ShiftOp, P: m_AnyOf(preds: m_Srl(L: m_Value(), R: m_TruncOrSelf(Op: m_Deferred(V&: ShiftAmt))),
7134 preds: m_Shl(L: m_Value(), R: m_TruncOrSelf(Op: m_Deferred(V&: ShiftAmt)))));
7135
7136 // shift_amt > BitWidth-1 ? 0 : shift_op
7137 bool MatchedUGT = sd_match(
7138 N, P: m_Select(Cond: m_SpecificSetCC(CC: ISD::SETUGT, LHS: m_Value(N&: ShiftAmt),
7139 RHS: m_SpecificInt(V: APInt(BitWidth, BitWidth - 1))),
7140 T: m_Zero(), F: LogicalShift));
7141 // shift_amt < BitWidth ? shift_op : 0
7142 bool MatchedULT =
7143 !MatchedUGT &&
7144 sd_match(
7145 N, P: m_Select(Cond: m_SpecificSetCC(CC: ISD::SETULT, LHS: m_Value(N&: ShiftAmt),
7146 RHS: m_SpecificInt(V: APInt(BitWidth, BitWidth))),
7147 T: LogicalShift, F: m_Zero()));
7148
7149 if (!MatchedUGT && !MatchedULT)
7150 return SDValue();
7151
7152 // In LLVM IR, the shift amount and the value-to-be-shifted are the same
7153 // type, whereas in PTX the shift amount is always i32. Therefore when
7154 // shifting types larger than i32, we can only do this transformation if we
7155 // know that the upper bits of the shift amount are known zero.
7156 SDValue ClampAmt = ShiftOp.getOperand(i: 1);
7157 unsigned ClampAmtBits = ClampAmt.getValueSizeInBits();
7158 if (ShiftAmt.getValueSizeInBits() > ClampAmtBits &&
7159 DCI.DAG.computeKnownBits(Op: ShiftAmt).countMaxActiveBits() > ClampAmtBits)
7160 return SDValue();
7161
7162 // Return a clamp shift operation, which has the same semantics as PTX shift.
7163 unsigned ClampOpc = ShiftOp.getOpcode() == ISD::SRL ? NVPTXISD::SRL_CLAMP
7164 : NVPTXISD::SHL_CLAMP;
7165 return DCI.DAG.getNode(Opcode: ClampOpc, DL: SDLoc(N), VT: ShiftOp.getValueType(),
7166 N1: ShiftOp.getOperand(i: 0), N2: ClampAmt);
7167}
7168
7169static SDValue PerformVSELECTCombine(SDNode *N,
7170 TargetLowering::DAGCombinerInfo &DCI) {
7171 SDValue VA = N->getOperand(Num: 1);
7172 EVT VectorVT = VA.getValueType();
7173 if (VectorVT != MVT::v4i8)
7174 return SDValue();
7175
7176 // We need to split vselect into individual per-element operations Because we
7177 // use BFE/BFI instruction for byte extraction/insertion, we do end up with
7178 // 32-bit values, so we may as well do comparison as i32 to avoid conversions
7179 // to/from i16 normally used for i8 values.
7180 SmallVector<SDValue, 4> E;
7181 SDLoc DL(N);
7182 SDValue VCond = N->getOperand(Num: 0);
7183 SDValue VB = N->getOperand(Num: 2);
7184 for (int I = 0; I < 4; ++I) {
7185 SDValue C = DCI.DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i1, N1: VCond,
7186 N2: DCI.DAG.getConstant(Val: I, DL, VT: MVT::i32));
7187 SDValue EA = DCI.DAG.getAnyExtOrTrunc(
7188 Op: DCI.DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i8, N1: VA,
7189 N2: DCI.DAG.getConstant(Val: I, DL, VT: MVT::i32)),
7190 DL, VT: MVT::i32);
7191 SDValue EB = DCI.DAG.getAnyExtOrTrunc(
7192 Op: DCI.DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i8, N1: VB,
7193 N2: DCI.DAG.getConstant(Val: I, DL, VT: MVT::i32)),
7194 DL, VT: MVT::i32);
7195 E.push_back(Elt: DCI.DAG.getAnyExtOrTrunc(
7196 Op: DCI.DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::i32, N1: C, N2: EA, N3: EB), DL, VT: MVT::i8));
7197 }
7198 return DCI.DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: MVT::v4i8, Ops: E);
7199}
7200
7201static SDValue
7202PerformBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
7203 auto VT = N->getValueType(ResNo: 0);
7204 if (!DCI.isAfterLegalizeDAG() ||
7205 // only process v2*16 types
7206 !(NVPTX::isPackedVectorTy(VT) && VT.is32BitVector() &&
7207 VT.getVectorNumElements() == 2))
7208 return SDValue();
7209
7210 auto Op0 = N->getOperand(Num: 0);
7211 auto Op1 = N->getOperand(Num: 1);
7212
7213 // Start out by assuming we want to take the lower 2 bytes of each i32
7214 // operand.
7215 uint64_t Op0Bytes = 0x10;
7216 uint64_t Op1Bytes = 0x54;
7217
7218 std::pair<SDValue *, uint64_t *> OpData[2] = {{&Op0, &Op0Bytes},
7219 {&Op1, &Op1Bytes}};
7220
7221 // Check that each operand is an i16, truncated from an i32 operand. We'll
7222 // select individual bytes from those original operands. Optionally, fold in a
7223 // shift right of that original operand.
7224 for (auto &[Op, OpBytes] : OpData) {
7225 // Eat up any bitcast
7226 if (Op->getOpcode() == ISD::BITCAST)
7227 *Op = Op->getOperand(i: 0);
7228
7229 if (!(Op->getValueType() == MVT::i16 && Op->getOpcode() == ISD::TRUNCATE &&
7230 Op->getOperand(i: 0).getValueType() == MVT::i32))
7231 return SDValue();
7232
7233 // If the truncate has multiple uses, this optimization can increase
7234 // register pressure
7235 if (!Op->hasOneUse())
7236 return SDValue();
7237
7238 *Op = Op->getOperand(i: 0);
7239
7240 // Optionally, fold in a shift-right of the original operand and let permute
7241 // pick the two higher bytes of the original value directly.
7242 if (Op->getOpcode() == ISD::SRL && isa<ConstantSDNode>(Val: Op->getOperand(i: 1))) {
7243 if (cast<ConstantSDNode>(Val: Op->getOperand(i: 1))->getZExtValue() == 16) {
7244 // Shift the PRMT byte selector to pick upper bytes from each respective
7245 // value, instead of the lower ones: 0x10 -> 0x32, 0x54 -> 0x76
7246 assert((*OpBytes == 0x10 || *OpBytes == 0x54) &&
7247 "PRMT selector values out of range");
7248 *OpBytes += 0x22;
7249 *Op = Op->getOperand(i: 0);
7250 }
7251 }
7252 }
7253
7254 SDLoc DL(N);
7255 auto &DAG = DCI.DAG;
7256
7257 auto PRMT =
7258 getPRMT(A: DAG.getBitcast(VT: MVT::i32, V: Op0), B: DAG.getBitcast(VT: MVT::i32, V: Op1),
7259 Selector: (Op1Bytes << 8) | Op0Bytes, DL, DAG);
7260 return DAG.getBitcast(VT, V: PRMT);
7261}
7262
7263static SDValue combineADDRSPACECAST(SDNode *N,
7264 TargetLowering::DAGCombinerInfo &DCI) {
7265 auto *ASCN1 = cast<AddrSpaceCastSDNode>(Val: N);
7266
7267 if (auto *ASCN2 = dyn_cast<AddrSpaceCastSDNode>(Val: ASCN1->getOperand(Num: 0))) {
7268 assert(ASCN2->getDestAddressSpace() == ASCN1->getSrcAddressSpace());
7269
7270 // Fold asc[B -> A](asc[A -> B](x)) -> x
7271 if (ASCN1->getDestAddressSpace() == ASCN2->getSrcAddressSpace())
7272 return ASCN2->getOperand(Num: 0);
7273 }
7274
7275 return SDValue();
7276}
7277
7278// Given a constant selector value and a prmt mode, return the selector value
7279// normalized to the generic prmt mode. See the PTX ISA documentation for more
7280// details:
7281// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-prmt
7282static APInt getPRMTSelector(const APInt &Selector, unsigned Mode) {
7283 assert(Selector.getBitWidth() == 32 && "PRMT must have i32 operands");
7284
7285 if (Mode == NVPTX::PTXPrmtMode::NONE)
7286 return Selector;
7287
7288 const unsigned V = Selector.trunc(width: 2).getZExtValue();
7289
7290 const auto GetSelector = [](unsigned S0, unsigned S1, unsigned S2,
7291 unsigned S3) {
7292 return APInt(32, S0 | (S1 << 4) | (S2 << 8) | (S3 << 12));
7293 };
7294
7295 switch (Mode) {
7296 case NVPTX::PTXPrmtMode::F4E:
7297 return GetSelector(V, V + 1, V + 2, V + 3);
7298 case NVPTX::PTXPrmtMode::B4E:
7299 return GetSelector(V, (V - 1) & 7, (V - 2) & 7, (V - 3) & 7);
7300 case NVPTX::PTXPrmtMode::RC8:
7301 return GetSelector(V, V, V, V);
7302 case NVPTX::PTXPrmtMode::ECL:
7303 return GetSelector(V, std::max(a: V, b: 1U), std::max(a: V, b: 2U), 3U);
7304 case NVPTX::PTXPrmtMode::ECR:
7305 return GetSelector(0, std::min(a: V, b: 1U), std::min(a: V, b: 2U), V);
7306 case NVPTX::PTXPrmtMode::RC16: {
7307 unsigned V1 = (V & 1) << 1;
7308 return GetSelector(V1, V1 + 1, V1, V1 + 1);
7309 }
7310 default:
7311 llvm_unreachable("Invalid PRMT mode");
7312 }
7313}
7314
7315static APInt computePRMT(APInt A, APInt B, APInt Selector, unsigned Mode) {
7316 assert(A.getBitWidth() == 32 && B.getBitWidth() == 32 &&
7317 Selector.getBitWidth() == 32 && "PRMT must have i32 operands");
7318 // {b, a} = {{b7, b6, b5, b4}, {b3, b2, b1, b0}}
7319 APInt BitField = B.concat(NewLSB: A);
7320 APInt SelectorVal = getPRMTSelector(Selector, Mode);
7321 APInt Result(32, 0);
7322 for (unsigned I : llvm::seq(Size: 4U)) {
7323 APInt Sel = SelectorVal.extractBits(numBits: 4, bitPosition: I * 4);
7324 unsigned Idx = Sel.getLoBits(numBits: 3).getZExtValue();
7325 unsigned Sign = Sel.getHiBits(numBits: 1).getZExtValue();
7326 APInt Byte = BitField.extractBits(numBits: 8, bitPosition: Idx * 8);
7327 if (Sign)
7328 Byte = Byte.ashr(ShiftAmt: 8);
7329 Result.insertBits(SubBits: Byte, bitPosition: I * 8);
7330 }
7331 return Result;
7332}
7333
7334static SDValue combinePRMT(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
7335 CodeGenOptLevel OptLevel) {
7336 if (OptLevel == CodeGenOptLevel::None)
7337 return SDValue();
7338
7339 // Constant fold PRMT
7340 if (isa<ConstantSDNode>(Val: N->getOperand(Num: 0)) &&
7341 isa<ConstantSDNode>(Val: N->getOperand(Num: 1)) &&
7342 isa<ConstantSDNode>(Val: N->getOperand(Num: 2)))
7343 return DCI.DAG.getConstant(Val: computePRMT(A: N->getConstantOperandAPInt(Num: 0),
7344 B: N->getConstantOperandAPInt(Num: 1),
7345 Selector: N->getConstantOperandAPInt(Num: 2),
7346 Mode: N->getConstantOperandVal(Num: 3)),
7347 DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
7348 return SDValue();
7349}
7350
7351// During call lowering we wrap the return values in a ProxyReg node which
7352// depend on the chain value produced by the completed call. This ensures that
7353// the full call is emitted in cases where libcalls are used to legalize
7354// operations. To improve the functioning of other DAG combines we pull all
7355// operations we can through one of these nodes, ensuring that the ProxyReg
7356// directly wraps a load. That is:
7357//
7358// (ProxyReg (zext (load retval0))) => (zext (ProxyReg (load retval0)))
7359//
7360static SDValue sinkProxyReg(SDValue R, SDValue Chain,
7361 TargetLowering::DAGCombinerInfo &DCI) {
7362 switch (R.getOpcode()) {
7363 case ISD::TRUNCATE:
7364 case ISD::ANY_EXTEND:
7365 case ISD::SIGN_EXTEND:
7366 case ISD::ZERO_EXTEND:
7367 case ISD::BITCAST: {
7368 if (SDValue V = sinkProxyReg(R: R.getOperand(i: 0), Chain, DCI))
7369 return DCI.DAG.getNode(Opcode: R.getOpcode(), DL: SDLoc(R), VT: R.getValueType(), Operand: V);
7370 return SDValue();
7371 }
7372 case ISD::SHL:
7373 case ISD::SRL:
7374 case ISD::SRA:
7375 case ISD::OR: {
7376 if (SDValue A = sinkProxyReg(R: R.getOperand(i: 0), Chain, DCI))
7377 if (SDValue B = sinkProxyReg(R: R.getOperand(i: 1), Chain, DCI))
7378 return DCI.DAG.getNode(Opcode: R.getOpcode(), DL: SDLoc(R), VT: R.getValueType(), N1: A, N2: B);
7379 return SDValue();
7380 }
7381 case ISD::Constant:
7382 return R;
7383 case ISD::LOAD:
7384 case NVPTXISD::LoadV2:
7385 case NVPTXISD::LoadV4: {
7386 return DCI.DAG.getNode(Opcode: NVPTXISD::ProxyReg, DL: SDLoc(R), VT: R.getValueType(),
7387 Ops: {Chain, R});
7388 }
7389 case ISD::BUILD_VECTOR: {
7390 if (DCI.isBeforeLegalize())
7391 return SDValue();
7392
7393 SmallVector<SDValue, 16> Ops;
7394 for (auto &Op : R->ops()) {
7395 SDValue V = sinkProxyReg(R: Op, Chain, DCI);
7396 if (!V)
7397 return SDValue();
7398 Ops.push_back(Elt: V);
7399 }
7400 return DCI.DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SDLoc(R), VT: R.getValueType(), Ops);
7401 }
7402 case ISD::EXTRACT_VECTOR_ELT: {
7403 if (DCI.isBeforeLegalize())
7404 return SDValue();
7405
7406 if (SDValue V = sinkProxyReg(R: R.getOperand(i: 0), Chain, DCI))
7407 return DCI.DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(R),
7408 VT: R.getValueType(), N1: V, N2: R.getOperand(i: 1));
7409 return SDValue();
7410 }
7411 default:
7412 return SDValue();
7413 }
7414}
7415
7416static unsigned getFAddWithNegOpcode(EVT VT, Intrinsic::ID IID,
7417 APFloat::roundingMode RoundingMode) {
7418 const bool IsFTZ = nvvm::FPArithShouldFTZ(IntrinsicID: IID);
7419 const bool IsSat = nvvm::FPArithIsSaturating(IntrinsicID: IID);
7420 switch (VT.getScalarType().getSimpleVT().SimpleTy) {
7421 case MVT::f16: {
7422 static constexpr unsigned SubRNOpcodes[2][2] = {
7423 {NVPTXISD::SUB_RN, NVPTXISD::SUB_RN_SAT},
7424 {NVPTXISD::SUB_RN_FTZ, NVPTXISD::SUB_RN_FTZ_SAT}};
7425 return SubRNOpcodes[IsFTZ][IsSat];
7426 }
7427 case MVT::bf16:
7428 return NVPTXISD::SUB_RN;
7429 case MVT::f32: {
7430 // for f32x2 inputs
7431 if (!VT.isVector() || IsSat)
7432 return 0;
7433 static constexpr unsigned SubF32x2Opcodes[4][2] = {
7434 {NVPTXISD::SUB_RZ, NVPTXISD::SUB_RZ_FTZ}, // RZ
7435 {NVPTXISD::SUB_RN, NVPTXISD::SUB_RN_FTZ}, // RN
7436 {NVPTXISD::SUB_RP, NVPTXISD::SUB_RP_FTZ}, // RP
7437 {NVPTXISD::SUB_RM, NVPTXISD::SUB_RM_FTZ}}; // RM
7438 return SubF32x2Opcodes[static_cast<unsigned>(RoundingMode)][IsFTZ];
7439 }
7440 default:
7441 return 0;
7442 }
7443}
7444
7445static SDValue combineFAddWithNeg(SDNode *N, SelectionDAG &DAG,
7446 Intrinsic::ID AddIntrinsicID,
7447 APFloat::roundingMode RoundingMode) {
7448 const EVT VT = N->getValueType(ResNo: 0);
7449 const unsigned Opc = getFAddWithNegOpcode(VT, IID: AddIntrinsicID, RoundingMode);
7450 if (!Opc)
7451 return SDValue();
7452
7453 SDValue Op1 = N->getOperand(Num: 1);
7454 SDValue Op2 = N->getOperand(Num: 2);
7455
7456 SDValue SubOp1, SubOp2;
7457
7458 if (Op1.getOpcode() == ISD::FNEG) {
7459 SubOp1 = Op2;
7460 SubOp2 = Op1.getOperand(i: 0);
7461 } else if (Op2.getOpcode() == ISD::FNEG) {
7462 SubOp1 = Op1;
7463 SubOp2 = Op2.getOperand(i: 0);
7464 } else {
7465 return SDValue();
7466 }
7467
7468 return DAG.getNode(Opcode: Opc, DL: SDLoc(N), VT, N1: SubOp1, N2: SubOp2);
7469}
7470
7471// TODO: Remove the type-legality checks here once
7472// https://github.com/llvm/llvm-project/pull/172442 lands, adding support for
7473// explicit type constraints for overloaded intrinsics in tablegen.
7474static bool isSupportedFPArith(SDNode *N, const NVPTXSubtarget &STI,
7475 unsigned ISDOpcode, Intrinsic::ID IID,
7476 APFloat::roundingMode RoundingMode) {
7477 const EVT VT = N->getValueType(ResNo: 0);
7478 if (VT.isVector() && VT.getVectorElementCount() != ElementCount::getFixed(MinVal: 2))
7479 return false;
7480
7481 const bool IsRN = RoundingMode == APFloat::rmNearestTiesToEven;
7482 const bool IsFTZ = nvvm::FPArithShouldFTZ(IntrinsicID: IID);
7483 const bool IsSat = nvvm::FPArithIsSaturating(IntrinsicID: IID);
7484 switch (VT.getScalarType().getSimpleVT().SimpleTy) {
7485 case MVT::f16:
7486 return IsRN;
7487 case MVT::bf16:
7488 return IsRN && !IsSat && !IsFTZ && STI.hasNativeBF16Support(Opcode: ISDOpcode);
7489 case MVT::f32:
7490 return !VT.isVector() || (!IsSat && STI.hasF32x2Instructions());
7491 case MVT::f64:
7492 return !VT.isVector() && !IsSat && !IsFTZ;
7493 default:
7494 return false;
7495 }
7496}
7497
7498static SDValue diagnoseUnsupportedFPArith(SDNode *N, SelectionDAG &DAG,
7499 Intrinsic::ID IID,
7500 APFloat::roundingMode RoundingMode) {
7501 const EVT VT = N->getValueType(ResNo: 0);
7502 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
7503 DAG.getMachineFunction().getFunction(),
7504 Twine(Intrinsic::getBaseName(id: IID)) + " with rounding mode " +
7505 nvvm::GetRoundingModeName(RM: RoundingMode) + " and operand type " +
7506 VT.getEVTString() + " is not supported on this target",
7507 SDLoc(N).getDebugLoc()));
7508 return DAG.getPOISON(VT);
7509}
7510
7511static SDValue combineIntrinsicWOChain(SDNode *N,
7512 TargetLowering::DAGCombinerInfo &DCI,
7513 const NVPTXSubtarget &STI) {
7514 const Intrinsic::ID IID =
7515 static_cast<Intrinsic::ID>(N->getConstantOperandVal(Num: 0));
7516
7517 switch (IID) {
7518 default:
7519 break;
7520 case Intrinsic::nvvm_fadd:
7521 case Intrinsic::nvvm_fadd_ftz:
7522 case Intrinsic::nvvm_fadd_sat:
7523 case Intrinsic::nvvm_fadd_ftz_sat: {
7524 const auto RoundingMode = static_cast<APFloat::roundingMode>(
7525 N->getConstantOperandAPInt(Num: 3).getSExtValue());
7526 if (!isSupportedFPArith(N, STI, ISDOpcode: ISD::FADD, IID, RoundingMode))
7527 return diagnoseUnsupportedFPArith(N, DAG&: DCI.DAG, IID, RoundingMode);
7528 return combineFAddWithNeg(N, DAG&: DCI.DAG, AddIntrinsicID: IID, RoundingMode);
7529 }
7530 case Intrinsic::nvvm_spcompress:
7531 case Intrinsic::nvvm_spdecompress:
7532 return lowerSPIntrinsic(Op: SDValue(N, 0), DAG&: DCI.DAG);
7533 case Intrinsic::nvvm_fmul:
7534 case Intrinsic::nvvm_fmul_ftz:
7535 case Intrinsic::nvvm_fmul_sat:
7536 case Intrinsic::nvvm_fmul_ftz_sat: {
7537 const auto RoundingMode = static_cast<APFloat::roundingMode>(
7538 N->getConstantOperandAPInt(Num: 3).getSExtValue());
7539 if (!isSupportedFPArith(N, STI, ISDOpcode: ISD::FMUL, IID, RoundingMode))
7540 return diagnoseUnsupportedFPArith(N, DAG&: DCI.DAG, IID, RoundingMode);
7541 break;
7542 }
7543 }
7544 return SDValue();
7545}
7546
7547static SDValue combineProxyReg(SDNode *N,
7548 TargetLowering::DAGCombinerInfo &DCI) {
7549
7550 SDValue Chain = N->getOperand(Num: 0);
7551 SDValue Reg = N->getOperand(Num: 1);
7552
7553 // If the ProxyReg is not wrapping a load, try to pull the operations through
7554 // the ProxyReg.
7555 if (Reg.getOpcode() != ISD::LOAD) {
7556 if (SDValue V = sinkProxyReg(R: Reg, Chain, DCI))
7557 return V;
7558 }
7559
7560 return SDValue();
7561}
7562
7563SDValue NVPTXTargetLowering::PerformDAGCombine(SDNode *N,
7564 DAGCombinerInfo &DCI) const {
7565 CodeGenOptLevel OptLevel = getTargetMachine().getOptLevel();
7566 switch (N->getOpcode()) {
7567 default:
7568 break;
7569 case ISD::ADD:
7570 return PerformADDCombine(N, DCI, OptLevel);
7571 case ISD::ADDRSPACECAST:
7572 return combineADDRSPACECAST(N, DCI);
7573 case ISD::SIGN_EXTEND:
7574 case ISD::ZERO_EXTEND:
7575 return combineSZExtToMulWide(N, DCI, OptLevel);
7576 case ISD::BUILD_VECTOR:
7577 return PerformBUILD_VECTORCombine(N, DCI);
7578 case ISD::EXTRACT_VECTOR_ELT:
7579 return PerformEXTRACTCombine(N, DCI);
7580 case ISD::FADD:
7581 return performFADDCombine(N, DCI, OptLevel);
7582 case ISD::FMA:
7583 case ISD::FMUL:
7584 case ISD::FSUB:
7585 return performScalarizeV2F32Op(N, DCI, OptLevel);
7586 case ISD::FMAXNUM:
7587 case ISD::FMINNUM:
7588 case ISD::FMAXIMUM:
7589 case ISD::FMINIMUM:
7590 case ISD::FMAXIMUMNUM:
7591 case ISD::FMINIMUMNUM:
7592 return PerformFMinMaxCombine(N, DCI, STI);
7593 case ISD::LOAD:
7594 case NVPTXISD::LoadV2:
7595 case NVPTXISD::LoadV4:
7596 return combineLOAD(N, DCI, STI);
7597 case ISD::MUL:
7598 return PerformMULCombine(N, DCI, OptLevel);
7599 case NVPTXISD::PRMT:
7600 return combinePRMT(N, DCI, OptLevel);
7601 case NVPTXISD::ProxyReg:
7602 return combineProxyReg(N, DCI);
7603 case ISD::SETCC:
7604 return PerformSETCCCombine(N, DCI, STI);
7605 case ISD::SHL:
7606 return PerformSHLCombine(N, DCI, OptLevel);
7607 case ISD::STORE:
7608 case NVPTXISD::StoreV2:
7609 case NVPTXISD::StoreV4:
7610 return combineSTORE(N, DCI, STI);
7611 case ISD::SELECT:
7612 return PerformSELECTShiftCombine(N, DCI);
7613 case ISD::VSELECT:
7614 return PerformVSELECTCombine(N, DCI);
7615 case ISD::INTRINSIC_WO_CHAIN:
7616 return combineIntrinsicWOChain(N, DCI, STI);
7617 }
7618 return SDValue();
7619}
7620
7621static void ReplaceBITCAST(SDNode *Node, SelectionDAG &DAG,
7622 SmallVectorImpl<SDValue> &Results) {
7623 // Handle bitcasting to v2i8 without hitting the default promotion
7624 // strategy which goes through stack memory.
7625 SDValue Op(Node, 0);
7626 EVT ToVT = Op->getValueType(ResNo: 0);
7627 if (ToVT != MVT::v2i8) {
7628 return;
7629 }
7630
7631 // Bitcast to i16 and unpack elements into a vector
7632 SDLoc DL(Node);
7633 SDValue AsInt = DAG.getBitcast(VT: MVT::i16, V: Op->getOperand(Num: 0));
7634 SDValue Vec0 = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i8, Operand: AsInt);
7635 SDValue Const8 = DAG.getConstant(Val: 8, DL, VT: MVT::i16);
7636 SDValue Vec1 =
7637 DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i8,
7638 Operand: DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i16, Ops: {AsInt, Const8}));
7639 Results.push_back(
7640 Elt: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: MVT::v2i8, Ops: {Vec0, Vec1}));
7641}
7642
7643static void ReplaceINTRINSIC_W_CHAIN(SDNode *N, SelectionDAG &DAG,
7644 SmallVectorImpl<SDValue> &Results) {
7645 SDValue Chain = N->getOperand(Num: 0);
7646 SDValue Intrin = N->getOperand(Num: 1);
7647 SDLoc DL(N);
7648
7649 // Get the intrinsic ID
7650 unsigned IntrinNo = Intrin.getNode()->getAsZExtVal();
7651 switch (IntrinNo) {
7652 default:
7653 return;
7654 case Intrinsic::nvvm_ldu_global_i:
7655 case Intrinsic::nvvm_ldu_global_f:
7656 case Intrinsic::nvvm_ldu_global_p: {
7657 EVT ResVT = N->getValueType(ResNo: 0);
7658
7659 if (ResVT.isVector()) {
7660 // Vector LDG/LDU
7661
7662 unsigned NumElts = ResVT.getVectorNumElements();
7663 EVT EltVT = ResVT.getVectorElementType();
7664
7665 // Since LDU/LDG are target nodes, we cannot rely on DAG type
7666 // legalization.
7667 // Therefore, we must ensure the type is legal. For i1 and i8, we set the
7668 // loaded type to i16 and propagate the "real" type as the memory type.
7669 bool NeedTrunc = false;
7670 if (EltVT.getSizeInBits() < 16) {
7671 EltVT = MVT::i16;
7672 NeedTrunc = true;
7673 }
7674
7675 unsigned Opcode = 0;
7676 SDVTList LdResVTs;
7677
7678 switch (NumElts) {
7679 default:
7680 return;
7681 case 2:
7682 Opcode = NVPTXISD::LDUV2;
7683 LdResVTs = DAG.getVTList(VT1: EltVT, VT2: EltVT, VT3: MVT::Other);
7684 break;
7685 case 4: {
7686 Opcode = NVPTXISD::LDUV4;
7687 EVT ListVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other };
7688 LdResVTs = DAG.getVTList(VTs: ListVTs);
7689 break;
7690 }
7691 }
7692
7693 SmallVector<SDValue, 8> OtherOps;
7694
7695 // Copy regular operands
7696
7697 OtherOps.push_back(Elt: Chain); // Chain
7698 // Skip operand 1 (intrinsic ID)
7699 // Others
7700 OtherOps.append(in_start: N->op_begin() + 2, in_end: N->op_end());
7701
7702 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(Val: N);
7703
7704 SDValue NewLD = DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList: LdResVTs, Ops: OtherOps,
7705 MemVT: MemSD->getMemoryVT(),
7706 MMO: MemSD->getMemOperand());
7707
7708 SmallVector<SDValue, 4> ScalarRes;
7709
7710 for (unsigned i = 0; i < NumElts; ++i) {
7711 SDValue Res = NewLD.getValue(R: i);
7712 if (NeedTrunc)
7713 Res =
7714 DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ResVT.getVectorElementType(), Operand: Res);
7715 ScalarRes.push_back(Elt: Res);
7716 }
7717
7718 SDValue LoadChain = NewLD.getValue(R: NumElts);
7719
7720 SDValue BuildVec =
7721 DAG.getBuildVector(VT: ResVT, DL, Ops: ScalarRes);
7722
7723 Results.push_back(Elt: BuildVec);
7724 Results.push_back(Elt: LoadChain);
7725 } else {
7726 // i8 LDG/LDU
7727 assert(ResVT.isSimple() && ResVT.getSimpleVT().SimpleTy == MVT::i8 &&
7728 "Custom handling of non-i8 ldu/ldg?");
7729
7730 // Just copy all operands as-is
7731 SmallVector<SDValue, 4> Ops(N->ops());
7732
7733 // Force output to i16
7734 SDVTList LdResVTs = DAG.getVTList(VT1: MVT::i16, VT2: MVT::Other);
7735
7736 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(Val: N);
7737
7738 // We make sure the memory type is i8, which will be used during isel
7739 // to select the proper instruction.
7740 SDValue NewLD =
7741 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: LdResVTs, Ops,
7742 MemVT: MVT::i8, MMO: MemSD->getMemOperand());
7743
7744 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i8,
7745 Operand: NewLD.getValue(R: 0)));
7746 Results.push_back(Elt: NewLD.getValue(R: 1));
7747 }
7748 return;
7749 }
7750
7751 case Intrinsic::nvvm_tcgen05_ld_16x64b_x1:
7752 case Intrinsic::nvvm_tcgen05_ld_16x64b_x4:
7753 case Intrinsic::nvvm_tcgen05_ld_16x64b_x8:
7754 case Intrinsic::nvvm_tcgen05_ld_16x64b_x16:
7755 case Intrinsic::nvvm_tcgen05_ld_16x64b_x32:
7756 case Intrinsic::nvvm_tcgen05_ld_16x64b_x64:
7757 case Intrinsic::nvvm_tcgen05_ld_16x64b_x128:
7758 case Intrinsic::nvvm_tcgen05_ld_32x32b_x1:
7759 case Intrinsic::nvvm_tcgen05_ld_32x32b_x4:
7760 case Intrinsic::nvvm_tcgen05_ld_32x32b_x8:
7761 case Intrinsic::nvvm_tcgen05_ld_32x32b_x16:
7762 case Intrinsic::nvvm_tcgen05_ld_32x32b_x32:
7763 case Intrinsic::nvvm_tcgen05_ld_32x32b_x64:
7764 case Intrinsic::nvvm_tcgen05_ld_32x32b_x128:
7765 case Intrinsic::nvvm_tcgen05_ld_16x128b_x2:
7766 case Intrinsic::nvvm_tcgen05_ld_16x128b_x4:
7767 case Intrinsic::nvvm_tcgen05_ld_16x128b_x8:
7768 case Intrinsic::nvvm_tcgen05_ld_16x128b_x16:
7769 case Intrinsic::nvvm_tcgen05_ld_16x128b_x32:
7770 case Intrinsic::nvvm_tcgen05_ld_16x128b_x64:
7771 case Intrinsic::nvvm_tcgen05_ld_16x256b_x1:
7772 case Intrinsic::nvvm_tcgen05_ld_16x256b_x2:
7773 case Intrinsic::nvvm_tcgen05_ld_16x256b_x4:
7774 case Intrinsic::nvvm_tcgen05_ld_16x256b_x8:
7775 case Intrinsic::nvvm_tcgen05_ld_16x256b_x16:
7776 case Intrinsic::nvvm_tcgen05_ld_16x256b_x32:
7777 if (auto Res = lowerTcgen05Ld(N, DAG)) {
7778 Results.push_back(Elt: Res->first);
7779 Results.push_back(Elt: Res->second);
7780 }
7781 return;
7782
7783 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x1:
7784 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x4:
7785 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x8:
7786 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x16:
7787 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x32:
7788 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x64:
7789 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x128:
7790 if (auto Res = lowerTcgen05Ld(N, DAG, /*HasOffset=*/true)) {
7791 Results.push_back(Elt: Res->first);
7792 Results.push_back(Elt: Res->second);
7793 }
7794 return;
7795
7796 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x8_i32:
7797 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x8_f32:
7798 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x64_i32:
7799 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x64_f32:
7800 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x4_i32:
7801 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x4_f32:
7802 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x32_i32:
7803 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x32_f32:
7804 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x16_i32:
7805 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x16_f32:
7806 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x128_i32:
7807 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x128_f32:
7808 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x8_i32:
7809 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x8_f32:
7810 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x64_i32:
7811 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x64_f32:
7812 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x4_i32:
7813 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x4_f32:
7814 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x32_i32:
7815 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x32_f32:
7816 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x16_i32:
7817 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x16_f32:
7818 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x128_i32:
7819 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x128_f32:
7820 if (auto Res = lowerTcgen05LdRed(N, DAG)) {
7821 Results.push_back(Elt: std::get<0>(t&: *Res));
7822 Results.push_back(Elt: std::get<1>(t&: *Res));
7823 Results.push_back(Elt: std::get<2>(t&: *Res));
7824 }
7825 return;
7826 }
7827}
7828
7829static void ReplaceCopyFromReg_128(SDNode *N, SelectionDAG &DAG,
7830 SmallVectorImpl<SDValue> &Results) {
7831 // Change the CopyFromReg to output 2 64-bit results instead of a 128-bit
7832 // result so that it can pass the legalization
7833 SDLoc DL(N);
7834 SDValue Chain = N->getOperand(Num: 0);
7835 SDValue Reg = N->getOperand(Num: 1);
7836 SDValue Glue = N->getOperand(Num: 2);
7837
7838 assert(Reg.getValueType() == MVT::i128 &&
7839 "Custom lowering for CopyFromReg with 128-bit reg only");
7840 SmallVector<EVT, 4> ResultsType = {MVT::i64, MVT::i64, N->getValueType(ResNo: 1),
7841 N->getValueType(ResNo: 2)};
7842 SmallVector<SDValue, 3> NewOps = {Chain, Reg, Glue};
7843
7844 SDValue NewValue = DAG.getNode(Opcode: ISD::CopyFromReg, DL, ResultTys: ResultsType, Ops: NewOps);
7845 SDValue Pair = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i128,
7846 Ops: {NewValue.getValue(R: 0), NewValue.getValue(R: 1)});
7847
7848 Results.push_back(Elt: Pair);
7849 Results.push_back(Elt: NewValue.getValue(R: 2));
7850 Results.push_back(Elt: NewValue.getValue(R: 3));
7851}
7852
7853static void replaceProxyReg(SDNode *N, SelectionDAG &DAG,
7854 const TargetLowering &TLI,
7855 SmallVectorImpl<SDValue> &Results) {
7856 SDValue Chain = N->getOperand(Num: 0);
7857 SDValue Reg = N->getOperand(Num: 1);
7858
7859 MVT VT = TLI.getRegisterType(Context&: *DAG.getContext(), VT: Reg.getValueType());
7860
7861 SDValue NewReg = DAG.getAnyExtOrTrunc(Op: Reg, DL: SDLoc(N), VT);
7862 SDValue NewProxy =
7863 DAG.getNode(Opcode: NVPTXISD::ProxyReg, DL: SDLoc(N), VT, Ops: {Chain, NewReg});
7864 SDValue Res = DAG.getAnyExtOrTrunc(Op: NewProxy, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
7865
7866 Results.push_back(Elt: Res);
7867}
7868
7869static void replaceAtomicSwap128(SDNode *N, SelectionDAG &DAG,
7870 const NVPTXSubtarget &STI,
7871 SmallVectorImpl<SDValue> &Results) {
7872 assert(N->getValueType(0) == MVT::i128 &&
7873 "Custom lowering for atomic128 only supports i128");
7874
7875 AtomicSDNode *AN = cast<AtomicSDNode>(Val: N);
7876 SDLoc dl(N);
7877
7878 if (!STI.hasAtomSwap128()) {
7879 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
7880 DAG.getMachineFunction().getFunction(),
7881 "Support for b128 atomics introduced in PTX ISA version 8.3 and "
7882 "requires target sm_90.",
7883 dl.getDebugLoc()));
7884
7885 Results.push_back(Elt: DAG.getUNDEF(VT: MVT::i128));
7886 Results.push_back(Elt: AN->getOperand(Num: 0)); // Chain
7887 return;
7888 }
7889
7890 SmallVector<SDValue, 6> Ops;
7891 Ops.push_back(Elt: AN->getOperand(Num: 0)); // Chain
7892 Ops.push_back(Elt: AN->getOperand(Num: 1)); // Ptr
7893 for (const auto &Op : AN->ops().drop_front(N: 2)) {
7894 // Low part
7895 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: dl, VT: MVT::i64, N1: Op,
7896 N2: DAG.getIntPtrConstant(Val: 0, DL: dl)));
7897 // High part
7898 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: dl, VT: MVT::i64, N1: Op,
7899 N2: DAG.getIntPtrConstant(Val: 1, DL: dl)));
7900 }
7901 unsigned Opcode = N->getOpcode() == ISD::ATOMIC_SWAP
7902 ? NVPTXISD::ATOMIC_SWAP_B128
7903 : NVPTXISD::ATOMIC_CMP_SWAP_B128;
7904 SDVTList Tys = DAG.getVTList(VT1: MVT::i64, VT2: MVT::i64, VT3: MVT::Other);
7905 SDValue Result = DAG.getMemIntrinsicNode(Opcode, dl, VTList: Tys, Ops, MemVT: MVT::i128,
7906 MMO: AN->getMemOperand());
7907 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i128,
7908 Ops: {Result.getValue(R: 0), Result.getValue(R: 1)}));
7909 Results.push_back(Elt: Result.getValue(R: 2));
7910}
7911
7912void NVPTXTargetLowering::ReplaceNodeResults(
7913 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
7914 switch (N->getOpcode()) {
7915 default:
7916 report_fatal_error(reason: "Unhandled custom legalization");
7917 case ISD::BITCAST:
7918 ReplaceBITCAST(Node: N, DAG, Results);
7919 return;
7920 case ISD::LOAD:
7921 case ISD::MLOAD:
7922 replaceLoadVector(N, DAG, Results, STI);
7923 return;
7924 case ISD::INTRINSIC_W_CHAIN:
7925 ReplaceINTRINSIC_W_CHAIN(N, DAG, Results);
7926 return;
7927 case ISD::CopyFromReg:
7928 ReplaceCopyFromReg_128(N, DAG, Results);
7929 return;
7930 case NVPTXISD::ProxyReg:
7931 replaceProxyReg(N, DAG, TLI: *this, Results);
7932 return;
7933 case ISD::ATOMIC_CMP_SWAP:
7934 case ISD::ATOMIC_SWAP:
7935 replaceAtomicSwap128(N, DAG, STI, Results);
7936 return;
7937 }
7938}
7939
7940/// Returns how NVPTX should expand a scalar atomicrmw.
7941static TargetLoweringBase::AtomicExpansionKind getScalarAtomicRMWExpansion(
7942 AtomicRMWInst::BinOp Op, Type *Ty, unsigned AddressSpace, const Function &F,
7943 bool IgnoreDenormalMode, const NVPTXSubtarget &STI) {
7944 using AtomicExpansionKind = TargetLoweringBase::AtomicExpansionKind;
7945 assert(!Ty->isVectorTy() && "Expected a scalar atomicrmw type");
7946
7947 // Try to lower LLVM atomicrmw fadd/fsub to PTX atomic.add. Fsub is first
7948 // expanded to an fadd with a negated operand. This is complicated by the
7949 // weird FTZ behavior PTX atom.add has:
7950 // - atom.add.f32 on global memory flushes denormals
7951 // - atom.add.f32 on shared memory does not flush denormals
7952 // - atom.add.f16 and atomic.add.bf16 never flush denormals
7953 //
7954 // We lower to atom.add only if the function's FTZ behavior matches that of
7955 // atom.add; otherwise, we lower to a CAS loop. But we always allow
7956 // atomic.add.bf16; even though it never flushes denormals, we never flush
7957 // bf16 denormals when doing regular arithmetic, even when FTZ is enabled.
7958 if (Op == AtomicRMWInst::BinOp::FAdd || Op == AtomicRMWInst::BinOp::FSub) {
7959 AtomicExpansionKind ExpansionKind = Op == AtomicRMWInst::BinOp::FSub
7960 ? AtomicExpansionKind::Expand
7961 : AtomicExpansionKind::None;
7962
7963 // Both the -nvptx-allow-ftz-atomics option and per-instruction
7964 // !atomic.ignore.denormal.mode say that denormal handling is insignificant
7965 // here, so atom.add may be used even when its FTZ behavior disagrees with
7966 // the function's.
7967 const bool IgnoreFTZMismatch = AllowFTZAtomics || IgnoreDenormalMode;
7968
7969 if (Ty->isFloatTy()) {
7970 const bool FTZ = F.getDenormalMode(FPType: APFloat::IEEEsingle()).Output ==
7971 DenormalMode::PreserveSign;
7972 bool UseNative = IgnoreFTZMismatch;
7973 switch (AddressSpace) {
7974 case llvm::ADDRESS_SPACE_GLOBAL:
7975 UseNative |= FTZ;
7976 break;
7977 case llvm::ADDRESS_SPACE_SHARED:
7978 case llvm::ADDRESS_SPACE_SHARED_CLUSTER:
7979 UseNative |= !FTZ;
7980 break;
7981 }
7982 if (UseNative)
7983 return ExpansionKind;
7984 }
7985
7986 if (Ty->isHalfTy()) {
7987 // atom.add.f16 never flushes denormals, so it only agrees with a
7988 // function that is not in FTZ mode for f16.
7989 const bool FTZ = F.getDenormalMode(FPType: APFloat::IEEEhalf()).Output ==
7990 DenormalMode::PreserveSign;
7991 if ((!FTZ || IgnoreFTZMismatch) && STI.hasFeature(Feature: NVPTX::SM70) &&
7992 STI.hasFeature(Feature: NVPTX::PTX63))
7993 return ExpansionKind;
7994 }
7995
7996 if (Ty->isBFloatTy() && STI.hasFeature(Feature: NVPTX::SM90))
7997 return ExpansionKind;
7998
7999 if (Ty->isDoubleTy() && STI.hasAtomAddF64())
8000 return ExpansionKind;
8001 }
8002
8003 // PTX's only atomic fp op is `add`; all other ops expand to a CAS loop.
8004 if (AtomicRMWInst::isFPOperation(Op))
8005 return AtomicExpansionKind::CmpXChg;
8006
8007 assert(Ty->isIntegerTy() && "Expected an integer atomicrmw type");
8008 const unsigned BitWidth = cast<IntegerType>(Val: Ty)->getBitWidth();
8009
8010 switch (Op) {
8011 default:
8012 return AtomicExpansionKind::CmpXChg;
8013 case AtomicRMWInst::Xchg:
8014 if (BitWidth == 128)
8015 return AtomicExpansionKind::None;
8016 [[fallthrough]];
8017 case AtomicRMWInst::BinOp::Add:
8018 case AtomicRMWInst::BinOp::Sub: {
8019 AtomicExpansionKind ExpansionKind = Op == AtomicRMWInst::BinOp::Sub
8020 ? AtomicExpansionKind::Expand
8021 : AtomicExpansionKind::None;
8022 switch (BitWidth) {
8023 case 8:
8024 case 16:
8025 return AtomicExpansionKind::CmpXChg;
8026 case 32:
8027 case 64:
8028 return ExpansionKind;
8029 case 128:
8030 return AtomicExpansionKind::CmpXChg;
8031 default:
8032 llvm_unreachable("unsupported width encountered");
8033 }
8034 }
8035 case AtomicRMWInst::BinOp::And:
8036 case AtomicRMWInst::BinOp::Or:
8037 case AtomicRMWInst::BinOp::Xor:
8038 case AtomicRMWInst::BinOp::Max:
8039 case AtomicRMWInst::BinOp::Min:
8040 case AtomicRMWInst::BinOp::UMax:
8041 case AtomicRMWInst::BinOp::UMin:
8042 switch (BitWidth) {
8043 case 8:
8044 case 16:
8045 return AtomicExpansionKind::CmpXChg;
8046 case 32:
8047 return AtomicExpansionKind::None;
8048 case 64:
8049 if (STI.hasAtomMinMaxAndOrXor())
8050 return AtomicExpansionKind::None;
8051 return AtomicExpansionKind::CmpXChg;
8052 case 128:
8053 return AtomicExpansionKind::CmpXChg;
8054 default:
8055 llvm_unreachable("unsupported width encountered");
8056 }
8057 case AtomicRMWInst::UIncWrap:
8058 case AtomicRMWInst::UDecWrap:
8059 switch (BitWidth) {
8060 case 32:
8061 return AtomicExpansionKind::None;
8062 case 8:
8063 case 16:
8064 case 64:
8065 case 128:
8066 return AtomicExpansionKind::CmpXChg;
8067 default:
8068 llvm_unreachable("unsupported width encountered");
8069 }
8070 }
8071
8072 return AtomicExpansionKind::CmpXChg;
8073}
8074
8075NVPTXTargetLowering::AtomicExpansionKind
8076NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
8077 const DataLayout &DL = AI->getDataLayout();
8078 AtomicRMWInst::BinOp Op = AI->getOperation();
8079 const bool IgnoreDenormalMode =
8080 AI->hasMetadata(KindID: LLVMContext::MD_atomic_ignore_denormal_mode);
8081
8082 // TODO: once we support native elementwise vector atoms,
8083 // return `AtomicExpansionKind::None` to preserve and lower them.
8084 if (AI->isElementwise()) {
8085 auto *VecTy = cast<FixedVectorType>(Val: AI->getType());
8086 Type *Ty = VecTy->getElementType();
8087
8088 // Collapse <1 x T> elementwise RMWs to scalar RMWs
8089 if (VecTy->getNumElements() == 1)
8090 return AtomicExpansionKind::Expand;
8091
8092 uint64_t VecBytes = DL.getTypeStoreSize(Ty: VecTy).getFixedValue();
8093 uint64_t VecBits = DL.getTypeStoreSizeInBits(Ty: VecTy).getFixedValue();
8094
8095 if (Op == AtomicRMWInst::BinOp::Xchg) {
8096 // Split exchanges that are too wide or insufficiently aligned until
8097 // each piece can be lowered atomically.
8098 if (VecBits > getMaxAtomicSizeInBitsSupported() ||
8099 AI->getAlign().value() < VecBytes)
8100 return AtomicExpansionKind::Expand;
8101
8102 // Expand subword exchanges to a masked cmpxchg loop.
8103 if (VecBits < getMinCmpXchgSizeInBits())
8104 return AtomicExpansionKind::CmpXChg;
8105
8106 return AtomicExpansionKind::None;
8107 }
8108
8109 // If the scalar lane op is natively supported or expands to one, return
8110 // Expand so halving eventually bottoms out at the scalar base case. The
8111 // latter includes sub and fsub, which expand to add and fadd, respectively.
8112 AtomicExpansionKind LaneExpansionKind = getScalarAtomicRMWExpansion(
8113 Op, Ty, AddressSpace: AI->getPointerAddressSpace(), F: *AI->getFunction(),
8114 IgnoreDenormalMode, STI);
8115 if (LaneExpansionKind == AtomicExpansionKind::None ||
8116 LaneExpansionKind == AtomicExpansionKind::Expand)
8117 return AtomicExpansionKind::Expand;
8118
8119 // If the whole vector fits a single native cmpxchg, emit one wide
8120 // cmpxchg loop at the current width.
8121 if (VecBits <= getMaxAtomicSizeInBitsSupported())
8122 return AtomicExpansionKind::CmpXChg;
8123
8124 // If the vector is too wide for a single native cmpxchg, halve further to
8125 // emit multiple cmpxchg loops.
8126 return AtomicExpansionKind::Expand;
8127 }
8128
8129 Type *Ty = AI->getValOperand()->getType();
8130 if (Ty->isVectorTy())
8131 return AtomicExpansionKind::CmpXChg;
8132 return getScalarAtomicRMWExpansion(Op, Ty, AddressSpace: AI->getPointerAddressSpace(),
8133 F: *AI->getFunction(), IgnoreDenormalMode,
8134 STI);
8135}
8136
8137bool NVPTXTargetLowering::shouldInsertFencesForAtomic(
8138 const Instruction *I) const {
8139 // This function returns true iff AtomicExpandPass should enforce the
8140 // instruction's ordering with fences instead of leaving that ordering on the
8141 // atomic instruction itself. This covers CAS-loop emulation, seq_cst
8142 // operations (which are not natively supported by the PTX `atom`
8143 // instruction), and elementwise atomicrmw expansion.
8144 //
8145 // atomicrmw and cmpxchg instructions not efficiently supported by PTX
8146 // are lowered to CAS emulation loops that preserve their memory order,
8147 // syncscope, and volatile semantics. For PTX, it is more efficient to use
8148 // atom.cas.relaxed.sco instructions within the loop, and fences before and
8149 // after the loop to restore order.
8150 //
8151 // For ordered elementwise atomicrmw, if we expand elementwise, it is also
8152 // more efficient to insert fences around the whole split sequence once and
8153 // downgrade the ordering for the entire sequence, rather than having the
8154 // stronger ordering on every expanded atomicrmw.
8155 //
8156 // On targets with memory-order qualifiers, atomic instructions efficiently
8157 // supported by PTX are lowered to `atom.<op>.<sem>.<scope>` instructions with
8158 // their corresponding memory order and scope. Since PTX does not support
8159 // seq_cst, we emulate it by lowering to a fence.sc followed by an atom
8160 // according to the PTX atomics ABI.
8161 // https://docs.nvidia.com/cuda/ptx-writers-guide-to-interoperability/atomic-abi.html
8162 if (auto *CI = dyn_cast<AtomicCmpXchgInst>(Val: I))
8163 return (cast<IntegerType>(Val: CI->getCompareOperand()->getType())
8164 ->getBitWidth() < STI.getMinCmpXchgSizeInBits()) ||
8165 !STI.hasMemoryOrdering() ||
8166 CI->getMergedOrdering() == AtomicOrdering::SequentiallyConsistent;
8167 if (auto *RI = dyn_cast<AtomicRMWInst>(Val: I)) {
8168 AtomicExpansionKind Expansion = shouldExpandAtomicRMWInIR(AI: RI);
8169 return Expansion == AtomicExpansionKind::CmpXChg ||
8170 (RI->isElementwise() && Expansion == AtomicExpansionKind::Expand) ||
8171 !STI.hasMemoryOrdering() ||
8172 RI->getOrdering() == AtomicOrdering::SequentiallyConsistent;
8173 }
8174 return false;
8175}
8176
8177AtomicOrdering NVPTXTargetLowering::atomicOperationOrderAfterFenceSplit(
8178 const Instruction *I) const {
8179 // If the operation is emulated by a CAS-loop, or the target does not support
8180 // memory-order qualifiers, we set its IR ordering to monotonic.
8181 // AtomicExpandPass inserts fences to enforce the original memory ordering.
8182 //
8183 // When the operation is not emulated, but the memory order is seq_cst,
8184 // we must lower to "fence.sc.<scope>; atom.<op>.acquire.<scope>;" to conform
8185 // to the PTX atomics ABI.
8186 // https://docs.nvidia.com/cuda/ptx-writers-guide-to-interoperability/atomic-abi.html
8187 // For such cases, emitLeadingFence() will separately insert the leading
8188 // "fence.sc.<scope>;". Here, we only set the memory order to acquire.
8189 //
8190 // Otherwise, the operation is not emulated, and the memory order is not
8191 // seq_cst. In this case, the LLVM memory order is natively supported by the
8192 // PTX `atom` instruction, and we just lower to the corresponding
8193 // `atom.<op>.relaxed|acquire|release|acq_rel". For such cases, this function
8194 // will NOT be called.
8195 // prerequisite: shouldInsertFencesForAtomic() should have returned `true` for
8196 // I before its memory order was modified.
8197 if (!STI.hasMemoryOrdering())
8198 return AtomicOrdering::Monotonic;
8199
8200 if (auto *CI = dyn_cast<AtomicCmpXchgInst>(Val: I);
8201 CI && CI->getMergedOrdering() == AtomicOrdering::SequentiallyConsistent &&
8202 cast<IntegerType>(Val: CI->getCompareOperand()->getType())->getBitWidth() >=
8203 STI.getMinCmpXchgSizeInBits())
8204 return AtomicOrdering::Acquire;
8205 else if (auto *RI = dyn_cast<AtomicRMWInst>(Val: I);
8206 RI && RI->getOrdering() == AtomicOrdering::SequentiallyConsistent) {
8207 AtomicExpansionKind ExpansionKind = shouldExpandAtomicRMWInIR(AI: RI);
8208 // A non-elementwise `Expand` means we are expanding sub(x) to add(-x).
8209 // The atomic add will have acquire ordering.
8210 if (ExpansionKind == AtomicExpansionKind::None ||
8211 (ExpansionKind == AtomicExpansionKind::Expand && !RI->isElementwise()))
8212 return AtomicOrdering::Acquire;
8213 }
8214
8215 return AtomicOrdering::Monotonic;
8216}
8217
8218Instruction *NVPTXTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
8219 Instruction *Inst,
8220 AtomicOrdering Ord) const {
8221 // prerequisite: shouldInsertFencesForAtomic() should have returned `true` for
8222 // `Inst` before its memory order was modified. We cannot enforce this with an
8223 // assert, because AtomicExpandPass will have modified the memory order
8224 // between the initial call to shouldInsertFencesForAtomic() and the call to
8225 // this function.
8226 if (!isa<AtomicCmpXchgInst>(Val: Inst) && !isa<AtomicRMWInst>(Val: Inst))
8227 return TargetLoweringBase::emitLeadingFence(Builder, Inst, Ord);
8228
8229 // Specialize for cmpxchg and atomicrmw
8230 auto SSID = getAtomicSyncScopeID(I: Inst);
8231 assert(SSID.has_value() && "Expected an atomic operation");
8232
8233 if (isReleaseOrStronger(AO: Ord))
8234 return Builder.CreateFence(Ordering: Ord == AtomicOrdering::SequentiallyConsistent
8235 ? AtomicOrdering::SequentiallyConsistent
8236 : AtomicOrdering::Release,
8237 SSID: SSID.value());
8238
8239 return nullptr;
8240}
8241
8242Instruction *NVPTXTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
8243 Instruction *Inst,
8244 AtomicOrdering Ord) const {
8245 // prerequisite: shouldInsertFencesForAtomic() should have returned `true` for
8246 // `Inst` before its memory order was modified. See `emitLeadingFence` for why
8247 // this cannot be enforced with an assert. Specialize for cmpxchg and
8248 // atomicrmw
8249 auto *CI = dyn_cast<AtomicCmpXchgInst>(Val: Inst);
8250 auto *RI = dyn_cast<AtomicRMWInst>(Val: Inst);
8251 if (!CI && !RI)
8252 return TargetLoweringBase::emitTrailingFence(Builder, Inst, Ord);
8253
8254 auto SSID = getAtomicSyncScopeID(I: Inst);
8255 assert(SSID.has_value() && "Expected an atomic operation");
8256
8257 bool IsEmulated = !STI.hasMemoryOrdering();
8258 if (CI)
8259 IsEmulated |=
8260 cast<IntegerType>(Val: CI->getCompareOperand()->getType())->getBitWidth() <
8261 STI.getMinCmpXchgSizeInBits();
8262 else {
8263 AtomicExpansionKind Expansion = shouldExpandAtomicRMWInIR(AI: RI);
8264 IsEmulated |=
8265 Expansion == AtomicExpansionKind::CmpXChg ||
8266 (RI->isElementwise() && Expansion == AtomicExpansionKind::Expand);
8267 }
8268
8269 if (isAcquireOrStronger(AO: Ord) && IsEmulated)
8270 return Builder.CreateFence(Ordering: AtomicOrdering::Acquire, SSID: SSID.value());
8271
8272 return nullptr;
8273}
8274
8275// Rather than default to SINT when both UINT and SINT are custom, we only
8276// change the opcode when UINT is not legal and SINT is. UINT is preferred when
8277// both are custom since unsigned CVT instructions can lead to slightly better
8278// SASS code with fewer instructions.
8279unsigned NVPTXTargetLowering::getPreferredFPToIntOpcode(unsigned Op, EVT FromVT,
8280 EVT ToVT) const {
8281 if (isOperationLegal(Op, VT: ToVT))
8282 return Op;
8283 switch (Op) {
8284 case ISD::FP_TO_UINT:
8285 if (isOperationLegal(Op: ISD::FP_TO_SINT, VT: ToVT))
8286 return ISD::FP_TO_SINT;
8287 break;
8288 case ISD::STRICT_FP_TO_UINT:
8289 if (isOperationLegal(Op: ISD::STRICT_FP_TO_SINT, VT: ToVT))
8290 return ISD::STRICT_FP_TO_SINT;
8291 break;
8292 default:
8293 break;
8294 }
8295 return Op;
8296}
8297
8298// Pin NVPTXTargetObjectFile's vtables to this file.
8299NVPTXTargetObjectFile::~NVPTXTargetObjectFile() = default;
8300
8301MCSection *NVPTXTargetObjectFile::SelectSectionForGlobal(
8302 const GlobalObject *GO, SectionKind Kind, const TargetMachine &TM) const {
8303 return getDataSection();
8304}
8305
8306static void computeKnownBitsForPRMT(const SDValue Op, KnownBits &Known,
8307 const SelectionDAG &DAG, unsigned Depth) {
8308 SDValue A = Op.getOperand(i: 0);
8309 SDValue B = Op.getOperand(i: 1);
8310 ConstantSDNode *Selector = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
8311 unsigned Mode = Op.getConstantOperandVal(i: 3);
8312
8313 if (!Selector)
8314 return;
8315
8316 KnownBits AKnown = DAG.computeKnownBits(Op: A, Depth);
8317 KnownBits BKnown = DAG.computeKnownBits(Op: B, Depth);
8318
8319 // {b, a} = {{b7, b6, b5, b4}, {b3, b2, b1, b0}}
8320 assert(AKnown.getBitWidth() == 32 && BKnown.getBitWidth() == 32 &&
8321 "PRMT must have i32 operands");
8322 assert(Known.getBitWidth() == 32 && "PRMT must have i32 result");
8323 KnownBits BitField = BKnown.concat(Lo: AKnown);
8324
8325 APInt SelectorVal = getPRMTSelector(Selector: Selector->getAPIntValue(), Mode);
8326 for (unsigned I : llvm::seq(Size: 4)) {
8327 APInt Sel = SelectorVal.extractBits(numBits: 4, bitPosition: I * 4);
8328 unsigned Idx = Sel.getLoBits(numBits: 3).getZExtValue();
8329 unsigned Sign = Sel.getHiBits(numBits: 1).getZExtValue();
8330 KnownBits Byte = BitField.extractBits(NumBits: 8, BitPosition: Idx * 8);
8331 if (Sign)
8332 Byte = KnownBits::ashr(LHS: Byte, RHS: KnownBits::makeConstant(C: APInt(8, 7)));
8333 Known.insertBits(SubBits: Byte, BitPosition: I * 8);
8334 }
8335}
8336
8337static void computeKnownBitsForLoadV(const SDValue Op, KnownBits &Known) {
8338 MemSDNode *LD = cast<MemSDNode>(Val: Op);
8339
8340 // We can't do anything without knowing the sign bit.
8341 auto ExtType = LD->getConstantOperandVal(Num: LD->getNumOperands() - 1);
8342 if (ExtType == ISD::SEXTLOAD)
8343 return;
8344
8345 // ExtLoading to vector types is weird and may not work well with known bits.
8346 auto DestVT = LD->getValueType(ResNo: 0);
8347 if (DestVT.isVector())
8348 return;
8349
8350 assert(Known.getBitWidth() == DestVT.getSizeInBits());
8351 auto ElementBitWidth = getFromTypeWidthForLoad(Mem: LD);
8352 Known.Zero.setHighBits(Known.getBitWidth() - ElementBitWidth);
8353}
8354
8355void NVPTXTargetLowering::computeKnownBitsForTargetNode(
8356 const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
8357 const SelectionDAG &DAG, unsigned Depth) const {
8358 Known.resetAll();
8359
8360 switch (Op.getOpcode()) {
8361 case NVPTXISD::MUL_WIDE_SIGNED:
8362 case NVPTXISD::MUL_WIDE_UNSIGNED: {
8363 KnownBits LHS = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
8364 KnownBits RHS = DAG.computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
8365 unsigned BitWidth = Known.getBitWidth();
8366 bool IsSigned = Op.getOpcode() == NVPTXISD::MUL_WIDE_SIGNED;
8367 // Forming mul.wide must not hide bits that let subsequent combines remove
8368 // unused parts of its operands, such as the high bytes of narrow loads.
8369 Known = KnownBits::mul(LHS: IsSigned ? LHS.sext(BitWidth) : LHS.zext(BitWidth),
8370 RHS: IsSigned ? RHS.sext(BitWidth) : RHS.zext(BitWidth));
8371 break;
8372 }
8373 case NVPTXISD::PRMT:
8374 computeKnownBitsForPRMT(Op, Known, DAG, Depth);
8375 break;
8376 case NVPTXISD::LoadV2:
8377 case NVPTXISD::LoadV4:
8378 case NVPTXISD::LoadV8:
8379 computeKnownBitsForLoadV(Op, Known);
8380 break;
8381 default:
8382 break;
8383 }
8384}
8385
8386static std::pair<APInt, APInt> getPRMTDemandedBits(const APInt &SelectorVal,
8387 const APInt &DemandedBits) {
8388 APInt DemandedLHS = APInt(32, 0);
8389 APInt DemandedRHS = APInt(32, 0);
8390
8391 for (unsigned I : llvm::seq(Size: 4)) {
8392 if (DemandedBits.extractBits(numBits: 8, bitPosition: I * 8).isZero())
8393 continue;
8394
8395 APInt Sel = SelectorVal.extractBits(numBits: 4, bitPosition: I * 4);
8396 unsigned Idx = Sel.getLoBits(numBits: 3).getZExtValue();
8397 unsigned Sign = Sel.getHiBits(numBits: 1).getZExtValue();
8398
8399 APInt &Src = Idx < 4 ? DemandedLHS : DemandedRHS;
8400 unsigned ByteStart = (Idx % 4) * 8;
8401 if (Sign)
8402 Src.setBit(ByteStart + 7);
8403 else
8404 Src.setBits(loBit: ByteStart, hiBit: ByteStart + 8);
8405 }
8406
8407 return {DemandedLHS, DemandedRHS};
8408}
8409
8410// Replace undef with 0 as this is easier for other optimizations such as
8411// known bits.
8412static SDValue canonicalizePRMTInput(SDValue Op, SelectionDAG &DAG) {
8413 if (!Op)
8414 return SDValue();
8415 if (Op.isUndef())
8416 return DAG.getConstant(Val: 0, DL: SDLoc(), VT: MVT::i32);
8417 return Op;
8418}
8419
8420static SDValue simplifyDemandedBitsForPRMT(SDValue PRMT,
8421 const APInt &DemandedBits,
8422 SelectionDAG &DAG,
8423 const TargetLowering &TLI,
8424 unsigned Depth) {
8425 assert(PRMT.getOpcode() == NVPTXISD::PRMT);
8426 SDValue Op0 = PRMT.getOperand(i: 0);
8427 SDValue Op1 = PRMT.getOperand(i: 1);
8428 auto *SelectorConst = dyn_cast<ConstantSDNode>(Val: PRMT.getOperand(i: 2));
8429 if (!SelectorConst)
8430 return SDValue();
8431
8432 unsigned Mode = PRMT.getConstantOperandVal(i: 3);
8433 const APInt Selector = getPRMTSelector(Selector: SelectorConst->getAPIntValue(), Mode);
8434
8435 // Try to simplify the PRMT to one of the inputs if the used bytes are all
8436 // from the same input in the correct order.
8437 const unsigned LeadingBytes = DemandedBits.countLeadingZeros() / 8;
8438 const unsigned SelBits = (4 - LeadingBytes) * 4;
8439 if (Selector.getLoBits(numBits: SelBits) == APInt(32, 0x3210).getLoBits(numBits: SelBits))
8440 return Op0;
8441 if (Selector.getLoBits(numBits: SelBits) == APInt(32, 0x7654).getLoBits(numBits: SelBits))
8442 return Op1;
8443
8444 auto [DemandedLHS, DemandedRHS] = getPRMTDemandedBits(SelectorVal: Selector, DemandedBits);
8445
8446 // Attempt to avoid multi-use ops if we don't need anything from them.
8447 SDValue DemandedOp0 =
8448 TLI.SimplifyMultipleUseDemandedBits(Op: Op0, DemandedBits: DemandedLHS, DAG, Depth: Depth + 1);
8449 SDValue DemandedOp1 =
8450 TLI.SimplifyMultipleUseDemandedBits(Op: Op1, DemandedBits: DemandedRHS, DAG, Depth: Depth + 1);
8451
8452 DemandedOp0 = canonicalizePRMTInput(Op: DemandedOp0, DAG);
8453 DemandedOp1 = canonicalizePRMTInput(Op: DemandedOp1, DAG);
8454 if ((DemandedOp0 && DemandedOp0 != Op0) ||
8455 (DemandedOp1 && DemandedOp1 != Op1)) {
8456 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
8457 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
8458 return getPRMT(A: Op0, B: Op1, Selector: Selector.getZExtValue(), DL: SDLoc(PRMT), DAG);
8459 }
8460
8461 return SDValue();
8462}
8463
8464bool NVPTXTargetLowering::SimplifyDemandedBitsForTargetNode(
8465 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
8466 KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
8467 Known.resetAll();
8468
8469 switch (Op.getOpcode()) {
8470 case NVPTXISD::PRMT:
8471 if (SDValue Result = simplifyDemandedBitsForPRMT(PRMT: Op, DemandedBits, DAG&: TLO.DAG,
8472 TLI: *this, Depth)) {
8473 TLO.CombineTo(O: Op, N: Result);
8474 return true;
8475 }
8476 break;
8477 default:
8478 break;
8479 }
8480
8481 computeKnownBitsForTargetNode(Op, Known, DemandedElts, DAG: TLO.DAG, Depth);
8482 return false;
8483}
8484