1//===-- NVPTXISelLowering.cpp - NVPTX DAG Lowering Implementation ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that NVPTX uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "NVPTXISelLowering.h"
15#include "MCTargetDesc/NVPTXBaseInfo.h"
16#include "NVPTX.h"
17#include "NVPTXMachineFunctionInfo.h"
18#include "NVPTXSelectionDAGInfo.h"
19#include "NVPTXSubtarget.h"
20#include "NVPTXTargetMachine.h"
21#include "NVPTXTargetObjectFile.h"
22#include "NVPTXUtilities.h"
23#include "NVVMProperties.h"
24#include "llvm/ADT/APFloat.h"
25#include "llvm/ADT/APInt.h"
26#include "llvm/ADT/STLExtras.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/StringRef.h"
29#include "llvm/CodeGen/Analysis.h"
30#include "llvm/CodeGen/ISDOpcodes.h"
31#include "llvm/CodeGen/MachineFrameInfo.h"
32#include "llvm/CodeGen/MachineFunction.h"
33#include "llvm/CodeGen/MachineJumpTableInfo.h"
34#include "llvm/CodeGen/MachineMemOperand.h"
35#include "llvm/CodeGen/SDPatternMatch.h"
36#include "llvm/CodeGen/SelectionDAG.h"
37#include "llvm/CodeGen/SelectionDAGNodes.h"
38#include "llvm/CodeGen/TargetCallingConv.h"
39#include "llvm/CodeGen/TargetLowering.h"
40#include "llvm/CodeGen/ValueTypes.h"
41#include "llvm/CodeGenTypes/MachineValueType.h"
42#include "llvm/IR/Argument.h"
43#include "llvm/IR/Attributes.h"
44#include "llvm/IR/Constants.h"
45#include "llvm/IR/DataLayout.h"
46#include "llvm/IR/DerivedTypes.h"
47#include "llvm/IR/DiagnosticInfo.h"
48#include "llvm/IR/FPEnv.h"
49#include "llvm/IR/Function.h"
50#include "llvm/IR/GlobalValue.h"
51#include "llvm/IR/IRBuilder.h"
52#include "llvm/IR/Instruction.h"
53#include "llvm/IR/Instructions.h"
54#include "llvm/IR/IntrinsicsNVPTX.h"
55#include "llvm/IR/LLVMContext.h"
56#include "llvm/IR/Module.h"
57#include "llvm/IR/NVVMIntrinsicUtils.h"
58#include "llvm/IR/Type.h"
59#include "llvm/IR/Value.h"
60#include "llvm/MC/MCContext.h"
61#include "llvm/MC/MCSymbol.h"
62#include "llvm/Support/Alignment.h"
63#include "llvm/Support/AtomicOrdering.h"
64#include "llvm/Support/Casting.h"
65#include "llvm/Support/CodeGen.h"
66#include "llvm/Support/ErrorHandling.h"
67#include "llvm/Support/KnownBits.h"
68#include "llvm/Support/NVPTXAddrSpace.h"
69#include "llvm/Target/TargetMachine.h"
70#include "llvm/Target/TargetOptions.h"
71#include <algorithm>
72#include <cassert>
73#include <cmath>
74#include <cstdint>
75#include <iterator>
76#include <optional>
77#include <tuple>
78#include <utility>
79#include <vector>
80
81#define DEBUG_TYPE "nvptx-lower"
82
83using namespace llvm;
84
85NVPTX::DivPrecisionLevel
86NVPTXTargetLowering::getDivF32Level(const MachineFunction &MF,
87 const SDNode &N) const {
88 // If nvptx-prec-div32=N is used on the command-line, always honor it
89 if (auto PrecDivF32 = STI.getCLOpts().prec_divf32)
90 return *PrecDivF32;
91
92 const SDNodeFlags Flags = N.getFlags();
93 if (Flags.hasApproximateFuncs())
94 return NVPTX::DivPrecisionLevel::Approx;
95
96 return NVPTX::DivPrecisionLevel::IEEE754;
97}
98
99bool NVPTXTargetLowering::usePrecSqrtF32(const SDNode *N) const {
100 // If nvptx-prec-sqrtf32 is used on the command-line, always honor it
101 if (STI.getCLOpts().prec_sqrtf32 != BoolOrDefault::Default)
102 return STI.getCLOpts().prec_sqrtf32 == BoolOrDefault::True;
103
104 if (N) {
105 const SDNodeFlags Flags = N->getFlags();
106 if (Flags.hasApproximateFuncs())
107 return false;
108 }
109
110 return true;
111}
112
113// Input denormals must be flushed when requested, whereas output flushing is
114// optional. Select FTZ instructions based on the input mode.
115// In practice, both input and output should match anyway.
116bool NVPTXTargetLowering::useF16FTZ(const MachineFunction &MF) const {
117 return MF.getDenormalMode(FPType: APFloat::IEEEhalf()).Input ==
118 DenormalMode::PreserveSign;
119}
120
121bool NVPTXTargetLowering::useF32FTZ(const MachineFunction &MF) const {
122 return MF.getDenormalMode(FPType: APFloat::IEEEsingle()).Input ==
123 DenormalMode::PreserveSign;
124}
125
126static bool IsPTXVectorType(MVT VT) {
127 switch (VT.SimpleTy) {
128 default:
129 return false;
130 case MVT::v2i1:
131 case MVT::v4i1:
132 case MVT::v2i8:
133 case MVT::v4i8:
134 case MVT::v8i8: // <2 x i8x4>
135 case MVT::v16i8: // <4 x i8x4>
136 case MVT::v2i16:
137 case MVT::v4i16:
138 case MVT::v8i16: // <4 x i16x2>
139 case MVT::v2i32:
140 case MVT::v4i32:
141 case MVT::v2i64:
142 case MVT::v2f16:
143 case MVT::v4f16:
144 case MVT::v8f16: // <4 x f16x2>
145 case MVT::v2bf16:
146 case MVT::v4bf16:
147 case MVT::v8bf16: // <4 x bf16x2>
148 case MVT::v2f32:
149 case MVT::v4f32:
150 case MVT::v2f64:
151 case MVT::v4i64:
152 case MVT::v4f64:
153 case MVT::v8i32:
154 case MVT::v8f32:
155 case MVT::v16f16: // <8 x f16x2>
156 case MVT::v16bf16: // <8 x bf16x2>
157 case MVT::v16i16: // <8 x i16x2>
158 case MVT::v32i8: // <8 x i8x4>
159 return true;
160 }
161}
162
163// When legalizing vector loads/stores, this function is called, which does two
164// things:
165// 1. Determines Whether the vector is something we want to custom lower,
166// std::nullopt is returned if we do not want to custom lower it.
167// 2. If we do want to handle it, returns two parameters:
168// - unsigned int NumElts - The number of elements in the final vector
169// - EVT EltVT - The type of the elements in the final vector
170static std::optional<std::pair<unsigned int, MVT>>
171getVectorLoweringShape(EVT VectorEVT, const NVPTXSubtarget &STI,
172 unsigned AddressSpace) {
173 const bool CanLowerTo256Bit = STI.has256BitVectorLoadStore(AS: AddressSpace);
174
175 if (CanLowerTo256Bit && VectorEVT.isScalarInteger() &&
176 VectorEVT.getSizeInBits() == 256)
177 return {{4, MVT::i64}};
178
179 if (!VectorEVT.isSimple())
180 return std::nullopt;
181 const MVT VectorVT = VectorEVT.getSimpleVT();
182
183 if (!VectorVT.isVector()) {
184 if (VectorVT == MVT::i128 || VectorVT == MVT::f128)
185 return {{2, MVT::i64}};
186 return std::nullopt;
187 }
188
189 const MVT EltVT = VectorVT.getVectorElementType();
190 const unsigned NumElts = VectorVT.getVectorNumElements();
191
192 // The size of the PTX virtual register that holds a packed type.
193 unsigned PackRegSize;
194
195 // We only handle "native" vector sizes for now, e.g. <4 x double> is not
196 // legal. We can (and should) split that into 2 stores of <2 x double> here
197 // but I'm leaving that as a TODO for now.
198 switch (VectorVT.SimpleTy) {
199 default:
200 return std::nullopt;
201
202 case MVT::v4i64:
203 case MVT::v4f64:
204 // This is a "native" vector type iff the address space is global and the
205 // target supports 256-bit loads/stores
206 if (!CanLowerTo256Bit)
207 return std::nullopt;
208 [[fallthrough]];
209 case MVT::v2i8:
210 case MVT::v2i64:
211 case MVT::v2f64:
212 // This is a "native" vector type
213 return std::pair(NumElts, EltVT);
214
215 case MVT::v16f16: // <8 x f16x2>
216 case MVT::v16bf16: // <8 x bf16x2>
217 case MVT::v16i16: // <8 x i16x2>
218 case MVT::v32i8: // <8 x i8x4>
219 // This can be upsized into a "native" vector type iff the address space is
220 // global and the target supports 256-bit loads/stores.
221 if (!CanLowerTo256Bit)
222 return std::nullopt;
223 [[fallthrough]];
224 case MVT::v2i16: // <1 x i16x2>
225 case MVT::v2f16: // <1 x f16x2>
226 case MVT::v2bf16: // <1 x bf16x2>
227 case MVT::v4i8: // <1 x i8x4>
228 case MVT::v4i16: // <2 x i16x2>
229 case MVT::v4f16: // <2 x f16x2>
230 case MVT::v4bf16: // <2 x bf16x2>
231 case MVT::v8i8: // <2 x i8x4>
232 case MVT::v8f16: // <4 x f16x2>
233 case MVT::v8bf16: // <4 x bf16x2>
234 case MVT::v8i16: // <4 x i16x2>
235 case MVT::v16i8: // <4 x i8x4>
236 PackRegSize = 32;
237 break;
238
239 case MVT::v8f32: // <4 x f32x2>
240 case MVT::v8i32: // <4 x i32x2>
241 // This is a "native" vector type iff the address space is global and the
242 // target supports 256-bit loads/stores
243 if (!CanLowerTo256Bit)
244 return std::nullopt;
245 [[fallthrough]];
246 case MVT::v2f32: // <1 x f32x2>
247 case MVT::v4f32: // <2 x f32x2>
248 case MVT::v2i32: // <1 x i32x2>
249 case MVT::v4i32: // <2 x i32x2>
250 if (!STI.hasF32x2Instructions())
251 return std::pair(NumElts, EltVT);
252 PackRegSize = 64;
253 break;
254 }
255
256 // If we reach here, then we can pack 2 or more elements into a single 32-bit
257 // or 64-bit PTX register and treat the vector as a new vector containing
258 // packed elements.
259
260 // Number of elements to pack in one word.
261 const unsigned NPerReg = PackRegSize / EltVT.getSizeInBits();
262
263 return std::pair(NumElts / NPerReg, MVT::getVectorVT(VT: EltVT, NumElements: NPerReg));
264}
265
266/// ComputePTXValueVTs - For the given Type \p Ty, returns the set of primitive
267/// legal-ish MVTs that compose it. Unlike ComputeValueVTs, this will legalize
268/// the types as required by the calling convention (with special handling for
269/// i8s).
270/// NOTE: This is a band-aid for code that expects ComputeValueVTs to return the
271/// same number of types as the Ins/Outs arrays in LowerFormalArguments,
272/// LowerCall, and LowerReturn.
273static void ComputePTXValueVTs(const TargetLowering &TLI, const DataLayout &DL,
274 LLVMContext &Ctx, CallingConv::ID CallConv,
275 Type *Ty, SmallVectorImpl<EVT> &ValueVTs,
276 SmallVectorImpl<uint64_t> &Offsets,
277 uint64_t StartingOffset = 0) {
278 SmallVector<EVT, 16> TempVTs;
279 SmallVector<uint64_t, 16> TempOffsets;
280 ComputeValueVTs(TLI, DL, Ty, ValueVTs&: TempVTs, /*MemVTs=*/nullptr, FixedOffsets: &TempOffsets,
281 StartingOffset);
282
283 for (const auto [VT, Off] : zip(t&: TempVTs, u&: TempOffsets)) {
284 MVT RegisterVT = TLI.getRegisterTypeForCallingConv(Context&: Ctx, CC: CallConv, VT);
285 unsigned NumRegs = TLI.getNumRegistersForCallingConv(Context&: Ctx, CC: CallConv, VT);
286
287 // Since we actually can load/store b8, we need to ensure that we'll use
288 // the original sized type for any i8s or i8 vectors.
289 if (VT.getScalarType() == MVT::i8) {
290 if (RegisterVT == MVT::i16)
291 RegisterVT = MVT::i8;
292 else if (RegisterVT == MVT::v2i16)
293 RegisterVT = MVT::v2i8;
294 else
295 assert(RegisterVT == MVT::v4i8 &&
296 "Expected v4i8, v2i16, or i16 for i8 RegisterVT");
297 }
298
299 // TODO: This is horribly incorrect for cases where the vector elements are
300 // not a multiple of bytes (ex i1) and legal or i8. However, this problem
301 // has existed for as long as NVPTX has and no one has complained, so we'll
302 // leave it for now.
303 for (unsigned I : seq(Size: NumRegs)) {
304 ValueVTs.push_back(Elt: RegisterVT);
305 Offsets.push_back(Elt: Off + I * RegisterVT.getStoreSize());
306 }
307 }
308}
309
310// We return an EVT that can hold N VTs
311// If the VT is a vector, the resulting EVT is a flat vector with the same
312// element type as VT's element type.
313static EVT getVectorizedVT(EVT VT, unsigned N, LLVMContext &C) {
314 if (N == 1)
315 return VT;
316
317 return VT.isVector() ? EVT::getVectorVT(Context&: C, VT: VT.getScalarType(),
318 NumElements: VT.getVectorNumElements() * N)
319 : EVT::getVectorVT(Context&: C, VT, NumElements: N);
320}
321
322static SDValue getExtractVectorizedValue(SDValue V, unsigned I, EVT VT,
323 const SDLoc &dl, SelectionDAG &DAG) {
324 if (V.getValueType() == VT) {
325 assert(I == 0 && "Index must be 0 for scalar value");
326 return V;
327 }
328
329 if (!VT.isVector())
330 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT, N1: V,
331 N2: DAG.getVectorIdxConstant(Val: I, DL: dl));
332
333 return DAG.getNode(
334 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT, N1: V,
335 N2: DAG.getVectorIdxConstant(Val: I * VT.getVectorNumElements(), DL: dl));
336}
337
338template <typename T>
339static inline SDValue getBuildVectorizedValue(unsigned N, const SDLoc &dl,
340 SelectionDAG &DAG, T GetElement) {
341 if (N == 1)
342 return GetElement(0);
343
344 SmallVector<SDValue, 8> Values;
345 for (const unsigned I : llvm::seq(Size: N)) {
346 SDValue Val = GetElement(I);
347 if (Val.getValueType().isVector())
348 DAG.ExtractVectorElements(Op: Val, Args&: Values);
349 else
350 Values.push_back(Elt: Val);
351 }
352
353 EVT VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: Values[0].getValueType(),
354 NumElements: Values.size());
355 return DAG.getBuildVector(VT, DL: dl, Ops: Values);
356}
357
358/// PromoteScalarIntegerPTX
359/// Used to make sure the arguments/returns are suitable for passing
360/// and promote them to a larger size if they're not.
361///
362/// The promoted type is placed in \p PromoteVT if the function returns true.
363static EVT promoteScalarIntegerPTX(const EVT VT) {
364 if (VT.isScalarInteger()) {
365 switch (PowerOf2Ceil(A: VT.getFixedSizeInBits())) {
366 default:
367 llvm_unreachable(
368 "Promotion is not suitable for scalars of size larger than 64-bits");
369 case 1:
370 return MVT::i1;
371 case 2:
372 case 4:
373 case 8:
374 return MVT::i8;
375 case 16:
376 return MVT::i16;
377 case 32:
378 return MVT::i32;
379 case 64:
380 return MVT::i64;
381 }
382 }
383 return VT;
384}
385
386// Check whether we can merge loads/stores of some of the pieces of a
387// flattened function parameter or return value into a single vector
388// load/store.
389//
390// The flattened parameter is represented as a list of EVTs and
391// offsets, and the whole structure is aligned to ParamAlignment. This
392// function determines whether we can load/store pieces of the
393// parameter starting at index Idx using a single vectorized op of
394// size AccessSize. If so, it returns the number of param pieces
395// covered by the vector op. Otherwise, it returns 1.
396template <typename T>
397static unsigned canMergeParamLoadStoresStartingAt(
398 unsigned Idx, uint32_t AccessSize, const SmallVectorImpl<EVT> &ValueVTs,
399 const SmallVectorImpl<T> &Offsets, Align ParamAlignment) {
400
401 // Can't vectorize if param alignment is not sufficient.
402 if (ParamAlignment < AccessSize)
403 return 1;
404 // Can't vectorize if offset is not aligned.
405 if (Offsets[Idx] & (AccessSize - 1))
406 return 1;
407
408 EVT EltVT = ValueVTs[Idx];
409 unsigned EltSize = EltVT.getStoreSize();
410
411 // Element is too large to vectorize.
412 if (EltSize >= AccessSize)
413 return 1;
414
415 unsigned NumElts = AccessSize / EltSize;
416 // Can't vectorize if AccessBytes if not a multiple of EltSize.
417 if (AccessSize != EltSize * NumElts)
418 return 1;
419
420 // We don't have enough elements to vectorize.
421 if (Idx + NumElts > ValueVTs.size())
422 return 1;
423
424 // PTX ISA can only deal with 2- and 4-element vector ops.
425 if (NumElts != 4 && NumElts != 2)
426 return 1;
427
428 for (unsigned j = Idx + 1; j < Idx + NumElts; ++j) {
429 // Types do not match.
430 if (ValueVTs[j] != EltVT)
431 return 1;
432
433 // Elements are not contiguous.
434 if (Offsets[j] - Offsets[j - 1] != EltSize)
435 return 1;
436 }
437 // OK. We can vectorize ValueVTs[i..i+NumElts)
438 return NumElts;
439}
440
441// Computes whether and how we can vectorize the loads/stores of a
442// flattened function parameter or return value.
443//
444// The flattened parameter is represented as the list of ValueVTs and
445// Offsets, and is aligned to ParamAlignment bytes. We return a vector
446// of the same size as ValueVTs indicating how each piece should be
447// loaded/stored (i.e. as a scalar, or as part of a vector
448// load/store).
449template <typename T>
450static SmallVector<unsigned, 16>
451VectorizePTXValueVTs(const SmallVectorImpl<EVT> &ValueVTs,
452 const SmallVectorImpl<T> &Offsets, Align ParamAlignment,
453 bool IsVAArg = false) {
454 // Set vector size to match ValueVTs and mark all elements as
455 // scalars by default.
456
457 if (IsVAArg)
458 return SmallVector<unsigned>(ValueVTs.size(), 1);
459
460 SmallVector<unsigned, 16> VectorInfo;
461
462 const auto GetNumElts = [&](unsigned I) -> unsigned {
463 for (const unsigned AccessSize : {16, 8, 4, 2}) {
464 const unsigned NumElts = canMergeParamLoadStoresStartingAt(
465 I, AccessSize, ValueVTs, Offsets, ParamAlignment);
466 assert((NumElts == 1 || NumElts == 2 || NumElts == 4) &&
467 "Unexpected vectorization size");
468 if (NumElts != 1)
469 return NumElts;
470 }
471 return 1;
472 };
473
474 // Check what we can vectorize using 128/64/32-bit accesses.
475 for (unsigned I = 0, E = ValueVTs.size(); I != E;) {
476 const unsigned NumElts = GetNumElts(I);
477 VectorInfo.push_back(Elt: NumElts);
478 I += NumElts;
479 }
480 assert(std::accumulate(VectorInfo.begin(), VectorInfo.end(), 0u) ==
481 ValueVTs.size());
482 return VectorInfo;
483}
484
485// NVPTXTargetLowering Constructor.
486NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
487 const NVPTXSubtarget &STI)
488 : TargetLowering(TM, STI), STI(STI), GlobalUniqueCallSite(0) {
489 // always lower memset, memcpy, and memmove intrinsics to load/store
490 // instructions, rather
491 // then generating calls to memset, mempcy or memmove.
492 MaxStoresPerMemset = MaxStoresPerMemsetOptSize = (unsigned)0xFFFFFFFF;
493 MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = (unsigned) 0xFFFFFFFF;
494 MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = (unsigned) 0xFFFFFFFF;
495
496 setBooleanContents(ZeroOrNegativeOneBooleanContent);
497 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
498
499 // Jump is Expensive. Don't create extra control flow for 'and', 'or'
500 // condition branches.
501 setJumpIsExpensive(true);
502
503 // Wide divides are _very_ slow. Try to reduce the width of the divide if
504 // possible.
505 addBypassSlowDiv(SlowBitWidth: 64, FastBitWidth: 32);
506
507 // By default, use the Source scheduling
508 if (STI.getCLOpts().sched4reg)
509 setSchedulingPreference(Sched::RegPressure);
510 else
511 setSchedulingPreference(Sched::Source);
512
513 auto setFP16OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action,
514 LegalizeAction NoF16Action) {
515 bool IsOpSupported = STI.allowFP16Math();
516 switch (Op) {
517 // Several FP16 instructions are available on sm_80 only.
518 case ISD::FMINNUM:
519 case ISD::FMAXNUM:
520 case ISD::FMAXNUM_IEEE:
521 case ISD::FMINNUM_IEEE:
522 case ISD::FMAXIMUM:
523 case ISD::FMINIMUM:
524 case ISD::FMAXIMUMNUM:
525 case ISD::FMINIMUMNUM:
526 IsOpSupported &= STI.hasFeature(Feature: NVPTX::SM80);
527 break;
528 case ISD::FEXP2:
529 case ISD::FTANH:
530 IsOpSupported &=
531 STI.hasFeature(Feature: NVPTX::SM75) && STI.hasFeature(Feature: NVPTX::PTX70);
532 break;
533 }
534 setOperationAction(Op, VT, Action: IsOpSupported ? Action : NoF16Action);
535 };
536
537 auto setBF16OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action,
538 LegalizeAction NoBF16Action) {
539 bool IsOpSupported = STI.hasNativeBF16Support(Opcode: Op);
540 setOperationAction(
541 Op, VT, Action: IsOpSupported ? Action : NoBF16Action);
542 };
543
544 auto setI16x2OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action,
545 LegalizeAction NoI16x2Action) {
546 bool IsOpSupported = false;
547 // instructions are available on sm_90 only
548 switch (Op) {
549 case ISD::ADD:
550 case ISD::SMAX:
551 case ISD::SMIN:
552 case ISD::UMIN:
553 case ISD::UMAX:
554 IsOpSupported =
555 STI.hasFeature(Feature: NVPTX::SM90) && STI.hasFeature(Feature: NVPTX::PTX80);
556 break;
557 }
558 setOperationAction(Op, VT, Action: IsOpSupported ? Action : NoI16x2Action);
559 };
560
561 addRegisterClass(VT: MVT::i1, RC: &NVPTX::B1RegClass);
562 addRegisterClass(VT: MVT::i16, RC: &NVPTX::B16RegClass);
563 addRegisterClass(VT: MVT::v2i16, RC: &NVPTX::B32RegClass);
564 addRegisterClass(VT: MVT::v4i8, RC: &NVPTX::B32RegClass);
565 addRegisterClass(VT: MVT::i32, RC: &NVPTX::B32RegClass);
566 addRegisterClass(VT: MVT::i64, RC: &NVPTX::B64RegClass);
567 addRegisterClass(VT: MVT::f32, RC: &NVPTX::B32RegClass);
568 addRegisterClass(VT: MVT::f64, RC: &NVPTX::B64RegClass);
569 addRegisterClass(VT: MVT::f16, RC: &NVPTX::B16RegClass);
570 addRegisterClass(VT: MVT::v2f16, RC: &NVPTX::B32RegClass);
571 addRegisterClass(VT: MVT::bf16, RC: &NVPTX::B16RegClass);
572 addRegisterClass(VT: MVT::v2bf16, RC: &NVPTX::B32RegClass);
573
574 if (STI.hasF32x2Instructions()) {
575 addRegisterClass(VT: MVT::v2f32, RC: &NVPTX::B64RegClass);
576 addRegisterClass(VT: MVT::v2i32, RC: &NVPTX::B64RegClass);
577 }
578
579 // Conversion to/from FP16/FP16x2 is always legal.
580 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v2f16, Action: Custom);
581 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2f16, Action: Custom);
582 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v2f16, Action: Expand);
583 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v2f16, Action: Expand);
584
585 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: Legal);
586 if (STI.hasFeature(Feature: NVPTX::SM30))
587 setOperationAction(Op: ISD::READSTEADYCOUNTER, VT: MVT::i64, Action: Legal);
588
589 setFP16OperationAction(ISD::SETCC, MVT::f16, Legal, Promote);
590 setFP16OperationAction(ISD::SETCC, MVT::v2f16, Legal, Expand);
591
592 // Conversion to/from BFP16/BFP16x2 is always legal.
593 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v2bf16, Action: Custom);
594 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2bf16, Action: Custom);
595 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v2bf16, Action: Expand);
596 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v2bf16, Action: Expand);
597
598 setBF16OperationAction(ISD::SETCC, MVT::v2bf16, Legal, Expand);
599 setBF16OperationAction(ISD::SETCC, MVT::bf16, Legal, Promote);
600 if (getOperationAction(Op: ISD::SETCC, VT: MVT::bf16) == Promote)
601 AddPromotedToType(Opc: ISD::SETCC, OrigVT: MVT::bf16, DestVT: MVT::f32);
602
603 // Conversion to/from i16/i16x2 is always legal.
604 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v2i16, Action: Custom);
605 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2i16, Action: Custom);
606 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v2i16, Action: Expand);
607 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v2i16, Action: Expand);
608
609 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v4i8, Action: Custom);
610 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v4i8, Action: Custom);
611 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v4i8, Action: Custom);
612 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v4i8, Action: Custom);
613
614 // No support for these operations with v2f32/v2i32
615 setOperationAction(Ops: ISD::INSERT_VECTOR_ELT, VTs: {MVT::v2f32, MVT::v2i32}, Action: Expand);
616 setOperationAction(Ops: ISD::VECTOR_SHUFFLE, VTs: {MVT::v2f32, MVT::v2i32}, Action: Expand);
617
618 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v2i16, Action: Expand);
619 setOperationAction(
620 Ops: {ISD::ANY_EXTEND, ISD::BSWAP, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
621 VT: MVT::v2i32, Action: Expand);
622
623 // Need custom lowering in case the index is dynamic.
624 if (STI.hasF32x2Instructions())
625 setOperationAction(Ops: ISD::EXTRACT_VECTOR_ELT, VTs: {MVT::v2f32, MVT::v2i32},
626 Action: Custom);
627
628 // Custom conversions to/from v2i8.
629 setOperationAction(Op: ISD::BITCAST, VT: MVT::v2i8, Action: Custom);
630
631 // Only logical ops can be done on v4i8/v2i32 directly, others must be done
632 // elementwise.
633 setOperationAction(
634 Ops: {ISD::ABS, ISD::ADD, ISD::ADDC, ISD::ADDE,
635 ISD::BITREVERSE, ISD::CTLZ, ISD::CTPOP, ISD::CTTZ,
636 ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::FSHL, ISD::FSHR,
637 ISD::MUL, ISD::MULHS, ISD::MULHU, ISD::PARITY,
638 ISD::ROTL, ISD::ROTR, ISD::SADDO, ISD::SADDO_CARRY,
639 ISD::SADDSAT, ISD::SDIV, ISD::SDIVREM, ISD::SELECT_CC,
640 ISD::SETCC, ISD::SHL, ISD::SINT_TO_FP, ISD::SMAX,
641 ISD::SMIN, ISD::SMULO, ISD::SMUL_LOHI, ISD::SRA,
642 ISD::SREM, ISD::SRL, ISD::SSHLSAT, ISD::SSUBO,
643 ISD::SSUBO_CARRY, ISD::SSUBSAT, ISD::SUB, ISD::SUBC,
644 ISD::SUBE, ISD::UADDO, ISD::UADDO_CARRY, ISD::UADDSAT,
645 ISD::UDIV, ISD::UDIVREM, ISD::UINT_TO_FP, ISD::UMAX,
646 ISD::UMIN, ISD::UMULO, ISD::UMUL_LOHI, ISD::UREM,
647 ISD::USHLSAT, ISD::USUBO, ISD::USUBO_CARRY, ISD::VSELECT,
648 ISD::USUBSAT},
649 VTs: {MVT::v4i8, MVT::v2i32}, Action: Expand);
650
651 // Operations not directly supported by NVPTX.
652 for (MVT VT : {MVT::bf16, MVT::f16, MVT::v2bf16, MVT::v2f16, MVT::f32,
653 MVT::v2f32, MVT::f64, MVT::i1, MVT::i8, MVT::i16, MVT::v2i16,
654 MVT::v4i8, MVT::i32, MVT::v2i32, MVT::i64}) {
655 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
656 setOperationAction(Op: ISD::BR_CC, VT, Action: Expand);
657 }
658
659 setOperationAction(Ops: ISD::SDIVREM, VTs: {MVT::i32, MVT::i64}, Action: Expand);
660 setOperationAction(Ops: ISD::UDIVREM, VTs: {MVT::i32, MVT::i64}, Action: Expand);
661
662 // We don't want ops like FMINIMUM or UMAX to be lowered to SETCC+VSELECT.
663 setOperationAction(Ops: ISD::VSELECT, VTs: {MVT::v2f32, MVT::v2i32}, Action: Expand);
664
665 // Some SIGN_EXTEND_INREG can be done using cvt instruction.
666 // For others we will expand to a SHL/SRA pair.
667 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i64, Action: Legal);
668 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i32, Action: Legal);
669 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i16, Action: Legal);
670 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i8 , Action: Legal);
671 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
672 setOperationAction(Ops: ISD::SIGN_EXTEND_INREG, VTs: {MVT::v2i16, MVT::v2i32}, Action: Expand);
673
674 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i32 , Action: Custom);
675 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i32 , Action: Custom);
676 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i32 , Action: Custom);
677 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i64 , Action: Custom);
678 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i64 , Action: Custom);
679 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i64 , Action: Custom);
680
681 if (STI.hasCLMAD())
682 setOperationAction(Ops: {ISD::CLMUL, ISD::CLMULH}, VT: MVT::i64, Action: Legal);
683 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i32, Action: Legal);
684 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i64, Action: Legal);
685
686 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR},
687 VTs: {MVT::i8, MVT::i16, MVT::v2i16, MVT::i32, MVT::i64},
688 Action: Expand);
689
690 if (STI.hasHWROT32()) {
691 setOperationAction(Ops: {ISD::FSHL, ISD::FSHR}, VT: MVT::i32, Action: Legal);
692 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR, ISD::FSHL, ISD::FSHR}, VT: MVT::i64,
693 Action: Custom);
694 }
695
696 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: STI.hasBrx() ? Legal : Expand);
697 setOperationAction(Op: ISD::BRIND, VT: MVT::Other, Action: Expand);
698
699 // We want to legalize constant related memmove and memcopy
700 // intrinsics.
701 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::Other, Action: Custom);
702
703 // FP extload/truncstore is not legal in PTX. We need to expand all these.
704 for (auto FloatVTs :
705 {MVT::fp_valuetypes(), MVT::fp_fixedlen_vector_valuetypes()}) {
706 for (MVT ValVT : FloatVTs) {
707 for (MVT MemVT : FloatVTs) {
708 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT, MemVT, Action: Expand);
709 setTruncStoreAction(ValVT, MemVT, Action: Expand);
710 }
711 }
712 }
713
714 // To improve CodeGen we'll legalize any-extend loads to zext loads. This is
715 // how they'll be lowered in ISel anyway, and by doing this a little earlier
716 // we allow for more DAG combine opportunities.
717 for (auto IntVTs :
718 {MVT::integer_valuetypes(), MVT::integer_fixedlen_vector_valuetypes()})
719 for (MVT ValVT : IntVTs)
720 for (MVT MemVT : IntVTs)
721 if (isTypeLegal(VT: ValVT))
722 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT, MemVT, Action: Custom);
723
724 // PTX does not support load / store predicate registers
725 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VT: MVT::i1, Action: Custom);
726 for (MVT VT : MVT::integer_valuetypes()) {
727 setLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}, ValVT: VT, MemVT: MVT::i1,
728 Action: Promote);
729 setTruncStoreAction(ValVT: VT, MemVT: MVT::i1, Action: Expand);
730 }
731
732 // Disable generations of extload/truncstore for v2i32/v2i16/v2i8. The generic
733 // expansion for these nodes when they are unaligned is incorrect if the
734 // type is a vector.
735 //
736 // TODO: Fix the generic expansion for these nodes found in
737 // TargetLowering::expandUnalignedLoad/Store.
738 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::v2i16,
739 MemVT: MVT::v2i8, Action: Expand);
740 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::v2i32,
741 MemVTs: {MVT::v2i8, MVT::v2i16}, Action: Expand);
742 setTruncStoreAction(ValVT: MVT::v2i16, MemVT: MVT::v2i8, Action: Expand);
743 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i16, Action: Expand);
744 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i1, Action: Expand);
745 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i8, Action: Expand);
746
747 // Register custom handling for illegal type loads/stores. We'll try to custom
748 // lower almost all illegal types and logic in the lowering will discard cases
749 // we can't handle.
750 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VTs: {MVT::i128, MVT::i256, MVT::f128},
751 Action: Custom);
752 for (MVT VT : MVT::fixedlen_vector_valuetypes())
753 if (!isTypeLegal(VT) && VT.getStoreSizeInBits() <= 256)
754 setOperationAction(Ops: {ISD::STORE, ISD::LOAD, ISD::MSTORE, ISD::MLOAD}, VT,
755 Action: Custom);
756
757 // Custom legalization for LDU intrinsics.
758 // TODO: The logic to lower these is not very robust and we should rewrite it.
759 // Perhaps LDU should not be represented as an intrinsic at all.
760 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::i8, Action: Custom);
761 for (MVT VT : MVT::fixedlen_vector_valuetypes())
762 if (IsPTXVectorType(VT))
763 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT, Action: Custom);
764
765 setCondCodeAction(CCs: {ISD::SETNE, ISD::SETEQ, ISD::SETUGE, ISD::SETULE,
766 ISD::SETUGT, ISD::SETULT, ISD::SETGT, ISD::SETLT,
767 ISD::SETGE, ISD::SETLE},
768 VT: MVT::i1, Action: Expand);
769
770 // This is legal in NVPTX
771 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f64, Action: Legal);
772 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f32, Action: Legal);
773 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f16, Action: Legal);
774 setOperationAction(Op: ISD::ConstantFP, VT: MVT::bf16, Action: Legal);
775
776 setOperationAction(Ops: ISD::DYNAMIC_STACKALLOC, VTs: {MVT::i32, MVT::i64}, Action: Custom);
777 setOperationAction(Ops: {ISD::STACKRESTORE, ISD::STACKSAVE}, VT: MVT::Other, Action: Custom);
778
779 // TRAP can be lowered to PTX trap
780 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
781 // DEBUGTRAP can be lowered to PTX brkpt
782 setOperationAction(Op: ISD::DEBUGTRAP, VT: MVT::Other, Action: Legal);
783
784 // Support varargs.
785 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
786 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Custom);
787 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Expand);
788 setOperationAction(Op: ISD::VAEND, VT: MVT::Other, Action: Expand);
789
790 setOperationAction(Ops: {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX},
791 VTs: {MVT::i16, MVT::i32, MVT::i64}, Action: Legal);
792 // PTX abs.s is undefined for INT_MIN, so ISD::ABS (which requires
793 // abs(INT_MIN) == INT_MIN) must be expanded. ABS_MIN_POISON matches
794 // PTX abs semantics since INT_MIN input is poison/undefined.
795 setOperationAction(Ops: ISD::ABS, VTs: {MVT::i16, MVT::i32, MVT::i64}, Action: Expand);
796 setOperationAction(Ops: ISD::ABS_MIN_POISON, VTs: {MVT::i16, MVT::i32, MVT::i64},
797 Action: Legal);
798
799 setOperationAction(Ops: {ISD::CTPOP, ISD::CTLZ, ISD::CTLZ_ZERO_POISON}, VT: MVT::i16,
800 Action: Promote);
801 setOperationAction(Ops: {ISD::CTPOP, ISD::CTLZ}, VT: MVT::i32, Action: Legal);
802 setOperationAction(Ops: {ISD::CTPOP, ISD::CTLZ}, VT: MVT::i64, Action: Custom);
803
804 setI16x2OperationAction(ISD::ABS_MIN_POISON, MVT::v2i16, Legal, Custom);
805 setI16x2OperationAction(ISD::SMIN, MVT::v2i16, Legal, Custom);
806 setI16x2OperationAction(ISD::SMAX, MVT::v2i16, Legal, Custom);
807 setI16x2OperationAction(ISD::UMIN, MVT::v2i16, Legal, Custom);
808 setI16x2OperationAction(ISD::UMAX, MVT::v2i16, Legal, Custom);
809 setI16x2OperationAction(ISD::CTPOP, MVT::v2i16, Legal, Expand);
810 setI16x2OperationAction(ISD::CTLZ, MVT::v2i16, Legal, Expand);
811
812 setI16x2OperationAction(ISD::ADD, MVT::v2i16, Legal, Custom);
813 setI16x2OperationAction(ISD::SUB, MVT::v2i16, Legal, Custom);
814 setI16x2OperationAction(ISD::MUL, MVT::v2i16, Legal, Custom);
815 setI16x2OperationAction(ISD::SHL, MVT::v2i16, Legal, Custom);
816 setI16x2OperationAction(ISD::SREM, MVT::v2i16, Legal, Custom);
817 setI16x2OperationAction(ISD::UREM, MVT::v2i16, Legal, Custom);
818
819 // Other arithmetic and logic ops are unsupported.
820 setOperationAction(Ops: {ISD::SDIV, ISD::UDIV, ISD::SRA, ISD::SRL, ISD::MULHS,
821 ISD::MULHU, ISD::SMUL_LOHI, ISD::UMUL_LOHI,
822 ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::SINT_TO_FP,
823 ISD::UINT_TO_FP, ISD::SETCC},
824 VT: MVT::v2i16, Action: Expand);
825
826 // v2i32 is not supported for any arithmetic operations
827 setOperationAction(Ops: {ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX,
828 ISD::CTPOP, ISD::CTLZ, ISD::ADD, ISD::SUB, ISD::MUL,
829 ISD::SHL, ISD::SRA, ISD::SRL, ISD::OR, ISD::AND, ISD::XOR,
830 ISD::SREM, ISD::UREM},
831 VT: MVT::v2i32, Action: Expand);
832
833 setOperationAction(Op: ISD::ADDC, VT: MVT::i32, Action: Legal);
834 setOperationAction(Op: ISD::ADDE, VT: MVT::i32, Action: Legal);
835 setOperationAction(Op: ISD::SUBC, VT: MVT::i32, Action: Legal);
836 setOperationAction(Op: ISD::SUBE, VT: MVT::i32, Action: Legal);
837 if (STI.hasFeature(Feature: NVPTX::PTX43)) {
838 setOperationAction(Op: ISD::ADDC, VT: MVT::i64, Action: Legal);
839 setOperationAction(Op: ISD::ADDE, VT: MVT::i64, Action: Legal);
840 setOperationAction(Op: ISD::SUBC, VT: MVT::i64, Action: Legal);
841 setOperationAction(Op: ISD::SUBE, VT: MVT::i64, Action: Legal);
842 }
843
844 setOperationAction(Op: ISD::CTTZ, VT: MVT::i16, Action: Expand);
845 setOperationAction(Ops: ISD::CTTZ, VTs: {MVT::v2i16, MVT::v2i32}, Action: Expand);
846 setOperationAction(Op: ISD::CTTZ, VT: MVT::i32, Action: Expand);
847 setOperationAction(Op: ISD::CTTZ, VT: MVT::i64, Action: Expand);
848
849 // PTX does not directly support SELP of i1, so promote to i32 first
850 setOperationAction(Op: ISD::SELECT, VT: MVT::i1, Action: Custom);
851
852 // PTX only exposes separate mul.lo and mul.hi forms. Lower i16 and i32 to
853 // use mul.wide. i64 has no mul.wide, so it expands to MUL and MULHS/MULHU.
854 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VTs: {MVT::i16, MVT::i32},
855 Action: Custom);
856 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT: MVT::i64, Action: Expand);
857
858 // We have some custom DAG combine patterns for these nodes
859 setTargetDAGCombine({ISD::ADD,
860 ISD::AND,
861 ISD::EXTRACT_VECTOR_ELT,
862 ISD::FADD,
863 ISD::FMAXNUM,
864 ISD::FMINNUM,
865 ISD::FMAXIMUM,
866 ISD::FMINIMUM,
867 ISD::FMAXIMUMNUM,
868 ISD::FMINIMUMNUM,
869 ISD::MUL,
870 ISD::SELECT,
871 ISD::SHL,
872 ISD::SREM,
873 ISD::UREM,
874 ISD::VSELECT,
875 ISD::BUILD_VECTOR,
876 ISD::ADDRSPACECAST,
877 ISD::LOAD,
878 ISD::STORE,
879 ISD::ZERO_EXTEND,
880 ISD::SIGN_EXTEND,
881 ISD::INTRINSIC_WO_CHAIN});
882
883 // If the vector operands require register coalescing, scalarize instead
884 if (STI.hasF32x2Instructions())
885 setTargetDAGCombine({ISD::FMA, ISD::FMUL, ISD::FSUB});
886
887 // setcc for f16x2 and bf16x2 needs special handling to prevent
888 // legalizer's attempt to scalarize it due to v2i1 not being legal.
889 if (STI.allowFP16Math() || STI.hasBF16Math())
890 setTargetDAGCombine(ISD::SETCC);
891
892 // Vector reduction operations. These may be turned into shuffle or tree
893 // reductions depending on what instructions are available for each type.
894 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
895 MVT EltVT = VT.getVectorElementType();
896 if (EltVT == MVT::f32 || EltVT == MVT::f64) {
897 setOperationAction(Ops: {ISD::VECREDUCE_FMAX, ISD::VECREDUCE_FMIN,
898 ISD::VECREDUCE_FMAXIMUM, ISD::VECREDUCE_FMINIMUM,
899 ISD::VECREDUCE_FMAXIMUMNUM,
900 ISD::VECREDUCE_FMINIMUMNUM},
901 VT, Action: Custom);
902 }
903 }
904
905 // Promote fp16 arithmetic if fp16 hardware isn't available or the
906 // user passed --nvptx-no-fp16-math. The flag is useful because,
907 // although sm_53+ GPUs have some sort of FP16 support in
908 // hardware, only sm_53 and sm_60 have full implementation. Others
909 // only have token amount of hardware and are likely to run faster
910 // by using fp32 units instead.
911 for (const auto &Op : {ISD::FADD, ISD::FMUL, ISD::FSUB, ISD::FMA}) {
912 setFP16OperationAction(Op, MVT::f16, Legal, Promote);
913 setFP16OperationAction(Op, MVT::v2f16, Legal, Expand);
914 setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);
915 // bf16 must be promoted to f32.
916 setBF16OperationAction(Op, MVT::bf16, Legal, Promote);
917 if (getOperationAction(Op, VT: MVT::bf16) == Promote)
918 AddPromotedToType(Opc: Op, OrigVT: MVT::bf16, DestVT: MVT::f32);
919 setOperationAction(Op, VT: MVT::v2f32,
920 Action: STI.hasF32x2Instructions() ? Legal : Expand);
921 }
922
923 // On SM80, we select add/mul/sub as fma to avoid promotion to float
924 for (const auto &Op : {ISD::FADD, ISD::FMUL, ISD::FSUB}) {
925 for (const auto &VT : {MVT::bf16, MVT::v2bf16}) {
926 if (!STI.hasNativeBF16Support(Opcode: Op) && STI.hasNativeBF16Support(Opcode: ISD::FMA)) {
927 setOperationAction(Op, VT, Action: Custom);
928 }
929 }
930 }
931
932 // f16/f16x2 neg was introduced in PTX 60, SM_53.
933 const bool IsFP16FP16x2NegAvailable = STI.hasFeature(Feature: NVPTX::SM53) &&
934 STI.hasFeature(Feature: NVPTX::PTX60) &&
935 STI.allowFP16Math();
936 for (const auto &VT : {MVT::f16, MVT::v2f16})
937 setOperationAction(Op: ISD::FNEG, VT,
938 Action: IsFP16FP16x2NegAvailable ? Legal : Expand);
939
940 setBF16OperationAction(ISD::FNEG, MVT::bf16, Legal, Expand);
941 setBF16OperationAction(ISD::FNEG, MVT::v2bf16, Legal, Expand);
942 setOperationAction(Op: ISD::FNEG, VT: MVT::v2f32, Action: Expand);
943 // (would be) Library functions.
944
945 // These map to conversion instructions for scalar FP types.
946 for (const auto &Op : {ISD::FCEIL, ISD::FFLOOR, ISD::FNEARBYINT, ISD::FRINT,
947 ISD::FROUNDEVEN, ISD::FTRUNC}) {
948 setOperationAction(Op, VT: MVT::f16, Action: Legal);
949 setOperationAction(Op, VT: MVT::f32, Action: Legal);
950 setOperationAction(Op, VT: MVT::f64, Action: Legal);
951 setOperationAction(Op, VT: MVT::v2f16, Action: Expand);
952 setOperationAction(Op, VT: MVT::v2bf16, Action: Expand);
953 setOperationAction(Op, VT: MVT::v2f32, Action: Expand);
954 setBF16OperationAction(Op, MVT::bf16, Legal, Promote);
955 if (getOperationAction(Op, VT: MVT::bf16) == Promote)
956 AddPromotedToType(Opc: Op, OrigVT: MVT::bf16, DestVT: MVT::f32);
957 }
958
959 if (!STI.hasFeature(Feature: NVPTX::SM80) || !STI.hasFeature(Feature: NVPTX::PTX71)) {
960 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f32, Action: Expand);
961 }
962 if (!STI.hasFeature(Feature: NVPTX::SM90)) {
963 for (MVT VT : {MVT::bf16, MVT::f32, MVT::f64}) {
964 setOperationAction(Op: ISD::FP_EXTEND, VT, Action: Custom);
965 setOperationAction(Op: ISD::FP_ROUND, VT, Action: Custom);
966 }
967 }
968
969 // Expand nearest-even rounding and diagnose unsupported conversions.
970 setOperationAction(Ops: ISD::FPTRUNC_ROUND, VTs: {MVT::f16, MVT::bf16, MVT::f32},
971 Action: Custom);
972
973 // Expand v2f32 = fp_extend
974 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v2f32, Action: Expand);
975 // Expand v2[b]f16 = fp_round v2f32
976 setOperationAction(Ops: ISD::FP_ROUND, VTs: {MVT::v2bf16, MVT::v2f16}, Action: Expand);
977
978 // sm_80 only has conversions between f32 and bf16. Custom lower all other
979 // bf16 conversions.
980 if (!STI.hasFeature(Feature: NVPTX::SM90)) {
981 for (MVT VT : {MVT::i1, MVT::i16, MVT::i32, MVT::i64}) {
982 setOperationAction(
983 Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT},
984 VT, Action: Custom);
985 }
986 setOperationAction(
987 Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT},
988 VT: MVT::bf16, Action: Custom);
989 }
990
991 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT: MVT::i1, Action: Custom);
992 setOperationAction(Op: ISD::FROUND, VT: MVT::f16, Action: Promote);
993 setOperationAction(Op: ISD::FROUND, VT: MVT::v2f16, Action: Expand);
994 setOperationAction(Op: ISD::FROUND, VT: MVT::v2bf16, Action: Expand);
995 setOperationAction(Op: ISD::FROUND, VT: MVT::f32, Action: Custom);
996 setOperationAction(Op: ISD::FROUND, VT: MVT::f64, Action: Custom);
997 setOperationAction(Op: ISD::FROUND, VT: MVT::bf16, Action: Promote);
998 AddPromotedToType(Opc: ISD::FROUND, OrigVT: MVT::bf16, DestVT: MVT::f32);
999
1000 setOperationAction(Ops: {ISD::LROUND, ISD::LLROUND}, VTs: {MVT::f32, MVT::f64}, Action: Expand);
1001
1002 setOperationAction(Ops: {ISD::LRINT, ISD::LLRINT}, VTs: {MVT::f32, MVT::f64}, Action: Expand);
1003
1004 // 'Expand' implements FCOPYSIGN without calling an external library.
1005 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f16, Action: Expand);
1006 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::v2f16, Action: Expand);
1007 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::bf16, Action: Expand);
1008 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::v2bf16, Action: Expand);
1009 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f32, Action: Custom);
1010 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f64, Action: Custom);
1011
1012 // These map to corresponding instructions for f32/f64. f16 must be
1013 // promoted to f32. v2f16 is expanded to f16, which is then promoted
1014 // to f32.
1015 for (const auto &Op : {ISD::FDIV, ISD::FSQRT, ISD::FSIN, ISD::FCOS}) {
1016 setOperationAction(Op, VT: MVT::f16, Action: Promote);
1017 setOperationAction(Op, VT: MVT::f32, Action: Legal);
1018 // Only div/sqrt are legal for f64.
1019 if (Op == ISD::FDIV || Op == ISD::FSQRT) {
1020 setOperationAction(Op, VT: MVT::f64, Action: Legal);
1021 }
1022 setOperationAction(Ops: Op, VTs: {MVT::v2f16, MVT::v2bf16, MVT::v2f32}, Action: Expand);
1023 setOperationAction(Op, VT: MVT::bf16, Action: Promote);
1024 AddPromotedToType(Opc: Op, OrigVT: MVT::bf16, DestVT: MVT::f32);
1025 }
1026 // Expand remainders at the IR level using ExpandIRInsts.
1027 setOperationAction(Ops: ISD::FREM, VTs: {MVT::f16, MVT::bf16, MVT::f32, MVT::f64},
1028 Action: Expand);
1029
1030 // FTANH support:
1031 // - f32 (sm_75+, PTX 7.0+)
1032 // - f16/f16x2 (sm_75+, PTX 7.0+)
1033 // - bf16/bf16x2 (sm_90+, PTX 7.8+)
1034 // When f16/bf16 types aren't supported, they are promoted/expanded to f32.
1035 if (STI.hasFeature(Feature: NVPTX::SM75) && STI.hasFeature(Feature: NVPTX::PTX70))
1036 setOperationAction(Op: ISD::FTANH, VT: MVT::f32, Action: Legal);
1037 setOperationAction(Op: ISD::FTANH, VT: MVT::v2f32, Action: Expand);
1038
1039 // Scalar f16/bf16: promote to f32 when not natively supported.
1040 setFP16OperationAction(ISD::FTANH, MVT::f16, Legal, Promote);
1041 setBF16OperationAction(ISD::FTANH, MVT::bf16, Legal, Promote);
1042 if (getOperationAction(Op: ISD::FTANH, VT: MVT::bf16) == Promote)
1043 AddPromotedToType(Opc: ISD::FTANH, OrigVT: MVT::bf16, DestVT: MVT::f32);
1044
1045 // Vector v2f16/v2bf16: expand when not natively supported.
1046 setFP16OperationAction(ISD::FTANH, MVT::v2f16, Legal, Expand);
1047 setBF16OperationAction(ISD::FTANH, MVT::v2bf16, Legal, Expand);
1048
1049 setOperationAction(Ops: ISD::FABS, VTs: {MVT::f32, MVT::f64}, Action: Legal);
1050 setOperationAction(Op: ISD::FABS, VT: MVT::v2f32, Action: Expand);
1051 if (STI.hasFeature(Feature: NVPTX::PTX65)) {
1052 setFP16OperationAction(ISD::FABS, MVT::f16, Legal, Promote);
1053 setFP16OperationAction(ISD::FABS, MVT::v2f16, Legal, Expand);
1054 } else {
1055 setOperationAction(Op: ISD::FABS, VT: MVT::f16, Action: Promote);
1056 setOperationAction(Op: ISD::FABS, VT: MVT::v2f16, Action: Expand);
1057 }
1058 setBF16OperationAction(ISD::FABS, MVT::v2bf16, Legal, Expand);
1059 setBF16OperationAction(ISD::FABS, MVT::bf16, Legal, Promote);
1060 if (getOperationAction(Op: ISD::FABS, VT: MVT::bf16) == Promote)
1061 AddPromotedToType(Opc: ISD::FABS, OrigVT: MVT::bf16, DestVT: MVT::f32);
1062
1063 for (const auto &Op :
1064 {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM}) {
1065 setOperationAction(Op, VT: MVT::f32, Action: Legal);
1066 setOperationAction(Op, VT: MVT::f64, Action: Legal);
1067 setFP16OperationAction(Op, MVT::f16, Legal, Promote);
1068 setFP16OperationAction(Op, MVT::v2f16, Legal, Expand);
1069 setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);
1070 setBF16OperationAction(Op, MVT::bf16, Legal, Promote);
1071 if (getOperationAction(Op, VT: MVT::bf16) == Promote)
1072 AddPromotedToType(Opc: Op, OrigVT: MVT::bf16, DestVT: MVT::f32);
1073 setOperationAction(Op, VT: MVT::v2f32, Action: Expand);
1074 }
1075 bool SupportsF32MinMaxNaN = STI.hasFeature(Feature: NVPTX::SM80);
1076 for (const auto &Op : {ISD::FMINIMUM, ISD::FMAXIMUM}) {
1077 setOperationAction(Op, VT: MVT::f32, Action: SupportsF32MinMaxNaN ? Legal : Expand);
1078 setFP16OperationAction(Op, MVT::f16, Legal, Expand);
1079 setFP16OperationAction(Op, MVT::v2f16, Legal, Expand);
1080 setBF16OperationAction(Op, MVT::bf16, Legal, Expand);
1081 setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);
1082 setOperationAction(Op, VT: MVT::v2f32, Action: Expand);
1083 }
1084
1085 // Custom lowering for inline asm with 128-bit operands
1086 setOperationAction(Op: ISD::CopyToReg, VT: MVT::i128, Action: Custom);
1087 setOperationAction(Op: ISD::CopyFromReg, VT: MVT::i128, Action: Custom);
1088
1089 // FEXP2 support:
1090 // - f32
1091 // - f16/f16x2 (sm_70+, PTX 7.0+)
1092 // - bf16/bf16x2 (sm_90+, PTX 7.8+)
1093 // When f16/bf16 types aren't supported, they are promoted/expanded to f32.
1094 setOperationAction(Op: ISD::FEXP2, VT: MVT::f32, Action: Legal);
1095 setOperationAction(Op: ISD::FEXP2, VT: MVT::v2f32, Action: Expand);
1096 setFP16OperationAction(ISD::FEXP2, MVT::f16, Legal, Promote);
1097 setFP16OperationAction(ISD::FEXP2, MVT::v2f16, Legal, Expand);
1098 setBF16OperationAction(ISD::FEXP2, MVT::bf16, Legal, Promote);
1099 setBF16OperationAction(ISD::FEXP2, MVT::v2bf16, Legal, Expand);
1100
1101 // FLOG2 supports f32 only
1102 // f16/bf16 types aren't supported, but they are promoted/expanded to f32.
1103 if (STI.getCLOpts().approx_log2f32) {
1104 setOperationAction(Op: ISD::FLOG2, VT: MVT::f32, Action: Legal);
1105 setOperationPromotedToType(Opc: ISD::FLOG2, OrigVT: MVT::f16, DestVT: MVT::f32);
1106 setOperationPromotedToType(Opc: ISD::FLOG2, OrigVT: MVT::bf16, DestVT: MVT::f32);
1107 setOperationAction(Ops: ISD::FLOG2, VTs: {MVT::v2f16, MVT::v2bf16, MVT::v2f32},
1108 Action: Expand);
1109 }
1110
1111 setOperationAction(Ops: ISD::ADDRSPACECAST, VTs: {MVT::i32, MVT::i64}, Action: Custom);
1112
1113 // 128-bit atomic instructions are legal in PTX, but since i128 is not a
1114 // legal type, we need to custom lower them.
1115 setOperationAction(Ops: {ISD::ATOMIC_CMP_SWAP, ISD::ATOMIC_SWAP, ISD::ATOMIC_LOAD,
1116 ISD::ATOMIC_STORE},
1117 VT: MVT::i128, Action: Custom);
1118
1119 // Now deduce the information based on the above mentioned
1120 // actions
1121 computeRegisterProperties(TRI: STI.getRegisterInfo());
1122
1123 // PTX support for 16-bit CAS is emulated. Only use 32+
1124 setMinCmpXchgSizeInBits(STI.getMinCmpXchgSizeInBits());
1125 // ld/st.b128 requires only sm_70, unlike the atom.*.b128 forms.
1126 setMaxAtomicSizeInBitsSupported(STI.hasLdStB128() ? 128 : 64);
1127 setMaxDivRemBitWidthSupported(64);
1128 setMaxLargeFPConvertBitWidthSupported(64);
1129
1130 // Custom lowering for tcgen05.ld vector operands
1131 setOperationAction(Ops: ISD::INTRINSIC_W_CHAIN,
1132 VTs: {MVT::v1i32, MVT::v2i32, MVT::v4i32, MVT::v8i32,
1133 MVT::v16i32, MVT::v32i32, MVT::v64i32, MVT::v128i32,
1134 MVT::v2f32, MVT::v4f32, MVT::v8f32, MVT::v16f32,
1135 MVT::v32f32, MVT::v64f32, MVT::v128f32},
1136 Action: Custom);
1137
1138 // Custom lowering for tcgen05.st vector operands and the st.async
1139 // i128 (.b128) operand. MVT::i8 is needed for the st.async.{sys,gpu} b8
1140 // variant.
1141 setOperationAction(Ops: ISD::INTRINSIC_VOID,
1142 VTs: {MVT::i8, MVT::v1i32, MVT::v2i32, MVT::v4i32, MVT::v8i32,
1143 MVT::v16i32, MVT::v32i32, MVT::v64i32, MVT::v128i32,
1144 MVT::i128, MVT::Other},
1145 Action: Custom);
1146
1147 // Enable custom lowering for the following:
1148 // * MVT::i128 - clusterlaunchcontrol
1149 // * MVT::i32 - prmt
1150 // * MVT::v4f32 - cvt_rs fp{4/6/8}x4 intrinsics
1151 // * MVT::Other - internal.addrspace.wrap
1152 setOperationAction(Ops: ISD::INTRINSIC_WO_CHAIN,
1153 VTs: {MVT::i32, MVT::i128, MVT::v4f32, MVT::Other}, Action: Custom);
1154
1155 // Custom lowering for bswap
1156 setOperationAction(Ops: ISD::BSWAP, VTs: {MVT::i16, MVT::i32, MVT::i64, MVT::v2i16},
1157 Action: Custom);
1158}
1159
1160TargetLoweringBase::LegalizeTypeAction
1161NVPTXTargetLowering::getPreferredVectorAction(MVT VT) const {
1162 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
1163 VT.getScalarType() == MVT::i1)
1164 return TypeSplitVector;
1165 return TargetLoweringBase::getPreferredVectorAction(VT);
1166}
1167
1168SDValue NVPTXTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,
1169 int Enabled, int &ExtraSteps,
1170 bool &UseOneConst,
1171 bool Reciprocal) const {
1172 if (!(Enabled == ReciprocalEstimate::Enabled ||
1173 (Enabled == ReciprocalEstimate::Unspecified && !usePrecSqrtF32())))
1174 return SDValue();
1175
1176 if (ExtraSteps == ReciprocalEstimate::Unspecified)
1177 ExtraSteps = 0;
1178
1179 SDLoc DL(Operand);
1180 EVT VT = Operand.getValueType();
1181 bool Ftz = useF32FTZ(MF: DAG.getMachineFunction());
1182
1183 auto MakeIntrinsicCall = [&](Intrinsic::ID IID) {
1184 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT,
1185 N1: DAG.getConstant(Val: IID, DL, VT: MVT::i32), N2: Operand);
1186 };
1187
1188 // The sqrt and rsqrt refinement processes assume we always start out with an
1189 // approximation of the rsqrt. Therefore, if we're going to do any refinement
1190 // (i.e. ExtraSteps > 0), we must return an rsqrt. But if we're *not* doing
1191 // any refinement, we must return a regular sqrt.
1192 if (Reciprocal || ExtraSteps > 0) {
1193 if (VT == MVT::f32)
1194 return MakeIntrinsicCall(Ftz ? Intrinsic::nvvm_rsqrt_approx_ftz_f
1195 : Intrinsic::nvvm_rsqrt_approx_f);
1196 else if (VT == MVT::f64)
1197 return MakeIntrinsicCall(Intrinsic::nvvm_rsqrt_approx_d);
1198 else
1199 return SDValue();
1200 } else {
1201 if (VT == MVT::f32)
1202 return MakeIntrinsicCall(Ftz ? Intrinsic::nvvm_sqrt_approx_ftz_f
1203 : Intrinsic::nvvm_sqrt_approx_f);
1204 else {
1205 // There's no sqrt.approx.f64 instruction, so we emit
1206 // reciprocal(rsqrt(x)). This is faster than
1207 // select(x == 0, 0, x * rsqrt(x)). (In fact, it's faster than plain
1208 // x * rsqrt(x).)
1209 return DAG.getNode(
1210 Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT,
1211 N1: DAG.getConstant(Val: Intrinsic::nvvm_rcp_approx_ftz_d, DL, VT: MVT::i32),
1212 N2: MakeIntrinsicCall(Intrinsic::nvvm_rsqrt_approx_d));
1213 }
1214 }
1215}
1216
1217static MachinePointerInfo refinePtrAS(SDValue &Ptr, SelectionDAG &DAG) {
1218 // Load directly from the source address space of a cast to generic.
1219 unsigned SrcAS = ADDRESS_SPACE_GENERIC;
1220 if (Ptr->getOpcode() == ISD::ADDRSPACECAST) {
1221 const auto *ASC = cast<AddrSpaceCastSDNode>(Val&: Ptr);
1222 if (ASC->getDestAddressSpace() == ADDRESS_SPACE_GENERIC) {
1223 Ptr = ASC->getOperand(Num: 0);
1224 SrcAS = ASC->getSrcAddressSpace();
1225 }
1226 }
1227
1228 // Preserve the alloca's address space through frame-index inference.
1229 if (const auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Ptr))
1230 if (const AllocaInst *AI =
1231 DAG.getMachineFunction().getFrameInfo().getObjectAllocation(
1232 ObjectIdx: FIN->getIndex()))
1233 return MachinePointerInfo(AI);
1234
1235 return MachinePointerInfo(SrcAS);
1236}
1237
1238static ISD::NodeType getExtOpcode(const ISD::ArgFlagsTy &Flags) {
1239 if (Flags.isSExt())
1240 return ISD::SIGN_EXTEND;
1241 if (Flags.isZExt())
1242 return ISD::ZERO_EXTEND;
1243 return ISD::ANY_EXTEND;
1244}
1245
1246static SDValue correctParamType(SDValue V, EVT ExpectedVT,
1247 ISD::ArgFlagsTy Flags, SelectionDAG &DAG,
1248 SDLoc dl) {
1249 const EVT ActualVT = V.getValueType();
1250 assert((ActualVT == ExpectedVT ||
1251 (ExpectedVT.isInteger() && ActualVT.isInteger())) &&
1252 "Non-integer argument type size mismatch");
1253 if (ExpectedVT.bitsGT(VT: ActualVT))
1254 return DAG.getNode(Opcode: getExtOpcode(Flags), DL: dl, VT: ExpectedVT, Operand: V);
1255 if (ExpectedVT.bitsLT(VT: ActualVT))
1256 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ExpectedVT, Operand: V);
1257
1258 return V;
1259}
1260
1261static SDValue getSymbolNode(SelectionDAG &DAG, MCSymbol *Sym, EVT T) {
1262 return DAG.getNode(Opcode: NVPTXISD::Symbol, DL: SDLoc(), VT: T, Operand: DAG.getMCSymbol(Sym, VT: T));
1263}
1264
1265static SDValue getSymbolNode(SelectionDAG &DAG, const Twine &Name, EVT T) {
1266 MCContext &Ctx = DAG.getMachineFunction().getContext();
1267 return getSymbolNode(DAG, Sym: Ctx.getOrCreateSymbol(Name), T);
1268}
1269
1270SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
1271 SmallVectorImpl<SDValue> &InVals) const {
1272
1273 if (CLI.IsVarArg &&
1274 (!STI.hasFeature(Feature: NVPTX::PTX60) || !STI.hasFeature(Feature: NVPTX::SM30)))
1275 report_fatal_error(
1276 reason: "Support for variadic functions (unsized array parameter) introduced "
1277 "in PTX ISA version 6.0 and requires target sm_30.");
1278
1279 SelectionDAG &DAG = CLI.DAG;
1280 SDLoc dl = CLI.DL;
1281 const SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
1282 SDValue Callee = CLI.Callee;
1283 ArgListTy &Args = CLI.getArgs();
1284 Type *RetTy = CLI.RetTy;
1285 const CallBase *CB = CLI.CB;
1286 const DataLayout &DL = DAG.getDataLayout();
1287 LLVMContext &Ctx = *DAG.getContext();
1288
1289 const auto GetI32 = [&](const unsigned I) {
1290 return DAG.getConstant(Val: I, DL: dl, VT: MVT::i32);
1291 };
1292
1293 const unsigned UniqueCallSite = GlobalUniqueCallSite++;
1294 const SDValue CallChain = CLI.Chain;
1295 const SDValue StartChain =
1296 DAG.getCALLSEQ_START(Chain: CallChain, InSize: UniqueCallSite, OutSize: 0, DL: dl);
1297 SDValue DeclareGlue = StartChain.getValue(R: 1);
1298
1299 SmallVector<SDValue, 16> CallPrereqs{StartChain};
1300
1301 const auto MakeDeclareScalarParam = [&](SDValue Symbol, unsigned Size) {
1302 // PTX ABI requires integral types to be at least 32 bits in size. FP16 is
1303 // loaded/stored using i16, so it's handled here as well.
1304 const unsigned SizeBits = promoteScalarArgumentSize(Size: Size * 8);
1305 SDValue Declare =
1306 DAG.getNode(Opcode: NVPTXISD::DeclareScalarParam, DL: dl, ResultTys: {MVT::Other, MVT::Glue},
1307 Ops: {StartChain, Symbol, GetI32(SizeBits), DeclareGlue});
1308 CallPrereqs.push_back(Elt: Declare);
1309 DeclareGlue = Declare.getValue(R: 1);
1310 return Declare;
1311 };
1312
1313 const auto MakeDeclareArrayParam = [&](SDValue Symbol, Align Align,
1314 unsigned Size) {
1315 SDValue Declare = DAG.getNode(
1316 Opcode: NVPTXISD::DeclareArrayParam, DL: dl, ResultTys: {MVT::Other, MVT::Glue},
1317 Ops: {StartChain, Symbol, GetI32(Align.value()), GetI32(Size), DeclareGlue});
1318 CallPrereqs.push_back(Elt: Declare);
1319 DeclareGlue = Declare.getValue(R: 1);
1320 return Declare;
1321 };
1322
1323 // Variadic arguments.
1324 //
1325 // Normally, for each argument, we declare a param scalar or a param
1326 // byte array in the .param space, and store the argument value to that
1327 // param scalar or array starting at offset 0.
1328 //
1329 // In the case of the first variadic argument, we declare a vararg byte array
1330 // with size 0. The exact size of this array isn't known at this point, so
1331 // it'll be patched later. All the variadic arguments will be stored to this
1332 // array at a certain offset (which gets tracked by 'VAOffset'). The offset is
1333 // initially set to 0, so it can be used for non-variadic arguments (which use
1334 // 0 offset) to simplify the code.
1335 //
1336 // After all vararg is processed, 'VAOffset' holds the size of the
1337 // vararg byte array.
1338 assert((CLI.IsVarArg || CLI.Args.size() <= CLI.NumFixedArgs) &&
1339 "Non-VarArg function with extra arguments");
1340
1341 const unsigned FirstVAArg = CLI.NumFixedArgs; // position of first variadic
1342 unsigned VAOffset = 0; // current offset in the param array
1343
1344 const SDValue VADeclareParam =
1345 CLI.Args.size() > FirstVAArg
1346 ? MakeDeclareArrayParam(
1347 getCallParamSymbolNode(DAG, I: FirstVAArg, T: MVT::i32),
1348 Align(STI.getMaxRequiredAlignment()), 0)
1349 : SDValue();
1350
1351 // Args.size() and Outs.size() need not match.
1352 // Outs.size() will be larger
1353 // * if there is an aggregate argument with multiple fields (each field
1354 // showing up separately in Outs)
1355 // * if there is a vector argument with more than typical vector-length
1356 // elements (generally if more than 4) where each vector element is
1357 // individually present in Outs.
1358 // So a different index should be used for indexing into Outs/OutVals.
1359 // See similar issue in LowerFormalArguments.
1360 auto AllOuts = ArrayRef(CLI.Outs);
1361 auto AllOutVals = ArrayRef(CLI.OutVals);
1362 assert(AllOuts.size() == AllOutVals.size() &&
1363 "Outs and OutVals must be the same size");
1364 // Declare the .params or .reg need to pass values
1365 // to the function
1366 for (const auto E : llvm::enumerate(First&: Args)) {
1367 const auto ArgI = E.index();
1368 const auto Arg = E.value();
1369 const auto ArgOuts =
1370 AllOuts.take_while(Pred: [&](auto O) { return O.OrigArgIndex == ArgI; });
1371 const auto ArgOutVals = AllOutVals.take_front(N: ArgOuts.size());
1372 AllOuts = AllOuts.drop_front(N: ArgOuts.size());
1373 AllOutVals = AllOutVals.drop_front(N: ArgOuts.size());
1374
1375 const bool IsVAArg = (ArgI >= FirstVAArg);
1376 const bool IsByVal = Arg.IsByVal;
1377
1378 const SDValue ParamSymbol =
1379 getCallParamSymbolNode(DAG, I: IsVAArg ? FirstVAArg : ArgI, T: MVT::i32);
1380
1381 assert((!IsByVal || Arg.IndirectType) &&
1382 "byval arg must have indirect type");
1383 Type *ETy = (IsByVal ? Arg.IndirectType : Arg.Ty);
1384
1385 const Align ArgAlign = [&]() {
1386 const unsigned ParamIdx = ArgI + AttributeList::FirstArgIndex;
1387 if (IsByVal)
1388 return getDeviceByValParamAlign(CB, ArgTy: ETy, AttrIdx: ParamIdx, DL);
1389 return getPTXParamAlign(CB, Ty: Arg.Ty, AttrIdx: ParamIdx, DL);
1390 }();
1391
1392 const unsigned TySize = DL.getTypeAllocSize(Ty: ETy);
1393 assert((!IsByVal || TySize == ArgOuts[0].Flags.getByValSize()) &&
1394 "type size mismatch");
1395
1396 const SDValue ArgDeclare = [&]() {
1397 if (IsVAArg)
1398 return VADeclareParam;
1399
1400 if (IsByVal || shouldPassAsArray(Ty: Arg.Ty))
1401 return MakeDeclareArrayParam(ParamSymbol, ArgAlign, TySize);
1402
1403 assert(ArgOuts.size() == 1 && "We must pass only one value as non-array");
1404 assert((ArgOuts[0].VT.isInteger() || ArgOuts[0].VT.isFloatingPoint()) &&
1405 "Only int and float types are supported as non-array arguments");
1406
1407 return MakeDeclareScalarParam(ParamSymbol, TySize);
1408 }();
1409
1410 if (IsByVal) {
1411 assert(ArgOutVals.size() == 1 && "We must pass only one value as byval");
1412 SDValue SrcPtr = ArgOutVals[0];
1413 const MachinePointerInfo SrcPtrInfo = refinePtrAS(Ptr&: SrcPtr, DAG);
1414 // Don't use Flags.getNonZeroByValAlign as this includes the stackalign,
1415 // which does not apply to the source pointer.
1416 const Align BaseSrcAlign = [&]() {
1417 // The align attribute on a byval argument indicates the known alignment
1418 // of the pointer passed to the function.
1419 if (CB)
1420 if (const MaybeAlign A = CB->getParamAlign(ArgNo: ArgI))
1421 return *A;
1422 // Fall back to the default alignment for the type.
1423 // TODO: This might be too aggressive but we haven't had a problem with
1424 // it yet.
1425 return getPTXParamTypeAlign(ArgTy: ETy, DL);
1426 }();
1427
1428 if (IsVAArg)
1429 VAOffset = alignTo(Size: VAOffset, A: ArgAlign);
1430
1431 SmallVector<EVT, 4> ValueVTs, MemVTs;
1432 SmallVector<TypeSize, 4> Offsets;
1433 ComputeValueVTs(TLI: *this, DL, Ty: ETy, ValueVTs, MemVTs: &MemVTs, Offsets: &Offsets);
1434
1435 unsigned J = 0;
1436 const auto VI = VectorizePTXValueVTs(ValueVTs: MemVTs, Offsets, ParamAlignment: ArgAlign, IsVAArg);
1437 for (const unsigned NumElts : VI) {
1438 EVT LoadVT = getVectorizedVT(VT: MemVTs[J], N: NumElts, C&: Ctx);
1439 Align SrcAlign = commonAlignment(A: BaseSrcAlign, Offset: Offsets[J]);
1440 SDValue SrcAddr = DAG.getObjectPtrOffset(SL: dl, Ptr: SrcPtr, Offset: Offsets[J]);
1441 SDValue SrcLoad =
1442 DAG.getLoad(VT: LoadVT, dl, Chain: CallChain, Ptr: SrcAddr,
1443 PtrInfo: SrcPtrInfo.getWithOffset(O: Offsets[J]), Alignment: SrcAlign);
1444
1445 TypeSize ParamOffset = Offsets[J].getWithIncrement(RHS: VAOffset);
1446 Align ParamAlign = commonAlignment(A: ArgAlign, Offset: ParamOffset);
1447 SDValue ParamAddr =
1448 DAG.getObjectPtrOffset(SL: dl, Ptr: ParamSymbol, Offset: ParamOffset);
1449 SDValue StoreParam = DAG.getStore(
1450 Chain: ArgDeclare, dl, Val: SrcLoad, Ptr: ParamAddr,
1451 PtrInfo: MachinePointerInfo(NVPTX::AddressSpace::DeviceParam), Alignment: ParamAlign);
1452 CallPrereqs.push_back(Elt: StoreParam);
1453
1454 J += NumElts;
1455 }
1456 if (IsVAArg)
1457 VAOffset += TySize;
1458 } else {
1459 SmallVector<EVT, 16> VTs;
1460 SmallVector<uint64_t, 16> Offsets;
1461 ComputePTXValueVTs(TLI: *this, DL, Ctx, CallConv: CLI.CallConv, Ty: Arg.Ty, ValueVTs&: VTs, Offsets,
1462 StartingOffset: VAOffset);
1463 assert(VTs.size() == Offsets.size() && "Size mismatch");
1464 assert(VTs.size() == ArgOuts.size() && "Size mismatch");
1465
1466 // PTX Interoperability Guide 3.3(A): [Integer] Values shorter
1467 // than 32-bits are sign extended or zero extended, depending on
1468 // whether they are signed or unsigned types. This case applies
1469 // only to scalar parameters and not to aggregate values.
1470 const bool ExtendIntegerParam =
1471 Arg.Ty->isIntegerTy() && DL.getTypeAllocSizeInBits(Ty: Arg.Ty) < 32;
1472
1473 const auto GetStoredValue = [&](const unsigned I) {
1474 SDValue StVal = ArgOutVals[I];
1475 assert(promoteScalarIntegerPTX(StVal.getValueType()) ==
1476 StVal.getValueType() &&
1477 "OutVal type should always be legal");
1478
1479 const EVT VTI = promoteScalarIntegerPTX(VT: VTs[I]);
1480 const EVT StoreVT =
1481 ExtendIntegerParam ? MVT::i32 : (VTI == MVT::i1 ? MVT::i8 : VTI);
1482
1483 return correctParamType(V: StVal, ExpectedVT: StoreVT, Flags: ArgOuts[I].Flags, DAG, dl);
1484 };
1485
1486 unsigned J = 0;
1487 const auto VI = VectorizePTXValueVTs(ValueVTs: VTs, Offsets, ParamAlignment: ArgAlign, IsVAArg);
1488 for (const unsigned NumElts : VI) {
1489 const EVT EltVT = promoteScalarIntegerPTX(VT: VTs[J]);
1490
1491 unsigned Offset;
1492 if (IsVAArg) {
1493 // TODO: We may need to support vector types that can be passed
1494 // as scalars in variadic arguments.
1495 assert(NumElts == 1 &&
1496 "Vectorization should be disabled for vaargs.");
1497
1498 // Align each part of the variadic argument to their type.
1499 VAOffset = alignTo(Size: VAOffset, A: DAG.getEVTAlign(MemoryVT: EltVT));
1500 Offset = VAOffset;
1501
1502 const EVT TheStoreType = ExtendIntegerParam ? MVT::i32 : EltVT;
1503 VAOffset += DL.getTypeAllocSize(Ty: TheStoreType.getTypeForEVT(Context&: Ctx));
1504 } else {
1505 assert(VAOffset == 0 && "VAOffset must be 0 for non-VA args");
1506 Offset = Offsets[J];
1507 }
1508
1509 SDValue Ptr =
1510 DAG.getObjectPtrOffset(SL: dl, Ptr: ParamSymbol, Offset: TypeSize::getFixed(ExactSize: Offset));
1511
1512 const MaybeAlign CurrentAlign = ExtendIntegerParam
1513 ? MaybeAlign(std::nullopt)
1514 : commonAlignment(A: ArgAlign, Offset);
1515
1516 SDValue Val =
1517 getBuildVectorizedValue(N: NumElts, dl, DAG, GetElement: [&](unsigned K) {
1518 return GetStoredValue(J + K);
1519 });
1520
1521 SDValue StoreParam = DAG.getStore(
1522 Chain: ArgDeclare, dl, Val, Ptr,
1523 PtrInfo: MachinePointerInfo(NVPTX::AddressSpace::DeviceParam), Alignment: CurrentAlign);
1524 CallPrereqs.push_back(Elt: StoreParam);
1525
1526 J += NumElts;
1527 }
1528 }
1529 }
1530
1531 // Handle Result
1532 if (!Ins.empty()) {
1533 const SDValue RetSymbol = getSymbolNode(DAG, Name: "retval0", T: MVT::i32);
1534 const unsigned ResultSize = DL.getTypeAllocSize(Ty: RetTy);
1535 if (shouldPassAsArray(Ty: RetTy)) {
1536 const Align RetAlign =
1537 getPTXParamAlign(CB, Ty: RetTy, AttrIdx: AttributeList::ReturnIndex, DL);
1538 MakeDeclareArrayParam(RetSymbol, RetAlign, ResultSize);
1539 } else {
1540 MakeDeclareScalarParam(RetSymbol, ResultSize);
1541 }
1542 }
1543
1544 // Set the size of the vararg param byte array if the callee is a variadic
1545 // function and the variadic part is not empty.
1546 if (VADeclareParam) {
1547 SDValue DeclareParamOps[] = {VADeclareParam.getOperand(i: 0),
1548 VADeclareParam.getOperand(i: 1),
1549 VADeclareParam.getOperand(i: 2), GetI32(VAOffset),
1550 VADeclareParam.getOperand(i: 4)};
1551 DAG.MorphNodeTo(N: VADeclareParam.getNode(), Opc: VADeclareParam.getOpcode(),
1552 VTs: VADeclareParam->getVTList(), Ops: DeclareParamOps);
1553 }
1554
1555 const auto *Func = dyn_cast<GlobalAddressSDNode>(Val: Callee.getNode());
1556 const auto *CalleeF = Func ? dyn_cast<Function>(Val: Func->getGlobal()) : nullptr;
1557
1558 // If the type of the callsite does not match that of the function, convert
1559 // the callsite to an indirect call.
1560 const bool ConvertToIndirectCall =
1561 CalleeF && CB->getFunctionType() != CalleeF->getFunctionType();
1562
1563 // Both indirect calls and libcalls have nullptr Func. In order to distinguish
1564 // between them we must rely on the call site value which is valid for
1565 // indirect calls but is always null for libcalls.
1566 const bool IsIndirectCall = (!Func && CB) || ConvertToIndirectCall;
1567
1568 if (isa<ExternalSymbolSDNode>(Val: Callee)) {
1569 Function* CalleeFunc = nullptr;
1570
1571 // Try to find the callee in the current module.
1572 Callee = DAG.getSymbolFunctionGlobalAddress(Op: Callee, TargetFunction: &CalleeFunc);
1573 assert(CalleeFunc != nullptr && "Libcall callee must be set.");
1574
1575 // Set the "libcall callee" attribute to indicate that the function
1576 // must always have a declaration.
1577 CalleeFunc->addFnAttr(Kind: "nvptx-libcall-callee", Val: "true");
1578 }
1579
1580 // In the indirect function call case, PTX requires a prototype of the form:
1581 // proto_0 : .callprototype(.param .b32 _) _ (.param .b32 _);
1582 // Where the label is to be used as the last arg of the call instruction.
1583 // We record the call site here and emit all prototypes at the
1584 // start of the function in the AsmPrinter.
1585 SDValue Proto = GetI32(0);
1586 if (IsIndirectCall) {
1587 auto *ProtoSymbol = DAG.getMachineFunction()
1588 .getInfo<NVPTXMachineFunctionInfo>()
1589 ->addCallPrototype(CB, MF&: DAG.getMachineFunction());
1590 Proto = DAG.getMCSymbol(Sym: ProtoSymbol, VT: MVT::i32);
1591 }
1592
1593 const bool IsUnknownIntrinsic =
1594 CalleeF && CalleeF->isIntrinsic() &&
1595 CalleeF->getIntrinsicID() == Intrinsic::not_intrinsic;
1596 if (IsUnknownIntrinsic) {
1597 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1598 DAG.getMachineFunction().getFunction(),
1599 "call to unknown intrinsic '" + CalleeF->getName() +
1600 "' cannot be lowered by the NVPTX backend",
1601 dl.getDebugLoc()));
1602 }
1603
1604 const unsigned NumArgs =
1605 std::min<unsigned>(a: CLI.NumFixedArgs + 1, b: Args.size());
1606 /// CALL(Chain, IsConvergent, IsIndirectCall/IsUniform, NumReturns,
1607 /// NumParams, Callee, Proto)
1608 const SDValue CallToken = DAG.getTokenFactor(DL: dl, Vals&: CallPrereqs);
1609 const SDValue Call = DAG.getNode(
1610 Opcode: NVPTXISD::CALL, DL: dl, VT: MVT::Other,
1611 Ops: {CallToken, GetI32(CLI.IsConvergent), GetI32(IsIndirectCall),
1612 GetI32(Ins.empty() ? 0 : 1), GetI32(NumArgs), Callee, Proto});
1613
1614 SmallVector<SDValue, 16> LoadChains{Call};
1615 SmallVector<SDValue, 16> ProxyRegOps;
1616 if (!Ins.empty()) {
1617 SmallVector<EVT, 16> VTs;
1618 SmallVector<uint64_t, 16> Offsets;
1619 ComputePTXValueVTs(TLI: *this, DL, Ctx, CallConv: CLI.CallConv, Ty: RetTy, ValueVTs&: VTs, Offsets);
1620 assert(VTs.size() == Ins.size() && "Bad value decomposition");
1621
1622 const Align RetAlign =
1623 getPTXParamAlign(CB, Ty: RetTy, AttrIdx: AttributeList::ReturnIndex, DL);
1624 const SDValue RetSymbol = getSymbolNode(DAG, Name: "retval0", T: MVT::i32);
1625
1626 // PTX Interoperability Guide 3.3(A): [Integer] Values shorter than
1627 // 32-bits are sign extended or zero extended, depending on whether
1628 // they are signed or unsigned types.
1629 const bool ExtendIntegerRetVal =
1630 RetTy->isIntegerTy() && DL.getTypeAllocSizeInBits(Ty: RetTy) < 32;
1631
1632 unsigned I = 0;
1633 const auto VI = VectorizePTXValueVTs(ValueVTs: VTs, Offsets, ParamAlignment: RetAlign);
1634 for (const unsigned NumElts : VI) {
1635 const MaybeAlign CurrentAlign =
1636 ExtendIntegerRetVal ? MaybeAlign(std::nullopt)
1637 : commonAlignment(A: RetAlign, Offset: Offsets[I]);
1638
1639 const EVT VTI = promoteScalarIntegerPTX(VT: VTs[I]);
1640 const EVT LoadVT =
1641 ExtendIntegerRetVal ? MVT::i32 : (VTI == MVT::i1 ? MVT::i8 : VTI);
1642 const EVT VecVT = getVectorizedVT(VT: LoadVT, N: NumElts, C&: Ctx);
1643 SDValue Ptr =
1644 DAG.getObjectPtrOffset(SL: dl, Ptr: RetSymbol, Offset: TypeSize::getFixed(ExactSize: Offsets[I]));
1645
1646 SDValue R = DAG.getLoad(
1647 VT: VecVT, dl, Chain: Call, Ptr,
1648 PtrInfo: MachinePointerInfo(NVPTX::AddressSpace::DeviceParam), Alignment: CurrentAlign);
1649
1650 LoadChains.push_back(Elt: R.getValue(R: 1));
1651 for (const unsigned J : llvm::seq(Size: NumElts))
1652 ProxyRegOps.push_back(Elt: getExtractVectorizedValue(V: R, I: J, VT: LoadVT, dl, DAG));
1653 I += NumElts;
1654 }
1655 }
1656
1657 const SDValue EndToken = DAG.getTokenFactor(DL: dl, Vals&: LoadChains);
1658 const SDValue CallEnd = DAG.getCALLSEQ_END(Chain: EndToken, Size1: UniqueCallSite,
1659 Size2: UniqueCallSite + 1, Glue: SDValue(), DL: dl);
1660
1661 // Append ProxyReg instructions to the chain to make sure that `callseq_end`
1662 // will not get lost. Otherwise, during libcalls expansion, the nodes can become
1663 // dangling.
1664 for (const auto [I, Reg] : llvm::enumerate(First&: ProxyRegOps)) {
1665 SDValue Proxy =
1666 DAG.getNode(Opcode: NVPTXISD::ProxyReg, DL: dl, VT: Reg.getValueType(), Ops: {CallEnd, Reg});
1667 SDValue Ret = correctParamType(V: Proxy, ExpectedVT: Ins[I].VT, Flags: Ins[I].Flags, DAG, dl);
1668 InVals.push_back(Elt: Ret);
1669 }
1670
1671 // set IsTailCall to false for now, until we figure out how to express
1672 // tail call optimization in PTX
1673 CLI.IsTailCall = false;
1674 return CallEnd;
1675}
1676
1677SDValue NVPTXTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
1678 SelectionDAG &DAG) const {
1679
1680 if (!STI.hasFeature(Feature: NVPTX::PTX73) || !STI.hasFeature(Feature: NVPTX::SM52)) {
1681 const Function &Fn = DAG.getMachineFunction().getFunction();
1682
1683 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1684 Fn,
1685 "Support for dynamic alloca introduced in PTX ISA version 7.3 and "
1686 "requires target sm_52.",
1687 SDLoc(Op).getDebugLoc()));
1688 auto Ops = {DAG.getConstant(Val: 0, DL: SDLoc(), VT: Op.getValueType()),
1689 Op.getOperand(i: 0)};
1690 return DAG.getMergeValues(Ops, dl: SDLoc());
1691 }
1692
1693 SDLoc DL(Op.getNode());
1694 SDValue Chain = Op.getOperand(i: 0);
1695 SDValue Size = Op.getOperand(i: 1);
1696 uint64_t Align = Op.getConstantOperandVal(i: 2);
1697
1698 // The alignment on a ISD::DYNAMIC_STACKALLOC node may be 0 to indicate that
1699 // the default stack alignment should be used.
1700 if (Align == 0)
1701 Align = DAG.getSubtarget().getFrameLowering()->getStackAlign().value();
1702
1703 // The size for ptx alloca instruction is 64-bit for m64 and 32-bit for m32.
1704 const MVT LocalVT = getPointerTy(DL: DAG.getDataLayout(), AS: ADDRESS_SPACE_LOCAL);
1705
1706 SDValue Alloc =
1707 DAG.getNode(Opcode: NVPTXISD::DYNAMIC_STACKALLOC, DL, ResultTys: {LocalVT, MVT::Other},
1708 Ops: {Chain, DAG.getZExtOrTrunc(Op: Size, DL, VT: LocalVT),
1709 DAG.getTargetConstant(Val: Align, DL, VT: MVT::i32)});
1710
1711 // NVPTXLowerAlloca puts allocas in the local address space, so a local
1712 // pointer is requested here; escapes are explicit addrspacecasts in the IR.
1713 assert(Op.getValueType() == LocalVT && "Unexpected alloca pointer size");
1714
1715 return DAG.getMergeValues(Ops: {Alloc, SDValue(Alloc.getNode(), 1)}, dl: DL);
1716}
1717
1718SDValue NVPTXTargetLowering::LowerSTACKRESTORE(SDValue Op,
1719 SelectionDAG &DAG) const {
1720 SDLoc DL(Op.getNode());
1721 if (!STI.hasFeature(Feature: NVPTX::PTX73) || !STI.hasFeature(Feature: NVPTX::SM52)) {
1722 const Function &Fn = DAG.getMachineFunction().getFunction();
1723
1724 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1725 Fn,
1726 "Support for stackrestore requires PTX ISA version >= 7.3 and target "
1727 ">= sm_52.",
1728 DL.getDebugLoc()));
1729 return Op.getOperand(i: 0);
1730 }
1731
1732 const MVT LocalVT = getPointerTy(DL: DAG.getDataLayout(), AS: ADDRESS_SPACE_LOCAL);
1733 SDValue Chain = Op.getOperand(i: 0);
1734 SDValue Ptr = Op.getOperand(i: 1);
1735 SDValue ASC = DAG.getAddrSpaceCast(dl: DL, VT: LocalVT, Ptr, SrcAS: ADDRESS_SPACE_GENERIC,
1736 DestAS: ADDRESS_SPACE_LOCAL);
1737 return DAG.getNode(Opcode: NVPTXISD::STACKRESTORE, DL, VT: MVT::Other, Ops: {Chain, ASC});
1738}
1739
1740SDValue NVPTXTargetLowering::LowerSTACKSAVE(SDValue Op,
1741 SelectionDAG &DAG) const {
1742 SDLoc DL(Op.getNode());
1743 if (!STI.hasFeature(Feature: NVPTX::PTX73) || !STI.hasFeature(Feature: NVPTX::SM52)) {
1744 const Function &Fn = DAG.getMachineFunction().getFunction();
1745
1746 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1747 Fn,
1748 "Support for stacksave requires PTX ISA version >= 7.3 and target >= "
1749 "sm_52.",
1750 DL.getDebugLoc()));
1751 auto Ops = {DAG.getConstant(Val: 0, DL, VT: Op.getValueType()), Op.getOperand(i: 0)};
1752 return DAG.getMergeValues(Ops, dl: DL);
1753 }
1754
1755 const MVT LocalVT = getPointerTy(DL: DAG.getDataLayout(), AS: ADDRESS_SPACE_LOCAL);
1756 SDValue Chain = Op.getOperand(i: 0);
1757 SDValue SS =
1758 DAG.getNode(Opcode: NVPTXISD::STACKSAVE, DL, ResultTys: {LocalVT, MVT::Other}, Ops: Chain);
1759 SDValue ASC = DAG.getAddrSpaceCast(
1760 dl: DL, VT: Op.getValueType(), Ptr: SS, SrcAS: ADDRESS_SPACE_LOCAL, DestAS: ADDRESS_SPACE_GENERIC);
1761 return DAG.getMergeValues(Ops: {ASC, SDValue(SS.getNode(), 1)}, dl: DL);
1762}
1763
1764// By default CONCAT_VECTORS is lowered by ExpandVectorBuildThroughStack()
1765// (see LegalizeDAG.cpp). This is slow and uses local memory.
1766// We use extract/insert/build vector just as what LegalizeOp() does in llvm 2.5
1767SDValue
1768NVPTXTargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const {
1769 SDNode *Node = Op.getNode();
1770 SDLoc dl(Node);
1771 SmallVector<SDValue, 8> Ops;
1772 unsigned NumOperands = Node->getNumOperands();
1773 for (unsigned i = 0; i < NumOperands; ++i) {
1774 SDValue SubOp = Node->getOperand(Num: i);
1775 EVT VVT = SubOp.getNode()->getValueType(ResNo: 0);
1776 EVT EltVT = VVT.getVectorElementType();
1777 unsigned NumSubElem = VVT.getVectorNumElements();
1778 for (unsigned j = 0; j < NumSubElem; ++j) {
1779 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: SubOp,
1780 N2: DAG.getIntPtrConstant(Val: j, DL: dl)));
1781 }
1782 }
1783 return DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL: dl, Ops);
1784}
1785
1786static SDValue getPRMT(SDValue A, SDValue B, SDValue Selector, SDLoc DL,
1787 SelectionDAG &DAG,
1788 unsigned Mode = NVPTX::PTXPrmtMode::NONE) {
1789 assert(A.getValueType() == MVT::i32 && B.getValueType() == MVT::i32 &&
1790 Selector.getValueType() == MVT::i32 && "PRMT must have i32 operands");
1791 return DAG.getNode(Opcode: NVPTXISD::PRMT, DL, VT: MVT::i32,
1792 Ops: {A, B, Selector, DAG.getConstant(Val: Mode, DL, VT: MVT::i32)});
1793}
1794
1795static SDValue getPRMT(SDValue A, SDValue B, uint64_t Selector, SDLoc DL,
1796 SelectionDAG &DAG,
1797 unsigned Mode = NVPTX::PTXPrmtMode::NONE) {
1798 return getPRMT(A, B, Selector: DAG.getConstant(Val: Selector, DL, VT: MVT::i32), DL, DAG, Mode);
1799}
1800
1801/// Reduces the elements using the scalar operations provided. The operations
1802/// are sorted descending in number of inputs they take. The flags on the
1803/// original reduction operation will be propagated to each scalar operation.
1804/// Nearby elements are grouped in tree reduction, unlike the shuffle reduction
1805/// used in ExpandReductions and SelectionDAG.
1806static SDValue buildTreeReduction(
1807 const SmallVector<SDValue> &Elements, EVT EltTy,
1808 ArrayRef<std::pair<unsigned /*NodeType*/, unsigned /*NumInputs*/>> Ops,
1809 const SDLoc &DL, const SDNodeFlags Flags, SelectionDAG &DAG) {
1810 // Build the reduction tree at each level, starting with all the elements.
1811 SmallVector<SDValue> Level = Elements;
1812
1813 unsigned OpIdx = 0;
1814 while (Level.size() > 1) {
1815 // Try to reduce this level using the current operator.
1816 const auto [Op, NumInputs] = Ops[OpIdx];
1817
1818 // Build the next level by partially reducing all elements.
1819 SmallVector<SDValue> ReducedLevel;
1820 unsigned I = 0, E = Level.size();
1821 for (; I + NumInputs <= E; I += NumInputs) {
1822 // Reduce elements in groups of [NumInputs], as much as possible.
1823 ReducedLevel.push_back(Elt: DAG.getNode(
1824 Opcode: Op, DL, VT: EltTy, Ops: ArrayRef<SDValue>(Level).slice(N: I, M: NumInputs), Flags));
1825 }
1826
1827 if (I < E) {
1828 // Handle leftover elements.
1829
1830 if (ReducedLevel.empty()) {
1831 // We didn't reduce anything at this level. We need to pick a smaller
1832 // operator.
1833 ++OpIdx;
1834 assert(OpIdx < Ops.size() && "no smaller operators for reduction");
1835 continue;
1836 }
1837
1838 // We reduced some things but there's still more left, meaning the
1839 // operator's number of inputs doesn't evenly divide this level size. Move
1840 // these elements to the next level.
1841 for (; I < E; ++I)
1842 ReducedLevel.push_back(Elt: Level[I]);
1843 }
1844
1845 // Process the next level.
1846 Level = ReducedLevel;
1847 }
1848
1849 return *Level.begin();
1850}
1851
1852// Get scalar reduction opcode
1853static ISD::NodeType getScalarOpcodeForReduction(unsigned ReductionOpcode) {
1854 switch (ReductionOpcode) {
1855 case ISD::VECREDUCE_FMAX:
1856 return ISD::FMAXNUM;
1857 case ISD::VECREDUCE_FMIN:
1858 return ISD::FMINNUM;
1859 case ISD::VECREDUCE_FMAXIMUM:
1860 return ISD::FMAXIMUM;
1861 case ISD::VECREDUCE_FMINIMUM:
1862 return ISD::FMINIMUM;
1863 case ISD::VECREDUCE_FMAXIMUMNUM:
1864 return ISD::FMAXIMUMNUM;
1865 case ISD::VECREDUCE_FMINIMUMNUM:
1866 return ISD::FMINIMUMNUM;
1867 default:
1868 llvm_unreachable("unhandled reduction opcode");
1869 }
1870}
1871
1872/// Get 3-input scalar reduction opcode
1873static std::optional<unsigned>
1874getScalar3OpcodeForReduction(unsigned ReductionOpcode) {
1875 switch (ReductionOpcode) {
1876 case ISD::VECREDUCE_FMAX:
1877 case ISD::VECREDUCE_FMAXIMUMNUM:
1878 return NVPTXISD::FMAXNUM3;
1879 case ISD::VECREDUCE_FMIN:
1880 case ISD::VECREDUCE_FMINIMUMNUM:
1881 return NVPTXISD::FMINNUM3;
1882 case ISD::VECREDUCE_FMAXIMUM:
1883 return NVPTXISD::FMAXIMUM3;
1884 case ISD::VECREDUCE_FMINIMUM:
1885 return NVPTXISD::FMINIMUM3;
1886 default:
1887 return std::nullopt;
1888 }
1889}
1890
1891/// Lower reductions to either a sequence of operations or a tree if
1892/// reassociations are allowed. This method will use larger operations like
1893/// max3/min3 when the target supports them.
1894SDValue NVPTXTargetLowering::LowerVECREDUCE(SDValue Op,
1895 SelectionDAG &DAG) const {
1896 SDLoc DL(Op);
1897 const SDNodeFlags Flags = Op->getFlags();
1898 SDValue Vector = Op.getOperand(i: 0);
1899
1900 const unsigned Opcode = Op->getOpcode();
1901 const EVT EltTy = Vector.getValueType().getVectorElementType();
1902
1903 // Whether we can use 3-input min/max when expanding the reduction.
1904 const bool CanUseMinMax3 =
1905 EltTy == MVT::f32 && STI.hasFeature(Feature: NVPTX::SM100) &&
1906 STI.hasFeature(Feature: NVPTX::PTX88) &&
1907 (Opcode == ISD::VECREDUCE_FMAX || Opcode == ISD::VECREDUCE_FMIN ||
1908 Opcode == ISD::VECREDUCE_FMAXIMUM || Opcode == ISD::VECREDUCE_FMINIMUM ||
1909 Opcode == ISD::VECREDUCE_FMAXIMUMNUM ||
1910 Opcode == ISD::VECREDUCE_FMINIMUMNUM);
1911
1912 // A list of SDNode opcodes with equivalent semantics, sorted descending by
1913 // number of inputs they take.
1914 SmallVector<std::pair<unsigned /*Op*/, unsigned /*NumIn*/>, 2> ScalarOps;
1915
1916 if (auto Opcode3Elem = getScalar3OpcodeForReduction(ReductionOpcode: Opcode);
1917 CanUseMinMax3 && Opcode3Elem)
1918 ScalarOps.push_back(Elt: {*Opcode3Elem, 3});
1919 ScalarOps.push_back(Elt: {getScalarOpcodeForReduction(ReductionOpcode: Opcode), 2});
1920
1921 SmallVector<SDValue> Elements;
1922 DAG.ExtractVectorElements(Op: Vector, Args&: Elements);
1923
1924 return buildTreeReduction(Elements, EltTy, Ops: ScalarOps, DL, Flags, DAG);
1925}
1926
1927SDValue NVPTXTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
1928 // Handle bitcasting from v2i8 without hitting the default promotion
1929 // strategy which goes through stack memory.
1930 EVT FromVT = Op->getOperand(Num: 0)->getValueType(ResNo: 0);
1931 if (FromVT != MVT::v2i8) {
1932 return Op;
1933 }
1934
1935 // Pack vector elements into i16 and bitcast to final type
1936 SDLoc DL(Op);
1937 SDValue Vec0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i8,
1938 N1: Op->getOperand(Num: 0), N2: DAG.getIntPtrConstant(Val: 0, DL));
1939 SDValue Vec1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i8,
1940 N1: Op->getOperand(Num: 0), N2: DAG.getIntPtrConstant(Val: 1, DL));
1941 SDValue Extend0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i16, Operand: Vec0);
1942 SDValue Extend1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i16, Operand: Vec1);
1943 SDValue Const8 = DAG.getConstant(Val: 8, DL, VT: MVT::i16);
1944 SDValue AsInt = DAG.getNode(
1945 Opcode: ISD::OR, DL, VT: MVT::i16,
1946 Ops: {Extend0, DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i16, Ops: {Extend1, Const8})});
1947 EVT ToVT = Op->getValueType(ResNo: 0);
1948 return DAG.getBitcast(VT: ToVT, V: AsInt);
1949}
1950
1951// We can init constant f16x2/v2i16/v4i8 with a single .b32 move. Normally it
1952// would get lowered as two constant loads and vector-packing move.
1953// Instead we want just a constant move:
1954// mov.b32 %r2, 0x40003C00
1955SDValue NVPTXTargetLowering::LowerBUILD_VECTOR(SDValue Op,
1956 SelectionDAG &DAG) const {
1957 EVT VT = Op->getValueType(ResNo: 0);
1958 if (!(NVPTX::isPackedVectorTy(VT) && VT.is32BitVector()))
1959 return Op;
1960 SDLoc DL(Op);
1961
1962 if (!llvm::all_of(Range: Op->ops(), P: [](SDValue Operand) {
1963 return Operand->isUndef() || isa<ConstantSDNode>(Val: Operand) ||
1964 isa<ConstantFPSDNode>(Val: Operand);
1965 })) {
1966 if (VT != MVT::v4i8)
1967 return Op;
1968 // Lower non-const v4i8 vector as byte-wise constructed i32, which allows us
1969 // to optimize calculation of constant parts.
1970 auto GetPRMT = [&](const SDValue Left, const SDValue Right, bool Cast,
1971 uint64_t SelectionValue) -> SDValue {
1972 SDValue L = Left;
1973 SDValue R = Right;
1974 if (Cast) {
1975 L = DAG.getAnyExtOrTrunc(Op: L, DL, VT: MVT::i32);
1976 R = DAG.getAnyExtOrTrunc(Op: R, DL, VT: MVT::i32);
1977 }
1978 return getPRMT(A: L, B: R, Selector: SelectionValue, DL, DAG);
1979 };
1980 auto PRMT__10 = GetPRMT(Op->getOperand(Num: 0), Op->getOperand(Num: 1), true, 0x3340);
1981 auto PRMT__32 = GetPRMT(Op->getOperand(Num: 2), Op->getOperand(Num: 3), true, 0x3340);
1982 auto PRMT3210 = GetPRMT(PRMT__10, PRMT__32, false, 0x5410);
1983 return DAG.getBitcast(VT, V: PRMT3210);
1984 }
1985
1986 // Get value or the Nth operand as an APInt(32). Undef values treated as 0.
1987 auto GetOperand = [](SDValue Op, int N) -> APInt {
1988 const SDValue &Operand = Op->getOperand(Num: N);
1989 EVT VT = Op->getValueType(ResNo: 0);
1990 if (Operand->isUndef())
1991 return APInt(32, 0);
1992 APInt Value;
1993 if (VT == MVT::v2f16 || VT == MVT::v2bf16)
1994 Value = cast<ConstantFPSDNode>(Val: Operand)->getValueAPF().bitcastToAPInt();
1995 else if (VT == MVT::v2i16 || VT == MVT::v4i8)
1996 Value = Operand->getAsAPIntVal();
1997 else
1998 llvm_unreachable("Unsupported type");
1999 // i8 values are carried around as i16, so we need to zero out upper bits,
2000 // so they do not get in the way of combining individual byte values
2001 if (VT == MVT::v4i8)
2002 Value = Value.trunc(width: 8);
2003 return Value.zext(width: 32);
2004 };
2005
2006 // Construct a 32-bit constant by shifting into place smaller values
2007 // (elements of the vector type VT).
2008 // For example, if VT has 2 elements, then N == 2:
2009 // ShiftAmount = 32 / N = 16
2010 // Value |= Op0 (b16) << 0
2011 // Value |= Op1 (b16) << 16
2012 // If N == 4:
2013 // ShiftAmount = 32 / N = 8
2014 // Value |= Op0 (b8) << 0
2015 // Value |= Op1 (b8) << 8
2016 // Value |= Op2 (b8) << 16
2017 // Value |= Op3 (b8) << 24
2018 // ...etc
2019 APInt Value(32, 0);
2020 const unsigned NumElements = VT.getVectorNumElements();
2021 assert(32 % NumElements == 0 && "must evenly divide bit length");
2022 const unsigned ShiftAmount = 32 / NumElements;
2023 for (unsigned ElementNo : seq(Size: NumElements))
2024 Value |= GetOperand(Op, ElementNo).shl(shiftAmt: ElementNo * ShiftAmount);
2025 SDValue Const = DAG.getConstant(Val: Value, DL, VT: MVT::i32);
2026 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Op->getValueType(ResNo: 0), Operand: Const);
2027}
2028
2029SDValue NVPTXTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
2030 SelectionDAG &DAG) const {
2031 SDValue Index = Op->getOperand(Num: 1);
2032 SDValue Vector = Op->getOperand(Num: 0);
2033 SDLoc DL(Op);
2034 EVT VectorVT = Vector.getValueType();
2035
2036 if (VectorVT == MVT::v4i8) {
2037 SDValue Selector = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32,
2038 N1: DAG.getZExtOrTrunc(Op: Index, DL, VT: MVT::i32),
2039 N2: DAG.getConstant(Val: 0x7770, DL, VT: MVT::i32));
2040 SDValue PRMT = getPRMT(A: DAG.getBitcast(VT: MVT::i32, V: Vector),
2041 B: DAG.getConstant(Val: 0, DL, VT: MVT::i32), Selector, DL, DAG);
2042 SDValue Ext = DAG.getAnyExtOrTrunc(Op: PRMT, DL, VT: Op->getValueType(ResNo: 0));
2043 SDNodeFlags Flags;
2044 Flags.setNoSignedWrap(Ext.getScalarValueSizeInBits() > 8);
2045 Flags.setNoUnsignedWrap(Ext.getScalarValueSizeInBits() >= 8);
2046 Ext->setFlags(Flags);
2047 return Ext;
2048 }
2049
2050 // Constant index will be matched by tablegen.
2051 if (isa<ConstantSDNode>(Val: Index.getNode()))
2052 return Op;
2053
2054 // Extract individual elements and select one of them.
2055 assert(NVPTX::isPackedVectorTy(VectorVT) &&
2056 VectorVT.getVectorNumElements() == 2 && "Unexpected vector type.");
2057 EVT EltVT = VectorVT.getVectorElementType();
2058
2059 SDLoc dl(Op.getNode());
2060 SDValue E0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Vector,
2061 N2: DAG.getIntPtrConstant(Val: 0, DL: dl));
2062 SDValue E1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Vector,
2063 N2: DAG.getIntPtrConstant(Val: 1, DL: dl));
2064 return DAG.getSelectCC(DL: dl, LHS: Index, RHS: DAG.getIntPtrConstant(Val: 0, DL: dl), True: E0, False: E1,
2065 Cond: ISD::CondCode::SETEQ);
2066}
2067
2068SDValue NVPTXTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
2069 SelectionDAG &DAG) const {
2070 SDValue Vector = Op->getOperand(Num: 0);
2071 EVT VectorVT = Vector.getValueType();
2072
2073 if (VectorVT != MVT::v4i8)
2074 return Op;
2075 SDLoc DL(Op);
2076 SDValue Value = Op->getOperand(Num: 1);
2077 if (Value->isUndef())
2078 return Vector;
2079
2080 SDValue Index = Op->getOperand(Num: 2);
2081
2082 SDValue BFI =
2083 DAG.getNode(Opcode: NVPTXISD::BFI, DL, VT: MVT::i32,
2084 Ops: {DAG.getZExtOrTrunc(Op: Value, DL, VT: MVT::i32),
2085 DAG.getBitcast(VT: MVT::i32, V: Vector),
2086 DAG.getNode(Opcode: ISD::MUL, DL, VT: MVT::i32,
2087 N1: DAG.getZExtOrTrunc(Op: Index, DL, VT: MVT::i32),
2088 N2: DAG.getConstant(Val: 8, DL, VT: MVT::i32)),
2089 DAG.getConstant(Val: 8, DL, VT: MVT::i32)});
2090 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: Op->getValueType(ResNo: 0), Operand: BFI);
2091}
2092
2093SDValue NVPTXTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
2094 SelectionDAG &DAG) const {
2095 SDValue V1 = Op.getOperand(i: 0);
2096 EVT VectorVT = V1.getValueType();
2097 if (VectorVT != MVT::v4i8 || Op.getValueType() != MVT::v4i8)
2098 return Op;
2099
2100 // Lower shuffle to PRMT instruction.
2101 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: Op.getNode());
2102 SDValue V2 = Op.getOperand(i: 1);
2103 uint32_t Selector = 0;
2104 for (auto I : llvm::enumerate(First: SVN->getMask())) {
2105 if (I.value() != -1) // -1 is a placeholder for undef.
2106 Selector |= (I.value() << (I.index() * 4));
2107 }
2108
2109 SDLoc DL(Op);
2110 SDValue PRMT = getPRMT(A: DAG.getBitcast(VT: MVT::i32, V: V1),
2111 B: DAG.getBitcast(VT: MVT::i32, V: V2), Selector, DL, DAG);
2112 return DAG.getBitcast(VT: Op.getValueType(), V: PRMT);
2113}
2114/// LowerShiftRightParts - Lower SRL_PARTS, SRA_PARTS, which
2115/// 1) returns two i32 values and take a 2 x i32 value to shift plus a shift
2116/// amount, or
2117/// 2) returns two i64 values and take a 2 x i64 value to shift plus a shift
2118/// amount.
2119SDValue NVPTXTargetLowering::LowerShiftRightParts(SDValue Op,
2120 SelectionDAG &DAG) const {
2121 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
2122 assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS);
2123
2124 EVT VT = Op.getValueType();
2125 unsigned VTBits = VT.getSizeInBits();
2126 SDLoc dl(Op);
2127 SDValue ShOpLo = Op.getOperand(i: 0);
2128 SDValue ShOpHi = Op.getOperand(i: 1);
2129 SDValue ShAmt = Op.getOperand(i: 2);
2130 unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;
2131
2132 if (VTBits == 32 && STI.hasFeature(Feature: NVPTX::SM35)) {
2133 // For 32bit and sm35, we can use the funnel shift 'shf' instruction.
2134 // {dHi, dLo} = {aHi, aLo} >> Amt
2135 // dHi = aHi >> Amt
2136 // dLo = shf.r.clamp aLo, aHi, Amt
2137
2138 SDValue Hi = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: ShOpHi, N2: ShAmt);
2139 SDValue Lo =
2140 DAG.getNode(Opcode: NVPTXISD::FSHR_CLAMP, DL: dl, VT, N1: ShOpHi, N2: ShOpLo, N3: ShAmt);
2141
2142 SDValue Ops[2] = { Lo, Hi };
2143 return DAG.getMergeValues(Ops, dl);
2144 } else {
2145 // {dHi, dLo} = {aHi, aLo} >> Amt
2146 // - if (Amt>=size) then
2147 // dLo = aHi >> (Amt-size)
2148 // dHi = aHi >> Amt (this is either all 0 or all 1)
2149 // else
2150 // dLo = (aLo >>logic Amt) | (aHi << (size-Amt))
2151 // dHi = aHi >> Amt
2152
2153 SDValue RevShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
2154 N1: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32),
2155 N2: ShAmt);
2156 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: ShOpLo, N2: ShAmt);
2157 SDValue ExtraShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: ShAmt,
2158 N2: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32));
2159 SDValue Tmp2 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpHi, N2: RevShAmt);
2160 SDValue FalseVal = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp1, N2: Tmp2);
2161 SDValue TrueVal = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: ShOpHi, N2: ExtraShAmt);
2162
2163 SDValue Cmp = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: ShAmt,
2164 RHS: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32),
2165 Cond: ISD::SETGE);
2166 SDValue Hi = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: ShOpHi, N2: ShAmt);
2167 SDValue Lo = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cmp, N2: TrueVal, N3: FalseVal);
2168
2169 SDValue Ops[2] = { Lo, Hi };
2170 return DAG.getMergeValues(Ops, dl);
2171 }
2172}
2173
2174/// LowerShiftLeftParts - Lower SHL_PARTS, which
2175/// 1) returns two i32 values and take a 2 x i32 value to shift plus a shift
2176/// amount, or
2177/// 2) returns two i64 values and take a 2 x i64 value to shift plus a shift
2178/// amount.
2179SDValue NVPTXTargetLowering::LowerShiftLeftParts(SDValue Op,
2180 SelectionDAG &DAG) const {
2181 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
2182 assert(Op.getOpcode() == ISD::SHL_PARTS);
2183
2184 EVT VT = Op.getValueType();
2185 unsigned VTBits = VT.getSizeInBits();
2186 SDLoc dl(Op);
2187 SDValue ShOpLo = Op.getOperand(i: 0);
2188 SDValue ShOpHi = Op.getOperand(i: 1);
2189 SDValue ShAmt = Op.getOperand(i: 2);
2190
2191 if (VTBits == 32 && STI.hasFeature(Feature: NVPTX::SM35)) {
2192 // For 32bit and sm35, we can use the funnel shift 'shf' instruction.
2193 // {dHi, dLo} = {aHi, aLo} << Amt
2194 // dHi = shf.l.clamp aLo, aHi, Amt
2195 // dLo = aLo << Amt
2196
2197 SDValue Hi =
2198 DAG.getNode(Opcode: NVPTXISD::FSHL_CLAMP, DL: dl, VT, N1: ShOpHi, N2: ShOpLo, N3: ShAmt);
2199 SDValue Lo = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpLo, N2: ShAmt);
2200
2201 SDValue Ops[2] = { Lo, Hi };
2202 return DAG.getMergeValues(Ops, dl);
2203 } else {
2204 // {dHi, dLo} = {aHi, aLo} << Amt
2205 // - if (Amt>=size) then
2206 // dLo = aLo << Amt (all 0)
2207 // dLo = aLo << (Amt-size)
2208 // else
2209 // dLo = aLo << Amt
2210 // dHi = (aHi << Amt) | (aLo >> (size-Amt))
2211
2212 SDValue RevShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
2213 N1: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32),
2214 N2: ShAmt);
2215 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpHi, N2: ShAmt);
2216 SDValue ExtraShAmt = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: ShAmt,
2217 N2: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32));
2218 SDValue Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: ShOpLo, N2: RevShAmt);
2219 SDValue FalseVal = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp1, N2: Tmp2);
2220 SDValue TrueVal = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpLo, N2: ExtraShAmt);
2221
2222 SDValue Cmp = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: ShAmt,
2223 RHS: DAG.getConstant(Val: VTBits, DL: dl, VT: MVT::i32),
2224 Cond: ISD::SETGE);
2225 SDValue Lo = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpLo, N2: ShAmt);
2226 SDValue Hi = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cmp, N2: TrueVal, N3: FalseVal);
2227
2228 SDValue Ops[2] = { Lo, Hi };
2229 return DAG.getMergeValues(Ops, dl);
2230 }
2231}
2232
2233/// If the types match, convert the generic copysign to the NVPTXISD version,
2234/// otherwise bail ensuring that mismatched cases are properly expaned.
2235SDValue NVPTXTargetLowering::LowerFCOPYSIGN(SDValue Op,
2236 SelectionDAG &DAG) const {
2237 EVT VT = Op.getValueType();
2238 SDLoc DL(Op);
2239
2240 SDValue In1 = Op.getOperand(i: 0);
2241 SDValue In2 = Op.getOperand(i: 1);
2242 EVT SrcVT = In2.getValueType();
2243
2244 if (!SrcVT.bitsEq(VT))
2245 return SDValue();
2246
2247 return DAG.getNode(Opcode: NVPTXISD::FCOPYSIGN, DL, VT, N1: In1, N2: In2);
2248}
2249
2250SDValue NVPTXTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const {
2251 EVT VT = Op.getValueType();
2252
2253 if (VT == MVT::f32)
2254 return LowerFROUND32(Op, DAG);
2255
2256 if (VT == MVT::f64)
2257 return LowerFROUND64(Op, DAG);
2258
2259 llvm_unreachable("unhandled type");
2260}
2261
2262// This is the the rounding method used in CUDA libdevice in C like code:
2263// float roundf(float A)
2264// {
2265// float RoundedA = (float) (int) ( A > 0 ? (A + 0.5f) : (A - 0.5f));
2266// RoundedA = abs(A) > 0x1.0p23 ? A : RoundedA;
2267// return abs(A) < 0.5 ? (float)(int)A : RoundedA;
2268// }
2269SDValue NVPTXTargetLowering::LowerFROUND32(SDValue Op,
2270 SelectionDAG &DAG) const {
2271 SDLoc SL(Op);
2272 SDValue A = Op.getOperand(i: 0);
2273 EVT VT = Op.getValueType();
2274
2275 SDValue AbsA = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT, Operand: A);
2276
2277 // RoundedA = (float) (int) ( A > 0 ? (A + 0.5f) : (A - 0.5f))
2278 SDValue Bitcast = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: A);
2279 const unsigned SignBitMask = 0x80000000;
2280 SDValue Sign = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: Bitcast,
2281 N2: DAG.getConstant(Val: SignBitMask, DL: SL, VT: MVT::i32));
2282 const unsigned PointFiveInBits = 0x3F000000;
2283 SDValue PointFiveWithSignRaw =
2284 DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: Sign,
2285 N2: DAG.getConstant(Val: PointFiveInBits, DL: SL, VT: MVT::i32));
2286 SDValue PointFiveWithSign =
2287 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: PointFiveWithSignRaw);
2288 SDValue AdjustedA = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: A, N2: PointFiveWithSign);
2289 SDValue RoundedA = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: AdjustedA);
2290
2291 // RoundedA = abs(A) > 0x1.0p23 ? A : RoundedA;
2292 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Ctx&: *DAG.getContext(), VT);
2293 SDValue IsLarge =
2294 DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: AbsA, RHS: DAG.getConstantFP(Val: pow(x: 2.0, y: 23.0), DL: SL, VT),
2295 Cond: ISD::SETOGT);
2296 RoundedA = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsLarge, N2: A, N3: RoundedA);
2297
2298 // return abs(A) < 0.5 ? (float)(int)A : RoundedA;
2299 SDValue IsSmall =DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: AbsA,
2300 RHS: DAG.getConstantFP(Val: 0.5, DL: SL, VT), Cond: ISD::SETOLT);
2301 SDValue RoundedAForSmallA = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: A);
2302 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsSmall, N2: RoundedAForSmallA, N3: RoundedA);
2303}
2304
2305// The implementation of round(double) is similar to that of round(float) in
2306// that they both separate the value range into three regions and use a method
2307// specific to the region to round the values. However, round(double) first
2308// calculates the round of the absolute value and then adds the sign back while
2309// round(float) directly rounds the value with sign.
2310SDValue NVPTXTargetLowering::LowerFROUND64(SDValue Op,
2311 SelectionDAG &DAG) const {
2312 SDLoc SL(Op);
2313 SDValue A = Op.getOperand(i: 0);
2314 EVT VT = Op.getValueType();
2315
2316 SDValue AbsA = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT, Operand: A);
2317
2318 // double RoundedA = (double) (int) (abs(A) + 0.5f);
2319 SDValue AdjustedA = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: AbsA,
2320 N2: DAG.getConstantFP(Val: 0.5, DL: SL, VT));
2321 SDValue RoundedA = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: AdjustedA);
2322
2323 // RoundedA = abs(A) < 0.5 ? (double)0 : RoundedA;
2324 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Ctx&: *DAG.getContext(), VT);
2325 SDValue IsSmall =DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: AbsA,
2326 RHS: DAG.getConstantFP(Val: 0.5, DL: SL, VT), Cond: ISD::SETOLT);
2327 RoundedA = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsSmall,
2328 N2: DAG.getConstantFP(Val: 0, DL: SL, VT),
2329 N3: RoundedA);
2330
2331 // Add sign to rounded_A
2332 RoundedA = DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT, N1: RoundedA, N2: A);
2333 DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: A);
2334
2335 // RoundedA = abs(A) > 0x1.0p52 ? A : RoundedA;
2336 SDValue IsLarge =
2337 DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: AbsA, RHS: DAG.getConstantFP(Val: pow(x: 2.0, y: 52.0), DL: SL, VT),
2338 Cond: ISD::SETOGT);
2339 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsLarge, N2: A, N3: RoundedA);
2340}
2341
2342static SDValue PromoteBinOpToF32(SDNode *N, SelectionDAG &DAG) {
2343 EVT VT = N->getValueType(ResNo: 0);
2344 EVT NVT = MVT::f32;
2345 if (VT.isVector()) {
2346 NVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NVT, EC: VT.getVectorElementCount());
2347 }
2348 SDLoc DL(N);
2349 SDValue Tmp0 = DAG.getFPExtendOrRound(Op: N->getOperand(Num: 0), DL, VT: NVT);
2350 SDValue Tmp1 = DAG.getFPExtendOrRound(Op: N->getOperand(Num: 1), DL, VT: NVT);
2351 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL, VT: NVT, N1: Tmp0, N2: Tmp1, Flags: N->getFlags());
2352 return DAG.getFPExtendOrRound(Op: Res, DL, VT);
2353}
2354
2355SDValue NVPTXTargetLowering::PromoteBinOpIfF32FTZ(SDValue Op,
2356 SelectionDAG &DAG) const {
2357 if (useF32FTZ(MF: DAG.getMachineFunction())) {
2358 return PromoteBinOpToF32(N: Op.getNode(), DAG);
2359 }
2360 return Op;
2361}
2362
2363SDValue NVPTXTargetLowering::LowerINT_TO_FP(SDValue Op,
2364 SelectionDAG &DAG) const {
2365 assert(!STI.hasFeature(NVPTX::SM90));
2366
2367 if (Op.getValueType() == MVT::bf16) {
2368 SDLoc Loc(Op);
2369 return DAG.getNode(
2370 Opcode: ISD::FP_ROUND, DL: Loc, VT: MVT::bf16,
2371 N1: DAG.getNode(Opcode: Op.getOpcode(), DL: Loc, VT: MVT::f32, Operand: Op.getOperand(i: 0)),
2372 N2: DAG.getIntPtrConstant(Val: 0, DL: Loc, /*isTarget=*/true));
2373 }
2374
2375 // Everything else is considered legal.
2376 return Op;
2377}
2378
2379SDValue NVPTXTargetLowering::LowerFP_TO_INT(SDValue Op,
2380 SelectionDAG &DAG) const {
2381 assert(!STI.hasFeature(NVPTX::SM90));
2382
2383 if (Op.getOperand(i: 0).getValueType() == MVT::bf16) {
2384 SDLoc Loc(Op);
2385 return DAG.getNode(
2386 Opcode: Op.getOpcode(), DL: Loc, VT: Op.getValueType(),
2387 Operand: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: Loc, VT: MVT::f32, Operand: Op.getOperand(i: 0)));
2388 }
2389
2390 // Everything else is considered legal.
2391 return Op;
2392}
2393
2394SDValue NVPTXTargetLowering::LowerFP_ROUND(SDValue Op,
2395 SelectionDAG &DAG) const {
2396 EVT NarrowVT = Op.getValueType();
2397 SDValue Wide = Op.getOperand(i: 0);
2398 EVT WideVT = Wide.getValueType();
2399 if (NarrowVT.getScalarType() == MVT::bf16) {
2400 const TargetLowering *TLI = STI.getTargetLowering();
2401 if (!STI.hasFeature(Feature: NVPTX::SM80)) {
2402 return TLI->expandFP_ROUND(Node: Op.getNode(), DAG);
2403 }
2404 if (!STI.hasFeature(Feature: NVPTX::SM90)) {
2405 // sm_80 was the first architecture to support f32 -> bf16.
2406 if (WideVT.getScalarType() == MVT::f32) {
2407 return Op;
2408 }
2409 if (WideVT.getScalarType() == MVT::f64) {
2410 SDLoc Loc(Op);
2411 // Round-inexact-to-odd f64 to f32, then do the final rounding using
2412 // the hardware f32 -> bf16 instruction.
2413 SDValue rod = TLI->expandRoundInexactToOdd(
2414 ResultVT: WideVT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::f32), Op: Wide, DL: Loc,
2415 DAG);
2416 return DAG.getFPExtendOrRound(Op: rod, DL: Loc, VT: NarrowVT);
2417 }
2418 return TLI->expandFP_ROUND(Node: Op.getNode(), DAG);
2419 }
2420 }
2421
2422 // Everything else is considered legal.
2423 return Op;
2424}
2425
2426static SDValue lowerMUL_LOHI(SDValue Op, SelectionDAG &DAG) {
2427 const SDLoc DL(Op);
2428 const EVT VT = Op.getValueType();
2429 const unsigned Opc = Op.getOpcode() == ISD::SMUL_LOHI
2430 ? NVPTXISD::MUL_WIDE_SIGNED
2431 : NVPTXISD::MUL_WIDE_UNSIGNED;
2432
2433 SDValue Wide =
2434 DAG.getNode(Opcode: Opc, DL, VT: VT.widenIntegerElementType(Context&: *DAG.getContext()),
2435 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1));
2436 auto [Lo, Hi] = DAG.SplitScalar(N: Wide, DL, LoVT: VT, HiVT: VT);
2437 return DAG.getMergeValues(Ops: {Lo, Hi}, dl: DL);
2438}
2439
2440static SDValue lowerFPTRUNC_ROUND(SDValue Op, SelectionDAG &DAG,
2441 const NVPTXSubtarget &STI) {
2442 EVT SrcVT = Op.getOperand(i: 0).getValueType();
2443 EVT DstVT = Op.getValueType();
2444 auto RM = static_cast<RoundingMode>(Op.getConstantOperandVal(i: 1));
2445 if (RM == RoundingMode::NearestTiesToEven) {
2446 // Reuse the native selection and fallback expansion for ordinary fptrunc.
2447 SDLoc DL(Op);
2448 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: DstVT, N1: Op.getOperand(i: 0),
2449 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true),
2450 Flags: Op.getNode()->getFlags());
2451 }
2452
2453 bool RoundToInfinity =
2454 RM == RoundingMode::TowardNegative || RM == RoundingMode::TowardPositive;
2455
2456 bool Supported = RM == RoundingMode::TowardZero || RoundToInfinity;
2457 if (DstVT == MVT::bf16) {
2458 if (SrcVT == MVT::f64 || RoundToInfinity)
2459 Supported &= STI.hasFeature(Feature: NVPTX::SM90);
2460 else
2461 Supported &= STI.hasFeature(Feature: NVPTX::SM80);
2462 }
2463
2464 if (Supported)
2465 return Op;
2466
2467 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
2468 DAG.getMachineFunction().getFunction(),
2469 "unsupported conversion or rounding mode for llvm.fptrunc.round",
2470 SDLoc(Op).getDebugLoc()));
2471 return DAG.getPOISON(VT: DstVT);
2472}
2473
2474SDValue NVPTXTargetLowering::LowerFP_EXTEND(SDValue Op,
2475 SelectionDAG &DAG) const {
2476 SDValue Narrow = Op.getOperand(i: 0);
2477 EVT NarrowVT = Narrow.getValueType();
2478 EVT WideVT = Op.getValueType();
2479 if (NarrowVT.getScalarType() == MVT::bf16) {
2480 if (WideVT.getScalarType() == MVT::f32 &&
2481 (!STI.hasFeature(Feature: NVPTX::SM80) || !STI.hasFeature(Feature: NVPTX::PTX71))) {
2482 SDLoc Loc(Op);
2483 return DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: Loc, VT: WideVT, Operand: Narrow);
2484 }
2485 if (WideVT.getScalarType() == MVT::f64 && !STI.hasFeature(Feature: NVPTX::SM90)) {
2486 EVT F32 = NarrowVT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::f32);
2487 SDLoc Loc(Op);
2488 if (STI.hasFeature(Feature: NVPTX::SM80) && STI.hasFeature(Feature: NVPTX::PTX71)) {
2489 Op = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: Loc, VT: F32, Operand: Narrow);
2490 } else {
2491 Op = DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: Loc, VT: F32, Operand: Narrow);
2492 }
2493 return DAG.getNode(Opcode: ISD::FP_EXTEND, DL: Loc, VT: WideVT, Operand: Op);
2494 }
2495 }
2496
2497 // Everything else is considered legal.
2498 return Op;
2499}
2500
2501static SDValue LowerVectorArith(SDValue Op, SelectionDAG &DAG) {
2502 SDLoc DL(Op);
2503 if (Op.getValueType() != MVT::v2i16)
2504 return Op;
2505 EVT EltVT = Op.getValueType().getVectorElementType();
2506 SmallVector<SDValue> VecElements;
2507 for (int I = 0, E = Op.getValueType().getVectorNumElements(); I < E; I++) {
2508 SmallVector<SDValue> ScalarArgs;
2509 llvm::transform(Range: Op->ops(), d_first: std::back_inserter(x&: ScalarArgs),
2510 F: [&](const SDUse &O) {
2511 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT,
2512 N1: O.get(), N2: DAG.getIntPtrConstant(Val: I, DL));
2513 });
2514 VecElements.push_back(Elt: DAG.getNode(Opcode: Op.getOpcode(), DL, VT: EltVT, Ops: ScalarArgs));
2515 }
2516 SDValue V =
2517 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: Op.getValueType(), Ops: VecElements);
2518 return V;
2519}
2520
2521static SDValue lowerTcgen05St(SDValue Op, SelectionDAG &DAG,
2522 bool hasOffset = false) {
2523 // skip lowering if the vector operand is already legalized
2524 if (!Op->getOperand(Num: hasOffset ? 4 : 3).getValueType().isVector())
2525 return Op;
2526
2527 SDNode *N = Op.getNode();
2528 SDLoc DL(N);
2529 SmallVector<SDValue, 32> Ops;
2530
2531 // split the vector argument
2532 for (size_t I = 0; I < N->getNumOperands(); I++) {
2533 SDValue Val = N->getOperand(Num: I);
2534 EVT ValVT = Val.getValueType();
2535 if (ValVT.isVector()) {
2536 EVT EltVT = ValVT.getVectorElementType();
2537 for (unsigned J = 0, NElts = ValVT.getVectorNumElements(); J < NElts; J++)
2538 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Val,
2539 N2: DAG.getIntPtrConstant(Val: J, DL)));
2540 } else
2541 Ops.push_back(Elt: Val);
2542 }
2543
2544 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(Val: N);
2545 SDValue Tcgen05StNode =
2546 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl: DL, VTList: N->getVTList(), Ops,
2547 MemVT: MemSD->getMemoryVT(), MMO: MemSD->getMemOperand());
2548
2549 return Tcgen05StNode;
2550}
2551
2552static SDValue lowerBSWAP(SDValue Op, SelectionDAG &DAG) {
2553 SDLoc DL(Op);
2554 SDValue Src = Op.getOperand(i: 0);
2555 EVT VT = Op.getValueType();
2556
2557 switch (VT.getSimpleVT().SimpleTy) {
2558 case MVT::i16: {
2559 SDValue Extended = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Src);
2560 SDValue Swapped =
2561 getPRMT(A: Extended, B: DAG.getConstant(Val: 0, DL, VT: MVT::i32), Selector: 0x7701, DL, DAG);
2562 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: Swapped);
2563 }
2564 case MVT::i32: {
2565 return getPRMT(A: Src, B: DAG.getConstant(Val: 0, DL, VT: MVT::i32), Selector: 0x0123, DL, DAG);
2566 }
2567 case MVT::v2i16: {
2568 SDValue Converted = DAG.getBitcast(VT: MVT::i32, V: Src);
2569 SDValue Swapped =
2570 getPRMT(A: Converted, B: DAG.getConstant(Val: 0, DL, VT: MVT::i32), Selector: 0x2301, DL, DAG);
2571 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i16, Operand: Swapped);
2572 }
2573 case MVT::i64: {
2574 SDValue UnpackSrc =
2575 DAG.getNode(Opcode: NVPTXISD::UNPACK_VECTOR, DL, ResultTys: {MVT::i32, MVT::i32}, Ops: Src);
2576 SDValue SwappedLow =
2577 getPRMT(A: UnpackSrc.getValue(R: 0), B: DAG.getConstant(Val: 0, DL, VT: MVT::i32), Selector: 0x0123,
2578 DL, DAG);
2579 SDValue SwappedHigh =
2580 getPRMT(A: UnpackSrc.getValue(R: 1), B: DAG.getConstant(Val: 0, DL, VT: MVT::i32), Selector: 0x0123,
2581 DL, DAG);
2582 return DAG.getNode(Opcode: NVPTXISD::BUILD_VECTOR, DL, VT: MVT::i64,
2583 Ops: {SwappedHigh, SwappedLow});
2584 }
2585 default:
2586 llvm_unreachable("unsupported type for bswap");
2587 }
2588}
2589
2590static SDValue lowerStAsyncWithMbarrier(SDValue Op, SelectionDAG &DAG) {
2591 const Function &Fn = DAG.getMachineFunction().getFunction();
2592 SDNode *N = Op.getNode();
2593 SDLoc DL(N);
2594 Intrinsic::ID IntrinsicID = N->getConstantOperandVal(Num: 1);
2595 SDValue DestAddr = N->getOperand(Num: 2);
2596 SDValue Value = N->getOperand(Num: 3);
2597 SDValue MbarAddr = N->getOperand(Num: 4);
2598
2599 MVT ValueVT = Value.getSimpleValueType();
2600
2601 if (ValueVT == MVT::i32 || ValueVT == MVT::i64)
2602 return Op;
2603
2604 if (ValueVT == MVT::i128) {
2605 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: Value);
2606 SDValue ValueLo = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i64, N1: Cast,
2607 N2: DAG.getIntPtrConstant(Val: 0, DL));
2608 SDValue ValueHi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i64, N1: Cast,
2609 N2: DAG.getIntPtrConstant(Val: 1, DL));
2610 SDValue Ops[] = {N->getOperand(Num: 0), DestAddr, ValueLo, ValueHi, MbarAddr};
2611 return DAG.getNode(Opcode: NVPTXISD::ST_ASYNC_MBARRIER_B128, DL, VT: MVT::Other, Ops);
2612 }
2613
2614 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
2615 Fn,
2616 Twine("unsupported argument type ") + llvm::EVT(ValueVT).getEVTString() +
2617 " for " + llvm::Intrinsic::getName(id: IntrinsicID) + " intrinsic",
2618 DiagnosticLocation(DL.getDebugLoc())));
2619 return Op.getOperand(i: 0); // Return only the chain
2620}
2621
2622static SDValue lowerStAsyncRelease(SDValue Op, SelectionDAG &DAG) {
2623 const Function &Fn = DAG.getMachineFunction().getFunction();
2624 SDNode *N = Op.getNode();
2625 SDLoc DL(N);
2626 Intrinsic::ID IntrinsicID = N->getConstantOperandVal(Num: 1);
2627 SDValue DestAddr = N->getOperand(Num: 2);
2628 SDValue Value = N->getOperand(Num: 3);
2629
2630 MVT ValueVT = Value.getSimpleValueType();
2631
2632 if (ValueVT == MVT::i16 || ValueVT == MVT::i32 || ValueVT == MVT::i64)
2633 return Op;
2634
2635 if (ValueVT == MVT::i8) {
2636 unsigned OpCode;
2637 switch (IntrinsicID) {
2638 case Intrinsic::nvvm_st_async_sys:
2639 OpCode = NVPTXISD::ST_ASYNC_SYS_B8;
2640 break;
2641 case Intrinsic::nvvm_st_async_gpu:
2642 OpCode = NVPTXISD::ST_ASYNC_GPU_B8;
2643 break;
2644 case Intrinsic::nvvm_st_async_mmio_sys:
2645 OpCode = NVPTXISD::ST_ASYNC_MMIO_SYS_B8;
2646 break;
2647 default:
2648 llvm_unreachable("unexpected intrinsic ID for st.async.release");
2649 }
2650
2651 Value = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i16, Operand: Value);
2652
2653 // The `.mmio` variant has no multimem form and therefore no `isMultimem`
2654 // operand.
2655 if (IntrinsicID == Intrinsic::nvvm_st_async_mmio_sys) {
2656 SDValue Ops[] = {N->getOperand(Num: 0), DestAddr, Value};
2657 return DAG.getNode(Opcode: OpCode, DL, VT: MVT::Other, Ops);
2658 }
2659
2660 SDValue IsMultimem =
2661 DAG.getTargetConstant(Val: N->getConstantOperandVal(Num: 4), DL, VT: MVT::i1);
2662 SDValue Ops[] = {N->getOperand(Num: 0), DestAddr, Value, IsMultimem};
2663 return DAG.getNode(Opcode: OpCode, DL, VT: MVT::Other, Ops);
2664 }
2665
2666 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
2667 Fn,
2668 Twine("unsupported argument type ") + llvm::EVT(ValueVT).getEVTString() +
2669 " for " + llvm::Intrinsic::getName(id: IntrinsicID) + " intrinsic",
2670 DiagnosticLocation(DL.getDebugLoc())));
2671 return Op.getOperand(i: 0); // Return only the chain
2672}
2673
2674static unsigned getTcgen05MMADisableOutputLane(unsigned IID) {
2675 switch (IID) {
2676 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg1:
2677 return NVPTXISD::TCGEN05_MMA_SHARED_DISABLE_OUTPUT_LANE_CG1;
2678 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg2:
2679 return NVPTXISD::TCGEN05_MMA_SHARED_DISABLE_OUTPUT_LANE_CG2;
2680 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg1:
2681 return NVPTXISD::TCGEN05_MMA_SHARED_SCALE_D_DISABLE_OUTPUT_LANE_CG1;
2682 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg2:
2683 return NVPTXISD::TCGEN05_MMA_SHARED_SCALE_D_DISABLE_OUTPUT_LANE_CG2;
2684 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1:
2685 return NVPTXISD::TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG1;
2686 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2:
2687 return NVPTXISD::TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG2;
2688 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1:
2689 return NVPTXISD::TCGEN05_MMA_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG1;
2690 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2:
2691 return NVPTXISD::TCGEN05_MMA_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG2;
2692 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1_ashift:
2693 return NVPTXISD::TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG1_ASHIFT;
2694 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2_ashift:
2695 return NVPTXISD::TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG2_ASHIFT;
2696 case Intrinsic::
2697 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1_ashift:
2698 return NVPTXISD::TCGEN05_MMA_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG1_ASHIFT;
2699 case Intrinsic::
2700 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2_ashift:
2701 return NVPTXISD::TCGEN05_MMA_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG2_ASHIFT;
2702 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg1:
2703 return NVPTXISD::TCGEN05_MMA_SP_SHARED_DISABLE_OUTPUT_LANE_CG1;
2704 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg2:
2705 return NVPTXISD::TCGEN05_MMA_SP_SHARED_DISABLE_OUTPUT_LANE_CG2;
2706 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg1:
2707 return NVPTXISD::TCGEN05_MMA_SP_SHARED_SCALE_D_DISABLE_OUTPUT_LANE_CG1;
2708 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg2:
2709 return NVPTXISD::TCGEN05_MMA_SP_SHARED_SCALE_D_DISABLE_OUTPUT_LANE_CG2;
2710 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1:
2711 return NVPTXISD::TCGEN05_MMA_SP_TENSOR_DISABLE_OUTPUT_LANE_CG1;
2712 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2:
2713 return NVPTXISD::TCGEN05_MMA_SP_TENSOR_DISABLE_OUTPUT_LANE_CG2;
2714 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1_ashift:
2715 return NVPTXISD::TCGEN05_MMA_SP_TENSOR_DISABLE_OUTPUT_LANE_CG1_ASHIFT;
2716 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2_ashift:
2717 return NVPTXISD::TCGEN05_MMA_SP_TENSOR_DISABLE_OUTPUT_LANE_CG2_ASHIFT;
2718 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1:
2719 return NVPTXISD::TCGEN05_MMA_SP_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG1;
2720 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2:
2721 return NVPTXISD::TCGEN05_MMA_SP_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG2;
2722 case Intrinsic::
2723 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1_ashift:
2724 return NVPTXISD::
2725 TCGEN05_MMA_SP_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG1_ASHIFT;
2726 case Intrinsic::
2727 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2_ashift:
2728 return NVPTXISD::
2729 TCGEN05_MMA_SP_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG2_ASHIFT;
2730 case Intrinsic::
2731 nvvm_tcgen05_mma_shared_f8f6f4_disable_output_lane_cg1_decompress_b:
2732 return NVPTXISD::TCGEN05_MMA_SHARED_DISABLE_OUTPUT_LANE_CG1_DECOMPRESS_B;
2733 case Intrinsic::
2734 nvvm_tcgen05_mma_shared_f8f6f4_disable_output_lane_cg2_decompress_b:
2735 return NVPTXISD::TCGEN05_MMA_SHARED_DISABLE_OUTPUT_LANE_CG2_DECOMPRESS_B;
2736 case Intrinsic::
2737 nvvm_tcgen05_mma_tensor_f8f6f4_disable_output_lane_cg1_decompress_b:
2738 return NVPTXISD::TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG1_DECOMPRESS_B;
2739 case Intrinsic::
2740 nvvm_tcgen05_mma_tensor_f8f6f4_disable_output_lane_cg2_decompress_b:
2741 return NVPTXISD::TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG2_DECOMPRESS_B;
2742 };
2743 llvm_unreachable("unhandled tcgen05.mma.disable_output_lane intrinsic");
2744}
2745
2746static SDValue LowerTcgen05MMADisableOutputLane(SDValue Op, SelectionDAG &DAG) {
2747 SDNode *N = Op.getNode();
2748 SDLoc DL(N);
2749 unsigned IID = cast<ConstantSDNode>(Val: N->getOperand(Num: 1))->getZExtValue();
2750
2751 SmallVector<SDValue, 16> Ops;
2752 // split the vector argument
2753 for (size_t I = 0; I < N->getNumOperands(); I++) {
2754 if (I == 1)
2755 continue; // skip IID
2756 SDValue Val = N->getOperand(Num: I);
2757 EVT ValVT = Val.getValueType();
2758 if (ValVT.isVector()) {
2759 EVT EltVT = ValVT.getVectorElementType();
2760 for (unsigned J = 0, NElts = ValVT.getVectorNumElements(); J < NElts; J++)
2761 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Val,
2762 N2: DAG.getIntPtrConstant(Val: J, DL)));
2763 } else
2764 Ops.push_back(Elt: Val);
2765 }
2766
2767 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(Val: N);
2768 SDValue Tcgen05MMANode = DAG.getMemIntrinsicNode(
2769 Opcode: getTcgen05MMADisableOutputLane(IID), dl: DL, VTList: N->getVTList(), Ops,
2770 MemVT: MemSD->getMemoryVT(), MMO: MemSD->getMemOperand());
2771
2772 return Tcgen05MMANode;
2773}
2774
2775// Lower vector return type of tcgen05.ld intrinsics
2776static std::optional<std::pair<SDValue, SDValue>>
2777lowerTcgen05Ld(SDNode *N, SelectionDAG &DAG, bool HasOffset = false) {
2778 SDLoc DL(N);
2779 EVT ResVT = N->getValueType(ResNo: 0);
2780 if (!ResVT.isVector())
2781 return {}; // already legalized.
2782
2783 const unsigned NumElts = ResVT.getVectorNumElements();
2784
2785 // Create the return type of the instructions
2786 SmallVector<EVT, 5> ListVTs;
2787 for (unsigned i = 0; i < NumElts; ++i)
2788 ListVTs.push_back(Elt: MVT::i32);
2789
2790 ListVTs.push_back(Elt: N->getValueType(ResNo: 1)); // Chain
2791
2792 SDVTList ResVTs = DAG.getVTList(VTs: ListVTs);
2793
2794 SmallVector<SDValue, 8> Ops{N->getOperand(Num: 0), N->getOperand(Num: 1),
2795 N->getOperand(Num: 2)};
2796
2797 if (HasOffset) {
2798 Ops.push_back(Elt: N->getOperand(Num: 3)); // offset
2799 Ops.push_back(Elt: N->getOperand(Num: 4)); // Pack flag
2800 } else
2801 Ops.push_back(Elt: N->getOperand(Num: 3)); // Pack flag
2802
2803 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(Val: N);
2804 SDValue NewNode =
2805 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: ResVTs, Ops,
2806 MemVT: MemSD->getMemoryVT(), MMO: MemSD->getMemOperand());
2807
2808 // split the vector result
2809 SmallVector<SDValue, 4> ScalarRes;
2810 for (unsigned i = 0; i < NumElts; ++i) {
2811 SDValue Res = NewNode.getValue(R: i);
2812 ScalarRes.push_back(Elt: Res);
2813 }
2814
2815 SDValue Chain = NewNode.getValue(R: NumElts);
2816 SDValue BuildVector = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: ResVT, Ops: ScalarRes);
2817 return {{BuildVector, Chain}};
2818}
2819
2820static SDValue reportInvalidTensormapReplaceUsage(SDValue Op, SelectionDAG &DAG,
2821 unsigned Val) {
2822 SDNode *N = Op.getNode();
2823 SDLoc DL(N);
2824
2825 const Function &Fn = DAG.getMachineFunction().getFunction();
2826
2827 unsigned AS = 0;
2828 if (auto *MemN = dyn_cast<MemIntrinsicSDNode>(Val: N))
2829 AS = MemN->getAddressSpace();
2830 Type *PtrTy = PointerType::get(C&: *DAG.getContext(), AddressSpace: AS);
2831 Module *M = DAG.getMachineFunction().getFunction().getParent();
2832
2833 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
2834 Fn,
2835 "Intrinsic " +
2836 Intrinsic::getName(Id: N->getConstantOperandVal(Num: 1), OverloadTys: {PtrTy}, M) +
2837 " with value " + Twine(Val) +
2838 " is not supported on the given target.",
2839 DL.getDebugLoc()));
2840 return Op.getOperand(i: 0);
2841}
2842
2843static SDValue lowerTensormapReplaceElemtype(SDValue Op, SelectionDAG &DAG) {
2844 SDNode *N = Op.getNode();
2845 SDLoc DL(N);
2846
2847 // immediate argument representing elemtype
2848 unsigned Val = N->getConstantOperandVal(Num: 3);
2849
2850 if (!DAG.getSubtarget<NVPTXSubtarget>().hasTensormapReplaceElemtypeSupport(
2851 ElemType: Val))
2852 return reportInvalidTensormapReplaceUsage(Op, DAG, Val);
2853
2854 return Op;
2855}
2856
2857static SDValue lowerTensormapReplaceSwizzleMode(SDValue Op, SelectionDAG &DAG) {
2858 SDNode *N = Op.getNode();
2859 SDLoc DL(N);
2860
2861 // immediate argument representing swizzle mode
2862 unsigned Val = N->getConstantOperandVal(Num: 3);
2863
2864 if (!DAG.getSubtarget<NVPTXSubtarget>().hasTensormapReplaceSwizzleModeSupport(
2865 SwizzleMode: Val))
2866 return reportInvalidTensormapReplaceUsage(Op, DAG, Val);
2867
2868 return Op;
2869}
2870
2871static SDValue lowerIntrinsicVoid(SDValue Op, SelectionDAG &DAG) {
2872 SDNode *N = Op.getNode();
2873 SDValue Intrin = N->getOperand(Num: 1);
2874
2875 // Get the intrinsic ID
2876 unsigned IntrinNo = cast<ConstantSDNode>(Val: Intrin.getNode())->getZExtValue();
2877 switch (IntrinNo) {
2878 default:
2879 break;
2880 case Intrinsic::nvvm_st_async:
2881 return lowerStAsyncWithMbarrier(Op, DAG);
2882 case Intrinsic::nvvm_st_async_sys:
2883 case Intrinsic::nvvm_st_async_gpu:
2884 case Intrinsic::nvvm_st_async_mmio_sys:
2885 return lowerStAsyncRelease(Op, DAG);
2886
2887 case Intrinsic::nvvm_tcgen05_st_16x64b_x1:
2888 case Intrinsic::nvvm_tcgen05_st_16x64b_x2:
2889 case Intrinsic::nvvm_tcgen05_st_16x64b_x4:
2890 case Intrinsic::nvvm_tcgen05_st_16x64b_x8:
2891 case Intrinsic::nvvm_tcgen05_st_16x64b_x16:
2892 case Intrinsic::nvvm_tcgen05_st_16x64b_x32:
2893 case Intrinsic::nvvm_tcgen05_st_16x64b_x128:
2894 case Intrinsic::nvvm_tcgen05_st_16x128b_x1:
2895 case Intrinsic::nvvm_tcgen05_st_16x128b_x2:
2896 case Intrinsic::nvvm_tcgen05_st_16x128b_x4:
2897 case Intrinsic::nvvm_tcgen05_st_16x128b_x8:
2898 case Intrinsic::nvvm_tcgen05_st_16x128b_x16:
2899 case Intrinsic::nvvm_tcgen05_st_16x128b_x32:
2900 case Intrinsic::nvvm_tcgen05_st_16x128b_x64:
2901 case Intrinsic::nvvm_tcgen05_st_16x256b_x1:
2902 case Intrinsic::nvvm_tcgen05_st_16x256b_x2:
2903 case Intrinsic::nvvm_tcgen05_st_16x256b_x4:
2904 case Intrinsic::nvvm_tcgen05_st_16x256b_x8:
2905 case Intrinsic::nvvm_tcgen05_st_16x256b_x16:
2906 case Intrinsic::nvvm_tcgen05_st_16x256b_x32:
2907 case Intrinsic::nvvm_tcgen05_st_32x32b_x1:
2908 case Intrinsic::nvvm_tcgen05_st_32x32b_x2:
2909 case Intrinsic::nvvm_tcgen05_st_32x32b_x4:
2910 case Intrinsic::nvvm_tcgen05_st_32x32b_x8:
2911 case Intrinsic::nvvm_tcgen05_st_32x32b_x16:
2912 case Intrinsic::nvvm_tcgen05_st_32x32b_x32:
2913 case Intrinsic::nvvm_tcgen05_st_16x64b_x64:
2914 case Intrinsic::nvvm_tcgen05_st_32x32b_x64:
2915 case Intrinsic::nvvm_tcgen05_st_32x32b_x128:
2916 return lowerTcgen05St(Op, DAG);
2917 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x1:
2918 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x2:
2919 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x4:
2920 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x8:
2921 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x16:
2922 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x32:
2923 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x64:
2924 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x128:
2925 return lowerTcgen05St(Op, DAG, /* hasOffset */ true);
2926 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg1:
2927 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg2:
2928 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg1:
2929 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg2:
2930 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg1:
2931 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg2:
2932 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg1:
2933 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg2:
2934 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1:
2935 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2:
2936 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1:
2937 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2:
2938 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1:
2939 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2:
2940 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1:
2941 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2:
2942 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1_ashift:
2943 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2_ashift:
2944 case Intrinsic::
2945 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1_ashift:
2946 case Intrinsic::
2947 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2_ashift:
2948 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1_ashift:
2949 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2_ashift:
2950 case Intrinsic::
2951 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1_ashift:
2952 case Intrinsic::
2953 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2_ashift:
2954 case Intrinsic::
2955 nvvm_tcgen05_mma_shared_f8f6f4_disable_output_lane_cg1_decompress_b:
2956 case Intrinsic::
2957 nvvm_tcgen05_mma_shared_f8f6f4_disable_output_lane_cg2_decompress_b:
2958 case Intrinsic::
2959 nvvm_tcgen05_mma_tensor_f8f6f4_disable_output_lane_cg1_decompress_b:
2960 case Intrinsic::
2961 nvvm_tcgen05_mma_tensor_f8f6f4_disable_output_lane_cg2_decompress_b:
2962 return LowerTcgen05MMADisableOutputLane(Op, DAG);
2963 case Intrinsic::nvvm_tensormap_replace_elemtype:
2964 return lowerTensormapReplaceElemtype(Op, DAG);
2965 case Intrinsic::nvvm_tensormap_replace_swizzle_mode:
2966 return lowerTensormapReplaceSwizzleMode(Op, DAG);
2967 }
2968 return Op;
2969}
2970
2971static SDValue LowerClusterLaunchControlQueryCancel(SDValue Op,
2972 SelectionDAG &DAG) {
2973
2974 SDNode *N = Op.getNode();
2975 if (N->getOperand(Num: 1).getValueType() != MVT::i128) {
2976 // return, if the operand is already lowered
2977 return SDValue();
2978 }
2979
2980 unsigned IID =
2981 cast<ConstantSDNode>(Val: N->getOperand(Num: 0).getNode())->getZExtValue();
2982 auto Opcode = [&]() {
2983 switch (IID) {
2984 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_is_canceled:
2985 return NVPTXISD::CLUSTERLAUNCHCONTROL_QUERY_CANCEL_IS_CANCELED;
2986 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_x:
2987 return NVPTXISD::CLUSTERLAUNCHCONTROL_QUERY_CANCEL_GET_FIRST_CTAID_X;
2988 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_y:
2989 return NVPTXISD::CLUSTERLAUNCHCONTROL_QUERY_CANCEL_GET_FIRST_CTAID_Y;
2990 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_z:
2991 return NVPTXISD::CLUSTERLAUNCHCONTROL_QUERY_CANCEL_GET_FIRST_CTAID_Z;
2992 default:
2993 llvm_unreachable("unsupported/unhandled intrinsic");
2994 }
2995 }();
2996
2997 SDLoc DL(N);
2998 SDValue TryCancelResponse = N->getOperand(Num: 1);
2999 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: TryCancelResponse);
3000 SDValue TryCancelResponse0 =
3001 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i64, N1: Cast,
3002 N2: DAG.getIntPtrConstant(Val: 0, DL));
3003 SDValue TryCancelResponse1 =
3004 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i64, N1: Cast,
3005 N2: DAG.getIntPtrConstant(Val: 1, DL));
3006
3007 return DAG.getNode(Opcode, DL, VTList: N->getVTList(),
3008 Ops: {TryCancelResponse0, TryCancelResponse1});
3009}
3010
3011static SDValue lowerCvtRSIntrinsics(SDValue Op, SelectionDAG &DAG) {
3012 SDNode *N = Op.getNode();
3013 SDLoc DL(N);
3014 SDValue F32Vec = N->getOperand(Num: 1);
3015 SDValue RBits = N->getOperand(Num: 2);
3016
3017 unsigned IntrinsicID = N->getConstantOperandVal(Num: 0);
3018
3019 // Extract the 4 float elements from the vector
3020 SmallVector<SDValue, 6> Ops;
3021 for (unsigned i = 0; i < 4; ++i)
3022 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: F32Vec,
3023 N2: DAG.getIntPtrConstant(Val: i, DL)));
3024
3025 using NVPTX::PTXCvtMode::CvtMode;
3026
3027 auto [OpCode, RetTy, CvtModeFlag] =
3028 [&]() -> std::tuple<unsigned, MVT::SimpleValueType, uint32_t> {
3029 switch (IntrinsicID) {
3030 case Intrinsic::nvvm_f32x4_to_e4m3x4_rs_relu_satfinite:
3031 return {NVPTXISD::CVT_E4M3X4_F32X4_RS_SF, MVT::v4i8,
3032 CvtMode::RS | CvtMode::RELU_FLAG};
3033 case Intrinsic::nvvm_f32x4_to_e4m3x4_rs_satfinite:
3034 return {NVPTXISD::CVT_E4M3X4_F32X4_RS_SF, MVT::v4i8, CvtMode::RS};
3035 case Intrinsic::nvvm_f32x4_to_e5m2x4_rs_relu_satfinite:
3036 return {NVPTXISD::CVT_E5M2X4_F32X4_RS_SF, MVT::v4i8,
3037 CvtMode::RS | CvtMode::RELU_FLAG};
3038 case Intrinsic::nvvm_f32x4_to_e5m2x4_rs_satfinite:
3039 return {NVPTXISD::CVT_E5M2X4_F32X4_RS_SF, MVT::v4i8, CvtMode::RS};
3040 case Intrinsic::nvvm_f32x4_to_e2m3x4_rs_relu_satfinite:
3041 return {NVPTXISD::CVT_E2M3X4_F32X4_RS_SF, MVT::v4i8,
3042 CvtMode::RS | CvtMode::RELU_FLAG};
3043 case Intrinsic::nvvm_f32x4_to_e2m3x4_rs_satfinite:
3044 return {NVPTXISD::CVT_E2M3X4_F32X4_RS_SF, MVT::v4i8, CvtMode::RS};
3045 case Intrinsic::nvvm_f32x4_to_e3m2x4_rs_relu_satfinite:
3046 return {NVPTXISD::CVT_E3M2X4_F32X4_RS_SF, MVT::v4i8,
3047 CvtMode::RS | CvtMode::RELU_FLAG};
3048 case Intrinsic::nvvm_f32x4_to_e3m2x4_rs_satfinite:
3049 return {NVPTXISD::CVT_E3M2X4_F32X4_RS_SF, MVT::v4i8, CvtMode::RS};
3050 case Intrinsic::nvvm_f32x4_to_e2m1x4_rs_relu_satfinite:
3051 return {NVPTXISD::CVT_E2M1X4_F32X4_RS_SF, MVT::i16,
3052 CvtMode::RS | CvtMode::RELU_FLAG};
3053 case Intrinsic::nvvm_f32x4_to_e2m1x4_rs_satfinite:
3054 return {NVPTXISD::CVT_E2M1X4_F32X4_RS_SF, MVT::i16, CvtMode::RS};
3055 default:
3056 llvm_unreachable("unsupported/unhandled intrinsic");
3057 }
3058 }();
3059
3060 Ops.push_back(Elt: RBits);
3061 Ops.push_back(Elt: DAG.getConstant(Val: CvtModeFlag, DL, VT: MVT::i32));
3062
3063 return DAG.getNode(Opcode: OpCode, DL, VT: RetTy, Ops);
3064}
3065
3066static SDValue lowerPrmtIntrinsic(SDValue Op, SelectionDAG &DAG) {
3067 const unsigned Mode = [&]() {
3068 switch (Op->getConstantOperandVal(Num: 0)) {
3069 case Intrinsic::nvvm_prmt:
3070 return NVPTX::PTXPrmtMode::NONE;
3071 case Intrinsic::nvvm_prmt_b4e:
3072 return NVPTX::PTXPrmtMode::B4E;
3073 case Intrinsic::nvvm_prmt_ecl:
3074 return NVPTX::PTXPrmtMode::ECL;
3075 case Intrinsic::nvvm_prmt_ecr:
3076 return NVPTX::PTXPrmtMode::ECR;
3077 case Intrinsic::nvvm_prmt_f4e:
3078 return NVPTX::PTXPrmtMode::F4E;
3079 case Intrinsic::nvvm_prmt_rc16:
3080 return NVPTX::PTXPrmtMode::RC16;
3081 case Intrinsic::nvvm_prmt_rc8:
3082 return NVPTX::PTXPrmtMode::RC8;
3083 default:
3084 llvm_unreachable("unsupported/unhandled intrinsic");
3085 }
3086 }();
3087 SDLoc DL(Op);
3088 SDValue A = Op->getOperand(Num: 1);
3089 SDValue B = Op.getNumOperands() == 4 ? Op.getOperand(i: 2)
3090 : DAG.getConstant(Val: 0, DL, VT: MVT::i32);
3091 SDValue Selector = (Op->op_end() - 1)->get();
3092 return getPRMT(A, B, Selector, DL, DAG, Mode);
3093}
3094
3095#define TCGEN05_LD_RED_INTR(SHAPE, NUM, TYPE) \
3096 Intrinsic::nvvm_tcgen05_ld_red_##SHAPE##_x##NUM##_##TYPE
3097
3098#define TCGEN05_LD_RED_INST(SHAPE, NUM, TYPE) \
3099 NVPTXISD::TCGEN05_LD_RED_##SHAPE##_X##NUM##_##TYPE
3100
3101static unsigned getTcgen05LdRedID(Intrinsic::ID IID) {
3102 switch (IID) {
3103 case TCGEN05_LD_RED_INTR(32x32b, 2, f32):
3104 return TCGEN05_LD_RED_INST(32x32b, 2, F32);
3105 case TCGEN05_LD_RED_INTR(32x32b, 4, f32):
3106 return TCGEN05_LD_RED_INST(32x32b, 4, F32);
3107 case TCGEN05_LD_RED_INTR(32x32b, 8, f32):
3108 return TCGEN05_LD_RED_INST(32x32b, 8, F32);
3109 case TCGEN05_LD_RED_INTR(32x32b, 16, f32):
3110 return TCGEN05_LD_RED_INST(32x32b, 16, F32);
3111 case TCGEN05_LD_RED_INTR(32x32b, 32, f32):
3112 return TCGEN05_LD_RED_INST(32x32b, 32, F32);
3113 case TCGEN05_LD_RED_INTR(32x32b, 64, f32):
3114 return TCGEN05_LD_RED_INST(32x32b, 64, F32);
3115 case TCGEN05_LD_RED_INTR(32x32b, 128, f32):
3116 return TCGEN05_LD_RED_INST(32x32b, 128, F32);
3117 case TCGEN05_LD_RED_INTR(16x32bx2, 2, f32):
3118 return TCGEN05_LD_RED_INST(16x32bx2, 2, F32);
3119 case TCGEN05_LD_RED_INTR(16x32bx2, 4, f32):
3120 return TCGEN05_LD_RED_INST(16x32bx2, 4, F32);
3121 case TCGEN05_LD_RED_INTR(16x32bx2, 8, f32):
3122 return TCGEN05_LD_RED_INST(16x32bx2, 8, F32);
3123 case TCGEN05_LD_RED_INTR(16x32bx2, 16, f32):
3124 return TCGEN05_LD_RED_INST(16x32bx2, 16, F32);
3125 case TCGEN05_LD_RED_INTR(16x32bx2, 32, f32):
3126 return TCGEN05_LD_RED_INST(16x32bx2, 32, F32);
3127 case TCGEN05_LD_RED_INTR(16x32bx2, 64, f32):
3128 return TCGEN05_LD_RED_INST(16x32bx2, 64, F32);
3129 case TCGEN05_LD_RED_INTR(16x32bx2, 128, f32):
3130 return TCGEN05_LD_RED_INST(16x32bx2, 128, F32);
3131 case TCGEN05_LD_RED_INTR(32x32b, 2, i32):
3132 return TCGEN05_LD_RED_INST(32x32b, 2, I32);
3133 case TCGEN05_LD_RED_INTR(32x32b, 4, i32):
3134 return TCGEN05_LD_RED_INST(32x32b, 4, I32);
3135 case TCGEN05_LD_RED_INTR(32x32b, 8, i32):
3136 return TCGEN05_LD_RED_INST(32x32b, 8, I32);
3137 case TCGEN05_LD_RED_INTR(32x32b, 16, i32):
3138 return TCGEN05_LD_RED_INST(32x32b, 16, I32);
3139 case TCGEN05_LD_RED_INTR(32x32b, 32, i32):
3140 return TCGEN05_LD_RED_INST(32x32b, 32, I32);
3141 case TCGEN05_LD_RED_INTR(32x32b, 64, i32):
3142 return TCGEN05_LD_RED_INST(32x32b, 64, I32);
3143 case TCGEN05_LD_RED_INTR(32x32b, 128, i32):
3144 return TCGEN05_LD_RED_INST(32x32b, 128, I32);
3145 case TCGEN05_LD_RED_INTR(16x32bx2, 2, i32):
3146 return TCGEN05_LD_RED_INST(16x32bx2, 2, I32);
3147 case TCGEN05_LD_RED_INTR(16x32bx2, 4, i32):
3148 return TCGEN05_LD_RED_INST(16x32bx2, 4, I32);
3149 case TCGEN05_LD_RED_INTR(16x32bx2, 8, i32):
3150 return TCGEN05_LD_RED_INST(16x32bx2, 8, I32);
3151 case TCGEN05_LD_RED_INTR(16x32bx2, 16, i32):
3152 return TCGEN05_LD_RED_INST(16x32bx2, 16, I32);
3153 case TCGEN05_LD_RED_INTR(16x32bx2, 32, i32):
3154 return TCGEN05_LD_RED_INST(16x32bx2, 32, I32);
3155 case TCGEN05_LD_RED_INTR(16x32bx2, 64, i32):
3156 return TCGEN05_LD_RED_INST(16x32bx2, 64, I32);
3157 case TCGEN05_LD_RED_INTR(16x32bx2, 128, i32):
3158 return TCGEN05_LD_RED_INST(16x32bx2, 128, I32);
3159 default:
3160 llvm_unreachable("Invalid tcgen05.ld.red intrinsic ID");
3161 }
3162}
3163
3164// Lower vector return type of tcgen05.ld intrinsics
3165static std::optional<std::tuple<SDValue, SDValue, SDValue>>
3166lowerTcgen05LdRed(SDNode *N, SelectionDAG &DAG) {
3167 SDLoc DL(N);
3168 EVT ResVT = N->getValueType(ResNo: 0);
3169 if (!ResVT.isVector())
3170 return {}; // already legalized.
3171
3172 const unsigned NumElts = ResVT.getVectorNumElements();
3173
3174 // Create the return type of the instructions
3175 // +1 represents the reduction value
3176 SmallVector<EVT, 132> ListVTs{
3177 NumElts + 1,
3178 ResVT.getVectorElementType().isFloatingPoint() ? MVT::f32 : MVT::i32};
3179
3180 ListVTs.push_back(Elt: MVT::Other); // Chain
3181
3182 SDVTList ResVTs = DAG.getVTList(VTs: ListVTs);
3183
3184 // Prepare the Operands
3185 SmallVector<SDValue, 8> Ops{N->getOperand(Num: 0)}; // Chain
3186
3187 // skip IID at index 1
3188 for (unsigned i = 2; i < N->getNumOperands(); i++)
3189 Ops.push_back(Elt: N->getOperand(Num: i));
3190
3191 unsigned IID = cast<ConstantSDNode>(Val: N->getOperand(Num: 1))->getZExtValue();
3192 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(Val: N);
3193 SDValue NewNode =
3194 DAG.getMemIntrinsicNode(Opcode: getTcgen05LdRedID(IID), dl: DL, VTList: ResVTs, Ops,
3195 MemVT: MemSD->getMemoryVT(), MMO: MemSD->getMemOperand());
3196
3197 // Split vector result
3198 SmallVector<SDValue, 132> ScalarRes;
3199 for (unsigned i = 0; i < NumElts; ++i) {
3200 SDValue Res = NewNode.getValue(R: i);
3201 ScalarRes.push_back(Elt: Res);
3202 }
3203
3204 SDValue BuildVector = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: ResVT, Ops: ScalarRes);
3205 SDValue RedResult = NewNode.getValue(R: NumElts);
3206 SDValue Chain = NewNode.getValue(R: NumElts + 1);
3207 return {{BuildVector, RedResult, Chain}};
3208}
3209
3210static SDValue lowerIntrinsicWChain(SDValue Op, SelectionDAG &DAG) {
3211 switch (Op->getConstantOperandVal(Num: 1)) {
3212 default:
3213 return Op;
3214
3215 // These tcgen05 intrinsics return a v2i32, which is legal, so we have to
3216 // lower them through LowerOperation() instead of ReplaceNodeResults().
3217 case Intrinsic::nvvm_tcgen05_ld_16x64b_x2:
3218 case Intrinsic::nvvm_tcgen05_ld_16x128b_x1:
3219 case Intrinsic::nvvm_tcgen05_ld_32x32b_x2:
3220 if (auto Res = lowerTcgen05Ld(N: Op.getNode(), DAG))
3221 return DAG.getMergeValues(Ops: {Res->first, Res->second}, dl: SDLoc(Op));
3222 return SDValue();
3223
3224 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x2:
3225 if (auto Res = lowerTcgen05Ld(N: Op.getNode(), DAG, /*HasOffset=*/true))
3226 return DAG.getMergeValues(Ops: {Res->first, Res->second}, dl: SDLoc(Op));
3227 return SDValue();
3228
3229 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x2_f32:
3230 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x2_i32:
3231 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x2_f32:
3232 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x2_i32:
3233 if (auto Res = lowerTcgen05LdRed(N: Op.getNode(), DAG))
3234 return DAG.getMergeValues(
3235 Ops: {std::get<0>(t&: *Res), std::get<1>(t&: *Res), std::get<2>(t&: *Res)}, dl: SDLoc(Op));
3236 return SDValue();
3237 }
3238}
3239
3240static unsigned getSPNumRegs(EVT VT) {
3241 return divideCeil(Numerator: VT.getSizeInBits(), Denominator: 32);
3242}
3243
3244static EVT getSPRegType(EVT VT) {
3245 assert(VT.isVector() && "Expected a vector type");
3246 EVT EltVT = VT.getVectorElementType();
3247 if (EltVT == MVT::i8)
3248 return MVT::v4i8;
3249 if (EltVT == MVT::i16)
3250 return MVT::v2i16;
3251 return EltVT;
3252}
3253
3254static void appendSplitSPOperands(ArrayRef<SDUse> Input,
3255 SmallVectorImpl<SDValue> &Output,
3256 SelectionDAG &DAG) {
3257 for (SDValue Operand : Input) {
3258 EVT VT = Operand.getValueType();
3259 SDLoc DL(Operand);
3260 if (!VT.isVector()) {
3261 Output.push_back(Elt: Operand);
3262 continue;
3263 }
3264
3265 // Pack partial .b32 operands at their natural width before extending them
3266 // to avoid redundant cvt/prmt sequences.
3267 if (VT.getSizeInBits() < 32) {
3268 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
3269 Output.push_back(
3270 Elt: DAG.getAnyExtOrTrunc(Op: DAG.getBitcast(VT: IntVT, V: Operand), DL, VT: MVT::i32));
3271 continue;
3272 }
3273
3274 EVT RegVT = getSPRegType(VT);
3275 if (!RegVT.isVector()) {
3276 DAG.ExtractVectorElements(Op: Operand, Args&: Output);
3277 continue;
3278 }
3279
3280 unsigned RegElts = RegVT.getVectorNumElements();
3281 for (unsigned I : llvm::seq(Size: getSPNumRegs(VT)))
3282 Output.push_back(Elt: DAG.getBitcast(
3283 VT: MVT::i32, V: DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: RegVT, N1: Operand,
3284 N2: DAG.getVectorIdxConstant(Val: I * RegElts, DL))));
3285 }
3286}
3287
3288static SDValue joinSPRegs(EVT VT, ArrayRef<SDValue> Regs, const SDLoc &DL,
3289 SelectionDAG &DAG) {
3290 assert(Regs.size() == getSPNumRegs(VT) && VT.getSizeInBits() % 32 == 0 &&
3291 "Incorrect number of SP vector registers");
3292 if (!VT.isVector())
3293 return Regs.front();
3294
3295 EVT RegVT = getSPRegType(VT);
3296 SmallVector<SDValue> Parts;
3297 for (SDValue Reg : Regs)
3298 Parts.push_back(Elt: DAG.getBitcast(VT: RegVT, V: Reg));
3299
3300 if (!RegVT.isVector())
3301 return DAG.getBuildVector(VT, DL, Ops: Parts);
3302 if (Parts.size() == 1)
3303 return Parts.front();
3304 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, Ops: Parts);
3305}
3306
3307static unsigned getSPCompressNodeOpcode(unsigned NumResults) {
3308#define SPCOMPRESS_NODE_CASE(NumResults) \
3309 case NumResults: \
3310 return NVPTXISD::SPCOMPRESS_R##NumResults
3311
3312 switch (NumResults) {
3313 SPCOMPRESS_NODE_CASE(2);
3314 SPCOMPRESS_NODE_CASE(3);
3315 SPCOMPRESS_NODE_CASE(5);
3316 SPCOMPRESS_NODE_CASE(6);
3317 SPCOMPRESS_NODE_CASE(9);
3318 SPCOMPRESS_NODE_CASE(10);
3319 SPCOMPRESS_NODE_CASE(12);
3320 SPCOMPRESS_NODE_CASE(18);
3321 SPCOMPRESS_NODE_CASE(20);
3322 SPCOMPRESS_NODE_CASE(24);
3323 SPCOMPRESS_NODE_CASE(36);
3324 SPCOMPRESS_NODE_CASE(40);
3325 SPCOMPRESS_NODE_CASE(48);
3326 SPCOMPRESS_NODE_CASE(72);
3327 SPCOMPRESS_NODE_CASE(80);
3328 SPCOMPRESS_NODE_CASE(96);
3329 default:
3330 llvm_unreachable("Invalid spcompress result count");
3331 }
3332
3333#undef SPCOMPRESS_NODE_CASE
3334}
3335
3336static unsigned getSPDecompressNodeOpcode(unsigned NumResults) {
3337#define SPDECOMPRESS_NODE_CASE(NumResults) \
3338 case NumResults: \
3339 return NVPTXISD::SPDECOMPRESS_R##NumResults
3340
3341 switch (NumResults) {
3342 SPDECOMPRESS_NODE_CASE(1);
3343 SPDECOMPRESS_NODE_CASE(2);
3344 SPDECOMPRESS_NODE_CASE(4);
3345 SPDECOMPRESS_NODE_CASE(8);
3346 SPDECOMPRESS_NODE_CASE(16);
3347 SPDECOMPRESS_NODE_CASE(32);
3348 SPDECOMPRESS_NODE_CASE(64);
3349 SPDECOMPRESS_NODE_CASE(128);
3350 default:
3351 llvm_unreachable("Invalid spdecompress result count");
3352 }
3353
3354#undef SPDECOMPRESS_NODE_CASE
3355}
3356
3357static SDValue lowerSPIntrinsic(SDValue Op, SelectionDAG &DAG) {
3358 SDNode *N = Op.getNode();
3359 SDLoc DL(N);
3360
3361 auto IID = static_cast<Intrinsic::ID>(N->getConstantOperandVal(Num: 0));
3362 bool IsCompress = IID == Intrinsic::nvvm_spcompress;
3363 assert((IsCompress || IID == Intrinsic::nvvm_spdecompress) &&
3364 "Unexpected SP intrinsic");
3365
3366 EVT DataVT =
3367 IsCompress ? N->getOperand(Num: 1).getValueType() : N->getValueType(ResNo: 0);
3368 EVT CDataVT =
3369 IsCompress ? N->getValueType(ResNo: 1) : N->getOperand(Num: 2).getValueType();
3370
3371 unsigned ElemSize = DataVT.getScalarSizeInBits();
3372 unsigned IdxSize = N->getConstantOperandVal(Num: 3);
3373 unsigned NumTgt = N->getConstantOperandVal(Num: 4);
3374 unsigned NumSrc =
3375 NumTgt * CDataVT.getVectorNumElements() / DataVT.getVectorNumElements();
3376 unsigned RepeatFactor = IsCompress ? getSPNumRegs(VT: DataVT) / 2
3377 : DataVT.getVectorNumElements() / NumTgt;
3378
3379 SmallVector<SDValue, 16> Ops;
3380 for (unsigned Qualifier : {ElemSize, IdxSize, RepeatFactor, NumSrc, NumTgt})
3381 Ops.push_back(Elt: DAG.getTargetConstant(Val: Qualifier, DL, VT: MVT::i32));
3382 appendSplitSPOperands(Input: N->ops().slice(N: 1, M: 2), Output&: Ops, DAG);
3383
3384 SmallVector<EVT, 16> ResultVTs;
3385 for (const EVT VT : N->values())
3386 ResultVTs.append(NumInputs: getSPNumRegs(VT), Elt: MVT::i32);
3387
3388 unsigned Opcode = IsCompress ? getSPCompressNodeOpcode(NumResults: ResultVTs.size())
3389 : getSPDecompressNodeOpcode(NumResults: ResultVTs.size());
3390 SDValue Lowered = DAG.getNode(Opcode, DL, ResultTys: ResultVTs, Ops);
3391
3392 SmallVector<SDValue, 2> Retvals;
3393 unsigned Reg = 0;
3394 for (const EVT VT : N->values()) {
3395 SmallVector<SDValue> Regs;
3396 for (unsigned R : llvm::seq(Size: getSPNumRegs(VT)))
3397 Regs.push_back(Elt: Lowered.getValue(R: Reg + R));
3398 Reg += Regs.size();
3399 Retvals.push_back(Elt: joinSPRegs(VT, Regs, DL, DAG));
3400 }
3401
3402 return DAG.getMergeValues(Ops: Retvals, dl: DL);
3403}
3404
3405static SDValue lowerIntrinsicWOChain(SDValue Op, SelectionDAG &DAG) {
3406 switch (Op->getConstantOperandVal(Num: 0)) {
3407 default:
3408 return Op;
3409 case Intrinsic::nvvm_prmt:
3410 case Intrinsic::nvvm_prmt_b4e:
3411 case Intrinsic::nvvm_prmt_ecl:
3412 case Intrinsic::nvvm_prmt_ecr:
3413 case Intrinsic::nvvm_prmt_f4e:
3414 case Intrinsic::nvvm_prmt_rc16:
3415 case Intrinsic::nvvm_prmt_rc8:
3416 return lowerPrmtIntrinsic(Op, DAG);
3417 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_is_canceled:
3418 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_x:
3419 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_y:
3420 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_z:
3421 return LowerClusterLaunchControlQueryCancel(Op, DAG);
3422 case Intrinsic::nvvm_f32x4_to_e4m3x4_rs_satfinite:
3423 case Intrinsic::nvvm_f32x4_to_e4m3x4_rs_relu_satfinite:
3424 case Intrinsic::nvvm_f32x4_to_e5m2x4_rs_satfinite:
3425 case Intrinsic::nvvm_f32x4_to_e5m2x4_rs_relu_satfinite:
3426 case Intrinsic::nvvm_f32x4_to_e2m3x4_rs_satfinite:
3427 case Intrinsic::nvvm_f32x4_to_e2m3x4_rs_relu_satfinite:
3428 case Intrinsic::nvvm_f32x4_to_e3m2x4_rs_satfinite:
3429 case Intrinsic::nvvm_f32x4_to_e3m2x4_rs_relu_satfinite:
3430 case Intrinsic::nvvm_f32x4_to_e2m1x4_rs_satfinite:
3431 case Intrinsic::nvvm_f32x4_to_e2m1x4_rs_relu_satfinite:
3432 return lowerCvtRSIntrinsics(Op, DAG);
3433
3434 case Intrinsic::nvvm_spcompress:
3435 case Intrinsic::nvvm_spdecompress:
3436 return lowerSPIntrinsic(Op, DAG);
3437 }
3438}
3439
3440// In PTX 64-bit CTLZ and CTPOP are supported, but they return a 32-bit value.
3441// Lower these into a node returning the correct type which is zero-extended
3442// back to the correct size. This has to be a target node: DAG combines assume
3443// that the result of ISD::CTLZ and ISD::CTPOP is as wide as their operand.
3444static SDValue lowerCTLZCTPOP(SDValue Op, SelectionDAG &DAG) {
3445 SDValue V = Op->getOperand(Num: 0);
3446 assert(V.getValueType() == MVT::i64 &&
3447 "Unexpected CTLZ/CTPOP type to legalize");
3448
3449 SDLoc DL(Op);
3450 unsigned Opcode =
3451 Op->getOpcode() == ISD::CTLZ ? NVPTXISD::CLZ_B64 : NVPTXISD::POPC_B64;
3452 SDValue CT = DAG.getNode(Opcode, DL, VT: MVT::i32, Operand: V);
3453 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: CT, Flags: SDNodeFlags::NonNeg);
3454}
3455
3456static SDValue expandFSH64(SDValue A, SDValue B, SDValue ShiftAmount, SDLoc DL,
3457 unsigned Opcode, SelectionDAG &DAG) {
3458 assert(A.getValueType() == MVT::i64 && B.getValueType() == MVT::i64);
3459
3460 const auto *AmtConst = dyn_cast<ConstantSDNode>(Val&: ShiftAmount);
3461 if (!AmtConst)
3462 return SDValue();
3463 const auto Amt = AmtConst->getZExtValue() & 63;
3464
3465 SDValue UnpackA =
3466 DAG.getNode(Opcode: NVPTXISD::UNPACK_VECTOR, DL, ResultTys: {MVT::i32, MVT::i32}, Ops: A);
3467 SDValue UnpackB =
3468 DAG.getNode(Opcode: NVPTXISD::UNPACK_VECTOR, DL, ResultTys: {MVT::i32, MVT::i32}, Ops: B);
3469
3470 // Arch is Little endiain: 0 = low bits, 1 = high bits
3471 SDValue ALo = UnpackA.getValue(R: 0);
3472 SDValue AHi = UnpackA.getValue(R: 1);
3473 SDValue BLo = UnpackB.getValue(R: 0);
3474 SDValue BHi = UnpackB.getValue(R: 1);
3475
3476 // The bitfeild consists of { AHi : ALo : BHi : BLo }
3477 //
3478 // * FSHL, Amt < 32 - The window will contain { AHi : ALo : BHi }
3479 // * FSHL, Amt >= 32 - The window will contain { ALo : BHi : BLo }
3480 // * FSHR, Amt < 32 - The window will contain { ALo : BHi : BLo }
3481 // * FSHR, Amt >= 32 - The window will contain { AHi : ALo : BHi }
3482 //
3483 // Note that Amt = 0 and Amt = 32 are special cases where 32-bit funnel shifts
3484 // are not needed at all. Amt = 0 is a no-op producing either A or B depending
3485 // on the direction. Amt = 32 can be implemented by a packing and unpacking
3486 // move to select and arrange the 32bit values. For simplicity, these cases
3487 // are not handled here explicitly and instead we rely on DAGCombiner to
3488 // remove the no-op funnel shifts we insert.
3489 auto [High, Mid, Low] = ((Opcode == ISD::FSHL) == (Amt < 32))
3490 ? std::make_tuple(args&: AHi, args&: ALo, args&: BHi)
3491 : std::make_tuple(args&: ALo, args&: BHi, args&: BLo);
3492
3493 SDValue NewAmt = DAG.getConstant(Val: Amt & 31, DL, VT: MVT::i32);
3494 SDValue RHi = DAG.getNode(Opcode, DL, VT: MVT::i32, Ops: {High, Mid, NewAmt});
3495 SDValue RLo = DAG.getNode(Opcode, DL, VT: MVT::i32, Ops: {Mid, Low, NewAmt});
3496
3497 return DAG.getNode(Opcode: NVPTXISD::BUILD_VECTOR, DL, VT: MVT::i64, Ops: {RLo, RHi});
3498}
3499
3500static SDValue lowerFSH(SDValue Op, SelectionDAG &DAG) {
3501 return expandFSH64(A: Op->getOperand(Num: 0), B: Op->getOperand(Num: 1), ShiftAmount: Op->getOperand(Num: 2),
3502 DL: SDLoc(Op), Opcode: Op->getOpcode(), DAG);
3503}
3504
3505static SDValue lowerROT(SDValue Op, SelectionDAG &DAG) {
3506 unsigned Opcode = Op->getOpcode() == ISD::ROTL ? ISD::FSHL : ISD::FSHR;
3507 return expandFSH64(A: Op->getOperand(Num: 0), B: Op->getOperand(Num: 0), ShiftAmount: Op->getOperand(Num: 1),
3508 DL: SDLoc(Op), Opcode, DAG);
3509}
3510
3511static SDValue lowerSELECT(SDValue Op, SelectionDAG &DAG) {
3512 assert(Op.getValueType() == MVT::i1 && "Custom lowering enabled only for i1");
3513
3514 SDValue Cond = Op->getOperand(Num: 0);
3515 SDValue TrueVal = Op->getOperand(Num: 1);
3516 SDValue FalseVal = Op->getOperand(Num: 2);
3517 SDLoc DL(Op);
3518
3519 // If both operands are truncated, we push the select through the truncates.
3520 if (TrueVal.getOpcode() == ISD::TRUNCATE &&
3521 FalseVal.getOpcode() == ISD::TRUNCATE) {
3522 TrueVal = TrueVal.getOperand(i: 0);
3523 FalseVal = FalseVal.getOperand(i: 0);
3524
3525 EVT VT = TrueVal.getSimpleValueType().bitsLE(VT: FalseVal.getSimpleValueType())
3526 ? TrueVal.getValueType()
3527 : FalseVal.getValueType();
3528 TrueVal = DAG.getAnyExtOrTrunc(Op: TrueVal, DL, VT);
3529 FalseVal = DAG.getAnyExtOrTrunc(Op: FalseVal, DL, VT);
3530 SDValue Select = DAG.getSelect(DL, VT, Cond, LHS: TrueVal, RHS: FalseVal);
3531 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Select);
3532 }
3533
3534 // Otherwise, expand the select into a series of logical operations. These
3535 // often can be folded into other operations either by us or ptxas.
3536 TrueVal = DAG.getFreeze(V: TrueVal);
3537 FalseVal = DAG.getFreeze(V: FalseVal);
3538 SDValue And1 = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i1, N1: Cond, N2: TrueVal);
3539 SDValue NotCond = DAG.getNOT(DL, Val: Cond, VT: MVT::i1);
3540 SDValue And2 = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i1, N1: NotCond, N2: FalseVal);
3541 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i1, N1: And1, N2: And2);
3542 return Or;
3543}
3544
3545static SDValue lowerMSTORE(SDValue Op, SelectionDAG &DAG) {
3546 SDNode *N = Op.getNode();
3547
3548 SDValue Chain = N->getOperand(Num: 0);
3549 SDValue Val = N->getOperand(Num: 1);
3550 SDValue BasePtr = N->getOperand(Num: 2);
3551 SDValue Offset = N->getOperand(Num: 3);
3552 SDValue Mask = N->getOperand(Num: 4);
3553
3554 SDLoc DL(N);
3555 EVT ValVT = Val.getValueType();
3556 MemSDNode *MemSD = cast<MemSDNode>(Val: N);
3557 assert(ValVT.isVector() && "Masked vector store must have vector type");
3558 assert(MemSD->getAlign() >= DAG.getEVTAlign(ValVT) &&
3559 "Unexpected alignment for masked store");
3560
3561 unsigned Opcode = 0;
3562 switch (ValVT.getSimpleVT().SimpleTy) {
3563 default:
3564 llvm_unreachable("Unexpected masked vector store type");
3565 case MVT::v4i64:
3566 case MVT::v4f64: {
3567 Opcode = NVPTXISD::StoreV4;
3568 break;
3569 }
3570 case MVT::v8i32:
3571 case MVT::v8f32: {
3572 Opcode = NVPTXISD::StoreV8;
3573 break;
3574 }
3575 }
3576
3577 SmallVector<SDValue, 8> Ops;
3578
3579 // Construct the new SDNode. First operand is the chain.
3580 Ops.push_back(Elt: Chain);
3581
3582 // The next N operands are the values to store. Encode the mask into the
3583 // values using the sentinel register 0 to represent a masked-off element.
3584 assert(Mask.getValueType().isVector() &&
3585 Mask.getValueType().getVectorElementType() == MVT::i1 &&
3586 "Mask must be a vector of i1");
3587 assert(Mask.getOpcode() == ISD::BUILD_VECTOR &&
3588 "Mask expected to be a BUILD_VECTOR");
3589 assert(Mask.getValueType().getVectorNumElements() ==
3590 ValVT.getVectorNumElements() &&
3591 "Mask size must be the same as the vector size");
3592 for (auto [I, Op] : enumerate(First: Mask->ops())) {
3593 // Mask elements must be constants.
3594 if (Op.getNode()->getAsZExtVal() == 0) {
3595 // Append a sentinel register 0 to the Ops vector to represent a masked
3596 // off element, this will be handled in tablegen
3597 Ops.push_back(Elt: DAG.getRegister(Reg: MCRegister::NoRegister,
3598 VT: ValVT.getVectorElementType()));
3599 } else {
3600 // Extract the element from the vector to store
3601 SDValue ExtVal =
3602 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ValVT.getVectorElementType(),
3603 N1: Val, N2: DAG.getIntPtrConstant(Val: I, DL));
3604 Ops.push_back(Elt: ExtVal);
3605 }
3606 }
3607
3608 // Next, the pointer operand.
3609 Ops.push_back(Elt: BasePtr);
3610
3611 // Finally, the offset operand. We expect this to always be undef, and it will
3612 // be ignored in lowering, but to mirror the handling of the other vector
3613 // store instructions we include it in the new SDNode.
3614 assert(Offset.isUndef() && "Offset operand expected to be undef or poison");
3615 Ops.push_back(Elt: Offset);
3616
3617 SDValue NewSt =
3618 DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops,
3619 MemVT: MemSD->getMemoryVT(), MMO: MemSD->getMemOperand());
3620
3621 return NewSt;
3622}
3623
3624static SDValue lowerAtomicStore128(SDNode *N, SelectionDAG &DAG,
3625 const NVPTXSubtarget &STI) {
3626 assert(STI.hasLdStB128() &&
3627 "b128 atomic stores require PTX ISA version 8.3 and target sm_70");
3628 AtomicSDNode *AN = cast<AtomicSDNode>(Val: N);
3629 SDLoc dl(N);
3630
3631 SDValue Val = AN->getVal();
3632 assert(Val.getValueType() == MVT::i128 &&
3633 "Custom lowering for atomic128 only supports i128");
3634
3635 SDValue Ops[] = {AN->getOperand(Num: 0), // Chain
3636 AN->getBasePtr(),
3637 // Low part
3638 DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: dl, VT: MVT::i64, N1: Val,
3639 N2: DAG.getIntPtrConstant(Val: 0, DL: dl)),
3640 // High part
3641 DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: dl, VT: MVT::i64, N1: Val,
3642 N2: DAG.getIntPtrConstant(Val: 1, DL: dl))};
3643 return DAG.getMemIntrinsicNode(Opcode: NVPTXISD::ATOMIC_STORE_B128, dl,
3644 VTList: DAG.getVTList(VT: MVT::Other), Ops, MemVT: MVT::i128,
3645 MMO: AN->getMemOperand());
3646}
3647
3648SDValue
3649NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
3650 switch (Op.getOpcode()) {
3651 case ISD::RETURNADDR:
3652 return SDValue();
3653 case ISD::FRAMEADDR:
3654 return SDValue();
3655 case ISD::ADDRSPACECAST:
3656 return LowerADDRSPACECAST(Op, DAG);
3657 case ISD::INTRINSIC_W_CHAIN:
3658 return lowerIntrinsicWChain(Op, DAG);
3659 case ISD::INTRINSIC_WO_CHAIN:
3660 return lowerIntrinsicWOChain(Op, DAG);
3661 case ISD::INTRINSIC_VOID:
3662 return lowerIntrinsicVoid(Op, DAG);
3663 case ISD::BUILD_VECTOR:
3664 return LowerBUILD_VECTOR(Op, DAG);
3665 case ISD::BITCAST:
3666 return LowerBITCAST(Op, DAG);
3667 case ISD::EXTRACT_SUBVECTOR:
3668 return Op;
3669 case ISD::EXTRACT_VECTOR_ELT:
3670 return LowerEXTRACT_VECTOR_ELT(Op, DAG);
3671 case ISD::INSERT_VECTOR_ELT:
3672 return LowerINSERT_VECTOR_ELT(Op, DAG);
3673 case ISD::VECTOR_SHUFFLE:
3674 return LowerVECTOR_SHUFFLE(Op, DAG);
3675 case ISD::CONCAT_VECTORS:
3676 return LowerCONCAT_VECTORS(Op, DAG);
3677 case ISD::VECREDUCE_FMAX:
3678 case ISD::VECREDUCE_FMIN:
3679 case ISD::VECREDUCE_FMAXIMUM:
3680 case ISD::VECREDUCE_FMINIMUM:
3681 case ISD::VECREDUCE_FMAXIMUMNUM:
3682 case ISD::VECREDUCE_FMINIMUMNUM:
3683 return LowerVECREDUCE(Op, DAG);
3684 case ISD::STORE:
3685 return LowerSTORE(Op, DAG);
3686 case ISD::ATOMIC_STORE:
3687 return lowerAtomicStore128(N: Op.getNode(), DAG, STI);
3688 case ISD::MSTORE: {
3689 assert(STI.has256BitVectorLoadStore(
3690 cast<MemSDNode>(Op.getNode())->getAddressSpace()) &&
3691 "Masked store vector not supported on subtarget.");
3692 return lowerMSTORE(Op, DAG);
3693 }
3694 case ISD::LOAD:
3695 return LowerLOAD(Op, DAG);
3696 case ISD::MLOAD:
3697 return LowerMLOAD(Op, DAG);
3698 case ISD::SHL_PARTS:
3699 return LowerShiftLeftParts(Op, DAG);
3700 case ISD::SRA_PARTS:
3701 case ISD::SRL_PARTS:
3702 return LowerShiftRightParts(Op, DAG);
3703 case ISD::SELECT:
3704 return lowerSELECT(Op, DAG);
3705 case ISD::SMUL_LOHI:
3706 case ISD::UMUL_LOHI:
3707 return lowerMUL_LOHI(Op, DAG);
3708 case ISD::FROUND:
3709 return LowerFROUND(Op, DAG);
3710 case ISD::FCOPYSIGN:
3711 return LowerFCOPYSIGN(Op, DAG);
3712 case ISD::SINT_TO_FP:
3713 case ISD::UINT_TO_FP:
3714 return LowerINT_TO_FP(Op, DAG);
3715 case ISD::FP_TO_SINT:
3716 case ISD::FP_TO_UINT:
3717 // fptosi/fptoui to i1 truncate toward zero, so the only defined results
3718 // are {0,-1} (signed) and {0,1} (unsigned); every other input results in
3719 // poison. Thus we can simply lower to `x <= -1.0` or `x >= 1.0`.
3720 if (Op.getValueType() == MVT::i1) {
3721 SDLoc DL(Op);
3722 SDValue X = Op.getOperand(i: 0);
3723 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT;
3724 return DAG.getSetCC(
3725 DL, VT: MVT::i1, LHS: X,
3726 RHS: DAG.getConstantFP(Val: IsSigned ? -1.0 : 1.0, DL, VT: X.getValueType()),
3727 Cond: IsSigned ? ISD::SETOLE : ISD::SETOGE);
3728 }
3729 return LowerFP_TO_INT(Op, DAG);
3730 case ISD::FP_ROUND:
3731 return LowerFP_ROUND(Op, DAG);
3732 case ISD::FPTRUNC_ROUND:
3733 return lowerFPTRUNC_ROUND(Op, DAG, STI);
3734 case ISD::FP_EXTEND:
3735 return LowerFP_EXTEND(Op, DAG);
3736 case ISD::VAARG:
3737 return LowerVAARG(Op, DAG);
3738 case ISD::VASTART:
3739 return LowerVASTART(Op, DAG);
3740 case ISD::FSHL:
3741 case ISD::FSHR:
3742 return lowerFSH(Op, DAG);
3743 case ISD::ROTL:
3744 case ISD::ROTR:
3745 return lowerROT(Op, DAG);
3746 case ISD::ABS:
3747 case ISD::ABS_MIN_POISON:
3748 case ISD::SMIN:
3749 case ISD::SMAX:
3750 case ISD::UMIN:
3751 case ISD::UMAX:
3752 case ISD::ADD:
3753 case ISD::SUB:
3754 case ISD::MUL:
3755 case ISD::SHL:
3756 case ISD::SREM:
3757 case ISD::UREM:
3758 return LowerVectorArith(Op, DAG);
3759 case ISD::DYNAMIC_STACKALLOC:
3760 return LowerDYNAMIC_STACKALLOC(Op, DAG);
3761 case ISD::STACKRESTORE:
3762 return LowerSTACKRESTORE(Op, DAG);
3763 case ISD::STACKSAVE:
3764 return LowerSTACKSAVE(Op, DAG);
3765 case ISD::CopyToReg:
3766 return LowerCopyToReg_128(Op, DAG);
3767 case ISD::FADD:
3768 case ISD::FSUB:
3769 case ISD::FMUL:
3770 // Used only for bf16 on SM80, where we select fma for non-ftz operation
3771 return PromoteBinOpIfF32FTZ(Op, DAG);
3772 case ISD::CTPOP:
3773 case ISD::CTLZ:
3774 return lowerCTLZCTPOP(Op, DAG);
3775 case ISD::BSWAP:
3776 return lowerBSWAP(Op, DAG);
3777 default:
3778 llvm_unreachable("Custom lowering not defined for operation");
3779 }
3780}
3781
3782// This will prevent AsmPrinter from trying to print the jump tables itself.
3783unsigned NVPTXTargetLowering::getJumpTableEncoding() const {
3784 return MachineJumpTableInfo::EK_Inline;
3785}
3786
3787SDValue NVPTXTargetLowering::LowerADDRSPACECAST(SDValue Op,
3788 SelectionDAG &DAG) const {
3789 AddrSpaceCastSDNode *N = cast<AddrSpaceCastSDNode>(Val: Op.getNode());
3790 unsigned SrcAS = N->getSrcAddressSpace();
3791 unsigned DestAS = N->getDestAddressSpace();
3792 if (SrcAS != llvm::ADDRESS_SPACE_GENERIC &&
3793 DestAS != llvm::ADDRESS_SPACE_GENERIC) {
3794 // Shared and SharedCluster can be converted to each other through generic
3795 // space
3796 if ((SrcAS == llvm::ADDRESS_SPACE_SHARED &&
3797 DestAS == llvm::ADDRESS_SPACE_SHARED_CLUSTER) ||
3798 (SrcAS == llvm::ADDRESS_SPACE_SHARED_CLUSTER &&
3799 DestAS == llvm::ADDRESS_SPACE_SHARED)) {
3800 SDLoc DL(Op.getNode());
3801 const MVT GenerictVT =
3802 getPointerTy(DL: DAG.getDataLayout(), AS: ADDRESS_SPACE_GENERIC);
3803 SDValue GenericConversion = DAG.getAddrSpaceCast(
3804 dl: DL, VT: GenerictVT, Ptr: Op.getOperand(i: 0), SrcAS, DestAS: ADDRESS_SPACE_GENERIC);
3805 SDValue SharedClusterConversion =
3806 DAG.getAddrSpaceCast(dl: DL, VT: Op.getValueType(), Ptr: GenericConversion,
3807 SrcAS: ADDRESS_SPACE_GENERIC, DestAS);
3808 return SharedClusterConversion;
3809 }
3810
3811 return DAG.getUNDEF(VT: Op.getValueType());
3812 }
3813
3814 return Op;
3815}
3816
3817// This function is almost a copy of SelectionDAG::expandVAArg().
3818// The only diff is that this one produces loads from local address space.
3819SDValue NVPTXTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
3820 const TargetLowering *TLI = STI.getTargetLowering();
3821 SDLoc DL(Op);
3822
3823 SDNode *Node = Op.getNode();
3824 const Value *V = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 2))->getValue();
3825 EVT VT = Node->getValueType(ResNo: 0);
3826 auto *Ty = VT.getTypeForEVT(Context&: *DAG.getContext());
3827 SDValue Tmp1 = Node->getOperand(Num: 0);
3828 SDValue Tmp2 = Node->getOperand(Num: 1);
3829 const MaybeAlign MA(Node->getConstantOperandVal(Num: 3));
3830
3831 SDValue VAListLoad = DAG.getLoad(VT: TLI->getPointerTy(DL: DAG.getDataLayout()), dl: DL,
3832 Chain: Tmp1, Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
3833 SDValue VAList = VAListLoad;
3834
3835 if (MA && *MA > TLI->getMinStackArgumentAlignment()) {
3836 VAList = DAG.getNode(
3837 Opcode: ISD::ADD, DL, VT: VAList.getValueType(), N1: VAList,
3838 N2: DAG.getConstant(Val: MA->value() - 1, DL, VT: VAList.getValueType()));
3839
3840 VAList = DAG.getNode(Opcode: ISD::AND, DL, VT: VAList.getValueType(), N1: VAList,
3841 N2: DAG.getSignedConstant(Val: -(int64_t)MA->value(), DL,
3842 VT: VAList.getValueType()));
3843 }
3844
3845 // Increment the pointer, VAList, to the next vaarg
3846 Tmp1 = DAG.getNode(Opcode: ISD::ADD, DL, VT: VAList.getValueType(), N1: VAList,
3847 N2: DAG.getConstant(Val: DAG.getDataLayout().getTypeAllocSize(Ty),
3848 DL, VT: VAList.getValueType()));
3849
3850 // Store the incremented VAList to the legalized pointer
3851 Tmp1 = DAG.getStore(Chain: VAListLoad.getValue(R: 1), dl: DL, Val: Tmp1, Ptr: Tmp2,
3852 PtrInfo: MachinePointerInfo(V));
3853
3854 const Value *SrcV = Constant::getNullValue(
3855 Ty: PointerType::get(C&: *DAG.getContext(), AddressSpace: ADDRESS_SPACE_LOCAL));
3856
3857 // Load the actual argument out of the pointer VAList
3858 return DAG.getLoad(VT, dl: DL, Chain: Tmp1, Ptr: VAList, PtrInfo: MachinePointerInfo(SrcV));
3859}
3860
3861SDValue NVPTXTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
3862 const TargetLowering *TLI = STI.getTargetLowering();
3863 SDLoc DL(Op);
3864 EVT PtrVT = TLI->getPointerTy(DL: DAG.getDataLayout());
3865
3866 // Store the address of unsized array <function>_vararg[] in the ap object.
3867 SDValue VAReg = getParamSymbolNode(DAG, /* vararg */ I: -1, T: PtrVT);
3868
3869 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
3870 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL, Val: VAReg, Ptr: Op.getOperand(i: 1),
3871 PtrInfo: MachinePointerInfo(SV));
3872}
3873
3874static std::pair<MemSDNode *, uint32_t>
3875convertMLOADToLoadWithUsedBytesMask(MemSDNode *N, SelectionDAG &DAG,
3876 const NVPTXSubtarget &STI) {
3877 SDValue Chain = N->getOperand(Num: 0);
3878 SDValue BasePtr = N->getOperand(Num: 1);
3879 SDValue Mask = N->getOperand(Num: 3);
3880 [[maybe_unused]] SDValue Passthru = N->getOperand(Num: 4);
3881
3882 SDLoc DL(N);
3883 EVT ResVT = N->getValueType(ResNo: 0);
3884 assert(ResVT.isVector() && "Masked vector load must have vector type");
3885 // While we only expect poison passthru vectors as an input to the backend,
3886 // when the legalization framework splits a poison vector in half, it creates
3887 // two undef vectors, so we can technically expect those too.
3888 assert((Passthru.getOpcode() == ISD::POISON ||
3889 Passthru.getOpcode() == ISD::UNDEF) &&
3890 "Passthru operand expected to be poison or undef");
3891
3892 // Extract the mask and convert it to a uint32_t representing the used bytes
3893 // of the entire vector load
3894 uint32_t UsedBytesMask = 0;
3895 uint32_t ElementSizeInBits = ResVT.getVectorElementType().getSizeInBits();
3896 assert(ElementSizeInBits % 8 == 0 && "Unexpected element size");
3897 uint32_t ElementSizeInBytes = ElementSizeInBits / 8;
3898 uint32_t ElementMask = (1u << ElementSizeInBytes) - 1u;
3899
3900 for (SDValue Op : reverse(C: Mask->ops())) {
3901 // We technically only want to do this shift for every
3902 // iteration *but* the first, but in the first iteration UsedBytesMask is 0,
3903 // so this shift is a no-op.
3904 UsedBytesMask <<= ElementSizeInBytes;
3905
3906 // Mask elements must be constants.
3907 if (Op->getAsZExtVal() != 0)
3908 UsedBytesMask |= ElementMask;
3909 }
3910
3911 assert(UsedBytesMask != 0 && UsedBytesMask != UINT32_MAX &&
3912 "Unexpected masked load with elements masked all on or all off");
3913
3914 // Create a new load sd node to be handled normally by ReplaceLoadVector.
3915 MemSDNode *NewLD = cast<MemSDNode>(
3916 Val: DAG.getLoad(VT: ResVT, dl: DL, Chain, Ptr: BasePtr, MMO: N->getMemOperand()).getNode());
3917
3918 // If our subtarget does not support the used bytes mask pragma, "drop" the
3919 // mask by setting it to UINT32_MAX
3920 if (!STI.hasUsedBytesMaskPragma())
3921 UsedBytesMask = UINT32_MAX;
3922
3923 return {NewLD, UsedBytesMask};
3924}
3925
3926/// replaceLoadVector - Convert vector loads into multi-output scalar loads.
3927static std::optional<std::pair<SDValue, SDValue>>
3928replaceLoadVector(SDNode *N, SelectionDAG &DAG, const NVPTXSubtarget &STI) {
3929 MemSDNode *LD = cast<MemSDNode>(Val: N);
3930 const EVT ResVT = LD->getValueType(ResNo: 0);
3931 const EVT MemVT = LD->getMemoryVT();
3932
3933 // If we're doing sign/zero extension as part of the load, avoid lowering to
3934 // a LoadV node. TODO: consider relaxing this restriction.
3935 if (ResVT != MemVT)
3936 return std::nullopt;
3937
3938 const auto NumEltsAndEltVT =
3939 getVectorLoweringShape(VectorEVT: ResVT, STI, AddressSpace: LD->getAddressSpace());
3940 if (!NumEltsAndEltVT)
3941 return std::nullopt;
3942 const auto [NumElts, EltVT] = NumEltsAndEltVT.value();
3943
3944 Align Alignment = LD->getAlign();
3945 const auto &TD = DAG.getDataLayout();
3946 Align PrefAlign = TD.getPrefTypeAlign(Ty: MemVT.getTypeForEVT(Context&: *DAG.getContext()));
3947 if (Alignment < PrefAlign) {
3948 // This load is not sufficiently aligned, so bail out and let this vector
3949 // load be scalarized. Note that we may still be able to emit smaller
3950 // vector loads. For example, if we are loading a <4 x float> with an
3951 // alignment of 8, this check will fail but the legalizer will try again
3952 // with 2 x <2 x float>, which will succeed with an alignment of 8.
3953 return std::nullopt;
3954 }
3955
3956 // If we have a masked load, convert it to a normal load now
3957 std::optional<uint32_t> UsedBytesMask = std::nullopt;
3958 if (LD->getOpcode() == ISD::MLOAD)
3959 std::tie(args&: LD, args&: UsedBytesMask) =
3960 convertMLOADToLoadWithUsedBytesMask(N: LD, DAG, STI);
3961
3962 // Since LoadV2 is a target node, we cannot rely on DAG type legalization.
3963 // Therefore, we must ensure the type is legal. For i1 and i8, we set the
3964 // loaded type to i16 and propagate the "real" type as the memory type.
3965 const MVT LoadEltVT = (EltVT.getSizeInBits() < 16) ? MVT::i16 : EltVT;
3966
3967 unsigned Opcode;
3968 switch (NumElts) {
3969 default:
3970 return std::nullopt;
3971 case 2:
3972 Opcode = NVPTXISD::LoadV2;
3973 break;
3974 case 4:
3975 Opcode = NVPTXISD::LoadV4;
3976 break;
3977 case 8:
3978 Opcode = NVPTXISD::LoadV8;
3979 break;
3980 }
3981 auto ListVTs = SmallVector<EVT, 9>(NumElts, LoadEltVT);
3982 ListVTs.push_back(Elt: MVT::Other);
3983 SDVTList LdResVTs = DAG.getVTList(VTs: ListVTs);
3984
3985 SDLoc DL(LD);
3986
3987 // Copy regular operands
3988 SmallVector<SDValue, 8> OtherOps(LD->ops());
3989
3990 OtherOps.push_back(
3991 Elt: DAG.getConstant(Val: UsedBytesMask.value_or(UINT32_MAX), DL, VT: MVT::i32));
3992
3993 // The select routine does not have access to the LoadSDNode instance, so
3994 // pass along the extension information
3995 OtherOps.push_back(
3996 Elt: DAG.getIntPtrConstant(Val: cast<LoadSDNode>(Val: LD)->getExtensionType(), DL));
3997
3998 SDValue NewLD = DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList: LdResVTs, Ops: OtherOps, MemVT,
3999 MMO: LD->getMemOperand());
4000
4001 SmallVector<SDValue> ScalarRes;
4002 if (EltVT.isVector()) {
4003 assert(EVT(EltVT.getVectorElementType()) == ResVT.getVectorElementType());
4004 assert(NumElts * EltVT.getVectorNumElements() ==
4005 ResVT.getVectorNumElements());
4006 // Generate EXTRACT_VECTOR_ELTs to split v2[i,f,bf]16/v4i8 subvectors back
4007 // into individual elements.
4008 for (const unsigned I : llvm::seq(Size: NumElts)) {
4009 SDValue SubVector = NewLD.getValue(R: I);
4010 DAG.ExtractVectorElements(Op: SubVector, Args&: ScalarRes);
4011 }
4012 } else {
4013 for (const unsigned I : llvm::seq(Size: NumElts)) {
4014 SDValue Res = NewLD.getValue(R: I);
4015 if (LoadEltVT != EltVT)
4016 Res = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Res);
4017 ScalarRes.push_back(Elt: Res);
4018 }
4019 }
4020
4021 SDValue LoadChain = NewLD.getValue(R: NumElts);
4022
4023 const MVT BuildVecVT =
4024 MVT::getVectorVT(VT: EltVT.getScalarType(), NumElements: ScalarRes.size());
4025 SDValue BuildVec = DAG.getBuildVector(VT: BuildVecVT, DL, Ops: ScalarRes);
4026 SDValue LoadValue = DAG.getBitcast(VT: ResVT, V: BuildVec);
4027
4028 return {{LoadValue, LoadChain}};
4029}
4030
4031static void replaceLoadVector(SDNode *N, SelectionDAG &DAG,
4032 SmallVectorImpl<SDValue> &Results,
4033 const NVPTXSubtarget &STI) {
4034 if (auto Res = replaceLoadVector(N, DAG, STI))
4035 Results.append(IL: {Res->first, Res->second});
4036}
4037
4038static SDValue lowerLoadVector(SDNode *N, SelectionDAG &DAG,
4039 const NVPTXSubtarget &STI) {
4040 if (auto Res = replaceLoadVector(N, DAG, STI))
4041 return DAG.getMergeValues(Ops: {Res->first, Res->second}, dl: SDLoc(N));
4042 return SDValue();
4043}
4044
4045// v = ld i1* addr
4046// =>
4047// v1 = ld i8* addr (-> i16)
4048// v = trunc i16 to i1
4049static SDValue lowerLOADi1(LoadSDNode *LD, SelectionDAG &DAG) {
4050 SDLoc dl(LD);
4051 assert(LD->getExtensionType() == ISD::NON_EXTLOAD);
4052 assert(LD->getValueType(0) == MVT::i1 && "Custom lowering for i1 load only");
4053 SDValue newLD = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: MVT::i16, Chain: LD->getChain(),
4054 Ptr: LD->getBasePtr(), PtrInfo: LD->getPointerInfo(),
4055 MemVT: MVT::i8, Alignment: LD->getAlign(),
4056 MMOFlags: LD->getMemOperand()->getFlags());
4057 SDValue result = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i1, Operand: newLD);
4058 // The legalizer (the caller) is expecting two values from the legalized
4059 // load, so we build a MergeValues node for it. See ExpandUnalignedLoad()
4060 // in LegalizeDAG.cpp which also uses MergeValues.
4061 return DAG.getMergeValues(Ops: {result, newLD.getValue(R: 1)}, dl);
4062}
4063
4064SDValue NVPTXTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
4065 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
4066
4067 if (Op.getValueType() == MVT::i1)
4068 return lowerLOADi1(LD, DAG);
4069
4070 // To improve CodeGen we'll legalize any-extend loads to zext loads. This is
4071 // how they'll be lowered in ISel anyway, and by doing this a little earlier
4072 // we allow for more DAG combine opportunities.
4073 if (LD->getExtensionType() == ISD::EXTLOAD) {
4074 assert(LD->getValueType(0).isInteger() && LD->getMemoryVT().isInteger() &&
4075 "Unexpected fpext-load");
4076 return DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl: SDLoc(Op), VT: Op.getValueType(),
4077 Chain: LD->getChain(), Ptr: LD->getBasePtr(), MemVT: LD->getMemoryVT(),
4078 MMO: LD->getMemOperand());
4079 }
4080
4081 llvm_unreachable("Unexpected custom lowering for load");
4082}
4083
4084SDValue NVPTXTargetLowering::LowerMLOAD(SDValue Op, SelectionDAG &DAG) const {
4085 // v2f16/v2bf16/v2i16/v4i8 are legal, so we can't rely on legalizer to handle
4086 // masked loads of these types and have to handle them here.
4087 // v2f32 also needs to be handled here if the subtarget has f32x2
4088 // instructions, making it legal.
4089 //
4090 // Note: misaligned masked loads should never reach this point
4091 // because the override of isLegalMaskedLoad in NVPTXTargetTransformInfo.cpp
4092 // will validate alignment. Therefore, we do not need to special case handle
4093 // them here.
4094 EVT VT = Op.getValueType();
4095 if (NVPTX::isPackedVectorTy(VT)) {
4096 auto Result = convertMLOADToLoadWithUsedBytesMask(
4097 N: cast<MemSDNode>(Val: Op.getNode()), DAG, STI);
4098 MemSDNode *LD = std::get<0>(in&: Result);
4099 uint32_t UsedBytesMask = std::get<1>(in&: Result);
4100
4101 SDLoc DL(LD);
4102
4103 // Copy regular operands
4104 SmallVector<SDValue, 8> OtherOps(LD->ops());
4105
4106 OtherOps.push_back(Elt: DAG.getConstant(Val: UsedBytesMask, DL, VT: MVT::i32));
4107
4108 // We currently are not lowering extending loads, but pass the extension
4109 // type anyway as later handling expects it.
4110 OtherOps.push_back(
4111 Elt: DAG.getIntPtrConstant(Val: cast<LoadSDNode>(Val: LD)->getExtensionType(), DL));
4112 SDValue NewLD =
4113 DAG.getMemIntrinsicNode(Opcode: NVPTXISD::MLoad, dl: DL, VTList: LD->getVTList(), Ops: OtherOps,
4114 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
4115 return NewLD;
4116 }
4117 return SDValue();
4118}
4119
4120static SDValue lowerSTOREVector(SDValue Op, SelectionDAG &DAG,
4121 const NVPTXSubtarget &STI) {
4122 MemSDNode *N = cast<MemSDNode>(Val: Op.getNode());
4123 SDValue Val = N->getOperand(Num: 1);
4124 SDLoc DL(N);
4125 const EVT ValVT = Val.getValueType();
4126 const EVT MemVT = N->getMemoryVT();
4127
4128 // If we're truncating as part of the store, avoid lowering to a StoreV node.
4129 // TODO: consider relaxing this restriction.
4130 if (ValVT != MemVT)
4131 return SDValue();
4132
4133 const auto NumEltsAndEltVT =
4134 getVectorLoweringShape(VectorEVT: ValVT, STI, AddressSpace: N->getAddressSpace());
4135 if (!NumEltsAndEltVT)
4136 return SDValue();
4137 const auto [NumElts, EltVT] = NumEltsAndEltVT.value();
4138
4139 const DataLayout &TD = DAG.getDataLayout();
4140
4141 Align Alignment = N->getAlign();
4142 Align PrefAlign = TD.getPrefTypeAlign(Ty: ValVT.getTypeForEVT(Context&: *DAG.getContext()));
4143 if (Alignment < PrefAlign) {
4144 // This store is not sufficiently aligned, so bail out and let this vector
4145 // store be scalarized. Note that we may still be able to emit smaller
4146 // vector stores. For example, if we are storing a <4 x float> with an
4147 // alignment of 8, this check will fail but the legalizer will try again
4148 // with 2 x <2 x float>, which will succeed with an alignment of 8.
4149 return SDValue();
4150 }
4151
4152 unsigned Opcode;
4153 switch (NumElts) {
4154 default:
4155 return SDValue();
4156 case 2:
4157 Opcode = NVPTXISD::StoreV2;
4158 break;
4159 case 4:
4160 Opcode = NVPTXISD::StoreV4;
4161 break;
4162 case 8:
4163 Opcode = NVPTXISD::StoreV8;
4164 break;
4165 }
4166
4167 SmallVector<SDValue, 8> Ops;
4168
4169 // First is the chain
4170 Ops.push_back(Elt: N->getOperand(Num: 0));
4171
4172 // Then the split values
4173 if (EltVT.isVector()) {
4174 assert(EVT(EltVT.getVectorElementType()) == ValVT.getVectorElementType());
4175 assert(NumElts * EltVT.getVectorNumElements() ==
4176 ValVT.getVectorNumElements());
4177 // Combine individual elements into v2[i,f,bf]16/v4i8 subvectors to be
4178 // stored as b32s
4179 const unsigned NumEltsPerSubVector = EltVT.getVectorNumElements();
4180 for (const unsigned I : llvm::seq(Size: NumElts)) {
4181 SmallVector<SDValue, 4> SubVectorElts;
4182 DAG.ExtractVectorElements(Op: Val, Args&: SubVectorElts, Start: I * NumEltsPerSubVector,
4183 Count: NumEltsPerSubVector);
4184 Ops.push_back(Elt: DAG.getBuildVector(VT: EltVT, DL, Ops: SubVectorElts));
4185 }
4186 } else {
4187 SDValue V = DAG.getBitcast(VT: MVT::getVectorVT(VT: EltVT, NumElements: NumElts), V: Val);
4188 for (const unsigned I : llvm::seq(Size: NumElts)) {
4189 SDValue ExtVal = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: V,
4190 N2: DAG.getIntPtrConstant(Val: I, DL));
4191
4192 // Since StoreV2 is a target node, we cannot rely on DAG type
4193 // legalization. Therefore, we must ensure the type is legal. For i1 and
4194 // i8, we set the stored type to i16 and propagate the "real" type as the
4195 // memory type.
4196 if (EltVT.getSizeInBits() < 16)
4197 ExtVal = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i16, Operand: ExtVal);
4198 Ops.push_back(Elt: ExtVal);
4199 }
4200 }
4201
4202 // Then any remaining arguments
4203 Ops.append(in_start: N->op_begin() + 2, in_end: N->op_end());
4204
4205 SDValue NewSt =
4206 DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops,
4207 MemVT: N->getMemoryVT(), MMO: N->getMemOperand());
4208
4209 // return DCI.CombineTo(N, NewSt, true);
4210 return NewSt;
4211}
4212
4213SDValue NVPTXTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
4214 StoreSDNode *Store = cast<StoreSDNode>(Val&: Op);
4215 EVT VT = Store->getMemoryVT();
4216
4217 // Leave truncating stores to i1 to the generic legalizer.
4218 if (VT == MVT::i1 && !Store->isTruncatingStore())
4219 return LowerSTOREi1(Op, DAG);
4220
4221 // Lower store of any other vector type, including v2f32 as we want to break
4222 // it apart since this is not a widely-supported type.
4223 return lowerSTOREVector(Op, DAG, STI);
4224}
4225
4226// st i1 v, addr
4227// =>
4228// v1 = zxt v to i16
4229// st.u8 i16, addr
4230SDValue NVPTXTargetLowering::LowerSTOREi1(SDValue Op, SelectionDAG &DAG) const {
4231 SDNode *Node = Op.getNode();
4232 SDLoc dl(Node);
4233 StoreSDNode *ST = cast<StoreSDNode>(Val: Node);
4234 SDValue Tmp1 = ST->getChain();
4235 SDValue Tmp2 = ST->getBasePtr();
4236 SDValue Tmp3 = ST->getValue();
4237 assert(Tmp3.getValueType() == MVT::i1 && "Custom lowering for i1 store only");
4238 Tmp3 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::i16, Operand: Tmp3);
4239 SDValue Result =
4240 DAG.getTruncStore(Chain: Tmp1, dl, Val: Tmp3, Ptr: Tmp2, PtrInfo: ST->getPointerInfo(), SVT: MVT::i8,
4241 Alignment: ST->getAlign(), MMOFlags: ST->getMemOperand()->getFlags());
4242 return Result;
4243}
4244
4245SDValue NVPTXTargetLowering::LowerCopyToReg_128(SDValue Op,
4246 SelectionDAG &DAG) const {
4247 // Change the CopyToReg to take in two 64-bit operands instead of a 128-bit
4248 // operand so that it can pass the legalization.
4249
4250 assert(Op.getOperand(1).getValueType() == MVT::i128 &&
4251 "Custom lowering for 128-bit CopyToReg only");
4252
4253 SDNode *Node = Op.getNode();
4254 SDLoc DL(Node);
4255
4256 SDValue Cast = DAG.getBitcast(VT: MVT::v2i64, V: Op->getOperand(Num: 2));
4257 SDValue Lo = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i64, N1: Cast,
4258 N2: DAG.getIntPtrConstant(Val: 0, DL));
4259 SDValue Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i64, N1: Cast,
4260 N2: DAG.getIntPtrConstant(Val: 1, DL));
4261
4262 SmallVector<SDValue, 5> NewOps(Op->getNumOperands() + 1);
4263 SmallVector<EVT, 3> ResultsType(Node->values());
4264
4265 NewOps[0] = Op->getOperand(Num: 0); // Chain
4266 NewOps[1] = Op->getOperand(Num: 1); // Dst Reg
4267 NewOps[2] = Lo; // Lower 64-bit
4268 NewOps[3] = Hi; // Higher 64-bit
4269 if (Op.getNumOperands() == 4)
4270 NewOps[4] = Op->getOperand(Num: 3); // Glue if exists
4271
4272 return DAG.getNode(Opcode: ISD::CopyToReg, DL, ResultTys: ResultsType, Ops: NewOps);
4273}
4274
4275unsigned NVPTXTargetLowering::getNumRegisters(
4276 LLVMContext &Context, EVT VT,
4277 std::optional<MVT> RegisterVT = std::nullopt) const {
4278 if (VT == MVT::i128 && RegisterVT == MVT::i128)
4279 return 1;
4280 return TargetLoweringBase::getNumRegisters(Context, VT, RegisterVT);
4281}
4282
4283bool NVPTXTargetLowering::splitValueIntoRegisterParts(
4284 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
4285 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
4286 if (Val.getValueType() == MVT::i128 && NumParts == 1) {
4287 Parts[0] = Val;
4288 return true;
4289 }
4290 return false;
4291}
4292
4293SDValue NVPTXTargetLowering::getParamSymbolNode(SelectionDAG &DAG, int I,
4294 EVT T) const {
4295 const MachineFunction &MF = DAG.getMachineFunction();
4296 return getSymbolNode(
4297 DAG, Sym: getParamSymbol(Ctx&: MF.getContext(), F: &MF.getFunction(), Idx: I), T);
4298}
4299
4300SDValue NVPTXTargetLowering::getCallParamSymbolNode(SelectionDAG &DAG, int I,
4301 EVT T) const {
4302 return getSymbolNode(DAG, Name: "param" + Twine(I), T);
4303}
4304
4305SDValue NVPTXTargetLowering::LowerFormalArguments(
4306 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4307 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
4308 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4309 const DataLayout &DL = DAG.getDataLayout();
4310 LLVMContext &Ctx = *DAG.getContext();
4311
4312 const Function &F = DAG.getMachineFunction().getFunction();
4313 const bool IsKernel = isKernelFunction(F);
4314
4315 const MVT PtrVT = getPointerTy(DL, AS: IsKernel ? ADDRESS_SPACE_ENTRY_PARAM
4316 : ADDRESS_SPACE_LOCAL);
4317
4318 SDValue Root = DAG.getRoot();
4319 SmallVector<SDValue, 16> OutChains;
4320
4321 // argTypes.size() (or theArgs.size()) and Ins.size() need not match.
4322 // Ins.size() will be larger
4323 // * if there is an aggregate argument with multiple fields (each field
4324 // showing up separately in Ins)
4325 // * if there is a vector argument with more than typical vector-length
4326 // elements (generally if more than 4) where each vector element is
4327 // individually present in Ins.
4328 // So a different index should be used for indexing into Ins.
4329 // See similar issue in LowerCall.
4330
4331 auto AllIns = ArrayRef(Ins);
4332 const auto NonEmptyArgs = make_filter_range(
4333 Range: F.args(), Pred: [](const Argument &A) { return !A.getType()->isEmptyTy(); });
4334 for (const auto &[ParamI, Arg] : enumerate(First: NonEmptyArgs)) {
4335 const unsigned ArgNo = Arg.getArgNo();
4336 const auto ArgIns =
4337 AllIns.take_while(Pred: [&](auto I) { return I.OrigArgIndex == ArgNo; });
4338 AllIns = AllIns.drop_front(N: ArgIns.size());
4339
4340 Type *Ty = Arg.getType();
4341 assert(!ArgIns.empty() &&
4342 "Non-empty argument produced no parameter values");
4343
4344 if (Arg.use_empty()) {
4345 // argument is dead
4346 for (const auto &In : ArgIns) {
4347 assert(!In.Used && "Arg.use_empty() is true but Arg is used?");
4348 InVals.push_back(Elt: DAG.getUNDEF(VT: In.VT));
4349 }
4350 continue;
4351 }
4352
4353 SDValue ArgSymbol = getParamSymbolNode(DAG, I: ParamI, T: PtrVT);
4354
4355 // In the following cases, assign a node order of "i+1"
4356 // to newly created nodes. The SDNodes for params have to
4357 // appear in the same order as their order of appearance
4358 // in the original function. "i+1" holds that order.
4359 if (Arg.hasByValAttr()) {
4360 // Param has ByVal attribute
4361 // Return MoveParam(param symbol).
4362 // Ideally, the param symbol can be returned directly,
4363 // but when SDNode builder decides to use it in a CopyToReg(),
4364 // machine instruction fails because TargetExternalSymbol
4365 // (not lowered) is target dependent, and CopyToReg assumes
4366 // the source is lowered.
4367 assert(ArgIns.size() == 1 && "ByVal argument must be a pointer");
4368 const auto &ByvalIn = ArgIns[0];
4369 assert(getValueType(DL, Ty) == ByvalIn.VT &&
4370 "Ins type did not match function type");
4371
4372 SDValue P;
4373 if (IsKernel) {
4374 assert(Ty->getPointerAddressSpace() == ADDRESS_SPACE_ENTRY_PARAM &&
4375 "Kernel ByVal argument must be lowered to the param address "
4376 "space by NVPTXLowerArgs");
4377 P = ArgSymbol;
4378 P.getNode()->setIROrder(Arg.getArgNo() + 1);
4379 } else {
4380 P = DAG.getNode(Opcode: NVPTXISD::MoveParam, DL: dl, VT: ArgSymbol.getValueType(),
4381 Operand: ArgSymbol);
4382 P.getNode()->setIROrder(Arg.getArgNo() + 1);
4383 P = DAG.getAddrSpaceCast(dl, VT: ByvalIn.VT, Ptr: P, SrcAS: ADDRESS_SPACE_LOCAL,
4384 DestAS: ADDRESS_SPACE_GENERIC);
4385 }
4386 InVals.push_back(Elt: P);
4387 } else {
4388 SmallVector<EVT, 16> VTs;
4389 SmallVector<uint64_t, 16> Offsets;
4390 ComputePTXValueVTs(TLI: *this, DL, Ctx, CallConv, Ty, ValueVTs&: VTs, Offsets);
4391 assert(VTs.size() == ArgIns.size() && "Size mismatch");
4392 assert(VTs.size() == Offsets.size() && "Size mismatch");
4393
4394 const Align ArgAlign = getPTXParamAlign(
4395 F: &F, Ty, AttrIdx: Arg.getArgNo() + AttributeList::FirstArgIndex, DL);
4396
4397 unsigned I = 0;
4398 const auto VI = VectorizePTXValueVTs(ValueVTs: VTs, Offsets, ParamAlignment: ArgAlign);
4399 for (const unsigned NumElts : VI) {
4400 // i1 is loaded/stored as i8
4401 const EVT LoadVT = VTs[I] == MVT::i1 ? MVT::i8 : VTs[I];
4402 const EVT VecVT = getVectorizedVT(VT: LoadVT, N: NumElts, C&: Ctx);
4403
4404 SDValue VecAddr = DAG.getObjectPtrOffset(
4405 SL: dl, Ptr: ArgSymbol, Offset: TypeSize::getFixed(ExactSize: Offsets[I]));
4406
4407 const Align PartAlign = commonAlignment(A: ArgAlign, Offset: Offsets[I]);
4408 const unsigned AS = IsKernel ? NVPTX::AddressSpace::EntryParam
4409 : NVPTX::AddressSpace::DeviceParam;
4410 SDValue P = DAG.getLoad(VT: VecVT, dl, Chain: Root, Ptr: VecAddr,
4411 PtrInfo: MachinePointerInfo(AS), Alignment: PartAlign,
4412 MMOFlags: MachineMemOperand::MODereferenceable |
4413 MachineMemOperand::MOInvariant);
4414 P.getNode()->setIROrder(Arg.getArgNo() + 1);
4415 for (const unsigned J : llvm::seq(Size: NumElts)) {
4416 SDValue Elt = getExtractVectorizedValue(V: P, I: J, VT: LoadVT, dl, DAG);
4417
4418 Elt = correctParamType(V: Elt, ExpectedVT: ArgIns[I + J].VT, Flags: ArgIns[I + J].Flags,
4419 DAG, dl);
4420 InVals.push_back(Elt);
4421 }
4422 I += NumElts;
4423 }
4424 }
4425 }
4426
4427 if (!OutChains.empty())
4428 DAG.setRoot(DAG.getTokenFactor(DL: dl, Vals&: OutChains));
4429
4430 return Chain;
4431}
4432
4433SDValue
4434NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
4435 bool isVarArg,
4436 const SmallVectorImpl<ISD::OutputArg> &Outs,
4437 const SmallVectorImpl<SDValue> &OutVals,
4438 const SDLoc &dl, SelectionDAG &DAG) const {
4439 const Function &F = DAG.getMachineFunction().getFunction();
4440 Type *RetTy = F.getReturnType();
4441
4442 if (RetTy->isVoidTy()) {
4443 assert(OutVals.empty() && Outs.empty() && "Return value expected for void");
4444 return DAG.getNode(Opcode: NVPTXISD::RET_GLUE, DL: dl, VT: MVT::Other, Operand: Chain);
4445 }
4446
4447 const DataLayout &DL = DAG.getDataLayout();
4448 LLVMContext &Ctx = *DAG.getContext();
4449
4450 const SDValue RetSymbol = getSymbolNode(DAG, Name: "func_retval0", T: MVT::i32);
4451 const auto RetAlign =
4452 getPTXParamAlign(F: &F, Ty: RetTy, AttrIdx: AttributeList::ReturnIndex, DL);
4453
4454 // PTX Interoperability Guide 3.3(A): [Integer] Values shorter than
4455 // 32-bits are sign extended or zero extended, depending on whether
4456 // they are signed or unsigned types.
4457 const bool ExtendIntegerRetVal =
4458 RetTy->isIntegerTy() && DL.getTypeAllocSizeInBits(Ty: RetTy) < 32;
4459
4460 SmallVector<EVT, 16> VTs;
4461 SmallVector<uint64_t, 16> Offsets;
4462 ComputePTXValueVTs(TLI: *this, DL, Ctx, CallConv, Ty: RetTy, ValueVTs&: VTs, Offsets);
4463 assert(VTs.size() == OutVals.size() && "Bad return value decomposition");
4464
4465 const auto GetRetVal = [&](unsigned I) -> SDValue {
4466 SDValue RetVal = OutVals[I];
4467 assert(promoteScalarIntegerPTX(RetVal.getValueType()) ==
4468 RetVal.getValueType() &&
4469 "OutVal type should always be legal");
4470
4471 const EVT VTI = promoteScalarIntegerPTX(VT: VTs[I]);
4472 const EVT StoreVT =
4473 ExtendIntegerRetVal ? MVT::i32 : (VTI == MVT::i1 ? MVT::i8 : VTI);
4474 return correctParamType(V: RetVal, ExpectedVT: StoreVT, Flags: Outs[I].Flags, DAG, dl);
4475 };
4476
4477 unsigned I = 0;
4478 const auto VI = VectorizePTXValueVTs(ValueVTs: VTs, Offsets, ParamAlignment: RetAlign);
4479 for (const unsigned NumElts : VI) {
4480 const MaybeAlign CurrentAlign = ExtendIntegerRetVal
4481 ? MaybeAlign(std::nullopt)
4482 : commonAlignment(A: RetAlign, Offset: Offsets[I]);
4483
4484 SDValue Val = getBuildVectorizedValue(
4485 N: NumElts, dl, DAG, GetElement: [&](unsigned K) { return GetRetVal(I + K); });
4486
4487 SDValue Ptr =
4488 DAG.getObjectPtrOffset(SL: dl, Ptr: RetSymbol, Offset: TypeSize::getFixed(ExactSize: Offsets[I]));
4489
4490 Chain = DAG.getStore(Chain, dl, Val, Ptr,
4491 PtrInfo: MachinePointerInfo(NVPTX::AddressSpace::DeviceParam),
4492 Alignment: CurrentAlign);
4493
4494 I += NumElts;
4495 }
4496
4497 return DAG.getNode(Opcode: NVPTXISD::RET_GLUE, DL: dl, VT: MVT::Other, Operand: Chain);
4498}
4499
4500void NVPTXTargetLowering::LowerAsmOperandForConstraint(
4501 SDValue Op, StringRef Constraint, std::vector<SDValue> &Ops,
4502 SelectionDAG &DAG) const {
4503 if (Constraint.size() > 1)
4504 return;
4505 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
4506}
4507
4508// llvm.ptx.memcpy.const and llvm.ptx.memmove.const need to be modeled as
4509// TgtMemIntrinsic
4510// because we need the information that is only available in the "Value" type
4511// of destination
4512// pointer. In particular, the address space information.
4513void NVPTXTargetLowering::getTgtMemIntrinsic(
4514 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
4515 MachineFunction &MF, unsigned Intrinsic) const {
4516 IntrinsicInfo Info;
4517 switch (Intrinsic) {
4518 default:
4519 return;
4520 case Intrinsic::nvvm_match_all_sync_i32p:
4521 case Intrinsic::nvvm_match_all_sync_i64p:
4522 Info.opc = ISD::INTRINSIC_W_CHAIN;
4523 // memVT is bogus. These intrinsics have IntrInaccessibleMemOnly attribute
4524 // in order to model data exchange with other threads, but perform no real
4525 // memory accesses.
4526 Info.memVT = MVT::i1;
4527
4528 // Our result depends on both our and other thread's arguments.
4529 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
4530 Infos.push_back(Elt: Info);
4531 return;
4532 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_col:
4533 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_row:
4534 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_col_stride:
4535 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_row_stride:
4536 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_col:
4537 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_row:
4538 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_col_stride:
4539 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_row_stride:
4540 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_col:
4541 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_row:
4542 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_col_stride:
4543 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_row_stride:
4544 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_col:
4545 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_row:
4546 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_col_stride:
4547 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_row_stride:
4548 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_col:
4549 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_row:
4550 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_col_stride:
4551 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_row_stride:
4552 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_col:
4553 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_row:
4554 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_col_stride:
4555 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_row_stride: {
4556 Info.opc = ISD::INTRINSIC_W_CHAIN;
4557 Info.memVT = MVT::v8f16;
4558 Info.ptrVal = I.getArgOperand(i: 0);
4559 Info.offset = 0;
4560 Info.flags = MachineMemOperand::MOLoad;
4561 Info.align = Align(16);
4562 Infos.push_back(Elt: Info);
4563 return;
4564 }
4565 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_col:
4566 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_col_stride:
4567 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_col_stride:
4568 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_col:
4569 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_row:
4570 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_row_stride:
4571 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_row_stride:
4572 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_row:
4573 case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_col:
4574 case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_col_stride:
4575 case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_row:
4576 case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_row_stride:
4577 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_col:
4578 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_col_stride:
4579 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_col_stride:
4580 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_col:
4581 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_row:
4582 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_row_stride:
4583 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_row_stride:
4584 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_row:
4585 case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_col:
4586 case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_col_stride:
4587 case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_row:
4588 case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_row_stride: {
4589 Info.opc = ISD::INTRINSIC_W_CHAIN;
4590 Info.memVT = MVT::v2i32;
4591 Info.ptrVal = I.getArgOperand(i: 0);
4592 Info.offset = 0;
4593 Info.flags = MachineMemOperand::MOLoad;
4594 Info.align = Align(8);
4595 Infos.push_back(Elt: Info);
4596 return;
4597 }
4598
4599 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_col:
4600 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_col_stride:
4601 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_col_stride:
4602 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_col:
4603 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_row:
4604 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_row_stride:
4605 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_row_stride:
4606 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_row:
4607 case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_col:
4608 case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_col_stride:
4609 case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_row:
4610 case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_row_stride:
4611 case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_col:
4612 case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_col_stride:
4613 case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_row:
4614 case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_row_stride:
4615
4616 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_col:
4617 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_col_stride:
4618 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_col_stride:
4619 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_col:
4620 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_row:
4621 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_row_stride:
4622 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_row_stride:
4623 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_row:
4624 case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_col:
4625 case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_col_stride:
4626 case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_row:
4627 case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_row_stride:
4628 case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_col:
4629 case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_col_stride:
4630 case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_row:
4631 case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_row_stride:
4632 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x4_b16:
4633 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x4_trans_b16:
4634 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x2_trans_b8:
4635 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x2_trans_b8x16_b4x16_p64:
4636 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x2_trans_b8x16_b6x16_p32:
4637 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x4_b8x16_b4x16_p64:
4638 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x4_b8x16_b6x16_p32:
4639 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x4_s8_s4: {
4640 Info.opc = ISD::INTRINSIC_W_CHAIN;
4641 Info.memVT = MVT::v4i32;
4642 Info.ptrVal = I.getArgOperand(i: 0);
4643 Info.offset = 0;
4644 Info.flags = MachineMemOperand::MOLoad;
4645 Info.align = Align(16);
4646 Infos.push_back(Elt: Info);
4647 return;
4648 }
4649
4650 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_col:
4651 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_col_stride:
4652 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_col_stride:
4653 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_col:
4654 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_row:
4655 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_row_stride:
4656 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_row_stride:
4657 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_row:
4658
4659 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_col:
4660 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_col_stride:
4661 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_col_stride:
4662 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_col:
4663 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_row:
4664 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_row_stride:
4665 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_row_stride:
4666 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_row:
4667 case Intrinsic::nvvm_wmma_m8n8k128_load_a_b1_row:
4668 case Intrinsic::nvvm_wmma_m8n8k128_load_a_b1_row_stride:
4669 case Intrinsic::nvvm_wmma_m8n8k128_load_b_b1_col:
4670 case Intrinsic::nvvm_wmma_m8n8k128_load_b_b1_col_stride:
4671 case Intrinsic::nvvm_wmma_m8n8k32_load_a_s4_row:
4672 case Intrinsic::nvvm_wmma_m8n8k32_load_a_s4_row_stride:
4673 case Intrinsic::nvvm_wmma_m8n8k32_load_a_u4_row_stride:
4674 case Intrinsic::nvvm_wmma_m8n8k32_load_a_u4_row:
4675 case Intrinsic::nvvm_wmma_m8n8k32_load_b_s4_col:
4676 case Intrinsic::nvvm_wmma_m8n8k32_load_b_s4_col_stride:
4677 case Intrinsic::nvvm_wmma_m8n8k32_load_b_u4_col_stride:
4678 case Intrinsic::nvvm_wmma_m8n8k32_load_b_u4_col:
4679 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x1_b16:
4680 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x1_trans_b16:
4681 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x1_b8x16_b4x16_p64:
4682 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x1_b8x16_b6x16_p32:
4683 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x1_s8_s4: {
4684 Info.opc = ISD::INTRINSIC_W_CHAIN;
4685 Info.memVT = MVT::i32;
4686 Info.ptrVal = I.getArgOperand(i: 0);
4687 Info.offset = 0;
4688 Info.flags = MachineMemOperand::MOLoad;
4689 Info.align = Align(4);
4690 Infos.push_back(Elt: Info);
4691 return;
4692 }
4693
4694 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_col:
4695 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_row:
4696 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_col_stride:
4697 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_row_stride:
4698 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_col:
4699 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_row:
4700 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_col_stride:
4701 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_row_stride:
4702 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_col:
4703 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_row:
4704 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_col_stride:
4705 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_row_stride: {
4706 Info.opc = ISD::INTRINSIC_W_CHAIN;
4707 Info.memVT = MVT::v4f16;
4708 Info.ptrVal = I.getArgOperand(i: 0);
4709 Info.offset = 0;
4710 Info.flags = MachineMemOperand::MOLoad;
4711 Info.align = Align(16);
4712 Infos.push_back(Elt: Info);
4713 return;
4714 }
4715
4716 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_col:
4717 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_row:
4718 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_col_stride:
4719 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_row_stride:
4720 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_col:
4721 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_row:
4722 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_col_stride:
4723 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_row_stride:
4724 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_col:
4725 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_row:
4726 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_col_stride:
4727 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_row_stride:
4728 case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_col:
4729 case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_row:
4730 case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_col_stride:
4731 case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_row_stride: {
4732 Info.opc = ISD::INTRINSIC_W_CHAIN;
4733 Info.memVT = MVT::v8f32;
4734 Info.ptrVal = I.getArgOperand(i: 0);
4735 Info.offset = 0;
4736 Info.flags = MachineMemOperand::MOLoad;
4737 Info.align = Align(16);
4738 Infos.push_back(Elt: Info);
4739 return;
4740 }
4741
4742 case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_col:
4743 case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_col_stride:
4744 case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_row:
4745 case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_row_stride:
4746
4747 case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_col:
4748 case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_col_stride:
4749 case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_row:
4750 case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_row_stride:
4751
4752 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_col:
4753 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_col_stride:
4754 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_row:
4755 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_row_stride:
4756 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_col:
4757 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_col_stride:
4758 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_row:
4759 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_row_stride:
4760 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_col:
4761 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_col_stride:
4762 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_row:
4763 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_row_stride: {
4764 Info.opc = ISD::INTRINSIC_W_CHAIN;
4765 Info.memVT = MVT::v8i32;
4766 Info.ptrVal = I.getArgOperand(i: 0);
4767 Info.offset = 0;
4768 Info.flags = MachineMemOperand::MOLoad;
4769 Info.align = Align(16);
4770 Infos.push_back(Elt: Info);
4771 return;
4772 }
4773
4774 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_col:
4775 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_col_stride:
4776 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_row:
4777 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_row_stride:
4778 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_col:
4779 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_col_stride:
4780 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_row:
4781 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_row_stride:
4782 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x2_b16:
4783 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x2_trans_b16:
4784 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x1_trans_b8:
4785 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x1_trans_b8x16_b4x16_p64:
4786 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x1_trans_b8x16_b6x16_p32:
4787 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x2_b8x16_b4x16_p64:
4788 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x2_b8x16_b6x16_p32:
4789 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x2_s8_s4: {
4790 Info.opc = ISD::INTRINSIC_W_CHAIN;
4791 Info.memVT = MVT::v2i32;
4792 Info.ptrVal = I.getArgOperand(i: 0);
4793 Info.offset = 0;
4794 Info.flags = MachineMemOperand::MOLoad;
4795 Info.align = Align(8);
4796 Infos.push_back(Elt: Info);
4797 return;
4798 }
4799
4800 case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_col:
4801 case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_col_stride:
4802 case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_row:
4803 case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_row_stride:
4804
4805 case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_col:
4806 case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_col_stride:
4807 case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_row:
4808 case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_row_stride: {
4809 Info.opc = ISD::INTRINSIC_W_CHAIN;
4810 Info.memVT = MVT::f64;
4811 Info.ptrVal = I.getArgOperand(i: 0);
4812 Info.offset = 0;
4813 Info.flags = MachineMemOperand::MOLoad;
4814 Info.align = Align(8);
4815 Infos.push_back(Elt: Info);
4816 return;
4817 }
4818
4819 case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_col:
4820 case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_col_stride:
4821 case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_row:
4822 case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_row_stride: {
4823 Info.opc = ISD::INTRINSIC_W_CHAIN;
4824 Info.memVT = MVT::v2f64;
4825 Info.ptrVal = I.getArgOperand(i: 0);
4826 Info.offset = 0;
4827 Info.flags = MachineMemOperand::MOLoad;
4828 Info.align = Align(16);
4829 Infos.push_back(Elt: Info);
4830 return;
4831 }
4832
4833 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_col:
4834 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_row:
4835 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_col_stride:
4836 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_row_stride:
4837 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_col:
4838 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_row:
4839 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_col_stride:
4840 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_row_stride:
4841 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_col:
4842 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_row:
4843 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_col_stride:
4844 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_row_stride: {
4845 Info.opc = ISD::INTRINSIC_VOID;
4846 Info.memVT = MVT::v4f16;
4847 Info.ptrVal = I.getArgOperand(i: 0);
4848 Info.offset = 0;
4849 Info.flags = MachineMemOperand::MOStore;
4850 Info.align = Align(16);
4851 Infos.push_back(Elt: Info);
4852 return;
4853 }
4854
4855 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_col:
4856 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_row:
4857 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_col_stride:
4858 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_row_stride:
4859 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_col:
4860 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_row:
4861 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_col_stride:
4862 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_row_stride:
4863 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_col:
4864 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_row:
4865 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_col_stride:
4866 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_row_stride:
4867 case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_col:
4868 case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_row:
4869 case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_col_stride:
4870 case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_row_stride: {
4871 Info.opc = ISD::INTRINSIC_VOID;
4872 Info.memVT = MVT::v8f32;
4873 Info.ptrVal = I.getArgOperand(i: 0);
4874 Info.offset = 0;
4875 Info.flags = MachineMemOperand::MOStore;
4876 Info.align = Align(16);
4877 Infos.push_back(Elt: Info);
4878 return;
4879 }
4880
4881 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_col:
4882 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_col_stride:
4883 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_row:
4884 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_row_stride:
4885 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_col:
4886 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_col_stride:
4887 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_row:
4888 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_row_stride:
4889 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_col:
4890 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_col_stride:
4891 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_row:
4892 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_row_stride: {
4893 Info.opc = ISD::INTRINSIC_VOID;
4894 Info.memVT = MVT::v8i32;
4895 Info.ptrVal = I.getArgOperand(i: 0);
4896 Info.offset = 0;
4897 Info.flags = MachineMemOperand::MOStore;
4898 Info.align = Align(16);
4899 Infos.push_back(Elt: Info);
4900 return;
4901 }
4902
4903 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_col:
4904 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_col_stride:
4905 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_row:
4906 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_row_stride:
4907 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_col:
4908 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_col_stride:
4909 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_row:
4910 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_row_stride:
4911 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x2_b16:
4912 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x2_trans_b16:
4913 case Intrinsic::nvvm_stmatrix_sync_aligned_m16n8_x2_trans_b8: {
4914 Info.opc = ISD::INTRINSIC_VOID;
4915 Info.memVT = MVT::v2i32;
4916 Info.ptrVal = I.getArgOperand(i: 0);
4917 Info.offset = 0;
4918 Info.flags = MachineMemOperand::MOStore;
4919 Info.align = Align(8);
4920 Infos.push_back(Elt: Info);
4921 return;
4922 }
4923
4924 case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_col:
4925 case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_col_stride:
4926 case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_row:
4927 case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_row_stride: {
4928 Info.opc = ISD::INTRINSIC_VOID;
4929 Info.memVT = MVT::v2f64;
4930 Info.ptrVal = I.getArgOperand(i: 0);
4931 Info.offset = 0;
4932 Info.flags = MachineMemOperand::MOStore;
4933 Info.align = Align(16);
4934 Infos.push_back(Elt: Info);
4935 return;
4936 }
4937
4938 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x1_b16:
4939 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x1_trans_b16:
4940 case Intrinsic::nvvm_stmatrix_sync_aligned_m16n8_x1_trans_b8: {
4941 Info.opc = ISD::INTRINSIC_VOID;
4942 Info.memVT = MVT::i32;
4943 Info.ptrVal = I.getArgOperand(i: 0);
4944 Info.offset = 0;
4945 Info.flags = MachineMemOperand::MOStore;
4946 Info.align = Align(4);
4947 Infos.push_back(Elt: Info);
4948 return;
4949 }
4950
4951 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x4_b16:
4952 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x4_trans_b16:
4953 case Intrinsic::nvvm_stmatrix_sync_aligned_m16n8_x4_trans_b8: {
4954 Info.opc = ISD::INTRINSIC_VOID;
4955 Info.memVT = MVT::v4i32;
4956 Info.ptrVal = I.getArgOperand(i: 0);
4957 Info.offset = 0;
4958 Info.flags = MachineMemOperand::MOStore;
4959 Info.align = Align(16);
4960 Infos.push_back(Elt: Info);
4961 return;
4962 }
4963
4964 case Intrinsic::nvvm_st_bulk: {
4965 Value *Dst = I.getArgOperand(i: 0);
4966 Value *Val = I.getArgOperand(i: 2);
4967 Info.opc = ISD::INTRINSIC_VOID;
4968 Info.memVT = MVT::getVT(Ty: Val->getType());
4969 Info.ptrVal = Dst;
4970 Info.offset = 0;
4971 Info.flags = MachineMemOperand::MOStore;
4972 Info.align.reset();
4973 Info.size = MemoryLocation::UnknownSize;
4974 Infos.push_back(Elt: Info);
4975 return;
4976 }
4977
4978 case Intrinsic::nvvm_prefetch_tensormap: {
4979 auto &DL = I.getDataLayout();
4980 Info.opc = ISD::INTRINSIC_VOID;
4981 Info.memVT = getPointerTy(DL);
4982 Info.ptrVal = I.getArgOperand(i: 0);
4983 Info.offset = 0;
4984 Info.flags =
4985 MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable;
4986 Info.align.reset();
4987 Infos.push_back(Elt: Info);
4988 return;
4989 }
4990
4991 case Intrinsic::nvvm_prefetch_L1_32B_valid_addr: {
4992 Info.opc = ISD::INTRINSIC_VOID;
4993 Info.memVT = MVT::i8;
4994 Info.ptrVal = I.getArgOperand(i: 0);
4995 Info.offset = 0;
4996 Info.flags = MachineMemOperand::MOLoad;
4997 Info.align = Align(1);
4998 Infos.push_back(Elt: Info);
4999 return;
5000 }
5001
5002 case Intrinsic::nvvm_mbarrier_init: {
5003 Info.opc = ISD::INTRINSIC_VOID;
5004 Info.memVT = MVT::i64;
5005 Info.ptrVal = I.getArgOperand(i: 0);
5006 Info.offset = 0;
5007 Info.flags = MachineMemOperand::MOStore;
5008 Info.align = Align(8);
5009 Infos.push_back(Elt: Info);
5010 return;
5011 }
5012
5013 case Intrinsic::nvvm_mbarrier_check_layout: {
5014 Info.opc = ISD::INTRINSIC_W_CHAIN;
5015 Info.memVT = MVT::i64;
5016 Info.ptrVal = I.getArgOperand(i: 0);
5017 Info.offset = 0;
5018 Info.flags = MachineMemOperand::MOLoad;
5019 Info.align = Align(8);
5020 Infos.push_back(Elt: Info);
5021 return;
5022 }
5023
5024 case Intrinsic::nvvm_tensormap_replace_global_address:
5025 case Intrinsic::nvvm_tensormap_replace_global_stride: {
5026 Info.opc = ISD::INTRINSIC_VOID;
5027 Info.memVT = MVT::i64;
5028 Info.ptrVal = I.getArgOperand(i: 0);
5029 Info.offset = 0;
5030 Info.flags = MachineMemOperand::MOStore;
5031 Info.align.reset();
5032 Infos.push_back(Elt: Info);
5033 return;
5034 }
5035
5036 case Intrinsic::nvvm_tensormap_replace_rank:
5037 case Intrinsic::nvvm_tensormap_replace_box_dim:
5038 case Intrinsic::nvvm_tensormap_replace_global_dim:
5039 case Intrinsic::nvvm_tensormap_replace_element_stride:
5040 case Intrinsic::nvvm_tensormap_replace_elemtype:
5041 case Intrinsic::nvvm_tensormap_replace_interleave_layout:
5042 case Intrinsic::nvvm_tensormap_replace_swizzle_mode:
5043 case Intrinsic::nvvm_tensormap_replace_swizzle_atomicity:
5044 case Intrinsic::nvvm_tensormap_replace_fill_mode:
5045 case Intrinsic::nvvm_tensormap_replace_alloc_class: {
5046 Info.opc = ISD::INTRINSIC_VOID;
5047 Info.memVT = MVT::i32;
5048 Info.ptrVal = I.getArgOperand(i: 0);
5049 Info.offset = 0;
5050 Info.flags = MachineMemOperand::MOStore;
5051 Info.align.reset();
5052 Infos.push_back(Elt: Info);
5053 return;
5054 }
5055
5056 case Intrinsic::nvvm_tensormap_query_alloc_class_large: {
5057 Info.opc = ISD::INTRINSIC_W_CHAIN;
5058 Info.memVT = MVT::i32;
5059 Info.ptrVal = I.getArgOperand(i: 0);
5060 Info.offset = 0;
5061 Info.flags = MachineMemOperand::MOLoad;
5062 Info.align.reset();
5063 Infos.push_back(Elt: Info);
5064 return;
5065 }
5066
5067 case Intrinsic::nvvm_ldu_global_i:
5068 case Intrinsic::nvvm_ldu_global_f:
5069 case Intrinsic::nvvm_ldu_global_p: {
5070 Info.opc = ISD::INTRINSIC_W_CHAIN;
5071 Info.memVT = getValueType(DL: I.getDataLayout(), Ty: I.getType());
5072 Info.ptrVal = I.getArgOperand(i: 0);
5073 Info.offset = 0;
5074 Info.flags = MachineMemOperand::MOLoad;
5075 Info.align = cast<ConstantInt>(Val: I.getArgOperand(i: 1))->getMaybeAlignValue();
5076
5077 Infos.push_back(Elt: Info);
5078 return;
5079 }
5080 case Intrinsic::nvvm_tex_1d_v4f32_s32:
5081 case Intrinsic::nvvm_tex_1d_v4f32_f32:
5082 case Intrinsic::nvvm_tex_1d_level_v4f32_f32:
5083 case Intrinsic::nvvm_tex_1d_grad_v4f32_f32:
5084 case Intrinsic::nvvm_tex_1d_array_v4f32_s32:
5085 case Intrinsic::nvvm_tex_1d_array_v4f32_f32:
5086 case Intrinsic::nvvm_tex_1d_array_level_v4f32_f32:
5087 case Intrinsic::nvvm_tex_1d_array_grad_v4f32_f32:
5088 case Intrinsic::nvvm_tex_2d_v4f32_s32:
5089 case Intrinsic::nvvm_tex_2d_v4f32_f32:
5090 case Intrinsic::nvvm_tex_2d_level_v4f32_f32:
5091 case Intrinsic::nvvm_tex_2d_grad_v4f32_f32:
5092 case Intrinsic::nvvm_tex_2d_array_v4f32_s32:
5093 case Intrinsic::nvvm_tex_2d_array_v4f32_f32:
5094 case Intrinsic::nvvm_tex_2d_array_level_v4f32_f32:
5095 case Intrinsic::nvvm_tex_2d_array_grad_v4f32_f32:
5096 case Intrinsic::nvvm_tex_3d_v4f32_s32:
5097 case Intrinsic::nvvm_tex_3d_v4f32_f32:
5098 case Intrinsic::nvvm_tex_3d_level_v4f32_f32:
5099 case Intrinsic::nvvm_tex_3d_grad_v4f32_f32:
5100 case Intrinsic::nvvm_tex_cube_v4f32_f32:
5101 case Intrinsic::nvvm_tex_cube_level_v4f32_f32:
5102 case Intrinsic::nvvm_tex_cube_array_v4f32_f32:
5103 case Intrinsic::nvvm_tex_cube_array_level_v4f32_f32:
5104 case Intrinsic::nvvm_tld4_r_2d_v4f32_f32:
5105 case Intrinsic::nvvm_tld4_g_2d_v4f32_f32:
5106 case Intrinsic::nvvm_tld4_b_2d_v4f32_f32:
5107 case Intrinsic::nvvm_tld4_a_2d_v4f32_f32:
5108 case Intrinsic::nvvm_tex_unified_1d_v4f32_s32:
5109 case Intrinsic::nvvm_tex_unified_1d_v4f32_f32:
5110 case Intrinsic::nvvm_tex_unified_1d_level_v4f32_f32:
5111 case Intrinsic::nvvm_tex_unified_1d_grad_v4f32_f32:
5112 case Intrinsic::nvvm_tex_unified_1d_array_v4f32_s32:
5113 case Intrinsic::nvvm_tex_unified_1d_array_v4f32_f32:
5114 case Intrinsic::nvvm_tex_unified_1d_array_level_v4f32_f32:
5115 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4f32_f32:
5116 case Intrinsic::nvvm_tex_unified_2d_v4f32_s32:
5117 case Intrinsic::nvvm_tex_unified_2d_v4f32_f32:
5118 case Intrinsic::nvvm_tex_unified_2d_level_v4f32_f32:
5119 case Intrinsic::nvvm_tex_unified_2d_grad_v4f32_f32:
5120 case Intrinsic::nvvm_tex_unified_2d_array_v4f32_s32:
5121 case Intrinsic::nvvm_tex_unified_2d_array_v4f32_f32:
5122 case Intrinsic::nvvm_tex_unified_2d_array_level_v4f32_f32:
5123 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4f32_f32:
5124 case Intrinsic::nvvm_tex_unified_3d_v4f32_s32:
5125 case Intrinsic::nvvm_tex_unified_3d_v4f32_f32:
5126 case Intrinsic::nvvm_tex_unified_3d_level_v4f32_f32:
5127 case Intrinsic::nvvm_tex_unified_3d_grad_v4f32_f32:
5128 case Intrinsic::nvvm_tex_unified_cube_v4f32_f32:
5129 case Intrinsic::nvvm_tex_unified_cube_level_v4f32_f32:
5130 case Intrinsic::nvvm_tex_unified_cube_array_v4f32_f32:
5131 case Intrinsic::nvvm_tex_unified_cube_array_level_v4f32_f32:
5132 case Intrinsic::nvvm_tex_unified_cube_grad_v4f32_f32:
5133 case Intrinsic::nvvm_tex_unified_cube_array_grad_v4f32_f32:
5134 case Intrinsic::nvvm_tld4_unified_r_2d_v4f32_f32:
5135 case Intrinsic::nvvm_tld4_unified_g_2d_v4f32_f32:
5136 case Intrinsic::nvvm_tld4_unified_b_2d_v4f32_f32:
5137 case Intrinsic::nvvm_tld4_unified_a_2d_v4f32_f32:
5138 Info.opc = ISD::INTRINSIC_W_CHAIN;
5139 Info.memVT = MVT::v4f32;
5140 Info.ptrVal = nullptr;
5141 Info.offset = 0;
5142 Info.flags = MachineMemOperand::MOLoad;
5143 Info.align = Align(16);
5144 Infos.push_back(Elt: Info);
5145 return;
5146
5147 case Intrinsic::nvvm_tex_1d_v4s32_s32:
5148 case Intrinsic::nvvm_tex_1d_v4s32_f32:
5149 case Intrinsic::nvvm_tex_1d_level_v4s32_f32:
5150 case Intrinsic::nvvm_tex_1d_grad_v4s32_f32:
5151 case Intrinsic::nvvm_tex_1d_array_v4s32_s32:
5152 case Intrinsic::nvvm_tex_1d_array_v4s32_f32:
5153 case Intrinsic::nvvm_tex_1d_array_level_v4s32_f32:
5154 case Intrinsic::nvvm_tex_1d_array_grad_v4s32_f32:
5155 case Intrinsic::nvvm_tex_2d_v4s32_s32:
5156 case Intrinsic::nvvm_tex_2d_v4s32_f32:
5157 case Intrinsic::nvvm_tex_2d_level_v4s32_f32:
5158 case Intrinsic::nvvm_tex_2d_grad_v4s32_f32:
5159 case Intrinsic::nvvm_tex_2d_array_v4s32_s32:
5160 case Intrinsic::nvvm_tex_2d_array_v4s32_f32:
5161 case Intrinsic::nvvm_tex_2d_array_level_v4s32_f32:
5162 case Intrinsic::nvvm_tex_2d_array_grad_v4s32_f32:
5163 case Intrinsic::nvvm_tex_3d_v4s32_s32:
5164 case Intrinsic::nvvm_tex_3d_v4s32_f32:
5165 case Intrinsic::nvvm_tex_3d_level_v4s32_f32:
5166 case Intrinsic::nvvm_tex_3d_grad_v4s32_f32:
5167 case Intrinsic::nvvm_tex_cube_v4s32_f32:
5168 case Intrinsic::nvvm_tex_cube_level_v4s32_f32:
5169 case Intrinsic::nvvm_tex_cube_array_v4s32_f32:
5170 case Intrinsic::nvvm_tex_cube_array_level_v4s32_f32:
5171 case Intrinsic::nvvm_tex_cube_v4u32_f32:
5172 case Intrinsic::nvvm_tex_cube_level_v4u32_f32:
5173 case Intrinsic::nvvm_tex_cube_array_v4u32_f32:
5174 case Intrinsic::nvvm_tex_cube_array_level_v4u32_f32:
5175 case Intrinsic::nvvm_tex_1d_v4u32_s32:
5176 case Intrinsic::nvvm_tex_1d_v4u32_f32:
5177 case Intrinsic::nvvm_tex_1d_level_v4u32_f32:
5178 case Intrinsic::nvvm_tex_1d_grad_v4u32_f32:
5179 case Intrinsic::nvvm_tex_1d_array_v4u32_s32:
5180 case Intrinsic::nvvm_tex_1d_array_v4u32_f32:
5181 case Intrinsic::nvvm_tex_1d_array_level_v4u32_f32:
5182 case Intrinsic::nvvm_tex_1d_array_grad_v4u32_f32:
5183 case Intrinsic::nvvm_tex_2d_v4u32_s32:
5184 case Intrinsic::nvvm_tex_2d_v4u32_f32:
5185 case Intrinsic::nvvm_tex_2d_level_v4u32_f32:
5186 case Intrinsic::nvvm_tex_2d_grad_v4u32_f32:
5187 case Intrinsic::nvvm_tex_2d_array_v4u32_s32:
5188 case Intrinsic::nvvm_tex_2d_array_v4u32_f32:
5189 case Intrinsic::nvvm_tex_2d_array_level_v4u32_f32:
5190 case Intrinsic::nvvm_tex_2d_array_grad_v4u32_f32:
5191 case Intrinsic::nvvm_tex_3d_v4u32_s32:
5192 case Intrinsic::nvvm_tex_3d_v4u32_f32:
5193 case Intrinsic::nvvm_tex_3d_level_v4u32_f32:
5194 case Intrinsic::nvvm_tex_3d_grad_v4u32_f32:
5195 case Intrinsic::nvvm_tld4_r_2d_v4s32_f32:
5196 case Intrinsic::nvvm_tld4_g_2d_v4s32_f32:
5197 case Intrinsic::nvvm_tld4_b_2d_v4s32_f32:
5198 case Intrinsic::nvvm_tld4_a_2d_v4s32_f32:
5199 case Intrinsic::nvvm_tld4_r_2d_v4u32_f32:
5200 case Intrinsic::nvvm_tld4_g_2d_v4u32_f32:
5201 case Intrinsic::nvvm_tld4_b_2d_v4u32_f32:
5202 case Intrinsic::nvvm_tld4_a_2d_v4u32_f32:
5203 case Intrinsic::nvvm_tex_unified_1d_v4s32_s32:
5204 case Intrinsic::nvvm_tex_unified_1d_v4s32_f32:
5205 case Intrinsic::nvvm_tex_unified_1d_level_v4s32_f32:
5206 case Intrinsic::nvvm_tex_unified_1d_grad_v4s32_f32:
5207 case Intrinsic::nvvm_tex_unified_1d_array_v4s32_s32:
5208 case Intrinsic::nvvm_tex_unified_1d_array_v4s32_f32:
5209 case Intrinsic::nvvm_tex_unified_1d_array_level_v4s32_f32:
5210 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4s32_f32:
5211 case Intrinsic::nvvm_tex_unified_2d_v4s32_s32:
5212 case Intrinsic::nvvm_tex_unified_2d_v4s32_f32:
5213 case Intrinsic::nvvm_tex_unified_2d_level_v4s32_f32:
5214 case Intrinsic::nvvm_tex_unified_2d_grad_v4s32_f32:
5215 case Intrinsic::nvvm_tex_unified_2d_array_v4s32_s32:
5216 case Intrinsic::nvvm_tex_unified_2d_array_v4s32_f32:
5217 case Intrinsic::nvvm_tex_unified_2d_array_level_v4s32_f32:
5218 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4s32_f32:
5219 case Intrinsic::nvvm_tex_unified_3d_v4s32_s32:
5220 case Intrinsic::nvvm_tex_unified_3d_v4s32_f32:
5221 case Intrinsic::nvvm_tex_unified_3d_level_v4s32_f32:
5222 case Intrinsic::nvvm_tex_unified_3d_grad_v4s32_f32:
5223 case Intrinsic::nvvm_tex_unified_1d_v4u32_s32:
5224 case Intrinsic::nvvm_tex_unified_1d_v4u32_f32:
5225 case Intrinsic::nvvm_tex_unified_1d_level_v4u32_f32:
5226 case Intrinsic::nvvm_tex_unified_1d_grad_v4u32_f32:
5227 case Intrinsic::nvvm_tex_unified_1d_array_v4u32_s32:
5228 case Intrinsic::nvvm_tex_unified_1d_array_v4u32_f32:
5229 case Intrinsic::nvvm_tex_unified_1d_array_level_v4u32_f32:
5230 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4u32_f32:
5231 case Intrinsic::nvvm_tex_unified_2d_v4u32_s32:
5232 case Intrinsic::nvvm_tex_unified_2d_v4u32_f32:
5233 case Intrinsic::nvvm_tex_unified_2d_level_v4u32_f32:
5234 case Intrinsic::nvvm_tex_unified_2d_grad_v4u32_f32:
5235 case Intrinsic::nvvm_tex_unified_2d_array_v4u32_s32:
5236 case Intrinsic::nvvm_tex_unified_2d_array_v4u32_f32:
5237 case Intrinsic::nvvm_tex_unified_2d_array_level_v4u32_f32:
5238 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4u32_f32:
5239 case Intrinsic::nvvm_tex_unified_3d_v4u32_s32:
5240 case Intrinsic::nvvm_tex_unified_3d_v4u32_f32:
5241 case Intrinsic::nvvm_tex_unified_3d_level_v4u32_f32:
5242 case Intrinsic::nvvm_tex_unified_3d_grad_v4u32_f32:
5243 case Intrinsic::nvvm_tex_unified_cube_v4s32_f32:
5244 case Intrinsic::nvvm_tex_unified_cube_level_v4s32_f32:
5245 case Intrinsic::nvvm_tex_unified_cube_array_v4s32_f32:
5246 case Intrinsic::nvvm_tex_unified_cube_array_level_v4s32_f32:
5247 case Intrinsic::nvvm_tex_unified_cube_v4u32_f32:
5248 case Intrinsic::nvvm_tex_unified_cube_level_v4u32_f32:
5249 case Intrinsic::nvvm_tex_unified_cube_array_v4u32_f32:
5250 case Intrinsic::nvvm_tex_unified_cube_array_level_v4u32_f32:
5251 case Intrinsic::nvvm_tex_unified_cube_grad_v4s32_f32:
5252 case Intrinsic::nvvm_tex_unified_cube_grad_v4u32_f32:
5253 case Intrinsic::nvvm_tex_unified_cube_array_grad_v4s32_f32:
5254 case Intrinsic::nvvm_tex_unified_cube_array_grad_v4u32_f32:
5255 case Intrinsic::nvvm_tld4_unified_r_2d_v4s32_f32:
5256 case Intrinsic::nvvm_tld4_unified_g_2d_v4s32_f32:
5257 case Intrinsic::nvvm_tld4_unified_b_2d_v4s32_f32:
5258 case Intrinsic::nvvm_tld4_unified_a_2d_v4s32_f32:
5259 case Intrinsic::nvvm_tld4_unified_r_2d_v4u32_f32:
5260 case Intrinsic::nvvm_tld4_unified_g_2d_v4u32_f32:
5261 case Intrinsic::nvvm_tld4_unified_b_2d_v4u32_f32:
5262 case Intrinsic::nvvm_tld4_unified_a_2d_v4u32_f32:
5263 Info.opc = ISD::INTRINSIC_W_CHAIN;
5264 Info.memVT = MVT::v4i32;
5265 Info.ptrVal = nullptr;
5266 Info.offset = 0;
5267 Info.flags = MachineMemOperand::MOLoad;
5268 Info.align = Align(16);
5269 Infos.push_back(Elt: Info);
5270 return;
5271
5272 case Intrinsic::nvvm_suld_1d_i8_clamp:
5273 case Intrinsic::nvvm_suld_1d_v2i8_clamp:
5274 case Intrinsic::nvvm_suld_1d_v4i8_clamp:
5275 case Intrinsic::nvvm_suld_1d_array_i8_clamp:
5276 case Intrinsic::nvvm_suld_1d_array_v2i8_clamp:
5277 case Intrinsic::nvvm_suld_1d_array_v4i8_clamp:
5278 case Intrinsic::nvvm_suld_2d_i8_clamp:
5279 case Intrinsic::nvvm_suld_2d_v2i8_clamp:
5280 case Intrinsic::nvvm_suld_2d_v4i8_clamp:
5281 case Intrinsic::nvvm_suld_2d_array_i8_clamp:
5282 case Intrinsic::nvvm_suld_2d_array_v2i8_clamp:
5283 case Intrinsic::nvvm_suld_2d_array_v4i8_clamp:
5284 case Intrinsic::nvvm_suld_3d_i8_clamp:
5285 case Intrinsic::nvvm_suld_3d_v2i8_clamp:
5286 case Intrinsic::nvvm_suld_3d_v4i8_clamp:
5287 case Intrinsic::nvvm_suld_1d_i8_trap:
5288 case Intrinsic::nvvm_suld_1d_v2i8_trap:
5289 case Intrinsic::nvvm_suld_1d_v4i8_trap:
5290 case Intrinsic::nvvm_suld_1d_array_i8_trap:
5291 case Intrinsic::nvvm_suld_1d_array_v2i8_trap:
5292 case Intrinsic::nvvm_suld_1d_array_v4i8_trap:
5293 case Intrinsic::nvvm_suld_2d_i8_trap:
5294 case Intrinsic::nvvm_suld_2d_v2i8_trap:
5295 case Intrinsic::nvvm_suld_2d_v4i8_trap:
5296 case Intrinsic::nvvm_suld_2d_array_i8_trap:
5297 case Intrinsic::nvvm_suld_2d_array_v2i8_trap:
5298 case Intrinsic::nvvm_suld_2d_array_v4i8_trap:
5299 case Intrinsic::nvvm_suld_3d_i8_trap:
5300 case Intrinsic::nvvm_suld_3d_v2i8_trap:
5301 case Intrinsic::nvvm_suld_3d_v4i8_trap:
5302 case Intrinsic::nvvm_suld_1d_i8_zero:
5303 case Intrinsic::nvvm_suld_1d_v2i8_zero:
5304 case Intrinsic::nvvm_suld_1d_v4i8_zero:
5305 case Intrinsic::nvvm_suld_1d_array_i8_zero:
5306 case Intrinsic::nvvm_suld_1d_array_v2i8_zero:
5307 case Intrinsic::nvvm_suld_1d_array_v4i8_zero:
5308 case Intrinsic::nvvm_suld_2d_i8_zero:
5309 case Intrinsic::nvvm_suld_2d_v2i8_zero:
5310 case Intrinsic::nvvm_suld_2d_v4i8_zero:
5311 case Intrinsic::nvvm_suld_2d_array_i8_zero:
5312 case Intrinsic::nvvm_suld_2d_array_v2i8_zero:
5313 case Intrinsic::nvvm_suld_2d_array_v4i8_zero:
5314 case Intrinsic::nvvm_suld_3d_i8_zero:
5315 case Intrinsic::nvvm_suld_3d_v2i8_zero:
5316 case Intrinsic::nvvm_suld_3d_v4i8_zero:
5317 Info.opc = ISD::INTRINSIC_W_CHAIN;
5318 Info.memVT = MVT::i8;
5319 Info.ptrVal = nullptr;
5320 Info.offset = 0;
5321 Info.flags = MachineMemOperand::MOLoad;
5322 Info.align = Align(16);
5323 Infos.push_back(Elt: Info);
5324 return;
5325
5326 case Intrinsic::nvvm_suld_1d_i16_clamp:
5327 case Intrinsic::nvvm_suld_1d_v2i16_clamp:
5328 case Intrinsic::nvvm_suld_1d_v4i16_clamp:
5329 case Intrinsic::nvvm_suld_1d_array_i16_clamp:
5330 case Intrinsic::nvvm_suld_1d_array_v2i16_clamp:
5331 case Intrinsic::nvvm_suld_1d_array_v4i16_clamp:
5332 case Intrinsic::nvvm_suld_2d_i16_clamp:
5333 case Intrinsic::nvvm_suld_2d_v2i16_clamp:
5334 case Intrinsic::nvvm_suld_2d_v4i16_clamp:
5335 case Intrinsic::nvvm_suld_2d_array_i16_clamp:
5336 case Intrinsic::nvvm_suld_2d_array_v2i16_clamp:
5337 case Intrinsic::nvvm_suld_2d_array_v4i16_clamp:
5338 case Intrinsic::nvvm_suld_3d_i16_clamp:
5339 case Intrinsic::nvvm_suld_3d_v2i16_clamp:
5340 case Intrinsic::nvvm_suld_3d_v4i16_clamp:
5341 case Intrinsic::nvvm_suld_1d_i16_trap:
5342 case Intrinsic::nvvm_suld_1d_v2i16_trap:
5343 case Intrinsic::nvvm_suld_1d_v4i16_trap:
5344 case Intrinsic::nvvm_suld_1d_array_i16_trap:
5345 case Intrinsic::nvvm_suld_1d_array_v2i16_trap:
5346 case Intrinsic::nvvm_suld_1d_array_v4i16_trap:
5347 case Intrinsic::nvvm_suld_2d_i16_trap:
5348 case Intrinsic::nvvm_suld_2d_v2i16_trap:
5349 case Intrinsic::nvvm_suld_2d_v4i16_trap:
5350 case Intrinsic::nvvm_suld_2d_array_i16_trap:
5351 case Intrinsic::nvvm_suld_2d_array_v2i16_trap:
5352 case Intrinsic::nvvm_suld_2d_array_v4i16_trap:
5353 case Intrinsic::nvvm_suld_3d_i16_trap:
5354 case Intrinsic::nvvm_suld_3d_v2i16_trap:
5355 case Intrinsic::nvvm_suld_3d_v4i16_trap:
5356 case Intrinsic::nvvm_suld_1d_i16_zero:
5357 case Intrinsic::nvvm_suld_1d_v2i16_zero:
5358 case Intrinsic::nvvm_suld_1d_v4i16_zero:
5359 case Intrinsic::nvvm_suld_1d_array_i16_zero:
5360 case Intrinsic::nvvm_suld_1d_array_v2i16_zero:
5361 case Intrinsic::nvvm_suld_1d_array_v4i16_zero:
5362 case Intrinsic::nvvm_suld_2d_i16_zero:
5363 case Intrinsic::nvvm_suld_2d_v2i16_zero:
5364 case Intrinsic::nvvm_suld_2d_v4i16_zero:
5365 case Intrinsic::nvvm_suld_2d_array_i16_zero:
5366 case Intrinsic::nvvm_suld_2d_array_v2i16_zero:
5367 case Intrinsic::nvvm_suld_2d_array_v4i16_zero:
5368 case Intrinsic::nvvm_suld_3d_i16_zero:
5369 case Intrinsic::nvvm_suld_3d_v2i16_zero:
5370 case Intrinsic::nvvm_suld_3d_v4i16_zero:
5371 Info.opc = ISD::INTRINSIC_W_CHAIN;
5372 Info.memVT = MVT::i16;
5373 Info.ptrVal = nullptr;
5374 Info.offset = 0;
5375 Info.flags = MachineMemOperand::MOLoad;
5376 Info.align = Align(16);
5377 Infos.push_back(Elt: Info);
5378 return;
5379
5380 case Intrinsic::nvvm_suld_1d_i32_clamp:
5381 case Intrinsic::nvvm_suld_1d_v2i32_clamp:
5382 case Intrinsic::nvvm_suld_1d_v4i32_clamp:
5383 case Intrinsic::nvvm_suld_1d_array_i32_clamp:
5384 case Intrinsic::nvvm_suld_1d_array_v2i32_clamp:
5385 case Intrinsic::nvvm_suld_1d_array_v4i32_clamp:
5386 case Intrinsic::nvvm_suld_2d_i32_clamp:
5387 case Intrinsic::nvvm_suld_2d_v2i32_clamp:
5388 case Intrinsic::nvvm_suld_2d_v4i32_clamp:
5389 case Intrinsic::nvvm_suld_2d_array_i32_clamp:
5390 case Intrinsic::nvvm_suld_2d_array_v2i32_clamp:
5391 case Intrinsic::nvvm_suld_2d_array_v4i32_clamp:
5392 case Intrinsic::nvvm_suld_3d_i32_clamp:
5393 case Intrinsic::nvvm_suld_3d_v2i32_clamp:
5394 case Intrinsic::nvvm_suld_3d_v4i32_clamp:
5395 case Intrinsic::nvvm_suld_1d_i32_trap:
5396 case Intrinsic::nvvm_suld_1d_v2i32_trap:
5397 case Intrinsic::nvvm_suld_1d_v4i32_trap:
5398 case Intrinsic::nvvm_suld_1d_array_i32_trap:
5399 case Intrinsic::nvvm_suld_1d_array_v2i32_trap:
5400 case Intrinsic::nvvm_suld_1d_array_v4i32_trap:
5401 case Intrinsic::nvvm_suld_2d_i32_trap:
5402 case Intrinsic::nvvm_suld_2d_v2i32_trap:
5403 case Intrinsic::nvvm_suld_2d_v4i32_trap:
5404 case Intrinsic::nvvm_suld_2d_array_i32_trap:
5405 case Intrinsic::nvvm_suld_2d_array_v2i32_trap:
5406 case Intrinsic::nvvm_suld_2d_array_v4i32_trap:
5407 case Intrinsic::nvvm_suld_3d_i32_trap:
5408 case Intrinsic::nvvm_suld_3d_v2i32_trap:
5409 case Intrinsic::nvvm_suld_3d_v4i32_trap:
5410 case Intrinsic::nvvm_suld_1d_i32_zero:
5411 case Intrinsic::nvvm_suld_1d_v2i32_zero:
5412 case Intrinsic::nvvm_suld_1d_v4i32_zero:
5413 case Intrinsic::nvvm_suld_1d_array_i32_zero:
5414 case Intrinsic::nvvm_suld_1d_array_v2i32_zero:
5415 case Intrinsic::nvvm_suld_1d_array_v4i32_zero:
5416 case Intrinsic::nvvm_suld_2d_i32_zero:
5417 case Intrinsic::nvvm_suld_2d_v2i32_zero:
5418 case Intrinsic::nvvm_suld_2d_v4i32_zero:
5419 case Intrinsic::nvvm_suld_2d_array_i32_zero:
5420 case Intrinsic::nvvm_suld_2d_array_v2i32_zero:
5421 case Intrinsic::nvvm_suld_2d_array_v4i32_zero:
5422 case Intrinsic::nvvm_suld_3d_i32_zero:
5423 case Intrinsic::nvvm_suld_3d_v2i32_zero:
5424 case Intrinsic::nvvm_suld_3d_v4i32_zero:
5425 Info.opc = ISD::INTRINSIC_W_CHAIN;
5426 Info.memVT = MVT::i32;
5427 Info.ptrVal = nullptr;
5428 Info.offset = 0;
5429 Info.flags = MachineMemOperand::MOLoad;
5430 Info.align = Align(16);
5431 Infos.push_back(Elt: Info);
5432 return;
5433
5434 case Intrinsic::nvvm_suld_1d_i64_clamp:
5435 case Intrinsic::nvvm_suld_1d_v2i64_clamp:
5436 case Intrinsic::nvvm_suld_1d_array_i64_clamp:
5437 case Intrinsic::nvvm_suld_1d_array_v2i64_clamp:
5438 case Intrinsic::nvvm_suld_2d_i64_clamp:
5439 case Intrinsic::nvvm_suld_2d_v2i64_clamp:
5440 case Intrinsic::nvvm_suld_2d_array_i64_clamp:
5441 case Intrinsic::nvvm_suld_2d_array_v2i64_clamp:
5442 case Intrinsic::nvvm_suld_3d_i64_clamp:
5443 case Intrinsic::nvvm_suld_3d_v2i64_clamp:
5444 case Intrinsic::nvvm_suld_1d_i64_trap:
5445 case Intrinsic::nvvm_suld_1d_v2i64_trap:
5446 case Intrinsic::nvvm_suld_1d_array_i64_trap:
5447 case Intrinsic::nvvm_suld_1d_array_v2i64_trap:
5448 case Intrinsic::nvvm_suld_2d_i64_trap:
5449 case Intrinsic::nvvm_suld_2d_v2i64_trap:
5450 case Intrinsic::nvvm_suld_2d_array_i64_trap:
5451 case Intrinsic::nvvm_suld_2d_array_v2i64_trap:
5452 case Intrinsic::nvvm_suld_3d_i64_trap:
5453 case Intrinsic::nvvm_suld_3d_v2i64_trap:
5454 case Intrinsic::nvvm_suld_1d_i64_zero:
5455 case Intrinsic::nvvm_suld_1d_v2i64_zero:
5456 case Intrinsic::nvvm_suld_1d_array_i64_zero:
5457 case Intrinsic::nvvm_suld_1d_array_v2i64_zero:
5458 case Intrinsic::nvvm_suld_2d_i64_zero:
5459 case Intrinsic::nvvm_suld_2d_v2i64_zero:
5460 case Intrinsic::nvvm_suld_2d_array_i64_zero:
5461 case Intrinsic::nvvm_suld_2d_array_v2i64_zero:
5462 case Intrinsic::nvvm_suld_3d_i64_zero:
5463 case Intrinsic::nvvm_suld_3d_v2i64_zero:
5464 Info.opc = ISD::INTRINSIC_W_CHAIN;
5465 Info.memVT = MVT::i64;
5466 Info.ptrVal = nullptr;
5467 Info.offset = 0;
5468 Info.flags = MachineMemOperand::MOLoad;
5469 Info.align = Align(16);
5470 Infos.push_back(Elt: Info);
5471 return;
5472
5473 case Intrinsic::nvvm_tcgen05_ld_16x64b_x1:
5474 case Intrinsic::nvvm_tcgen05_ld_32x32b_x1:
5475 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x1: {
5476 Info.opc = ISD::INTRINSIC_W_CHAIN;
5477 Info.memVT = MVT::v1i32;
5478 Info.ptrVal = I.getArgOperand(i: 0);
5479 Info.offset = 0;
5480 Info.flags = MachineMemOperand::MOLoad;
5481 Info.align.reset();
5482 Infos.push_back(Elt: Info);
5483 return;
5484 }
5485
5486 case Intrinsic::nvvm_tcgen05_ld_16x64b_x2:
5487 case Intrinsic::nvvm_tcgen05_ld_16x128b_x1:
5488 case Intrinsic::nvvm_tcgen05_ld_32x32b_x2:
5489 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x2:
5490 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x2_i32:
5491 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x2_i32: {
5492 Info.opc = ISD::INTRINSIC_W_CHAIN;
5493 Info.memVT = MVT::v2i32;
5494 Info.ptrVal = I.getArgOperand(i: 0);
5495 Info.offset = 0;
5496 Info.flags = MachineMemOperand::MOLoad;
5497 Info.align.reset();
5498 Infos.push_back(Elt: Info);
5499 return;
5500 }
5501
5502 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x2_f32:
5503 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x2_f32: {
5504 Info.opc = ISD::INTRINSIC_W_CHAIN;
5505 Info.memVT = MVT::v2f32;
5506 Info.ptrVal = I.getArgOperand(i: 0);
5507 Info.offset = 0;
5508 Info.flags = MachineMemOperand::MOLoad;
5509 Info.align.reset();
5510 Infos.push_back(Elt: Info);
5511 return;
5512 }
5513
5514 case Intrinsic::nvvm_tcgen05_ld_16x64b_x4:
5515 case Intrinsic::nvvm_tcgen05_ld_16x128b_x2:
5516 case Intrinsic::nvvm_tcgen05_ld_32x32b_x4:
5517 case Intrinsic::nvvm_tcgen05_ld_16x256b_x1:
5518 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x4:
5519 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x4_i32:
5520 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x4_i32: {
5521 Info.opc = ISD::INTRINSIC_W_CHAIN;
5522 Info.memVT = MVT::v4i32;
5523 Info.ptrVal = I.getArgOperand(i: 0);
5524 Info.offset = 0;
5525 Info.flags = MachineMemOperand::MOLoad;
5526 Info.align.reset();
5527 Infos.push_back(Elt: Info);
5528 return;
5529 }
5530
5531 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x4_f32:
5532 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x4_f32: {
5533 Info.opc = ISD::INTRINSIC_W_CHAIN;
5534 Info.memVT = MVT::v4f32;
5535 Info.ptrVal = I.getArgOperand(i: 0);
5536 Info.offset = 0;
5537 Info.flags = MachineMemOperand::MOLoad;
5538 Info.align.reset();
5539 Infos.push_back(Elt: Info);
5540 return;
5541 }
5542
5543 case Intrinsic::nvvm_tcgen05_ld_16x64b_x8:
5544 case Intrinsic::nvvm_tcgen05_ld_16x128b_x4:
5545 case Intrinsic::nvvm_tcgen05_ld_16x256b_x2:
5546 case Intrinsic::nvvm_tcgen05_ld_32x32b_x8:
5547 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x8:
5548 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x8_i32:
5549 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x8_i32: {
5550 Info.opc = ISD::INTRINSIC_W_CHAIN;
5551 Info.memVT = MVT::v8i32;
5552 Info.ptrVal = I.getArgOperand(i: 0);
5553 Info.offset = 0;
5554 Info.flags = MachineMemOperand::MOLoad;
5555 Info.align.reset();
5556 Infos.push_back(Elt: Info);
5557 return;
5558 }
5559
5560 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x8_f32:
5561 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x8_f32: {
5562 Info.opc = ISD::INTRINSIC_W_CHAIN;
5563 Info.memVT = MVT::v8f32;
5564 Info.ptrVal = I.getArgOperand(i: 0);
5565 Info.offset = 0;
5566 Info.flags = MachineMemOperand::MOLoad;
5567 Info.align.reset();
5568 Infos.push_back(Elt: Info);
5569 return;
5570 }
5571
5572 case Intrinsic::nvvm_tcgen05_ld_16x64b_x16:
5573 case Intrinsic::nvvm_tcgen05_ld_16x128b_x8:
5574 case Intrinsic::nvvm_tcgen05_ld_16x256b_x4:
5575 case Intrinsic::nvvm_tcgen05_ld_32x32b_x16:
5576 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x16:
5577 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x16_i32:
5578 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x16_i32: {
5579 Info.opc = ISD::INTRINSIC_W_CHAIN;
5580 Info.memVT = MVT::v16i32;
5581 Info.ptrVal = I.getArgOperand(i: 0);
5582 Info.offset = 0;
5583 Info.flags = MachineMemOperand::MOLoad;
5584 Info.align.reset();
5585 Infos.push_back(Elt: Info);
5586 return;
5587 }
5588
5589 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x16_f32:
5590 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x16_f32: {
5591 Info.opc = ISD::INTRINSIC_W_CHAIN;
5592 Info.memVT = MVT::v16f32;
5593 Info.ptrVal = I.getArgOperand(i: 0);
5594 Info.offset = 0;
5595 Info.flags = MachineMemOperand::MOLoad;
5596 Info.align.reset();
5597 Infos.push_back(Elt: Info);
5598 return;
5599 }
5600
5601 case Intrinsic::nvvm_tcgen05_ld_16x64b_x32:
5602 case Intrinsic::nvvm_tcgen05_ld_16x128b_x16:
5603 case Intrinsic::nvvm_tcgen05_ld_16x256b_x8:
5604 case Intrinsic::nvvm_tcgen05_ld_32x32b_x32:
5605 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x32:
5606 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x32_i32:
5607 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x32_i32: {
5608 Info.opc = ISD::INTRINSIC_W_CHAIN;
5609 Info.memVT = MVT::v32i32;
5610 Info.ptrVal = I.getArgOperand(i: 0);
5611 Info.offset = 0;
5612 Info.flags = MachineMemOperand::MOLoad;
5613 Info.align.reset();
5614 Infos.push_back(Elt: Info);
5615 return;
5616 }
5617
5618 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x32_f32:
5619 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x32_f32: {
5620 Info.opc = ISD::INTRINSIC_W_CHAIN;
5621 Info.memVT = MVT::v32f32;
5622 Info.ptrVal = I.getArgOperand(i: 0);
5623 Info.offset = 0;
5624 Info.flags = MachineMemOperand::MOLoad;
5625 Info.align.reset();
5626 Infos.push_back(Elt: Info);
5627 return;
5628 }
5629
5630 case Intrinsic::nvvm_tcgen05_ld_16x64b_x64:
5631 case Intrinsic::nvvm_tcgen05_ld_16x128b_x32:
5632 case Intrinsic::nvvm_tcgen05_ld_16x256b_x16:
5633 case Intrinsic::nvvm_tcgen05_ld_32x32b_x64:
5634 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x64:
5635 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x64_i32:
5636 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x64_i32: {
5637 Info.opc = ISD::INTRINSIC_W_CHAIN;
5638 Info.memVT = MVT::v64i32;
5639 Info.ptrVal = I.getArgOperand(i: 0);
5640 Info.offset = 0;
5641 Info.flags = MachineMemOperand::MOLoad;
5642 Info.align.reset();
5643 Infos.push_back(Elt: Info);
5644 return;
5645 }
5646
5647 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x64_f32:
5648 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x64_f32: {
5649 Info.opc = ISD::INTRINSIC_W_CHAIN;
5650 Info.memVT = MVT::v64f32;
5651 Info.ptrVal = I.getArgOperand(i: 0);
5652 Info.offset = 0;
5653 Info.flags = MachineMemOperand::MOLoad;
5654 Info.align.reset();
5655 Infos.push_back(Elt: Info);
5656 return;
5657 }
5658
5659 case Intrinsic::nvvm_tcgen05_ld_16x64b_x128:
5660 case Intrinsic::nvvm_tcgen05_ld_16x128b_x64:
5661 case Intrinsic::nvvm_tcgen05_ld_16x256b_x32:
5662 case Intrinsic::nvvm_tcgen05_ld_32x32b_x128:
5663 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x128:
5664 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x128_i32:
5665 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x128_i32: {
5666 Info.opc = ISD::INTRINSIC_W_CHAIN;
5667 Info.memVT = MVT::v128i32;
5668 Info.ptrVal = I.getArgOperand(i: 0);
5669 Info.offset = 0;
5670 Info.flags = MachineMemOperand::MOLoad;
5671 Info.align.reset();
5672 Infos.push_back(Elt: Info);
5673 return;
5674 }
5675
5676 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x128_f32:
5677 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x128_f32: {
5678 Info.opc = ISD::INTRINSIC_W_CHAIN;
5679 Info.memVT = MVT::v128f32;
5680 Info.ptrVal = I.getArgOperand(i: 0);
5681 Info.offset = 0;
5682 Info.flags = MachineMemOperand::MOLoad;
5683 Info.align.reset();
5684 Infos.push_back(Elt: Info);
5685 return;
5686 }
5687
5688 case Intrinsic::nvvm_tcgen05_st_16x64b_x1:
5689 case Intrinsic::nvvm_tcgen05_st_32x32b_x1:
5690 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x1: {
5691 Info.opc = ISD::INTRINSIC_VOID;
5692 Info.memVT = MVT::v1i32;
5693 Info.ptrVal = I.getArgOperand(i: 0);
5694 Info.offset = 0;
5695 Info.flags = MachineMemOperand::MOStore;
5696 Info.align.reset();
5697 Infos.push_back(Elt: Info);
5698 return;
5699 }
5700
5701 case Intrinsic::nvvm_tcgen05_st_16x64b_x2:
5702 case Intrinsic::nvvm_tcgen05_st_16x128b_x1:
5703 case Intrinsic::nvvm_tcgen05_st_32x32b_x2:
5704 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x2: {
5705 Info.opc = ISD::INTRINSIC_VOID;
5706 Info.memVT = MVT::v2i32;
5707 Info.ptrVal = I.getArgOperand(i: 0);
5708 Info.offset = 0;
5709 Info.flags = MachineMemOperand::MOStore;
5710 Info.align.reset();
5711 Infos.push_back(Elt: Info);
5712 return;
5713 }
5714
5715 case Intrinsic::nvvm_tcgen05_st_16x64b_x4:
5716 case Intrinsic::nvvm_tcgen05_st_16x128b_x2:
5717 case Intrinsic::nvvm_tcgen05_st_16x256b_x1:
5718 case Intrinsic::nvvm_tcgen05_st_32x32b_x4:
5719 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x4: {
5720 Info.opc = ISD::INTRINSIC_VOID;
5721 Info.memVT = MVT::v4i32;
5722 Info.ptrVal = I.getArgOperand(i: 0);
5723 Info.offset = 0;
5724 Info.flags = MachineMemOperand::MOStore;
5725 Info.align.reset();
5726 Infos.push_back(Elt: Info);
5727 return;
5728 }
5729
5730 case Intrinsic::nvvm_tcgen05_st_16x64b_x8:
5731 case Intrinsic::nvvm_tcgen05_st_16x128b_x4:
5732 case Intrinsic::nvvm_tcgen05_st_16x256b_x2:
5733 case Intrinsic::nvvm_tcgen05_st_32x32b_x8:
5734 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x8: {
5735 Info.opc = ISD::INTRINSIC_VOID;
5736 Info.memVT = MVT::v8i32;
5737 Info.ptrVal = I.getArgOperand(i: 0);
5738 Info.offset = 0;
5739 Info.flags = MachineMemOperand::MOStore;
5740 Info.align.reset();
5741 Infos.push_back(Elt: Info);
5742 return;
5743 }
5744
5745 case Intrinsic::nvvm_tcgen05_st_16x64b_x16:
5746 case Intrinsic::nvvm_tcgen05_st_16x128b_x8:
5747 case Intrinsic::nvvm_tcgen05_st_16x256b_x4:
5748 case Intrinsic::nvvm_tcgen05_st_32x32b_x16:
5749 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x16: {
5750 Info.opc = ISD::INTRINSIC_VOID;
5751 Info.memVT = MVT::v16i32;
5752 Info.ptrVal = I.getArgOperand(i: 0);
5753 Info.offset = 0;
5754 Info.flags = MachineMemOperand::MOStore;
5755 Info.align.reset();
5756 Infos.push_back(Elt: Info);
5757 return;
5758 }
5759
5760 case Intrinsic::nvvm_tcgen05_st_16x64b_x32:
5761 case Intrinsic::nvvm_tcgen05_st_16x128b_x16:
5762 case Intrinsic::nvvm_tcgen05_st_16x256b_x8:
5763 case Intrinsic::nvvm_tcgen05_st_32x32b_x32:
5764 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x32: {
5765 Info.opc = ISD::INTRINSIC_VOID;
5766 Info.memVT = MVT::v32i32;
5767 Info.ptrVal = I.getArgOperand(i: 0);
5768 Info.offset = 0;
5769 Info.flags = MachineMemOperand::MOStore;
5770 Info.align.reset();
5771 Infos.push_back(Elt: Info);
5772 return;
5773 }
5774
5775 case Intrinsic::nvvm_tcgen05_st_16x64b_x64:
5776 case Intrinsic::nvvm_tcgen05_st_16x128b_x32:
5777 case Intrinsic::nvvm_tcgen05_st_16x256b_x16:
5778 case Intrinsic::nvvm_tcgen05_st_32x32b_x64:
5779 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x64: {
5780 Info.opc = ISD::INTRINSIC_VOID;
5781 Info.memVT = MVT::v64i32;
5782 Info.ptrVal = I.getArgOperand(i: 0);
5783 Info.offset = 0;
5784 Info.flags = MachineMemOperand::MOStore;
5785 Info.align.reset();
5786 Infos.push_back(Elt: Info);
5787 return;
5788 }
5789
5790 case Intrinsic::nvvm_tcgen05_st_16x64b_x128:
5791 case Intrinsic::nvvm_tcgen05_st_16x128b_x64:
5792 case Intrinsic::nvvm_tcgen05_st_16x256b_x32:
5793 case Intrinsic::nvvm_tcgen05_st_32x32b_x128:
5794 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x128: {
5795 Info.opc = ISD::INTRINSIC_VOID;
5796 Info.memVT = MVT::v128i32;
5797 Info.ptrVal = I.getArgOperand(i: 0);
5798 Info.offset = 0;
5799 Info.flags = MachineMemOperand::MOStore;
5800 Info.align.reset();
5801 Infos.push_back(Elt: Info);
5802 return;
5803 }
5804 case Intrinsic::
5805 nvvm_tcgen05_mma_shared_f8f6f4_disable_output_lane_cg1_decompress_b:
5806 case Intrinsic::
5807 nvvm_tcgen05_mma_tensor_f8f6f4_disable_output_lane_cg1_decompress_b:
5808 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg1:
5809 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg1:
5810 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg1:
5811 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg1:
5812 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1:
5813 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1:
5814 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1_ashift:
5815 case Intrinsic::
5816 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1_ashift:
5817 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1:
5818 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1:
5819 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1_ashift:
5820 case Intrinsic::
5821 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1_ashift: {
5822 // We are reading and writing back to TMem
5823 Info.opc = ISD::INTRINSIC_VOID;
5824 Info.memVT = MVT::v4i32;
5825 Info.ptrVal = I.getArgOperand(i: 0);
5826 Info.offset = 0;
5827 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
5828 Info.align = Align(16);
5829 Infos.push_back(Elt: Info);
5830 return;
5831 }
5832
5833 case Intrinsic::
5834 nvvm_tcgen05_mma_shared_f8f6f4_disable_output_lane_cg2_decompress_b:
5835 case Intrinsic::
5836 nvvm_tcgen05_mma_tensor_f8f6f4_disable_output_lane_cg2_decompress_b:
5837 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg2:
5838 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg2:
5839 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg2:
5840 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg2:
5841 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2:
5842 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2:
5843 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2:
5844 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2:
5845 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2_ashift:
5846 case Intrinsic::
5847 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2_ashift:
5848 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2_ashift:
5849 case Intrinsic::
5850 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2_ashift: {
5851 // We are reading and writing back to TMem
5852 Info.opc = ISD::INTRINSIC_VOID;
5853 Info.memVT = MVT::v8i32;
5854 Info.ptrVal = I.getArgOperand(i: 0);
5855 Info.offset = 0;
5856 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
5857 Info.align = Align(16);
5858 Infos.push_back(Elt: Info);
5859 return;
5860 }
5861 case Intrinsic::nvvm_tcgen05_alloc_cg1:
5862 case Intrinsic::nvvm_tcgen05_alloc_cg2:
5863 Info.opc = ISD::INTRINSIC_VOID;
5864 Info.memVT = MVT::i32;
5865 Info.ptrVal = I.getArgOperand(i: 0);
5866 Info.offset = 0;
5867 Info.flags = MachineMemOperand::MOStore;
5868 Info.align = Align(4);
5869 Infos.push_back(Elt: Info);
5870 return;
5871 }
5872}
5873
5874// Helper for getting a function parameter symbol. Its name is composed from
5875// the function name and the parameter index. Negative index corresponds to the
5876// special parameter (unsized array) used for passing variable arguments.
5877MCSymbol *NVPTXTargetLowering::getParamSymbol(MCContext &Ctx, const Function *F,
5878 int Idx) const {
5879 const StringRef FuncName = getTargetMachine().getSymbol(GV: F)->getName();
5880 if (Idx < 0)
5881 return Ctx.getOrCreateSymbol(Name: FuncName + "_vararg");
5882 return Ctx.getOrCreateSymbol(Name: FuncName + "_param_" + Twine(Idx));
5883}
5884
5885/// isLegalAddressingMode - Return true if the addressing mode represented
5886/// by AM is legal for this target, for a load/store of the specified type.
5887/// Used to guide target specific optimizations, like loop strength reduction
5888/// (LoopStrengthReduce.cpp) and memory optimization for address mode
5889/// (CodeGenPrepare.cpp)
5890bool NVPTXTargetLowering::isLegalAddressingMode(const DataLayout &DL,
5891 const AddrMode &AM, Type *Ty,
5892 unsigned AS, Instruction *I) const {
5893 // AddrMode - This represents an addressing mode of:
5894 // BaseGV + BaseOffs + BaseReg + Scale*ScaleReg
5895 //
5896 // The legal address modes are
5897 // - [avar]
5898 // - [areg]
5899 // - [areg+immoff]
5900 // - [immAddr]
5901
5902 // immoff must fit in a signed 32-bit int
5903 if (!APInt(64, AM.BaseOffs).isSignedIntN(N: 32))
5904 return false;
5905
5906 if (AM.BaseGV)
5907 return !AM.BaseOffs && !AM.HasBaseReg && !AM.Scale;
5908
5909 switch (AM.Scale) {
5910 case 0: // "r", "r+i" or "i" is allowed
5911 break;
5912 case 1:
5913 if (AM.HasBaseReg) // "r+r+i" or "r+r" is not allowed.
5914 return false;
5915 // Otherwise we have r+i.
5916 break;
5917 default:
5918 // No scale > 1 is allowed
5919 return false;
5920 }
5921 return true;
5922}
5923
5924//===----------------------------------------------------------------------===//
5925// NVPTX Inline Assembly Support
5926//===----------------------------------------------------------------------===//
5927
5928/// getConstraintType - Given a constraint letter, return the type of
5929/// constraint it is for this target.
5930NVPTXTargetLowering::ConstraintType
5931NVPTXTargetLowering::getConstraintType(StringRef Constraint) const {
5932 if (Constraint.size() == 1) {
5933 switch (Constraint[0]) {
5934 default:
5935 break;
5936 case 'b':
5937 case 'r':
5938 case 'h':
5939 case 'c':
5940 case 'l':
5941 case 'f':
5942 case 'd':
5943 case 'q':
5944 case '0':
5945 case 'N':
5946 return C_RegisterClass;
5947 }
5948 }
5949 return TargetLowering::getConstraintType(Constraint);
5950}
5951
5952std::pair<unsigned, const TargetRegisterClass *>
5953NVPTXTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
5954 StringRef Constraint,
5955 MVT VT) const {
5956 if (Constraint.size() == 1) {
5957 switch (Constraint[0]) {
5958 case 'b':
5959 return std::make_pair(x: 0U, y: &NVPTX::B1RegClass);
5960 case 'c':
5961 case 'h':
5962 return std::make_pair(x: 0U, y: &NVPTX::B16RegClass);
5963 case 'r':
5964 case 'f':
5965 return std::make_pair(x: 0U, y: &NVPTX::B32RegClass);
5966 case 'l':
5967 case 'N':
5968 case 'd':
5969 return std::make_pair(x: 0U, y: &NVPTX::B64RegClass);
5970 case 'q': {
5971 if (!STI.hasFeature(Feature: NVPTX::SM70))
5972 report_fatal_error(reason: "Inline asm with 128 bit operands is only "
5973 "supported for sm_70 and higher!");
5974 return std::make_pair(x: 0U, y: &NVPTX::B128RegClass);
5975 }
5976 }
5977 }
5978 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
5979}
5980
5981//===----------------------------------------------------------------------===//
5982// NVPTX DAG Combining
5983//===----------------------------------------------------------------------===//
5984
5985bool NVPTXTargetLowering::allowFMA(MachineFunction &MF,
5986 CodeGenOptLevel OptLevel) const {
5987 // Always honor command-line argument
5988 if (auto FMAContractLevel = STI.getCLOpts().fma_level)
5989 return *FMAContractLevel > 0;
5990
5991 // Do not contract if we're not optimizing the code.
5992 if (OptLevel == CodeGenOptLevel::None)
5993 return false;
5994
5995 return false;
5996}
5997
5998static bool isConstZero(const SDValue &Operand) {
5999 const auto *Const = dyn_cast<ConstantSDNode>(Val: Operand);
6000 return Const && Const->getZExtValue() == 0;
6001}
6002
6003/// PerformADDCombineWithOperands - Try DAG combinations for an ADD with
6004/// operands N0 and N1. This is a helper for PerformADDCombine that is
6005/// called with the default operands, and if that fails, with commuted
6006/// operands.
6007static SDValue
6008PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
6009 TargetLowering::DAGCombinerInfo &DCI) {
6010 EVT VT = N0.getValueType();
6011
6012 // Since integer multiply-add costs the same as integer multiply
6013 // but is more costly than integer add, do the fusion only when
6014 // the mul is only used in the add.
6015 // TODO: this may not be true for later architectures, consider relaxing this
6016 if (!N0.getNode()->hasOneUse())
6017 return SDValue();
6018
6019 // fold (add (select cond, 0, (mul a, b)), c)
6020 // -> (select cond, c, (add (mul a, b), c))
6021 //
6022 if (N0.getOpcode() == ISD::SELECT) {
6023 unsigned ZeroOpNum;
6024 if (isConstZero(Operand: N0->getOperand(Num: 1)))
6025 ZeroOpNum = 1;
6026 else if (isConstZero(Operand: N0->getOperand(Num: 2)))
6027 ZeroOpNum = 2;
6028 else
6029 return SDValue();
6030
6031 SDValue M = N0->getOperand(Num: (ZeroOpNum == 1) ? 2 : 1);
6032 if (M->getOpcode() != ISD::MUL || !M.getNode()->hasOneUse())
6033 return SDValue();
6034
6035 SDLoc DL(N);
6036 SDValue Mul =
6037 DCI.DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: M->getOperand(Num: 0), N2: M->getOperand(Num: 1));
6038 SDValue MAD = DCI.DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Mul, N2: N1);
6039 return DCI.DAG.getSelect(DL: SDLoc(N), VT, Cond: N0->getOperand(Num: 0),
6040 LHS: ((ZeroOpNum == 1) ? N1 : MAD),
6041 RHS: ((ZeroOpNum == 1) ? MAD : N1));
6042 }
6043
6044 return SDValue();
6045}
6046
6047SDValue NVPTXTargetLowering::performFADDCombineWithOperands(
6048 SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI,
6049 CodeGenOptLevel OptLevel) const {
6050 EVT VT = N0.getValueType();
6051 if (N0.getOpcode() == ISD::FMUL) {
6052 if (!(allowFMA(MF&: DCI.DAG.getMachineFunction(), OptLevel) ||
6053 (N->getFlags().hasAllowContract() &&
6054 N0->getFlags().hasAllowContract())))
6055 return SDValue();
6056
6057 // For floating point:
6058 // Do the fusion only when the mul has less than 5 uses and all
6059 // are add.
6060 // The heuristic is that if a use is not an add, then that use
6061 // cannot be fused into fma, therefore mul is still needed anyway.
6062 // If there are more than 4 uses, even if they are all add, fusing
6063 // them will increase register pressue.
6064 //
6065 int numUses = 0;
6066 int nonAddCount = 0;
6067 for (const SDNode *User : N0.getNode()->users()) {
6068 numUses++;
6069 if (User->getOpcode() != ISD::FADD)
6070 ++nonAddCount;
6071 if (numUses >= 5)
6072 return SDValue();
6073 }
6074 if (nonAddCount) {
6075 int orderNo = N->getIROrder();
6076 int orderNo2 = N0.getNode()->getIROrder();
6077 // simple heuristics here for considering potential register
6078 // pressure, the logics here is that the differnce are used
6079 // to measure the distance between def and use, the longer distance
6080 // more likely cause register pressure.
6081 if (orderNo - orderNo2 < 500)
6082 return SDValue();
6083
6084 // Now, check if at least one of the FMUL's operands is live beyond the
6085 // node N, which guarantees that the FMA will not increase register
6086 // pressure at node N.
6087 bool opIsLive = false;
6088 const SDNode *left = N0.getOperand(i: 0).getNode();
6089 const SDNode *right = N0.getOperand(i: 1).getNode();
6090
6091 if (isa<ConstantSDNode>(Val: left) || isa<ConstantSDNode>(Val: right))
6092 opIsLive = true;
6093
6094 if (!opIsLive)
6095 for (const SDNode *User : left->users()) {
6096 int orderNo3 = User->getIROrder();
6097 if (orderNo3 > orderNo) {
6098 opIsLive = true;
6099 break;
6100 }
6101 }
6102
6103 if (!opIsLive)
6104 for (const SDNode *User : right->users()) {
6105 int orderNo3 = User->getIROrder();
6106 if (orderNo3 > orderNo) {
6107 opIsLive = true;
6108 break;
6109 }
6110 }
6111
6112 if (!opIsLive)
6113 return SDValue();
6114 }
6115
6116 return DCI.DAG.getNode(Opcode: ISD::FMA, DL: SDLoc(N), VT, N1: N0.getOperand(i: 0),
6117 N2: N0.getOperand(i: 1), N3: N1);
6118 }
6119
6120 return SDValue();
6121}
6122
6123/// Fold unpacking movs into a load by increasing the number of return values.
6124///
6125/// ex:
6126/// L: v2f16,ch = load <p>
6127/// a: f16 = extractelt L:0, 0
6128/// b: f16 = extractelt L:0, 1
6129/// use(a, b)
6130///
6131/// ...is turned into...
6132///
6133/// L: f16,f16,ch = LoadV2 <p>
6134/// use(L:0, L:1)
6135static SDValue
6136combineUnpackingMovIntoLoad(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
6137 // Don't run this optimization before the legalizer
6138 if (!DCI.isAfterLegalizeDAG())
6139 return SDValue();
6140
6141 EVT ElementVT = N->getValueType(ResNo: 0);
6142 // Avoid non-packed types and v4i8
6143 if (!NVPTX::isPackedVectorTy(VT: ElementVT) || ElementVT == MVT::v4i8)
6144 return SDValue();
6145
6146 // Check whether all outputs are either used by an extractelt or are
6147 // glue/chain nodes
6148 if (!all_of(Range: N->uses(), P: [&](SDUse &U) {
6149 // Skip glue, chain nodes
6150 if (U.getValueType() == MVT::Glue || U.getValueType() == MVT::Other)
6151 return true;
6152 if (U.getUser()->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
6153 if (N->getOpcode() != ISD::LOAD)
6154 return true;
6155 // Since this is an ISD::LOAD, check all extractelts are used. If
6156 // any are not used, we don't want to defeat another optimization that
6157 // will narrow the load.
6158 //
6159 // For example:
6160 //
6161 // L: v2f16,ch = load <p>
6162 // e0: f16 = extractelt L:0, 0
6163 // e1: f16 = extractelt L:0, 1 <-- unused
6164 // store e0
6165 //
6166 // Can be optimized by DAGCombiner to:
6167 //
6168 // L: f16,ch = load <p>
6169 // store L:0
6170 return !U.getUser()->use_empty();
6171 }
6172
6173 // Otherwise, this use prevents us from splitting a value.
6174 return false;
6175 }))
6176 return SDValue();
6177
6178 auto *LD = cast<MemSDNode>(Val: N);
6179 SDLoc DL(LD);
6180
6181 // the new opcode after we double the number of operands
6182 unsigned Opcode;
6183 SmallVector<SDValue> Operands(LD->ops());
6184 unsigned OldNumOutputs; // non-glue, non-chain outputs
6185 switch (LD->getOpcode()) {
6186 case ISD::LOAD:
6187 OldNumOutputs = 1;
6188 // Any packed type is legal, so the legalizer will not have lowered
6189 // ISD::LOAD -> NVPTXISD::Load (unless it's under-aligned). We have to do it
6190 // here.
6191 Opcode = NVPTXISD::LoadV2;
6192 // append a "full" used bytes mask operand right before the extension type
6193 // operand, signifying that all bytes are used.
6194 Operands.push_back(Elt: DCI.DAG.getConstant(UINT32_MAX, DL, VT: MVT::i32));
6195 Operands.push_back(Elt: DCI.DAG.getIntPtrConstant(
6196 Val: cast<LoadSDNode>(Val: LD)->getExtensionType(), DL));
6197 break;
6198 case NVPTXISD::LoadV2:
6199 OldNumOutputs = 2;
6200 Opcode = NVPTXISD::LoadV4;
6201 break;
6202 case NVPTXISD::LoadV4:
6203 // V8 is only supported for f32/i32. Don't forget, we're not changing the
6204 // load size here. This is already a 256-bit load.
6205 if (ElementVT != MVT::v2f32 && ElementVT != MVT::v2i32)
6206 return SDValue();
6207 OldNumOutputs = 4;
6208 Opcode = NVPTXISD::LoadV8;
6209 break;
6210 case NVPTXISD::LoadV8:
6211 // PTX doesn't support the next doubling of outputs
6212 return SDValue();
6213 }
6214
6215 // the non-glue, non-chain outputs in the new load
6216 const unsigned NewNumOutputs = OldNumOutputs * 2;
6217 SmallVector<EVT> NewVTs(NewNumOutputs, ElementVT.getVectorElementType());
6218 // add remaining chain and glue values
6219 NewVTs.append(in_start: LD->value_begin() + OldNumOutputs, in_end: LD->value_end());
6220
6221 // Create the new load
6222 SDValue NewLoad = DCI.DAG.getMemIntrinsicNode(
6223 Opcode, dl: DL, VTList: DCI.DAG.getVTList(VTs: NewVTs), Ops: Operands, MemVT: LD->getMemoryVT(),
6224 MMO: LD->getMemOperand());
6225
6226 // Now we use a combination of BUILD_VECTORs and a MERGE_VALUES node to keep
6227 // the outputs the same. These nodes will be optimized away in later
6228 // DAGCombiner iterations.
6229 SmallVector<SDValue> Results;
6230 for (unsigned I : seq(Size: OldNumOutputs))
6231 Results.push_back(Elt: DCI.DAG.getBuildVector(
6232 VT: ElementVT, DL, Ops: {NewLoad.getValue(R: I * 2), NewLoad.getValue(R: I * 2 + 1)}));
6233 // Add remaining chain and glue nodes
6234 for (unsigned I : seq(Size: NewLoad->getNumValues() - NewNumOutputs))
6235 Results.push_back(Elt: NewLoad.getValue(R: NewNumOutputs + I));
6236
6237 return DCI.DAG.getMergeValues(Ops: Results, dl: DL);
6238}
6239
6240/// Fold packing movs into a store.
6241///
6242/// ex:
6243/// v1: v2f16 = BUILD_VECTOR a:f16, b:f16
6244/// v2: v2f16 = BUILD_VECTOR c:f16, d:f16
6245/// StoreV2 v1, v2
6246///
6247/// ...is turned into...
6248///
6249/// StoreV4 a, b, c, d
6250static SDValue combinePackingMovIntoStore(SDNode *N,
6251 TargetLowering::DAGCombinerInfo &DCI,
6252 unsigned Front, unsigned Back) {
6253 // We want to run this as late as possible since other optimizations may
6254 // eliminate the BUILD_VECTORs.
6255 if (!DCI.isAfterLegalizeDAG())
6256 return SDValue();
6257
6258 // Get the type of the operands being stored.
6259 EVT ElementVT = N->getOperand(Num: Front).getValueType();
6260
6261 // Avoid non-packed types and v4i8
6262 if (!NVPTX::isPackedVectorTy(VT: ElementVT) || ElementVT == MVT::v4i8)
6263 return SDValue();
6264
6265 auto *ST = cast<MemSDNode>(Val: N);
6266
6267 // The new opcode after we double the number of operands.
6268 unsigned Opcode;
6269 switch (N->getOpcode()) {
6270 case ISD::STORE:
6271 // Any packed type is legal, so the legalizer will not have lowered
6272 // ISD::STORE -> NVPTXISD::Store (unless it's under-aligned). We have to do
6273 // it here.
6274 Opcode = NVPTXISD::StoreV2;
6275 break;
6276 case NVPTXISD::StoreV2:
6277 Opcode = NVPTXISD::StoreV4;
6278 break;
6279 case NVPTXISD::StoreV4:
6280 // V8 is only supported for f32/i32. Don't forget, we're not changing the
6281 // store size here. This is already a 256-bit store.
6282 if (ElementVT != MVT::v2f32 && ElementVT != MVT::v2i32)
6283 return SDValue();
6284 Opcode = NVPTXISD::StoreV8;
6285 break;
6286 case NVPTXISD::StoreV8:
6287 // PTX doesn't support the next doubling of operands
6288 return SDValue();
6289 default:
6290 llvm_unreachable("Unhandled store opcode");
6291 }
6292
6293 // Scan the operands and if they're all BUILD_VECTORs, we'll have gathered
6294 // their elements.
6295 SmallVector<SDValue, 4> Operands(N->ops().take_front(N: Front));
6296 for (SDValue BV : N->ops().drop_front(N: Front).drop_back(N: Back)) {
6297 if (BV.getOpcode() != ISD::BUILD_VECTOR)
6298 return SDValue();
6299
6300 // If the operand has multiple uses, this optimization can increase register
6301 // pressure.
6302 if (!BV.hasOneUse())
6303 return SDValue();
6304
6305 // DAGCombiner visits nodes bottom-up. Check the BUILD_VECTOR operands for
6306 // any signs they may be folded by some other pattern or rule.
6307 for (SDValue Op : BV->ops()) {
6308 // Peek through bitcasts
6309 if (Op.getOpcode() == ISD::BITCAST)
6310 Op = Op.getOperand(i: 0);
6311
6312 // This may be folded into a PRMT.
6313 if (Op.getValueType() == MVT::i16 && Op.getOpcode() == ISD::TRUNCATE &&
6314 Op->getOperand(Num: 0).getValueType() == MVT::i32)
6315 return SDValue();
6316
6317 // This may be folded into cvt.bf16x2
6318 if (Op.getOpcode() == ISD::FP_ROUND)
6319 return SDValue();
6320 }
6321 Operands.append(IL: {BV.getOperand(i: 0), BV.getOperand(i: 1)});
6322 }
6323 Operands.append(in_start: N->op_end() - Back, in_end: N->op_end());
6324
6325 // Now we replace the store
6326 return DCI.DAG.getMemIntrinsicNode(Opcode, dl: SDLoc(N), VTList: N->getVTList(), Ops: Operands,
6327 MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand());
6328}
6329
6330static SDValue combineSTORE(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
6331 const NVPTXSubtarget &STI) {
6332
6333 if (DCI.isBeforeLegalize() && N->getOpcode() == ISD::STORE) {
6334 // Here is our chance to custom lower a store with a non-simple type.
6335 // Unfortunately, we can't do this in the legalizer because there is no
6336 // way to setOperationAction for an non-simple type.
6337 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
6338 if (!ST->getValue().getValueType().isSimple())
6339 return lowerSTOREVector(Op: SDValue(ST, 0), DAG&: DCI.DAG, STI);
6340 }
6341
6342 return combinePackingMovIntoStore(N, DCI, Front: 1, Back: 2);
6343}
6344
6345static SDValue combineLOAD(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
6346 const NVPTXSubtarget &STI) {
6347 if (DCI.isBeforeLegalize() && N->getOpcode() == ISD::LOAD) {
6348 // Here is our chance to custom lower a load with a non-simple type.
6349 // Unfortunately, we can't do this in the legalizer because there is no
6350 // way to setOperationAction for an non-simple type.
6351 if (!N->getValueType(ResNo: 0).isSimple())
6352 return lowerLoadVector(N, DAG&: DCI.DAG, STI);
6353 }
6354
6355 return combineUnpackingMovIntoLoad(N, DCI);
6356}
6357
6358/// PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
6359///
6360static SDValue PerformADDCombine(SDNode *N,
6361 TargetLowering::DAGCombinerInfo &DCI,
6362 CodeGenOptLevel OptLevel) {
6363 if (OptLevel == CodeGenOptLevel::None)
6364 return SDValue();
6365
6366 SDValue N0 = N->getOperand(Num: 0);
6367 SDValue N1 = N->getOperand(Num: 1);
6368
6369 // Skip non-integer, non-scalar case
6370 EVT VT = N0.getValueType();
6371 if (VT.isVector() || VT != MVT::i32)
6372 return SDValue();
6373
6374 // First try with the default operand order.
6375 if (SDValue Result = PerformADDCombineWithOperands(N, N0, N1, DCI))
6376 return Result;
6377
6378 // If that didn't work, try again with the operands commuted.
6379 return PerformADDCombineWithOperands(N, N0: N1, N1: N0, DCI);
6380}
6381
6382/// Check if a v2f32 BUILD_VECTOR provably packs values from non-adjacent
6383/// register pairs (non-coalescable).
6384static bool isNonCoalescableBuildVector(const SDValue &BV) {
6385 if (BV.getOpcode() != ISD::BUILD_VECTOR || BV.getValueType() != MVT::v2f32)
6386 return false;
6387
6388 SDValue Elt0 = BV.getOperand(i: 0);
6389 SDValue Elt1 = BV.getOperand(i: 1);
6390
6391 bool IsExt0 = Elt0.getOpcode() == ISD::EXTRACT_VECTOR_ELT;
6392 bool IsExt1 = Elt1.getOpcode() == ISD::EXTRACT_VECTOR_ELT;
6393
6394 // If neither element is an EXTRACT_VECTOR_ELT they are free-standing
6395 // scalars and the register allocator can still place them side-by-side.
6396 if (!IsExt0 && !IsExt1)
6397 return false;
6398
6399 // If exactly one element is an EXTRACT_VECTOR_ELT, the other is a scalar
6400 // that cannot generally occupy the adjacent register slot.
6401 if (IsExt0 != IsExt1)
6402 return true;
6403
6404 // At this point both sources are extracting from vectors. If they are from
6405 // different vectors, then the BUILD_VECTOR is non-coalescable.
6406 SDValue Src0 = Elt0.getOperand(i: 0);
6407 SDValue Src1 = Elt1.getOperand(i: 0);
6408 if (Src0 != Src1)
6409 return true;
6410
6411 auto *Idx0 = dyn_cast<ConstantSDNode>(Val: Elt0.getOperand(i: 1));
6412 auto *Idx1 = dyn_cast<ConstantSDNode>(Val: Elt1.getOperand(i: 1));
6413 // If both indices are dynamic they will be lowered to
6414 // loads and the vector will be spilled to local memory. The register
6415 // allocator can easily place the results in adjacent registers.
6416 if (!Idx0 && !Idx1)
6417 return false;
6418
6419 // If one index is dynamic and the other is constant, the value from the
6420 // constant load will result in an additional register to pair with the result
6421 // from the dynamic load. We consider this non-coalescable.
6422 if ((Idx0 && !Idx1) || (!Idx0 && Idx1))
6423 return true;
6424
6425 // Both are constant, adjacent pairs are coalescable
6426 return std::abs(i: Idx0->getSExtValue() - Idx1->getSExtValue()) != 1;
6427}
6428
6429/// Return true if FMUL v2f32 node \p N may be scalarized to fold each lane's
6430/// product into a scalar FMA.
6431bool NVPTXTargetLowering::mayFoldFMULIntoFMA(SDNode *N, MachineFunction &MF,
6432 CodeGenOptLevel OptLevel) const {
6433 if (N->getOpcode() != ISD::FMUL || N->getValueType(ResNo: 0) != MVT::v2f32)
6434 return false;
6435 const bool GlobalFMA = allowFMA(MF, OptLevel);
6436 if (!N->getFlags().hasAllowContract() && !GlobalFMA)
6437 return false;
6438
6439 const SDNode *FirstFAdd = nullptr;
6440 unsigned NumScalarFAdd = 0;
6441
6442 // Both lanes must feed unique FADDs
6443 for (SDNode *EE : N->users()) {
6444 if (NumScalarFAdd == 2)
6445 return false;
6446
6447 if (EE->getOpcode() != ISD::EXTRACT_VECTOR_ELT || !EE->hasOneUse() ||
6448 !isa<ConstantSDNode>(Val: EE->getOperand(Num: 1)))
6449 return false;
6450
6451 const SDNode *const FAdd = *EE->users().begin();
6452 if (FAdd->getOpcode() != ISD::FADD ||
6453 (!GlobalFMA && !FAdd->getFlags().hasAllowContract()))
6454 return false;
6455
6456 if (!FirstFAdd)
6457 FirstFAdd = FAdd;
6458 else if (FAdd == FirstFAdd)
6459 return false;
6460
6461 NumScalarFAdd++;
6462 }
6463
6464 return NumScalarFAdd == 2;
6465}
6466
6467/// Scalarize a v2f32 arithmetic node (FADD, FMUL, FSUB, FMA) when at least
6468/// one operand is a BUILD_VECTOR that repacks values from non-adjacent register
6469/// pairs. Without this combine the BUILD_VECTOR forces allocation of a
6470/// temporary 64-bit register, increasing register pressure.
6471///
6472/// Example - before:
6473/// t0: v2f32,v2f32,ch = LoadV2 ...
6474/// t1: f32 = extract_vector_elt t0, 0
6475/// t2: f32 = extract_vector_elt t0:1, 0
6476/// t3: v2f32 = BUILD_VECTOR t1, t2 ;; non-coalescable repack
6477/// t4: v2f32 = fma t_a, t3, t_c
6478///
6479/// After:
6480/// t0: v2f32,v2f32,ch = LoadV2 ...
6481/// t1: f32 = extract_vector_elt t0, 0
6482/// t2: f32 = extract_vector_elt t0:1, 0
6483/// a0: f32 = extract_vector_elt t_a, 0
6484/// a1: f32 = extract_vector_elt t_a, 1
6485/// c0: f32 = extract_vector_elt t_c, 0
6486/// c1: f32 = extract_vector_elt t_c, 1
6487/// r0: f32 = fma a0, t1, c0
6488/// r1: f32 = fma a1, t2, c1
6489/// t4: v2f32 = BUILD_VECTOR r0, r1
6490///
6491/// Also scalarizes an FMUL when all output lanes feed into scalar FADDs
6492/// to enable scalar FMA combining.
6493SDValue NVPTXTargetLowering::performScalarizeV2F32Op(
6494 SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
6495 CodeGenOptLevel OptLevel) const {
6496 EVT VT = N->getValueType(ResNo: 0);
6497 if (VT != MVT::v2f32)
6498 return SDValue();
6499
6500 if (none_of(Range: N->ops(), P: isNonCoalescableBuildVector) &&
6501 !mayFoldFMULIntoFMA(N, MF&: DCI.DAG.getMachineFunction(), OptLevel))
6502 return SDValue();
6503
6504 SelectionDAG &DAG = DCI.DAG;
6505 SDLoc DL(N);
6506 EVT EltVT = VT.getVectorElementType();
6507 unsigned Opc = N->getOpcode();
6508
6509 // For each operand, get the scalar element at the given index: if the operand
6510 // is a BUILD_VECTOR, grab the element directly; otherwise, emit an
6511 // EXTRACT_VECTOR_ELT.
6512 auto GetElement = [&](SDValue Op, unsigned Index) -> SDValue {
6513 if (Op.getOpcode() == ISD::BUILD_VECTOR)
6514 return Op.getOperand(i: Index);
6515 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Op,
6516 N2: DAG.getVectorIdxConstant(Val: Index, DL));
6517 };
6518
6519 // Build scalar operand lists for element 0 and element 1.
6520 SmallVector<SDValue, 3> Ops0, Ops1;
6521 for (const SDValue &Op : N->ops()) {
6522 Ops0.push_back(Elt: GetElement(Op, 0));
6523 Ops1.push_back(Elt: GetElement(Op, 1));
6524 }
6525
6526 SDValue Res0 = DAG.getNode(Opcode: Opc, DL, VT: EltVT, Ops: Ops0, Flags: N->getFlags());
6527 SDValue Res1 = DAG.getNode(Opcode: Opc, DL, VT: EltVT, Ops: Ops1, Flags: N->getFlags());
6528
6529 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT, N1: Res0, N2: Res1);
6530}
6531
6532/// Target-specific dag combine xforms for ISD::FADD.
6533SDValue
6534NVPTXTargetLowering::performFADDCombine(SDNode *N,
6535 TargetLowering::DAGCombinerInfo &DCI,
6536 CodeGenOptLevel OptLevel) const {
6537 if (SDValue Result = performScalarizeV2F32Op(N, DCI, OptLevel))
6538 return Result;
6539
6540 SDValue N0 = N->getOperand(Num: 0);
6541 SDValue N1 = N->getOperand(Num: 1);
6542
6543 EVT VT = N0.getValueType();
6544 if (VT.isVector() || !(VT == MVT::f32 || VT == MVT::f64))
6545 return SDValue();
6546
6547 // First try with the default operand order.
6548 if (SDValue Result = performFADDCombineWithOperands(N, N0, N1, DCI, OptLevel))
6549 return Result;
6550
6551 // If that didn't work, try again with the operands commuted.
6552 return performFADDCombineWithOperands(N, N0: N1, N1: N0, DCI, OptLevel);
6553}
6554
6555/// Get 3-input version of a 2-input min/max opcode
6556static unsigned getMinMax3Opcode(unsigned MinMax2Opcode) {
6557 switch (MinMax2Opcode) {
6558 case ISD::FMAXNUM:
6559 case ISD::FMAXIMUMNUM:
6560 return NVPTXISD::FMAXNUM3;
6561 case ISD::FMINNUM:
6562 case ISD::FMINIMUMNUM:
6563 return NVPTXISD::FMINNUM3;
6564 case ISD::FMAXIMUM:
6565 return NVPTXISD::FMAXIMUM3;
6566 case ISD::FMINIMUM:
6567 return NVPTXISD::FMINIMUM3;
6568 default:
6569 llvm_unreachable("Invalid 2-input min/max opcode");
6570 }
6571}
6572
6573/// PerformFMinMaxCombine - Combine (fmaxnum (fmaxnum a, b), c) into
6574/// (fmaxnum3 a, b, c). Also covers other llvm min/max intrinsics.
6575static SDValue PerformFMinMaxCombine(SDNode *N,
6576 TargetLowering::DAGCombinerInfo &DCI,
6577 const NVPTXSubtarget &STI) {
6578
6579 // 3-input min/max requires PTX 8.8+ and SM_100+, and only supports f32s
6580 EVT VT = N->getValueType(ResNo: 0);
6581 if (VT != MVT::f32 || !STI.hasFeature(Feature: NVPTX::PTX88) ||
6582 !STI.hasFeature(Feature: NVPTX::SM100))
6583 return SDValue();
6584
6585 SDValue Op0 = N->getOperand(Num: 0);
6586 SDValue Op1 = N->getOperand(Num: 1);
6587 unsigned MinMaxOp2 = N->getOpcode();
6588 unsigned MinMaxOp3 = getMinMax3Opcode(MinMax2Opcode: MinMaxOp2);
6589
6590 if (Op0.getOpcode() == MinMaxOp2 && Op0.hasOneUse()) {
6591 // (maxnum (maxnum a, b), c) -> (maxnum3 a, b, c)
6592 SDValue A = Op0.getOperand(i: 0);
6593 SDValue B = Op0.getOperand(i: 1);
6594 SDValue C = Op1;
6595 return DCI.DAG.getNode(Opcode: MinMaxOp3, DL: SDLoc(N), VT, N1: A, N2: B, N3: C, Flags: N->getFlags());
6596 } else if (Op1.getOpcode() == MinMaxOp2 && Op1.hasOneUse()) {
6597 // (maxnum a, (maxnum b, c)) -> (maxnum3 a, b, c)
6598 SDValue A = Op0;
6599 SDValue B = Op1.getOperand(i: 0);
6600 SDValue C = Op1.getOperand(i: 1);
6601 return DCI.DAG.getNode(Opcode: MinMaxOp3, DL: SDLoc(N), VT, N1: A, N2: B, N3: C, Flags: N->getFlags());
6602 }
6603 return SDValue();
6604}
6605
6606// sext (mul.iN nsw x, y) => mul.wide.sN x, y
6607// zext (mul.iN nuw x, y) => mul.wide.uN x, y
6608// sext (shl.iN nsw x, const) => mul.wide.sN x, (1 << const)
6609// zext (shl.iN nuw x, const) => mul.wide.uN x, (1 << const)
6610static SDValue combineSZExtToMulWide(SDNode *N,
6611 TargetLowering::DAGCombinerInfo &DCI,
6612 CodeGenOptLevel OptLevel) {
6613 assert(N->getOpcode() == ISD::SIGN_EXTEND ||
6614 N->getOpcode() == ISD::ZERO_EXTEND);
6615
6616 if (OptLevel == CodeGenOptLevel::None)
6617 return SDValue();
6618
6619 SDValue Op = N->getOperand(Num: 0);
6620 if (!Op.hasOneUse())
6621 return SDValue();
6622
6623 EVT ToVT = N->getValueType(ResNo: 0);
6624 EVT FromVT = Op.getValueType();
6625 if (!((ToVT == MVT::i32 && FromVT == MVT::i16) ||
6626 (ToVT == MVT::i64 && FromVT == MVT::i32)))
6627 return SDValue();
6628
6629 bool IsSigned = N->getOpcode() == ISD::SIGN_EXTEND;
6630 if ((IsSigned && !Op->getFlags().hasNoSignedWrap()) ||
6631 (!IsSigned && !Op->getFlags().hasNoUnsignedWrap()))
6632 return SDValue();
6633
6634 SDLoc DL(N);
6635 SDValue LHS = Op.getOperand(i: 0);
6636 SDValue RHS = Op.getOperand(i: 1);
6637 unsigned MulWideOpcode =
6638 IsSigned ? NVPTXISD::MUL_WIDE_SIGNED : NVPTXISD::MUL_WIDE_UNSIGNED;
6639 if (Op.getOpcode() == ISD::MUL) {
6640 return DCI.DAG.getNode(Opcode: MulWideOpcode, DL, VT: ToVT, N1: LHS, N2: RHS);
6641 } else if (Op.getOpcode() == ISD::SHL && isa<ConstantSDNode>(Val: RHS)) {
6642 const auto ShiftAmt = Op.getConstantOperandVal(i: 1);
6643 const auto MulVal = APInt(FromVT.getSizeInBits(), 1) << ShiftAmt;
6644
6645 // Note that the sext (shl nsw ...) case doesn't work if 1 << const
6646 // overflows to a negative value! The only valid input values in this
6647 // case are 0 and -1 (all other values yield poison because of the nsw),
6648 // and mul.wide.sN would give us the wrong sign for -1. We could use
6649 // mul.wide.uN, but since this is a weird case anyway, we might as well not
6650 // apply this transformation at all.
6651 if (IsSigned && MulVal.isNegative())
6652 return SDValue();
6653
6654 RHS = DCI.DAG.getConstant(Val: MulVal, DL, VT: FromVT);
6655 return DCI.DAG.getNode(Opcode: MulWideOpcode, DL, VT: ToVT, N1: LHS, N2: RHS);
6656 }
6657
6658 return SDValue();
6659}
6660
6661/// Check whether Op is explicitly extended from at most \p OptSize bits,
6662/// setting \p IsSigned to the signedness of the extension.
6663static bool IsExtendedFromLte(SDValue Op, unsigned OptSize, bool &IsSigned) {
6664 if (Op.getOpcode() == ISD::SIGN_EXTEND ||
6665 Op.getOpcode() == ISD::SIGN_EXTEND_INREG) {
6666 EVT OrigVT = Op.getOperand(i: 0).getValueType();
6667 if (OrigVT.getFixedSizeInBits() <= OptSize) {
6668 IsSigned = true;
6669 return true;
6670 }
6671 } else if (Op.getOpcode() == ISD::ZERO_EXTEND) {
6672 EVT OrigVT = Op.getOperand(i: 0).getValueType();
6673 if (OrigVT.getFixedSizeInBits() <= OptSize) {
6674 IsSigned = false;
6675 return true;
6676 }
6677 }
6678
6679 return false;
6680}
6681
6682/// AreMulWideOperandsDemotable - Checks if the given LHS and RHS operands can
6683/// be demoted to \p OptSize bits without loss of information using the same
6684/// signedness. The signedness of the operands is placed in \p IsSigned.
6685static bool AreMulWideOperandsDemotable(SDValue LHS, SDValue RHS,
6686 unsigned OptSize,
6687 const SelectionDAG &DAG,
6688 bool &IsSigned) {
6689 // The native register size is 32 bits, so extending i16 to i32 is often
6690 // cheap, whereas truncating i64 to i32 is free. Use known bits to narrow
6691 // i64 products, and require explicit extensions for i32 products.
6692 if (LHS.getValueType() != MVT::i64) {
6693 if (!IsExtendedFromLte(Op: LHS, OptSize, IsSigned))
6694 return false;
6695
6696 if (auto *CI = dyn_cast<ConstantSDNode>(Val&: RHS)) {
6697 const APInt &Val = CI->getAPIntValue();
6698 return IsSigned ? Val.isSignedIntN(N: OptSize) : Val.isIntN(N: OptSize);
6699 }
6700
6701 bool RHSSigned;
6702 return IsExtendedFromLte(Op: RHS, OptSize, IsSigned&: RHSSigned) && IsSigned == RHSSigned;
6703 }
6704
6705 // Extensions may have been folded into masks, shifts, or other operations.
6706 // Check the values' widths rather than requiring explicit extension nodes.
6707 if (DAG.computeKnownBits(Op: LHS).countMaxActiveBits() <= OptSize &&
6708 DAG.computeKnownBits(Op: RHS).countMaxActiveBits() <= OptSize) {
6709 IsSigned = false;
6710 return true;
6711 }
6712 if (DAG.ComputeMaxSignificantBits(Op: LHS) <= OptSize &&
6713 DAG.ComputeMaxSignificantBits(Op: RHS) <= OptSize) {
6714 IsSigned = true;
6715 return true;
6716 }
6717 return false;
6718}
6719
6720/// TryMULWIDECombine - Attempt to replace a multiply of M bits with a multiply
6721/// of M/2 bits that produces an M-bit result (i.e. mul.wide). This transform
6722/// works on both multiply DAG nodes and SHL DAG nodes with a constant shift
6723/// amount.
6724static SDValue TryMULWIDECombine(SDNode *N,
6725 TargetLowering::DAGCombinerInfo &DCI) {
6726 EVT MulType = N->getValueType(ResNo: 0);
6727 if (MulType != MVT::i32 && MulType != MVT::i64) {
6728 return SDValue();
6729 }
6730
6731 SDLoc DL(N);
6732 unsigned OptSize = MulType.getSizeInBits() >> 1;
6733 SDValue LHS = N->getOperand(Num: 0);
6734 SDValue RHS = N->getOperand(Num: 1);
6735
6736 // Canonicalize the multiply so the constant (if any) is on the right
6737 if (N->getOpcode() == ISD::MUL) {
6738 if (isa<ConstantSDNode>(Val: LHS)) {
6739 std::swap(a&: LHS, b&: RHS);
6740 }
6741 }
6742
6743 bool Signed;
6744 // If we have a SHL, determine the actual multiply amount.
6745 if (N->getOpcode() == ISD::SHL) {
6746 // A plain shift is already one instruction. Only fold it when doing so
6747 // can also remove an explicit extension.
6748 if (!IsExtendedFromLte(Op: LHS, OptSize, IsSigned&: Signed))
6749 return SDValue();
6750
6751 ConstantSDNode *ShlRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
6752 if (!ShlRHS) {
6753 return SDValue();
6754 }
6755
6756 APInt ShiftAmt = ShlRHS->getAPIntValue();
6757 unsigned BitWidth = MulType.getSizeInBits();
6758 unsigned Limit = Signed ? OptSize - 1 : OptSize;
6759 if (ShiftAmt.sge(RHS: 0) && ShiftAmt.slt(RHS: Limit)) {
6760 APInt MulVal = APInt(BitWidth, 1) << ShiftAmt;
6761 RHS = DCI.DAG.getConstant(Val: MulVal, DL, VT: MulType);
6762 } else {
6763 return SDValue();
6764 }
6765 } else if (!AreMulWideOperandsDemotable(LHS, RHS, OptSize, DAG: DCI.DAG, IsSigned&: Signed)) {
6766 return SDValue();
6767 }
6768
6769 if (N->getOpcode() == ISD::MUL) {
6770 // Keep matching low/high products together so ptxas can share their
6771 // partial products. Widening only the low product can duplicate work and
6772 // increase register pressure, even when its operands fit the narrow type.
6773 for (unsigned Opc : {ISD::MULHU, ISD::MULHS}) {
6774 if (SDNode *Hi = DCI.DAG.getNodeIfExists(Opcode: Opc, VTList: N->getVTList(), Ops: {LHS, RHS}))
6775 if (!Hi->use_empty())
6776 return SDValue();
6777 if (SDNode *Hi = DCI.DAG.getNodeIfExists(Opcode: Opc, VTList: N->getVTList(), Ops: {RHS, LHS}))
6778 if (!Hi->use_empty())
6779 return SDValue();
6780 }
6781 }
6782
6783 EVT DemotedVT;
6784 if (MulType == MVT::i32) {
6785 DemotedVT = MVT::i16;
6786 } else {
6787 DemotedVT = MVT::i32;
6788 }
6789
6790 // Truncate the operands to the correct size. Note that these are just for
6791 // type consistency and will (likely) be eliminated in later phases.
6792 SDValue TruncLHS =
6793 DCI.DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DemotedVT, Operand: LHS);
6794 SDValue TruncRHS =
6795 DCI.DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DemotedVT, Operand: RHS);
6796
6797 unsigned Opc;
6798 if (Signed) {
6799 Opc = NVPTXISD::MUL_WIDE_SIGNED;
6800 } else {
6801 Opc = NVPTXISD::MUL_WIDE_UNSIGNED;
6802 }
6803
6804 return DCI.DAG.getNode(Opcode: Opc, DL, VT: MulType, N1: TruncLHS, N2: TruncRHS);
6805}
6806
6807static bool isConstOne(const SDValue &Operand) {
6808 const auto *Const = dyn_cast<ConstantSDNode>(Val: Operand);
6809 return Const && Const->getZExtValue() == 1;
6810}
6811
6812static SDValue matchMADConstOnePattern(SDValue Add) {
6813 if (Add->getOpcode() != ISD::ADD)
6814 return SDValue();
6815
6816 if (isConstOne(Operand: Add->getOperand(Num: 0)))
6817 return Add->getOperand(Num: 1);
6818
6819 if (isConstOne(Operand: Add->getOperand(Num: 1)))
6820 return Add->getOperand(Num: 0);
6821
6822 return SDValue();
6823}
6824
6825static SDValue combineMADConstOne(SDValue X, SDValue Add, EVT VT, SDLoc DL,
6826 TargetLowering::DAGCombinerInfo &DCI) {
6827
6828 if (SDValue Y = matchMADConstOnePattern(Add)) {
6829 SDValue Mul = DCI.DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: X, N2: Y);
6830 return DCI.DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Mul, N2: X);
6831 }
6832
6833 return SDValue();
6834}
6835
6836static SDValue combineMulSelectConstOne(SDValue X, SDValue Select, EVT VT,
6837 SDLoc DL,
6838 TargetLowering::DAGCombinerInfo &DCI) {
6839 if (Select->getOpcode() != ISD::SELECT)
6840 return SDValue();
6841
6842 SDValue Cond = Select->getOperand(Num: 0);
6843
6844 unsigned ConstOpNo;
6845 if (isConstOne(Operand: Select->getOperand(Num: 1)))
6846 ConstOpNo = 1;
6847 else if (isConstOne(Operand: Select->getOperand(Num: 2)))
6848 ConstOpNo = 2;
6849 else
6850 return SDValue();
6851
6852 SDValue Y = Select->getOperand(Num: (ConstOpNo == 1) ? 2 : 1);
6853
6854 // Do not combine if the resulting sequence is not obviously profitable.
6855 if (!matchMADConstOnePattern(Add: Y))
6856 return SDValue();
6857
6858 SDValue NewMul = DCI.DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: X, N2: Y);
6859
6860 return DCI.DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
6861 N2: (ConstOpNo == 1) ? X : NewMul,
6862 N3: (ConstOpNo == 1) ? NewMul : X);
6863}
6864
6865static SDValue
6866PerformMULCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
6867 TargetLowering::DAGCombinerInfo &DCI) {
6868
6869 EVT VT = N0.getValueType();
6870 if (VT.isVector())
6871 return SDValue();
6872
6873 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)
6874 return SDValue();
6875
6876 SDLoc DL(N);
6877
6878 // (mul x, (add y, 1)) -> (add (mul x, y), x)
6879 if (SDValue Res = combineMADConstOne(X: N0, Add: N1, VT, DL, DCI))
6880 return Res;
6881 if (SDValue Res = combineMADConstOne(X: N1, Add: N0, VT, DL, DCI))
6882 return Res;
6883
6884 // (mul x, (select y, 1)) -> (select (mul x, y), x)
6885 if (SDValue Res = combineMulSelectConstOne(X: N0, Select: N1, VT, DL, DCI))
6886 return Res;
6887 if (SDValue Res = combineMulSelectConstOne(X: N1, Select: N0, VT, DL, DCI))
6888 return Res;
6889
6890 return SDValue();
6891}
6892
6893/// PerformMULCombine - Runs PTX-specific DAG combine patterns on MUL nodes.
6894static SDValue PerformMULCombine(SDNode *N,
6895 TargetLowering::DAGCombinerInfo &DCI,
6896 CodeGenOptLevel OptLevel) {
6897 if (OptLevel == CodeGenOptLevel::None)
6898 return SDValue();
6899
6900 if (SDValue Ret = TryMULWIDECombine(N, DCI))
6901 return Ret;
6902
6903 SDValue N0 = N->getOperand(Num: 0);
6904 SDValue N1 = N->getOperand(Num: 1);
6905 return PerformMULCombineWithOperands(N, N0, N1, DCI);
6906}
6907
6908/// Commute SHL with a bitwise logic operation when doing so exposes a common
6909/// shifted operand. For example:
6910///
6911/// Before:
6912/// N = shl (zext (LogicOp X, C)), ShiftAmount
6913/// OtherShift = shl (zext (OtherLogicOp X, OtherC)), ShiftAmount
6914///
6915/// After:
6916/// ShiftedX = shl (zext X), ShiftAmount
6917/// N = LogicOp ShiftedX, ShiftedC
6918/// OtherShift = OtherLogicOp ShiftedX, ShiftedOtherC
6919///
6920/// ShiftedC = (zext C) << ShiftAmount and ShiftedOtherC =
6921/// (zext OtherC) << ShiftAmount are folded constants. This replaces two
6922/// variable shifts with the single shared ShiftedX. Requiring another matching
6923/// shift avoids disrupting isolated address calculations where a shift may be
6924/// folded into the addressing mode.
6925static SDValue combineShiftOfLogicOp(SDNode *N,
6926 TargetLowering::DAGCombinerInfo &DCI) {
6927 using namespace SDPatternMatch;
6928
6929 struct ShiftOfLogicOp {
6930 SDNode *Shift;
6931 SDValue LogicOp;
6932 SDValue X;
6933 SDValue Constant;
6934 unsigned ExtendOpcode;
6935 };
6936
6937 // Match a logic operation, with an optional extension, inside a SHL.
6938 auto matchShiftOfLogicOp =
6939 [&](SDNode *Shift) -> std::optional<ShiftOfLogicOp> {
6940 if (Shift->getOpcode() != ISD::SHL || !Shift->getOperand(Num: 0).hasOneUse())
6941 return std::nullopt;
6942 ShiftOfLogicOp Match;
6943 Match.Shift = Shift;
6944 Match.LogicOp = Shift->getOperand(Num: 0);
6945 Match.ExtendOpcode = 0;
6946 if (ISD::isExtOpcode(Opcode: Match.LogicOp.getOpcode())) {
6947 Match.ExtendOpcode = Match.LogicOp.getOpcode();
6948 Match.LogicOp = Match.LogicOp.getOperand(i: 0);
6949 }
6950
6951 if (!sd_match(N: Match.LogicOp, P: m_OneUse(P: m_BitwiseLogic(
6952 L: m_Value(N&: Match.X),
6953 R: m_Value(N&: Match.Constant, P: m_ConstInt())))))
6954 return std::nullopt;
6955
6956 return Match;
6957 };
6958
6959 // Match N as the root shift-of-logic; bail if it does not fit the pattern.
6960 const std::optional<ShiftOfLogicOp> Root = matchShiftOfLogicOp(N);
6961 if (!Root)
6962 return SDValue();
6963
6964 // Only profitable for a constant shift amount: the per-op constant shift then
6965 // folds away instead of becoming an extra variable shift.
6966 if (!isConstOrConstSplat(N: N->getOperand(Num: 1)))
6967 return SDValue();
6968
6969 // Collect candidate shifts that share X. Reached through another user of X,
6970 // the logic result feeds the shift directly or through an optional extend.
6971 SmallVector<SDNode *, 4> CandidateShifts;
6972 for (const SDNode *CandidateLogicOp : Root->X->users()) {
6973 if (CandidateLogicOp == Root->LogicOp.getNode())
6974 continue;
6975 for (SDNode *LogicUser : CandidateLogicOp->users()) {
6976 if (ISD::isExtOpcode(Opcode: LogicUser->getOpcode())) {
6977 // shl (ext (logic X, C)): step through the extend to find the shift.
6978 for (SDNode *ExtendUser : LogicUser->users())
6979 if (ExtendUser->getOpcode() == ISD::SHL)
6980 CandidateShifts.push_back(Elt: ExtendUser);
6981 } else if (LogicUser->getOpcode() == ISD::SHL) {
6982 // shl (logic X, C): the user is already the shift.
6983 CandidateShifts.push_back(Elt: LogicUser);
6984 }
6985 }
6986 }
6987
6988 // Verify each candidate against the root's pattern: the same X, extension,
6989 // type, and shift amount.
6990 const EVT VT = N->getValueType(ResNo: 0);
6991 const SDValue ShiftAmount = N->getOperand(Num: 1);
6992 SmallVector<ShiftOfLogicOp, 4> Matches;
6993 for (SDNode *CandidateShift : CandidateShifts) {
6994 const std::optional<ShiftOfLogicOp> Candidate =
6995 matchShiftOfLogicOp(CandidateShift);
6996 if (Candidate && Candidate->X == Root->X &&
6997 Candidate->ExtendOpcode == Root->ExtendOpcode &&
6998 CandidateShift->getValueType(ResNo: 0) == VT &&
6999 CandidateShift->getOperand(Num: 1) == ShiftAmount)
7000 Matches.push_back(Elt: *Candidate);
7001 }
7002 if (Matches.empty())
7003 return SDValue();
7004
7005 // Build the shared shifted X once, then rewrite the root and every match
7006 // into a logic op over it so the shift is CSE'd.
7007 SelectionDAG &DAG = DCI.DAG;
7008 const SDValue ShiftedX =
7009 DAG.getNode(Opcode: ISD::SHL, DL: SDLoc(N), VT,
7010 N1: Root->ExtendOpcode
7011 ? DAG.getNode(Opcode: Root->ExtendOpcode, DL: SDLoc(N), VT, Operand: Root->X)
7012 : Root->X,
7013 N2: ShiftAmount);
7014
7015 // Rebuild the logic op from shared ShiftedX and a folded constant shift.
7016 auto buildCommutedLogicOp = [&](const SDValue LogicOp, SDValue C,
7017 const SDLoc &DL) {
7018 if (Root->ExtendOpcode)
7019 C = DAG.getNode(Opcode: Root->ExtendOpcode, DL, VT, Operand: C);
7020 const SDValue ShiftedC = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: C, N2: ShiftAmount);
7021 return DAG.getNode(Opcode: LogicOp.getOpcode(), DL, VT, N1: ShiftedX, N2: ShiftedC,
7022 Flags: LogicOp->getFlags());
7023 };
7024
7025 for (const ShiftOfLogicOp &Match : Matches)
7026 DCI.CombineTo(N: Match.Shift,
7027 Res: buildCommutedLogicOp(Match.LogicOp, Match.Constant,
7028 SDLoc(Match.Shift)));
7029 return buildCommutedLogicOp(Root->LogicOp, Root->Constant, SDLoc(N));
7030}
7031
7032/// PerformSHLCombine - Runs PTX-specific DAG combine patterns on SHL nodes.
7033static SDValue PerformSHLCombine(SDNode *N,
7034 TargetLowering::DAGCombinerInfo &DCI,
7035 CodeGenOptLevel OptLevel) {
7036 if (OptLevel > CodeGenOptLevel::None) {
7037 // Expose a shared shifted operand for CSE before mul.wide folding, which
7038 // would otherwise consume the shift.
7039 if (SDValue Ret = combineShiftOfLogicOp(N, DCI))
7040 return Ret;
7041
7042 // Try mul.wide combining at OptLevel > 0
7043 if (SDValue Ret = TryMULWIDECombine(N, DCI))
7044 return Ret;
7045 }
7046
7047 return SDValue();
7048}
7049
7050static SDValue PerformSETCCCombine(SDNode *N,
7051 TargetLowering::DAGCombinerInfo &DCI,
7052 const NVPTXSubtarget &STI) {
7053 EVT CCType = N->getValueType(ResNo: 0);
7054 SDValue A = N->getOperand(Num: 0);
7055 SDValue B = N->getOperand(Num: 1);
7056
7057 EVT AType = A.getValueType();
7058 if (!(CCType == MVT::v2i1 && (AType == MVT::v2f16 || AType == MVT::v2bf16)))
7059 return SDValue();
7060
7061 if (A.getValueType() == MVT::v2bf16 && !STI.hasFeature(Feature: NVPTX::SM90))
7062 return SDValue();
7063
7064 SDLoc DL(N);
7065 // setp.f16x2 returns two scalar predicates, which we need to
7066 // convert back to v2i1. The returned result will be scalarized by
7067 // the legalizer, but the comparison will remain a single vector
7068 // instruction.
7069 SDValue CCNode = DCI.DAG.getNode(
7070 Opcode: A.getValueType() == MVT::v2f16 ? NVPTXISD::SETP_F16X2
7071 : NVPTXISD::SETP_BF16X2,
7072 DL, VTList: DCI.DAG.getVTList(VT1: MVT::i1, VT2: MVT::i1), Ops: {A, B, N->getOperand(Num: 2)});
7073 return DCI.DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: CCType, N1: CCNode.getValue(R: 0),
7074 N2: CCNode.getValue(R: 1));
7075}
7076
7077static SDValue PerformEXTRACTCombine(SDNode *N,
7078 TargetLowering::DAGCombinerInfo &DCI) {
7079 SDValue Vector = peekThroughFreeze(V: N->getOperand(Num: 0));
7080 SDLoc DL(N);
7081 EVT VectorVT = Vector.getValueType();
7082 if (Vector->getOpcode() == ISD::LOAD && VectorVT.isSimple() &&
7083 IsPTXVectorType(VT: VectorVT.getSimpleVT()))
7084 return SDValue(); // Native vector loads already combine nicely w/
7085 // extract_vector_elt.
7086 // Don't mess with singletons or packed types (v2*32, v2*16, v4i8 and v8i8),
7087 // we already handle them OK.
7088 if (VectorVT.getVectorNumElements() == 1 ||
7089 NVPTX::isPackedVectorTy(VT: VectorVT) || VectorVT == MVT::v8i8)
7090 return SDValue();
7091
7092 // Don't mess with undef values as sra may be simplified to 0, not undef.
7093 if (Vector->isUndef() || ISD::allOperandsUndef(V: Vector))
7094 return SDValue();
7095
7096 uint64_t VectorBits = VectorVT.getSizeInBits();
7097 // We only handle the types we can extract in-register.
7098 if (!(VectorBits == 16 || VectorBits == 32 || VectorBits == 64))
7099 return SDValue();
7100
7101 ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
7102 // Index == 0 is handled by generic DAG combiner.
7103 if (!Index || Index->getZExtValue() == 0)
7104 return SDValue();
7105
7106 MVT IVT = MVT::getIntegerVT(BitWidth: VectorBits);
7107 EVT EltVT = VectorVT.getVectorElementType();
7108 EVT EltIVT = EltVT.changeTypeToInteger();
7109 uint64_t EltBits = EltVT.getScalarSizeInBits();
7110
7111 SDValue Result = DCI.DAG.getNode(
7112 Opcode: ISD::TRUNCATE, DL, VT: EltIVT,
7113 Operand: DCI.DAG.getNode(
7114 Opcode: ISD::SRA, DL, VT: IVT, N1: DCI.DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IVT, Operand: Vector),
7115 N2: DCI.DAG.getConstant(Val: Index->getZExtValue() * EltBits, DL, VT: IVT)));
7116
7117 // If element has non-integer type, bitcast it back to the expected type.
7118 if (EltVT != EltIVT)
7119 Result = DCI.DAG.getNode(Opcode: ISD::BITCAST, DL, VT: EltVT, Operand: Result);
7120 // Past legalizer, we may need to extent i8 -> i16 to match the register type.
7121 if (EltVT != N->getValueType(ResNo: 0))
7122 Result = DCI.DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: N->getValueType(ResNo: 0), Operand: Result);
7123
7124 return Result;
7125}
7126
7127/// Transform patterns like:
7128/// (select (ugt shift_amt, BitWidth-1), 0, (srl/shl x, shift_amt))
7129/// (select (ult shift_amt, BitWidth), (srl/shl x, shift_amt), 0)
7130/// Into:
7131/// (NVPTXISD::SRL_CLAMP x, shift_amt) or (NVPTXISD::SHL_CLAMP x, shift_amt)
7132///
7133/// These patterns arise from code like `s >= 32 ? 0 : x >> s`. In LLVM,
7134/// over-shifting a value results in poison, but PTX shr/shl instructions clamp
7135/// the shift amount to BitWidth, making the guard redundant.
7136///
7137/// Note: We only handle SRL and SHL, not SRA, because arithmetic right shifts
7138/// can produce 0 or -1 when shift >= BitWidth.
7139/// Note: We don't handle uge or ule. These don't appear because of
7140/// canonicalization.
7141static SDValue PerformSELECTShiftCombine(SDNode *N,
7142 TargetLowering::DAGCombinerInfo &DCI) {
7143 if (!DCI.isAfterLegalizeDAG())
7144 return SDValue();
7145
7146 using namespace SDPatternMatch;
7147 unsigned BitWidth = N->getValueType(ResNo: 0).getSizeInBits();
7148 SDValue ShiftAmt, ShiftOp;
7149
7150 // Match logical shifts where the shift amount in the guard matches the shift
7151 // amount in the operation.
7152 auto LogicalShift = m_Value(
7153 N&: ShiftOp, P: m_AnyOf(preds: m_Srl(L: m_Value(), R: m_TruncOrSelf(Op: m_Deferred(V&: ShiftAmt))),
7154 preds: m_Shl(L: m_Value(), R: m_TruncOrSelf(Op: m_Deferred(V&: ShiftAmt)))));
7155
7156 // shift_amt > BitWidth-1 ? 0 : shift_op
7157 bool MatchedUGT = sd_match(
7158 N, P: m_Select(Cond: m_SpecificSetCC(CC: ISD::SETUGT, LHS: m_Value(N&: ShiftAmt),
7159 RHS: m_SpecificInt(V: APInt(BitWidth, BitWidth - 1))),
7160 T: m_Zero(), F: LogicalShift));
7161 // shift_amt < BitWidth ? shift_op : 0
7162 bool MatchedULT =
7163 !MatchedUGT &&
7164 sd_match(
7165 N, P: m_Select(Cond: m_SpecificSetCC(CC: ISD::SETULT, LHS: m_Value(N&: ShiftAmt),
7166 RHS: m_SpecificInt(V: APInt(BitWidth, BitWidth))),
7167 T: LogicalShift, F: m_Zero()));
7168
7169 if (!MatchedUGT && !MatchedULT)
7170 return SDValue();
7171
7172 // In LLVM IR, the shift amount and the value-to-be-shifted are the same
7173 // type, whereas in PTX the shift amount is always i32. Therefore when
7174 // shifting types larger than i32, we can only do this transformation if we
7175 // know that the upper bits of the shift amount are known zero.
7176 SDValue ClampAmt = ShiftOp.getOperand(i: 1);
7177 unsigned ClampAmtBits = ClampAmt.getValueSizeInBits();
7178 if (ShiftAmt.getValueSizeInBits() > ClampAmtBits &&
7179 DCI.DAG.computeKnownBits(Op: ShiftAmt).countMaxActiveBits() > ClampAmtBits)
7180 return SDValue();
7181
7182 // Return a clamp shift operation, which has the same semantics as PTX shift.
7183 unsigned ClampOpc = ShiftOp.getOpcode() == ISD::SRL ? NVPTXISD::SRL_CLAMP
7184 : NVPTXISD::SHL_CLAMP;
7185 return DCI.DAG.getNode(Opcode: ClampOpc, DL: SDLoc(N), VT: ShiftOp.getValueType(),
7186 N1: ShiftOp.getOperand(i: 0), N2: ClampAmt);
7187}
7188
7189static SDValue PerformVSELECTCombine(SDNode *N,
7190 TargetLowering::DAGCombinerInfo &DCI) {
7191 SDValue VA = N->getOperand(Num: 1);
7192 EVT VectorVT = VA.getValueType();
7193 if (VectorVT != MVT::v4i8)
7194 return SDValue();
7195
7196 // We need to split vselect into individual per-element operations Because we
7197 // use BFE/BFI instruction for byte extraction/insertion, we do end up with
7198 // 32-bit values, so we may as well do comparison as i32 to avoid conversions
7199 // to/from i16 normally used for i8 values.
7200 SmallVector<SDValue, 4> E;
7201 SDLoc DL(N);
7202 SDValue VCond = N->getOperand(Num: 0);
7203 SDValue VB = N->getOperand(Num: 2);
7204 for (int I = 0; I < 4; ++I) {
7205 SDValue C = DCI.DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i1, N1: VCond,
7206 N2: DCI.DAG.getConstant(Val: I, DL, VT: MVT::i32));
7207 SDValue EA = DCI.DAG.getAnyExtOrTrunc(
7208 Op: DCI.DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i8, N1: VA,
7209 N2: DCI.DAG.getConstant(Val: I, DL, VT: MVT::i32)),
7210 DL, VT: MVT::i32);
7211 SDValue EB = DCI.DAG.getAnyExtOrTrunc(
7212 Op: DCI.DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i8, N1: VB,
7213 N2: DCI.DAG.getConstant(Val: I, DL, VT: MVT::i32)),
7214 DL, VT: MVT::i32);
7215 E.push_back(Elt: DCI.DAG.getAnyExtOrTrunc(
7216 Op: DCI.DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::i32, N1: C, N2: EA, N3: EB), DL, VT: MVT::i8));
7217 }
7218 return DCI.DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: MVT::v4i8, Ops: E);
7219}
7220
7221static SDValue
7222PerformBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
7223 auto VT = N->getValueType(ResNo: 0);
7224 if (!DCI.isAfterLegalizeDAG() ||
7225 // only process v2*16 types
7226 !(NVPTX::isPackedVectorTy(VT) && VT.is32BitVector() &&
7227 VT.getVectorNumElements() == 2))
7228 return SDValue();
7229
7230 auto Op0 = N->getOperand(Num: 0);
7231 auto Op1 = N->getOperand(Num: 1);
7232
7233 // Start out by assuming we want to take the lower 2 bytes of each i32
7234 // operand.
7235 uint64_t Op0Bytes = 0x10;
7236 uint64_t Op1Bytes = 0x54;
7237
7238 std::pair<SDValue *, uint64_t *> OpData[2] = {{&Op0, &Op0Bytes},
7239 {&Op1, &Op1Bytes}};
7240
7241 // Check that each operand is an i16, truncated from an i32 operand. We'll
7242 // select individual bytes from those original operands. Optionally, fold in a
7243 // shift right of that original operand.
7244 for (auto &[Op, OpBytes] : OpData) {
7245 // Eat up any bitcast
7246 if (Op->getOpcode() == ISD::BITCAST)
7247 *Op = Op->getOperand(i: 0);
7248
7249 if (!(Op->getValueType() == MVT::i16 && Op->getOpcode() == ISD::TRUNCATE &&
7250 Op->getOperand(i: 0).getValueType() == MVT::i32))
7251 return SDValue();
7252
7253 // If the truncate has multiple uses, this optimization can increase
7254 // register pressure
7255 if (!Op->hasOneUse())
7256 return SDValue();
7257
7258 *Op = Op->getOperand(i: 0);
7259
7260 // Optionally, fold in a shift-right of the original operand and let permute
7261 // pick the two higher bytes of the original value directly.
7262 if (Op->getOpcode() == ISD::SRL && isa<ConstantSDNode>(Val: Op->getOperand(i: 1))) {
7263 if (cast<ConstantSDNode>(Val: Op->getOperand(i: 1))->getZExtValue() == 16) {
7264 // Shift the PRMT byte selector to pick upper bytes from each respective
7265 // value, instead of the lower ones: 0x10 -> 0x32, 0x54 -> 0x76
7266 assert((*OpBytes == 0x10 || *OpBytes == 0x54) &&
7267 "PRMT selector values out of range");
7268 *OpBytes += 0x22;
7269 *Op = Op->getOperand(i: 0);
7270 }
7271 }
7272 }
7273
7274 SDLoc DL(N);
7275 auto &DAG = DCI.DAG;
7276
7277 auto PRMT =
7278 getPRMT(A: DAG.getBitcast(VT: MVT::i32, V: Op0), B: DAG.getBitcast(VT: MVT::i32, V: Op1),
7279 Selector: (Op1Bytes << 8) | Op0Bytes, DL, DAG);
7280 return DAG.getBitcast(VT, V: PRMT);
7281}
7282
7283static SDValue combineADDRSPACECAST(SDNode *N,
7284 TargetLowering::DAGCombinerInfo &DCI) {
7285 auto *ASCN1 = cast<AddrSpaceCastSDNode>(Val: N);
7286
7287 if (auto *ASCN2 = dyn_cast<AddrSpaceCastSDNode>(Val: ASCN1->getOperand(Num: 0))) {
7288 assert(ASCN2->getDestAddressSpace() == ASCN1->getSrcAddressSpace());
7289
7290 // Fold asc[B -> A](asc[A -> B](x)) -> x
7291 if (ASCN1->getDestAddressSpace() == ASCN2->getSrcAddressSpace())
7292 return ASCN2->getOperand(Num: 0);
7293 }
7294
7295 return SDValue();
7296}
7297
7298// Given a constant selector value and a prmt mode, return the selector value
7299// normalized to the generic prmt mode. See the PTX ISA documentation for more
7300// details:
7301// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-prmt
7302static APInt getPRMTSelector(const APInt &Selector, unsigned Mode) {
7303 assert(Selector.getBitWidth() == 32 && "PRMT must have i32 operands");
7304
7305 if (Mode == NVPTX::PTXPrmtMode::NONE)
7306 return Selector;
7307
7308 const unsigned V = Selector.trunc(width: 2).getZExtValue();
7309
7310 const auto GetSelector = [](unsigned S0, unsigned S1, unsigned S2,
7311 unsigned S3) {
7312 return APInt(32, S0 | (S1 << 4) | (S2 << 8) | (S3 << 12));
7313 };
7314
7315 switch (Mode) {
7316 case NVPTX::PTXPrmtMode::F4E:
7317 return GetSelector(V, V + 1, V + 2, V + 3);
7318 case NVPTX::PTXPrmtMode::B4E:
7319 return GetSelector(V, (V - 1) & 7, (V - 2) & 7, (V - 3) & 7);
7320 case NVPTX::PTXPrmtMode::RC8:
7321 return GetSelector(V, V, V, V);
7322 case NVPTX::PTXPrmtMode::ECL:
7323 return GetSelector(V, std::max(a: V, b: 1U), std::max(a: V, b: 2U), 3U);
7324 case NVPTX::PTXPrmtMode::ECR:
7325 return GetSelector(0, std::min(a: V, b: 1U), std::min(a: V, b: 2U), V);
7326 case NVPTX::PTXPrmtMode::RC16: {
7327 unsigned V1 = (V & 1) << 1;
7328 return GetSelector(V1, V1 + 1, V1, V1 + 1);
7329 }
7330 default:
7331 llvm_unreachable("Invalid PRMT mode");
7332 }
7333}
7334
7335static APInt computePRMT(APInt A, APInt B, APInt Selector, unsigned Mode) {
7336 assert(A.getBitWidth() == 32 && B.getBitWidth() == 32 &&
7337 Selector.getBitWidth() == 32 && "PRMT must have i32 operands");
7338 // {b, a} = {{b7, b6, b5, b4}, {b3, b2, b1, b0}}
7339 APInt BitField = B.concat(NewLSB: A);
7340 APInt SelectorVal = getPRMTSelector(Selector, Mode);
7341 APInt Result(32, 0);
7342 for (unsigned I : llvm::seq(Size: 4U)) {
7343 APInt Sel = SelectorVal.extractBits(numBits: 4, bitPosition: I * 4);
7344 unsigned Idx = Sel.getLoBits(numBits: 3).getZExtValue();
7345 unsigned Sign = Sel.getHiBits(numBits: 1).getZExtValue();
7346 APInt Byte = BitField.extractBits(numBits: 8, bitPosition: Idx * 8);
7347 if (Sign)
7348 Byte = Byte.ashr(ShiftAmt: 8);
7349 Result.insertBits(SubBits: Byte, bitPosition: I * 8);
7350 }
7351 return Result;
7352}
7353
7354static SDValue combinePRMT(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
7355 CodeGenOptLevel OptLevel) {
7356 if (OptLevel == CodeGenOptLevel::None)
7357 return SDValue();
7358
7359 // Constant fold PRMT
7360 if (isa<ConstantSDNode>(Val: N->getOperand(Num: 0)) &&
7361 isa<ConstantSDNode>(Val: N->getOperand(Num: 1)) &&
7362 isa<ConstantSDNode>(Val: N->getOperand(Num: 2)))
7363 return DCI.DAG.getConstant(Val: computePRMT(A: N->getConstantOperandAPInt(Num: 0),
7364 B: N->getConstantOperandAPInt(Num: 1),
7365 Selector: N->getConstantOperandAPInt(Num: 2),
7366 Mode: N->getConstantOperandVal(Num: 3)),
7367 DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
7368 return SDValue();
7369}
7370
7371// During call lowering we wrap the return values in a ProxyReg node which
7372// depend on the chain value produced by the completed call. This ensures that
7373// the full call is emitted in cases where libcalls are used to legalize
7374// operations. To improve the functioning of other DAG combines we pull all
7375// operations we can through one of these nodes, ensuring that the ProxyReg
7376// directly wraps a load. That is:
7377//
7378// (ProxyReg (zext (load retval0))) => (zext (ProxyReg (load retval0)))
7379//
7380static SDValue sinkProxyReg(SDValue R, SDValue Chain,
7381 TargetLowering::DAGCombinerInfo &DCI) {
7382 switch (R.getOpcode()) {
7383 case ISD::TRUNCATE:
7384 case ISD::ANY_EXTEND:
7385 case ISD::SIGN_EXTEND:
7386 case ISD::ZERO_EXTEND:
7387 case ISD::BITCAST: {
7388 if (SDValue V = sinkProxyReg(R: R.getOperand(i: 0), Chain, DCI))
7389 return DCI.DAG.getNode(Opcode: R.getOpcode(), DL: SDLoc(R), VT: R.getValueType(), Operand: V);
7390 return SDValue();
7391 }
7392 case ISD::SHL:
7393 case ISD::SRL:
7394 case ISD::SRA:
7395 case ISD::OR: {
7396 if (SDValue A = sinkProxyReg(R: R.getOperand(i: 0), Chain, DCI))
7397 if (SDValue B = sinkProxyReg(R: R.getOperand(i: 1), Chain, DCI))
7398 return DCI.DAG.getNode(Opcode: R.getOpcode(), DL: SDLoc(R), VT: R.getValueType(), N1: A, N2: B);
7399 return SDValue();
7400 }
7401 case ISD::Constant:
7402 return R;
7403 case ISD::LOAD:
7404 case NVPTXISD::LoadV2:
7405 case NVPTXISD::LoadV4: {
7406 return DCI.DAG.getNode(Opcode: NVPTXISD::ProxyReg, DL: SDLoc(R), VT: R.getValueType(),
7407 Ops: {Chain, R});
7408 }
7409 case ISD::BUILD_VECTOR: {
7410 if (DCI.isBeforeLegalize())
7411 return SDValue();
7412
7413 SmallVector<SDValue, 16> Ops;
7414 for (auto &Op : R->ops()) {
7415 SDValue V = sinkProxyReg(R: Op, Chain, DCI);
7416 if (!V)
7417 return SDValue();
7418 Ops.push_back(Elt: V);
7419 }
7420 return DCI.DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SDLoc(R), VT: R.getValueType(), Ops);
7421 }
7422 case ISD::EXTRACT_VECTOR_ELT: {
7423 if (DCI.isBeforeLegalize())
7424 return SDValue();
7425
7426 if (SDValue V = sinkProxyReg(R: R.getOperand(i: 0), Chain, DCI))
7427 return DCI.DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(R),
7428 VT: R.getValueType(), N1: V, N2: R.getOperand(i: 1));
7429 return SDValue();
7430 }
7431 default:
7432 return SDValue();
7433 }
7434}
7435
7436static unsigned getFAddWithNegOpcode(EVT VT, Intrinsic::ID IID,
7437 APFloat::roundingMode RoundingMode) {
7438 const bool IsFTZ = nvvm::FPArithShouldFTZ(IntrinsicID: IID);
7439 const bool IsSat = nvvm::FPArithIsSaturating(IntrinsicID: IID);
7440 switch (VT.getScalarType().getSimpleVT().SimpleTy) {
7441 case MVT::f16: {
7442 static constexpr unsigned SubRNOpcodes[2][2] = {
7443 {NVPTXISD::SUB_RN, NVPTXISD::SUB_RN_SAT},
7444 {NVPTXISD::SUB_RN_FTZ, NVPTXISD::SUB_RN_FTZ_SAT}};
7445 return SubRNOpcodes[IsFTZ][IsSat];
7446 }
7447 case MVT::bf16:
7448 return NVPTXISD::SUB_RN;
7449 case MVT::f32: {
7450 // for f32x2 inputs
7451 if (!VT.isVector() || IsSat)
7452 return 0;
7453 static constexpr unsigned SubF32x2Opcodes[4][2] = {
7454 {NVPTXISD::SUB_RZ, NVPTXISD::SUB_RZ_FTZ}, // RZ
7455 {NVPTXISD::SUB_RN, NVPTXISD::SUB_RN_FTZ}, // RN
7456 {NVPTXISD::SUB_RP, NVPTXISD::SUB_RP_FTZ}, // RP
7457 {NVPTXISD::SUB_RM, NVPTXISD::SUB_RM_FTZ}}; // RM
7458 return SubF32x2Opcodes[static_cast<unsigned>(RoundingMode)][IsFTZ];
7459 }
7460 default:
7461 return 0;
7462 }
7463}
7464
7465static SDValue combineFAddWithNeg(SDNode *N, SelectionDAG &DAG,
7466 Intrinsic::ID AddIntrinsicID,
7467 APFloat::roundingMode RoundingMode) {
7468 const EVT VT = N->getValueType(ResNo: 0);
7469 const unsigned Opc = getFAddWithNegOpcode(VT, IID: AddIntrinsicID, RoundingMode);
7470 if (!Opc)
7471 return SDValue();
7472
7473 SDValue Op1 = N->getOperand(Num: 1);
7474 SDValue Op2 = N->getOperand(Num: 2);
7475
7476 SDValue SubOp1, SubOp2;
7477
7478 if (Op1.getOpcode() == ISD::FNEG) {
7479 SubOp1 = Op2;
7480 SubOp2 = Op1.getOperand(i: 0);
7481 } else if (Op2.getOpcode() == ISD::FNEG) {
7482 SubOp1 = Op1;
7483 SubOp2 = Op2.getOperand(i: 0);
7484 } else {
7485 return SDValue();
7486 }
7487
7488 return DAG.getNode(Opcode: Opc, DL: SDLoc(N), VT, N1: SubOp1, N2: SubOp2);
7489}
7490
7491static bool isSupportedFPArith(SDNode *N, const NVPTXSubtarget &STI,
7492 unsigned ISDOpcode, Intrinsic::ID IID) {
7493 const EVT VT = N->getValueType(ResNo: 0);
7494 switch (VT.getScalarType().getSimpleVT().SimpleTy) {
7495 case MVT::f16:
7496 return STI.hasFP16Math() &&
7497 (!nvvm::FPArithIsRelu(IntrinsicID: IID) || STI.hasFeature(Feature: NVPTX::SM80));
7498 case MVT::bf16:
7499 return STI.hasNativeBF16Support(Opcode: ISDOpcode);
7500 case MVT::f32:
7501 return !VT.isVector() || STI.hasF32x2Instructions();
7502 default:
7503 return true;
7504 }
7505}
7506
7507static SDValue diagnoseUnsupportedFPArith(SDNode *N, SelectionDAG &DAG,
7508 Intrinsic::ID IID,
7509 APFloat::roundingMode RoundingMode) {
7510 const EVT VT = N->getValueType(ResNo: 0);
7511 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
7512 DAG.getMachineFunction().getFunction(),
7513 Twine(Intrinsic::getBaseName(id: IID)) + " with rounding mode " +
7514 nvvm::GetRoundingModeName(RM: RoundingMode) + " and operand type " +
7515 VT.getEVTString() + " is not supported on this target",
7516 SDLoc(N).getDebugLoc()));
7517 return DAG.getPOISON(VT);
7518}
7519
7520static SDValue combineIntrinsicWOChain(SDNode *N,
7521 TargetLowering::DAGCombinerInfo &DCI,
7522 const NVPTXSubtarget &STI) {
7523 const Intrinsic::ID IID =
7524 static_cast<Intrinsic::ID>(N->getConstantOperandVal(Num: 0));
7525
7526 switch (IID) {
7527 default:
7528 break;
7529 case Intrinsic::nvvm_fadd:
7530 case Intrinsic::nvvm_fadd_ftz:
7531 case Intrinsic::nvvm_fadd_sat:
7532 case Intrinsic::nvvm_fadd_ftz_sat: {
7533 const auto RoundingMode = static_cast<APFloat::roundingMode>(
7534 N->getConstantOperandAPInt(Num: 3).getSExtValue());
7535 if (!isSupportedFPArith(N, STI, ISDOpcode: ISD::FADD, IID))
7536 return diagnoseUnsupportedFPArith(N, DAG&: DCI.DAG, IID, RoundingMode);
7537 return combineFAddWithNeg(N, DAG&: DCI.DAG, AddIntrinsicID: IID, RoundingMode);
7538 }
7539 case Intrinsic::nvvm_spcompress:
7540 case Intrinsic::nvvm_spdecompress:
7541 return lowerSPIntrinsic(Op: SDValue(N, 0), DAG&: DCI.DAG);
7542 case Intrinsic::nvvm_fma:
7543 case Intrinsic::nvvm_fma_ftz:
7544 case Intrinsic::nvvm_fma_sat:
7545 case Intrinsic::nvvm_fma_ftz_sat:
7546 case Intrinsic::nvvm_fma_relu:
7547 case Intrinsic::nvvm_fma_ftz_relu: {
7548 const auto RoundingMode = static_cast<APFloat::roundingMode>(
7549 N->getConstantOperandAPInt(Num: 4).getSExtValue());
7550 if (!isSupportedFPArith(N, STI, ISDOpcode: ISD::FMA, IID))
7551 return diagnoseUnsupportedFPArith(N, DAG&: DCI.DAG, IID, RoundingMode);
7552 break;
7553 }
7554 case Intrinsic::nvvm_fmul:
7555 case Intrinsic::nvvm_fmul_ftz:
7556 case Intrinsic::nvvm_fmul_sat:
7557 case Intrinsic::nvvm_fmul_ftz_sat: {
7558 const auto RoundingMode = static_cast<APFloat::roundingMode>(
7559 N->getConstantOperandAPInt(Num: 3).getSExtValue());
7560 if (!isSupportedFPArith(N, STI, ISDOpcode: ISD::FMUL, IID))
7561 return diagnoseUnsupportedFPArith(N, DAG&: DCI.DAG, IID, RoundingMode);
7562 break;
7563 }
7564 }
7565 return SDValue();
7566}
7567
7568static SDValue combineProxyReg(SDNode *N,
7569 TargetLowering::DAGCombinerInfo &DCI) {
7570
7571 SDValue Chain = N->getOperand(Num: 0);
7572 SDValue Reg = N->getOperand(Num: 1);
7573
7574 // If the ProxyReg is not wrapping a load, try to pull the operations through
7575 // the ProxyReg.
7576 if (Reg.getOpcode() != ISD::LOAD) {
7577 if (SDValue V = sinkProxyReg(R: Reg, Chain, DCI))
7578 return V;
7579 }
7580
7581 return SDValue();
7582}
7583
7584SDValue NVPTXTargetLowering::PerformDAGCombine(SDNode *N,
7585 DAGCombinerInfo &DCI) const {
7586 CodeGenOptLevel OptLevel = getTargetMachine().getOptLevel();
7587 switch (N->getOpcode()) {
7588 default:
7589 break;
7590 case ISD::ADD:
7591 return PerformADDCombine(N, DCI, OptLevel);
7592 case ISD::ADDRSPACECAST:
7593 return combineADDRSPACECAST(N, DCI);
7594 case ISD::SIGN_EXTEND:
7595 case ISD::ZERO_EXTEND:
7596 return combineSZExtToMulWide(N, DCI, OptLevel);
7597 case ISD::BUILD_VECTOR:
7598 return PerformBUILD_VECTORCombine(N, DCI);
7599 case ISD::EXTRACT_VECTOR_ELT:
7600 return PerformEXTRACTCombine(N, DCI);
7601 case ISD::FADD:
7602 return performFADDCombine(N, DCI, OptLevel);
7603 case ISD::FMA:
7604 case ISD::FMUL:
7605 case ISD::FSUB:
7606 return performScalarizeV2F32Op(N, DCI, OptLevel);
7607 case ISD::FMAXNUM:
7608 case ISD::FMINNUM:
7609 case ISD::FMAXIMUM:
7610 case ISD::FMINIMUM:
7611 case ISD::FMAXIMUMNUM:
7612 case ISD::FMINIMUMNUM:
7613 return PerformFMinMaxCombine(N, DCI, STI);
7614 case ISD::LOAD:
7615 case NVPTXISD::LoadV2:
7616 case NVPTXISD::LoadV4:
7617 return combineLOAD(N, DCI, STI);
7618 case ISD::MUL:
7619 return PerformMULCombine(N, DCI, OptLevel);
7620 case NVPTXISD::PRMT:
7621 return combinePRMT(N, DCI, OptLevel);
7622 case NVPTXISD::ProxyReg:
7623 return combineProxyReg(N, DCI);
7624 case ISD::SETCC:
7625 return PerformSETCCCombine(N, DCI, STI);
7626 case ISD::SHL:
7627 return PerformSHLCombine(N, DCI, OptLevel);
7628 case ISD::STORE:
7629 case NVPTXISD::StoreV2:
7630 case NVPTXISD::StoreV4:
7631 return combineSTORE(N, DCI, STI);
7632 case ISD::SELECT:
7633 return PerformSELECTShiftCombine(N, DCI);
7634 case ISD::VSELECT:
7635 return PerformVSELECTCombine(N, DCI);
7636 case ISD::INTRINSIC_WO_CHAIN:
7637 return combineIntrinsicWOChain(N, DCI, STI);
7638 }
7639 return SDValue();
7640}
7641
7642static void ReplaceBITCAST(SDNode *Node, SelectionDAG &DAG,
7643 SmallVectorImpl<SDValue> &Results) {
7644 // Handle bitcasting to v2i8 without hitting the default promotion
7645 // strategy which goes through stack memory.
7646 SDValue Op(Node, 0);
7647 EVT ToVT = Op->getValueType(ResNo: 0);
7648 if (ToVT != MVT::v2i8) {
7649 return;
7650 }
7651
7652 // Bitcast to i16 and unpack elements into a vector
7653 SDLoc DL(Node);
7654 SDValue AsInt = DAG.getBitcast(VT: MVT::i16, V: Op->getOperand(Num: 0));
7655 SDValue Vec0 = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i8, Operand: AsInt);
7656 SDValue Const8 = DAG.getConstant(Val: 8, DL, VT: MVT::i16);
7657 SDValue Vec1 =
7658 DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i8,
7659 Operand: DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i16, Ops: {AsInt, Const8}));
7660 Results.push_back(
7661 Elt: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: MVT::v2i8, Ops: {Vec0, Vec1}));
7662}
7663
7664static void ReplaceINTRINSIC_W_CHAIN(SDNode *N, SelectionDAG &DAG,
7665 SmallVectorImpl<SDValue> &Results) {
7666 SDValue Chain = N->getOperand(Num: 0);
7667 SDValue Intrin = N->getOperand(Num: 1);
7668 SDLoc DL(N);
7669
7670 // Get the intrinsic ID
7671 unsigned IntrinNo = Intrin.getNode()->getAsZExtVal();
7672 switch (IntrinNo) {
7673 default:
7674 return;
7675 case Intrinsic::nvvm_ldu_global_i:
7676 case Intrinsic::nvvm_ldu_global_f:
7677 case Intrinsic::nvvm_ldu_global_p: {
7678 EVT ResVT = N->getValueType(ResNo: 0);
7679
7680 if (ResVT.isVector()) {
7681 // Vector LDG/LDU
7682
7683 unsigned NumElts = ResVT.getVectorNumElements();
7684 EVT EltVT = ResVT.getVectorElementType();
7685
7686 // Since LDU/LDG are target nodes, we cannot rely on DAG type
7687 // legalization.
7688 // Therefore, we must ensure the type is legal. For i1 and i8, we set the
7689 // loaded type to i16 and propagate the "real" type as the memory type.
7690 bool NeedTrunc = false;
7691 if (EltVT.getSizeInBits() < 16) {
7692 EltVT = MVT::i16;
7693 NeedTrunc = true;
7694 }
7695
7696 unsigned Opcode = 0;
7697 SDVTList LdResVTs;
7698
7699 switch (NumElts) {
7700 default:
7701 return;
7702 case 2:
7703 Opcode = NVPTXISD::LDUV2;
7704 LdResVTs = DAG.getVTList(VT1: EltVT, VT2: EltVT, VT3: MVT::Other);
7705 break;
7706 case 4: {
7707 Opcode = NVPTXISD::LDUV4;
7708 EVT ListVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other };
7709 LdResVTs = DAG.getVTList(VTs: ListVTs);
7710 break;
7711 }
7712 }
7713
7714 SmallVector<SDValue, 8> OtherOps;
7715
7716 // Copy regular operands
7717
7718 OtherOps.push_back(Elt: Chain); // Chain
7719 // Skip operand 1 (intrinsic ID)
7720 // Others
7721 OtherOps.append(in_start: N->op_begin() + 2, in_end: N->op_end());
7722
7723 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(Val: N);
7724
7725 SDValue NewLD = DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList: LdResVTs, Ops: OtherOps,
7726 MemVT: MemSD->getMemoryVT(),
7727 MMO: MemSD->getMemOperand());
7728
7729 SmallVector<SDValue, 4> ScalarRes;
7730
7731 for (unsigned i = 0; i < NumElts; ++i) {
7732 SDValue Res = NewLD.getValue(R: i);
7733 if (NeedTrunc)
7734 Res =
7735 DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ResVT.getVectorElementType(), Operand: Res);
7736 ScalarRes.push_back(Elt: Res);
7737 }
7738
7739 SDValue LoadChain = NewLD.getValue(R: NumElts);
7740
7741 SDValue BuildVec =
7742 DAG.getBuildVector(VT: ResVT, DL, Ops: ScalarRes);
7743
7744 Results.push_back(Elt: BuildVec);
7745 Results.push_back(Elt: LoadChain);
7746 } else {
7747 // i8 LDG/LDU
7748 assert(ResVT.isSimple() && ResVT.getSimpleVT().SimpleTy == MVT::i8 &&
7749 "Custom handling of non-i8 ldu/ldg?");
7750
7751 // Just copy all operands as-is
7752 SmallVector<SDValue, 4> Ops(N->ops());
7753
7754 // Force output to i16
7755 SDVTList LdResVTs = DAG.getVTList(VT1: MVT::i16, VT2: MVT::Other);
7756
7757 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(Val: N);
7758
7759 // We make sure the memory type is i8, which will be used during isel
7760 // to select the proper instruction.
7761 SDValue NewLD =
7762 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: LdResVTs, Ops,
7763 MemVT: MVT::i8, MMO: MemSD->getMemOperand());
7764
7765 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i8,
7766 Operand: NewLD.getValue(R: 0)));
7767 Results.push_back(Elt: NewLD.getValue(R: 1));
7768 }
7769 return;
7770 }
7771
7772 case Intrinsic::nvvm_tcgen05_ld_16x64b_x1:
7773 case Intrinsic::nvvm_tcgen05_ld_16x64b_x4:
7774 case Intrinsic::nvvm_tcgen05_ld_16x64b_x8:
7775 case Intrinsic::nvvm_tcgen05_ld_16x64b_x16:
7776 case Intrinsic::nvvm_tcgen05_ld_16x64b_x32:
7777 case Intrinsic::nvvm_tcgen05_ld_16x64b_x64:
7778 case Intrinsic::nvvm_tcgen05_ld_16x64b_x128:
7779 case Intrinsic::nvvm_tcgen05_ld_32x32b_x1:
7780 case Intrinsic::nvvm_tcgen05_ld_32x32b_x4:
7781 case Intrinsic::nvvm_tcgen05_ld_32x32b_x8:
7782 case Intrinsic::nvvm_tcgen05_ld_32x32b_x16:
7783 case Intrinsic::nvvm_tcgen05_ld_32x32b_x32:
7784 case Intrinsic::nvvm_tcgen05_ld_32x32b_x64:
7785 case Intrinsic::nvvm_tcgen05_ld_32x32b_x128:
7786 case Intrinsic::nvvm_tcgen05_ld_16x128b_x2:
7787 case Intrinsic::nvvm_tcgen05_ld_16x128b_x4:
7788 case Intrinsic::nvvm_tcgen05_ld_16x128b_x8:
7789 case Intrinsic::nvvm_tcgen05_ld_16x128b_x16:
7790 case Intrinsic::nvvm_tcgen05_ld_16x128b_x32:
7791 case Intrinsic::nvvm_tcgen05_ld_16x128b_x64:
7792 case Intrinsic::nvvm_tcgen05_ld_16x256b_x1:
7793 case Intrinsic::nvvm_tcgen05_ld_16x256b_x2:
7794 case Intrinsic::nvvm_tcgen05_ld_16x256b_x4:
7795 case Intrinsic::nvvm_tcgen05_ld_16x256b_x8:
7796 case Intrinsic::nvvm_tcgen05_ld_16x256b_x16:
7797 case Intrinsic::nvvm_tcgen05_ld_16x256b_x32:
7798 if (auto Res = lowerTcgen05Ld(N, DAG)) {
7799 Results.push_back(Elt: Res->first);
7800 Results.push_back(Elt: Res->second);
7801 }
7802 return;
7803
7804 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x1:
7805 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x4:
7806 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x8:
7807 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x16:
7808 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x32:
7809 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x64:
7810 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x128:
7811 if (auto Res = lowerTcgen05Ld(N, DAG, /*HasOffset=*/true)) {
7812 Results.push_back(Elt: Res->first);
7813 Results.push_back(Elt: Res->second);
7814 }
7815 return;
7816
7817 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x8_i32:
7818 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x8_f32:
7819 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x64_i32:
7820 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x64_f32:
7821 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x4_i32:
7822 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x4_f32:
7823 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x32_i32:
7824 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x32_f32:
7825 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x16_i32:
7826 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x16_f32:
7827 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x128_i32:
7828 case Intrinsic::nvvm_tcgen05_ld_red_32x32b_x128_f32:
7829 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x8_i32:
7830 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x8_f32:
7831 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x64_i32:
7832 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x64_f32:
7833 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x4_i32:
7834 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x4_f32:
7835 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x32_i32:
7836 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x32_f32:
7837 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x16_i32:
7838 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x16_f32:
7839 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x128_i32:
7840 case Intrinsic::nvvm_tcgen05_ld_red_16x32bx2_x128_f32:
7841 if (auto Res = lowerTcgen05LdRed(N, DAG)) {
7842 Results.push_back(Elt: std::get<0>(t&: *Res));
7843 Results.push_back(Elt: std::get<1>(t&: *Res));
7844 Results.push_back(Elt: std::get<2>(t&: *Res));
7845 }
7846 return;
7847 }
7848}
7849
7850static void ReplaceCopyFromReg_128(SDNode *N, SelectionDAG &DAG,
7851 SmallVectorImpl<SDValue> &Results) {
7852 // Change the CopyFromReg to output 2 64-bit results instead of a 128-bit
7853 // result so that it can pass the legalization
7854 SDLoc DL(N);
7855 SDValue Chain = N->getOperand(Num: 0);
7856 SDValue Reg = N->getOperand(Num: 1);
7857 SDValue Glue = N->getOperand(Num: 2);
7858
7859 assert(Reg.getValueType() == MVT::i128 &&
7860 "Custom lowering for CopyFromReg with 128-bit reg only");
7861 SmallVector<EVT, 4> ResultsType = {MVT::i64, MVT::i64, N->getValueType(ResNo: 1),
7862 N->getValueType(ResNo: 2)};
7863 SmallVector<SDValue, 3> NewOps = {Chain, Reg, Glue};
7864
7865 SDValue NewValue = DAG.getNode(Opcode: ISD::CopyFromReg, DL, ResultTys: ResultsType, Ops: NewOps);
7866 SDValue Pair = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i128,
7867 Ops: {NewValue.getValue(R: 0), NewValue.getValue(R: 1)});
7868
7869 Results.push_back(Elt: Pair);
7870 Results.push_back(Elt: NewValue.getValue(R: 2));
7871 Results.push_back(Elt: NewValue.getValue(R: 3));
7872}
7873
7874static void replaceProxyReg(SDNode *N, SelectionDAG &DAG,
7875 const TargetLowering &TLI,
7876 SmallVectorImpl<SDValue> &Results) {
7877 SDValue Chain = N->getOperand(Num: 0);
7878 SDValue Reg = N->getOperand(Num: 1);
7879
7880 MVT VT = TLI.getRegisterType(Context&: *DAG.getContext(), VT: Reg.getValueType());
7881
7882 SDValue NewReg = DAG.getAnyExtOrTrunc(Op: Reg, DL: SDLoc(N), VT);
7883 SDValue NewProxy =
7884 DAG.getNode(Opcode: NVPTXISD::ProxyReg, DL: SDLoc(N), VT, Ops: {Chain, NewReg});
7885 SDValue Res = DAG.getAnyExtOrTrunc(Op: NewProxy, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
7886
7887 Results.push_back(Elt: Res);
7888}
7889
7890static void replaceAtomicSwap128(SDNode *N, SelectionDAG &DAG,
7891 const NVPTXSubtarget &STI,
7892 SmallVectorImpl<SDValue> &Results) {
7893 assert(N->getValueType(0) == MVT::i128 &&
7894 "Custom lowering for atomic128 only supports i128");
7895
7896 AtomicSDNode *AN = cast<AtomicSDNode>(Val: N);
7897 SDLoc dl(N);
7898
7899 if (!STI.hasAtomSwap128()) {
7900 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
7901 DAG.getMachineFunction().getFunction(),
7902 "Support for b128 atomics introduced in PTX ISA version 8.3 and "
7903 "requires target sm_90.",
7904 dl.getDebugLoc()));
7905
7906 Results.push_back(Elt: DAG.getUNDEF(VT: MVT::i128));
7907 Results.push_back(Elt: AN->getOperand(Num: 0)); // Chain
7908 return;
7909 }
7910
7911 SmallVector<SDValue, 6> Ops;
7912 Ops.push_back(Elt: AN->getOperand(Num: 0)); // Chain
7913 Ops.push_back(Elt: AN->getOperand(Num: 1)); // Ptr
7914 for (const auto &Op : AN->ops().drop_front(N: 2)) {
7915 // Low part
7916 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: dl, VT: MVT::i64, N1: Op,
7917 N2: DAG.getIntPtrConstant(Val: 0, DL: dl)));
7918 // High part
7919 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: dl, VT: MVT::i64, N1: Op,
7920 N2: DAG.getIntPtrConstant(Val: 1, DL: dl)));
7921 }
7922 unsigned Opcode = N->getOpcode() == ISD::ATOMIC_SWAP
7923 ? NVPTXISD::ATOMIC_SWAP_B128
7924 : NVPTXISD::ATOMIC_CMP_SWAP_B128;
7925 SDVTList Tys = DAG.getVTList(VT1: MVT::i64, VT2: MVT::i64, VT3: MVT::Other);
7926 SDValue Result = DAG.getMemIntrinsicNode(Opcode, dl, VTList: Tys, Ops, MemVT: MVT::i128,
7927 MMO: AN->getMemOperand());
7928 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i128,
7929 Ops: {Result.getValue(R: 0), Result.getValue(R: 1)}));
7930 Results.push_back(Elt: Result.getValue(R: 2));
7931}
7932
7933static void replaceAtomicLoad128(SDNode *N, SelectionDAG &DAG,
7934 const NVPTXSubtarget &STI,
7935 SmallVectorImpl<SDValue> &Results) {
7936 assert(STI.hasLdStB128() &&
7937 "b128 atomic loads require PTX ISA version 8.3 and target sm_70");
7938 assert(N->getValueType(0) == MVT::i128 &&
7939 "Custom lowering for atomic128 only supports i128");
7940
7941 AtomicSDNode *AN = cast<AtomicSDNode>(Val: N);
7942 SDLoc dl(N);
7943
7944 SDValue Ops[] = {AN->getOperand(Num: 0), // Chain
7945 AN->getBasePtr()};
7946 SDVTList Tys = DAG.getVTList(VT1: MVT::i64, VT2: MVT::i64, VT3: MVT::Other);
7947 SDValue Result = DAG.getMemIntrinsicNode(Opcode: NVPTXISD::ATOMIC_LOAD_B128, dl, VTList: Tys,
7948 Ops, MemVT: MVT::i128, MMO: AN->getMemOperand());
7949
7950 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i128,
7951 Ops: {Result.getValue(R: 0), Result.getValue(R: 1)}));
7952 Results.push_back(Elt: Result.getValue(R: 2));
7953}
7954
7955void NVPTXTargetLowering::ReplaceNodeResults(
7956 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
7957 switch (N->getOpcode()) {
7958 default:
7959 report_fatal_error(reason: "Unhandled custom legalization");
7960 case ISD::BITCAST:
7961 ReplaceBITCAST(Node: N, DAG, Results);
7962 return;
7963 case ISD::LOAD:
7964 case ISD::MLOAD:
7965 replaceLoadVector(N, DAG, Results, STI);
7966 return;
7967 case ISD::INTRINSIC_W_CHAIN:
7968 ReplaceINTRINSIC_W_CHAIN(N, DAG, Results);
7969 return;
7970 case ISD::CopyFromReg:
7971 ReplaceCopyFromReg_128(N, DAG, Results);
7972 return;
7973 case NVPTXISD::ProxyReg:
7974 replaceProxyReg(N, DAG, TLI: *this, Results);
7975 return;
7976 case ISD::ATOMIC_CMP_SWAP:
7977 case ISD::ATOMIC_SWAP:
7978 replaceAtomicSwap128(N, DAG, STI, Results);
7979 return;
7980 case ISD::ATOMIC_LOAD:
7981 replaceAtomicLoad128(N, DAG, STI, Results);
7982 return;
7983 }
7984}
7985
7986/// Returns how NVPTX should expand a scalar atomicrmw.
7987static TargetLoweringBase::AtomicExpansionKind getScalarAtomicRMWExpansion(
7988 AtomicRMWInst::BinOp Op, Type *Ty, unsigned AddressSpace, const Function &F,
7989 bool IgnoreDenormalMode, const NVPTXSubtarget &STI) {
7990 using AtomicExpansionKind = TargetLoweringBase::AtomicExpansionKind;
7991 assert(!Ty->isVectorTy() && "Expected a scalar atomicrmw type");
7992
7993 // Try to lower LLVM atomicrmw fadd/fsub to PTX atomic.add. Fsub is first
7994 // expanded to an fadd with a negated operand. This is complicated by the
7995 // weird FTZ behavior PTX atom.add has:
7996 // - atom.add.f32 on global memory flushes denormals
7997 // - atom.add.f32 on shared memory does not flush denormals
7998 // - atom.add.f16 and atomic.add.bf16 never flush denormals
7999 //
8000 // We lower to atom.add only if the function's FTZ behavior matches that of
8001 // atom.add; otherwise, we lower to a CAS loop. But we always allow
8002 // atomic.add.bf16; even though it never flushes denormals, we never flush
8003 // bf16 denormals when doing regular arithmetic, even when FTZ is enabled.
8004 if (Op == AtomicRMWInst::BinOp::FAdd || Op == AtomicRMWInst::BinOp::FSub) {
8005 AtomicExpansionKind ExpansionKind = Op == AtomicRMWInst::BinOp::FSub
8006 ? AtomicExpansionKind::Expand
8007 : AtomicExpansionKind::None;
8008
8009 // Both the -nvptx-allow-ftz-atomics option and per-instruction
8010 // !atomic.ignore.denormal.mode say that denormal handling is insignificant
8011 // here, so atom.add may be used even when its FTZ behavior disagrees with
8012 // the function's.
8013 const bool IgnoreFTZMismatch =
8014 STI.getCLOpts().allow_ftz_atomics || IgnoreDenormalMode;
8015
8016 if (Ty->isFloatTy()) {
8017 const bool FTZ = F.getDenormalMode(FPType: APFloat::IEEEsingle()).Input ==
8018 DenormalMode::PreserveSign;
8019 bool UseNative = IgnoreFTZMismatch;
8020 switch (AddressSpace) {
8021 case llvm::ADDRESS_SPACE_GLOBAL:
8022 UseNative |= FTZ;
8023 break;
8024 case llvm::ADDRESS_SPACE_SHARED:
8025 case llvm::ADDRESS_SPACE_SHARED_CLUSTER:
8026 UseNative |= !FTZ;
8027 break;
8028 }
8029 if (UseNative)
8030 return ExpansionKind;
8031 }
8032
8033 if (Ty->isHalfTy()) {
8034 // atom.add.f16 never flushes denormals, so it only agrees with a
8035 // function that is not in FTZ mode for f16.
8036 const bool FTZ = F.getDenormalMode(FPType: APFloat::IEEEhalf()).Input ==
8037 DenormalMode::PreserveSign;
8038 if ((!FTZ || IgnoreFTZMismatch) && STI.hasFeature(Feature: NVPTX::SM70) &&
8039 STI.hasFeature(Feature: NVPTX::PTX63))
8040 return ExpansionKind;
8041 }
8042
8043 if (Ty->isBFloatTy() && STI.hasFeature(Feature: NVPTX::SM90))
8044 return ExpansionKind;
8045
8046 if (Ty->isDoubleTy() && STI.hasAtomAddF64())
8047 return ExpansionKind;
8048 }
8049
8050 // PTX's only atomic fp op is `add`; all other ops expand to a CAS loop.
8051 if (AtomicRMWInst::isFPOperation(Op))
8052 return AtomicExpansionKind::CmpXChg;
8053
8054 assert(Ty->isIntegerTy() && "Expected an integer atomicrmw type");
8055 const unsigned BitWidth = cast<IntegerType>(Val: Ty)->getBitWidth();
8056
8057 switch (Op) {
8058 default:
8059 return AtomicExpansionKind::CmpXChg;
8060 case AtomicRMWInst::Xchg:
8061 if (BitWidth == 128)
8062 return AtomicExpansionKind::None;
8063 [[fallthrough]];
8064 case AtomicRMWInst::BinOp::Add:
8065 case AtomicRMWInst::BinOp::Sub: {
8066 AtomicExpansionKind ExpansionKind = Op == AtomicRMWInst::BinOp::Sub
8067 ? AtomicExpansionKind::Expand
8068 : AtomicExpansionKind::None;
8069 switch (BitWidth) {
8070 case 8:
8071 case 16:
8072 return AtomicExpansionKind::CmpXChg;
8073 case 32:
8074 case 64:
8075 return ExpansionKind;
8076 case 128:
8077 return AtomicExpansionKind::CmpXChg;
8078 default:
8079 llvm_unreachable("unsupported width encountered");
8080 }
8081 }
8082 case AtomicRMWInst::BinOp::And:
8083 case AtomicRMWInst::BinOp::Or:
8084 case AtomicRMWInst::BinOp::Xor:
8085 case AtomicRMWInst::BinOp::Max:
8086 case AtomicRMWInst::BinOp::Min:
8087 case AtomicRMWInst::BinOp::UMax:
8088 case AtomicRMWInst::BinOp::UMin:
8089 switch (BitWidth) {
8090 case 8:
8091 case 16:
8092 return AtomicExpansionKind::CmpXChg;
8093 case 32:
8094 return AtomicExpansionKind::None;
8095 case 64:
8096 if (STI.hasAtomMinMaxAndOrXor())
8097 return AtomicExpansionKind::None;
8098 return AtomicExpansionKind::CmpXChg;
8099 case 128:
8100 return AtomicExpansionKind::CmpXChg;
8101 default:
8102 llvm_unreachable("unsupported width encountered");
8103 }
8104 case AtomicRMWInst::UIncWrap:
8105 case AtomicRMWInst::UDecWrap:
8106 switch (BitWidth) {
8107 case 32:
8108 return AtomicExpansionKind::None;
8109 case 8:
8110 case 16:
8111 case 64:
8112 case 128:
8113 return AtomicExpansionKind::CmpXChg;
8114 default:
8115 llvm_unreachable("unsupported width encountered");
8116 }
8117 }
8118
8119 return AtomicExpansionKind::CmpXChg;
8120}
8121
8122NVPTXTargetLowering::AtomicExpansionKind
8123NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
8124 const DataLayout &DL = AI->getDataLayout();
8125 AtomicRMWInst::BinOp Op = AI->getOperation();
8126 const bool IgnoreDenormalMode =
8127 AI->hasMetadata(KindID: LLVMContext::MD_atomic_ignore_denormal_mode);
8128
8129 // TODO: once we support native elementwise vector atoms,
8130 // return `AtomicExpansionKind::None` to preserve and lower them.
8131 if (AI->isElementwise()) {
8132 auto *VecTy = cast<FixedVectorType>(Val: AI->getType());
8133 Type *Ty = VecTy->getElementType();
8134
8135 // Collapse <1 x T> elementwise RMWs to scalar RMWs
8136 if (VecTy->getNumElements() == 1)
8137 return AtomicExpansionKind::Expand;
8138
8139 uint64_t VecBytes = DL.getTypeStoreSize(Ty: VecTy).getFixedValue();
8140 uint64_t VecBits = DL.getTypeStoreSizeInBits(Ty: VecTy).getFixedValue();
8141 // RMWs need SM 90 for 128-bit operations, unlike atomic loads/stores.
8142 const unsigned MaxRMWBits = STI.hasAtomSwap128() ? 128 : 64;
8143
8144 if (Op == AtomicRMWInst::BinOp::Xchg) {
8145 // Split exchanges that are too wide or insufficiently aligned until
8146 // each piece can be lowered atomically.
8147 if (VecBits > MaxRMWBits || AI->getAlign().value() < VecBytes)
8148 return AtomicExpansionKind::Expand;
8149
8150 // Expand subword exchanges to a masked cmpxchg loop.
8151 if (VecBits < getMinCmpXchgSizeInBits())
8152 return AtomicExpansionKind::CmpXChg;
8153
8154 return AtomicExpansionKind::None;
8155 }
8156
8157 // If the scalar lane op is natively supported or expands to one, return
8158 // Expand so halving eventually bottoms out at the scalar base case. The
8159 // latter includes sub and fsub, which expand to add and fadd, respectively.
8160 AtomicExpansionKind LaneExpansionKind = getScalarAtomicRMWExpansion(
8161 Op, Ty, AddressSpace: AI->getPointerAddressSpace(), F: *AI->getFunction(),
8162 IgnoreDenormalMode, STI);
8163 if (LaneExpansionKind == AtomicExpansionKind::None ||
8164 LaneExpansionKind == AtomicExpansionKind::Expand)
8165 return AtomicExpansionKind::Expand;
8166
8167 // If the whole vector fits a single native cmpxchg, emit one wide
8168 // cmpxchg loop at the current width.
8169 if (VecBits <= MaxRMWBits)
8170 return AtomicExpansionKind::CmpXChg;
8171
8172 // If the vector is too wide for a single native cmpxchg, halve further to
8173 // emit multiple cmpxchg loops.
8174 return AtomicExpansionKind::Expand;
8175 }
8176
8177 Type *Ty = AI->getValOperand()->getType();
8178 if (Ty->isVectorTy())
8179 return AtomicExpansionKind::CmpXChg;
8180 return getScalarAtomicRMWExpansion(Op, Ty, AddressSpace: AI->getPointerAddressSpace(),
8181 F: *AI->getFunction(), IgnoreDenormalMode,
8182 STI);
8183}
8184
8185// clang-format off
8186// Atomic load lowering grid, one (LLVM ordering, native support) combination
8187// per line:
8188//
8189// seq_cst, ld.acquire supported: fence seq_cst; load acquire
8190// seq_cst, ld.acquire unsupported: fence seq_cst; load monotonic; fence acquire
8191// acquire, ld.acquire supported: load acquire
8192// acquire, ld.acquire unsupported: load monotonic; fence acquire
8193//
8194// Atomic store lowering grid, in the same form:
8195//
8196// seq_cst, st.release supported: fence seq_cst; store monotonic
8197// seq_cst, st.release unsupported: fence seq_cst; store monotonic
8198// release, st.release supported: store release
8199// release, st.release unsupported: fence release; store monotonic
8200//
8201// Note that ordered fences are only supported on architectures where
8202// acquire/release loads/stores are supported. On lower architectures, all
8203// LLVM fences are lowered "membar". This is correct, because membar provides
8204// all the guarantees of "fence.sc".
8205// Lowering is based on the PTX Atomics ABI:
8206// https://docs.nvidia.com/cuda/ptx-writers-guide-to-interoperability/atomic-abi.html
8207// clang-format on
8208
8209bool NVPTXTargetLowering::shouldInsertFencesForAtomic(
8210 const Instruction *I) const {
8211 // atomicrmw and cmpxchg instructions not efficiently supported by PTX
8212 // are lowered to CAS emulation loops that preserve their memory order,
8213 // syncscope, and volatile semantics. For PTX, it is more efficient to use
8214 // atom.cas.relaxed.sco instructions within the loop, and fences before and
8215 // after the loop to restore order.
8216 //
8217 // For ordered elementwise atomicrmw, if we expand elementwise, it is also
8218 // more efficient to insert fences around the whole split sequence once and
8219 // downgrade the ordering for the entire sequence, rather than having the
8220 // stronger ordering on every expanded atomicrmw.
8221 //
8222 // On targets with memory-order qualifiers, atomic instructions efficiently
8223 // supported by PTX are lowered to `atom.<op>.<sem>.<scope>` instructions with
8224 // their corresponding memory order and scope. Since PTX does not support
8225 // seq_cst, we emulate it by lowering to a fence.sc followed by an atom
8226 // according to the PTX atomics ABI.
8227 // https://docs.nvidia.com/cuda/ptx-writers-guide-to-interoperability/atomic-abi.html
8228 //
8229 // Atomic loads with a memory order not supported on the SM can be
8230 // inefficiently emulated by appending a `fence.acquire` after a relaxed load.
8231 if (auto *CI = dyn_cast<AtomicCmpXchgInst>(Val: I))
8232 return (cast<IntegerType>(Val: CI->getCompareOperand()->getType())
8233 ->getBitWidth() < STI.getMinCmpXchgSizeInBits()) ||
8234 !STI.hasMemoryOrdering() ||
8235 CI->getMergedOrdering() == AtomicOrdering::SequentiallyConsistent;
8236 if (auto *RI = dyn_cast<AtomicRMWInst>(Val: I)) {
8237 AtomicExpansionKind Expansion = shouldExpandAtomicRMWInIR(AI: RI);
8238 return Expansion == AtomicExpansionKind::CmpXChg ||
8239 (RI->isElementwise() && Expansion == AtomicExpansionKind::Expand) ||
8240 !STI.hasMemoryOrdering() ||
8241 RI->getOrdering() == AtomicOrdering::SequentiallyConsistent;
8242 }
8243 if (auto *LI = dyn_cast<LoadInst>(Val: I))
8244 return LI->getOrdering() == AtomicOrdering::SequentiallyConsistent ||
8245 (!STI.hasMemoryOrdering() && isAcquireOrStronger(AO: LI->getOrdering()));
8246 if (auto *SI = dyn_cast<StoreInst>(Val: I))
8247 return SI->getOrdering() == AtomicOrdering::SequentiallyConsistent ||
8248 (!STI.hasMemoryOrdering() && isReleaseOrStronger(AO: SI->getOrdering()));
8249 return false;
8250}
8251
8252AtomicOrdering NVPTXTargetLowering::atomicOperationOrderAfterFenceSplit(
8253 const Instruction *I) const {
8254 // Returns the new AtomicOrdering for the original op once AtomicExpandPass
8255 // has split it into leading/trailing fences + the op itself. Per the grids
8256 // above:
8257 // - cmpxchg / atomicrmw at seq_cst (not CAS-emulated): Acquire
8258 // ("fence.sc; atom.<op>.acquire" pattern).
8259 // - load at any ordering when hasMemoryOrdering: Acquire
8260 // ("fence.sc; ld.acquire" for seq_cst; the acquire load is already typed
8261 // on supported targets).
8262 // - everything else (CAS-emulated ops, seq_cst on !hasMemoryOrdering,
8263 // release-or-stronger store on !hasMemoryOrdering, etc.): Monotonic
8264 // (i.e., relaxed) -- the surrounding fences carry the ordering.
8265 //
8266 // If the operation is emulated by a CAS-loop, we lower the instruction to
8267 // atom.<op>.relaxed, since AtomicExpandPass will insert fences for enforcing
8268 // the correct memory ordering around the CAS loop.
8269 //
8270 // When the operation is not emulated, but the memory order is seq_cst,
8271 // we must lower to "fence.sc.<scope>; atom.<op>.acquire.<scope>;" to conform
8272 // to the PTX atomics ABI.
8273 // https://docs.nvidia.com/cuda/ptx-writers-guide-to-interoperability/atomic-abi.html
8274 // For such cases, emitLeadingFence() will separately insert the leading
8275 // "fence.sc.<scope>;". Here, we only set the memory order to acquire.
8276 //
8277 // Otherwise, the operation is not emulated, and the memory order is not
8278 // seq_cst. In this case, the LLVM memory order is natively supported by the
8279 // PTX `atom` instruction, and we just lower to the corresponding
8280 // `atom.<op>.relaxed|acquire|release|acq_rel". For such cases, this function
8281 // will NOT be called.
8282 // prerequisite: shouldInsertFencesForAtomic() should have returned `true` for
8283 // I before its memory order was modified.
8284 if (!STI.hasMemoryOrdering())
8285 return AtomicOrdering::Monotonic;
8286
8287 if (auto *CI = dyn_cast<AtomicCmpXchgInst>(Val: I);
8288 CI && CI->getMergedOrdering() == AtomicOrdering::SequentiallyConsistent &&
8289 cast<IntegerType>(Val: CI->getCompareOperand()->getType())->getBitWidth() >=
8290 STI.getMinCmpXchgSizeInBits()) {
8291 return AtomicOrdering::Acquire;
8292 } else if (auto *RI = dyn_cast<AtomicRMWInst>(Val: I);
8293 RI &&
8294 RI->getOrdering() == AtomicOrdering::SequentiallyConsistent) {
8295 AtomicExpansionKind ExpansionKind = shouldExpandAtomicRMWInIR(AI: RI);
8296 // A non-elementwise `Expand` means we are expanding sub(x) to add(-x).
8297 // The atomic add will have acquire ordering.
8298 if (ExpansionKind == AtomicExpansionKind::None ||
8299 (ExpansionKind == AtomicExpansionKind::Expand && !RI->isElementwise()))
8300 return AtomicOrdering::Acquire;
8301 } else if (isa<LoadInst>(Val: I)) {
8302 return AtomicOrdering::Acquire;
8303 }
8304
8305 return AtomicOrdering::Monotonic;
8306}
8307
8308Instruction *NVPTXTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
8309 Instruction *Inst,
8310 AtomicOrdering Ord) const {
8311 // Emits the leading fence per the grids above:
8312 // - cmpxchg / atomicrmw: fence.sc iff Ord == seq_cst, else fence.release
8313 // if Ord is release-or-stronger.
8314 // - load: fence.sc iff Ord == seq_cst.
8315 // - store: same shape as cmpxchg/rmw (fence.sc or fence.release based on
8316 // Ord being seq_cst vs release-or-stronger).
8317 // Returns nullptr if no leading fence is needed for this Inst/Ord.
8318 //
8319 // prerequisite: shouldInsertFencesForAtomic() should have returned `true` for
8320 // `Inst` before its memory order was modified. We cannot enforce this with an
8321 // assert, because AtomicExpandPass will have modified the memory order
8322 // between the initial call to shouldInsertFencesForAtomic() and the call to
8323 // this function.
8324 auto SSID = getAtomicSyncScopeID(I: Inst);
8325 assert(SSID.has_value() && "Expected an atomic operation");
8326 if (isa<AtomicCmpXchgInst>(Val: Inst) || isa<AtomicRMWInst>(Val: Inst)) {
8327 // Specialize for cmpxchg and atomicrmw
8328 if (isReleaseOrStronger(AO: Ord))
8329 return Builder.CreateFence(Ordering: Ord == AtomicOrdering::SequentiallyConsistent
8330 ? AtomicOrdering::SequentiallyConsistent
8331 : AtomicOrdering::Release,
8332 SSID: SSID.value());
8333 } else if (auto *LI = dyn_cast<LoadInst>(Val: Inst)) {
8334 // volatile operations have sys scope by default
8335 SSID = LI->isVolatile() ? Builder.getContext().getOrInsertSyncScopeID(SSN: "")
8336 : SSID;
8337 if (Ord == AtomicOrdering::SequentiallyConsistent)
8338 return Builder.CreateFence(Ordering: AtomicOrdering::SequentiallyConsistent,
8339 SSID: SSID.value());
8340 } else if (auto *SI = dyn_cast<StoreInst>(Val: Inst)) {
8341 // volatile operations have sys scope by default
8342 SSID = SI->isVolatile() ? Builder.getContext().getOrInsertSyncScopeID(SSN: "")
8343 : SSID;
8344 if (isReleaseOrStronger(AO: Ord))
8345 return Builder.CreateFence(Ordering: Ord == AtomicOrdering::SequentiallyConsistent
8346 ? AtomicOrdering::SequentiallyConsistent
8347 : AtomicOrdering::Release,
8348 SSID: SSID.value());
8349 }
8350
8351 return nullptr;
8352}
8353
8354Instruction *NVPTXTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
8355 Instruction *Inst,
8356 AtomicOrdering Ord) const {
8357 // Emits the trailing fence per the grids above:
8358 // - cmpxchg / atomicrmw: fence.acquire iff Ord is acquire-or-stronger AND
8359 // the op is being CAS-emulated (i.e., the inner CAS uses relaxed).
8360 // - load: fence.acquire iff !hasMemoryOrdering (covers both the acquire
8361 // and seq_cst cells of the load grid).
8362 // - store: never (release-side ordering for stores comes from the leading
8363 // fence or the typed `st.release` instruction).
8364 // Returns nullptr if no trailing fence is needed for this Inst/Ord.
8365 //
8366 // prerequisite: shouldInsertFencesForAtomic() should have returned `true` for
8367 // `Inst` before its memory order was modified. See `emitLeadingFence` for why
8368 // this cannot be enforced with an assert. Specialize for cmpxchg and
8369 // atomicrmw
8370 auto *CI = dyn_cast<AtomicCmpXchgInst>(Val: Inst);
8371 auto *RI = dyn_cast<AtomicRMWInst>(Val: Inst);
8372 auto SSID = getAtomicSyncScopeID(I: Inst);
8373 assert(SSID.has_value() && "Expected an atomic operation");
8374 if (CI || RI) {
8375 bool IsEmulated = !STI.hasMemoryOrdering();
8376 if (CI)
8377 IsEmulated |=
8378 cast<IntegerType>(Val: CI->getCompareOperand()->getType())->getBitWidth() <
8379 STI.getMinCmpXchgSizeInBits();
8380 else {
8381 AtomicExpansionKind Expansion = shouldExpandAtomicRMWInIR(AI: RI);
8382 IsEmulated |=
8383 Expansion == AtomicExpansionKind::CmpXChg ||
8384 (RI->isElementwise() && Expansion == AtomicExpansionKind::Expand);
8385 }
8386
8387 if (isAcquireOrStronger(AO: Ord) && IsEmulated)
8388 return Builder.CreateFence(Ordering: AtomicOrdering::Acquire, SSID: SSID.value());
8389
8390 } else if (isa<LoadInst>(Val: Inst) && !STI.hasMemoryOrdering()) {
8391 return Builder.CreateFence(Ordering: AtomicOrdering::Acquire, SSID: SSID.value());
8392 }
8393 return nullptr;
8394}
8395
8396// Rather than default to SINT when both UINT and SINT are custom, we only
8397// change the opcode when UINT is not legal and SINT is. UINT is preferred when
8398// both are custom since unsigned CVT instructions can lead to slightly better
8399// SASS code with fewer instructions.
8400unsigned NVPTXTargetLowering::getPreferredFPToIntOpcode(unsigned Op, EVT FromVT,
8401 EVT ToVT) const {
8402 if (isOperationLegal(Op, VT: ToVT))
8403 return Op;
8404 switch (Op) {
8405 case ISD::FP_TO_UINT:
8406 if (isOperationLegal(Op: ISD::FP_TO_SINT, VT: ToVT))
8407 return ISD::FP_TO_SINT;
8408 break;
8409 case ISD::STRICT_FP_TO_UINT:
8410 if (isOperationLegal(Op: ISD::STRICT_FP_TO_SINT, VT: ToVT))
8411 return ISD::STRICT_FP_TO_SINT;
8412 break;
8413 default:
8414 break;
8415 }
8416 return Op;
8417}
8418
8419// Pin NVPTXTargetObjectFile's vtables to this file.
8420NVPTXTargetObjectFile::~NVPTXTargetObjectFile() = default;
8421
8422MCSection *NVPTXTargetObjectFile::SelectSectionForGlobal(
8423 const GlobalObject *GO, SectionKind Kind, const TargetMachine &TM) const {
8424 return getDataSection();
8425}
8426
8427static void computeKnownBitsForPRMT(const SDValue Op, KnownBits &Known,
8428 const SelectionDAG &DAG, unsigned Depth) {
8429 SDValue A = Op.getOperand(i: 0);
8430 SDValue B = Op.getOperand(i: 1);
8431 ConstantSDNode *Selector = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
8432 unsigned Mode = Op.getConstantOperandVal(i: 3);
8433
8434 if (!Selector)
8435 return;
8436
8437 KnownBits AKnown = DAG.computeKnownBits(Op: A, Depth);
8438 KnownBits BKnown = DAG.computeKnownBits(Op: B, Depth);
8439
8440 // {b, a} = {{b7, b6, b5, b4}, {b3, b2, b1, b0}}
8441 assert(AKnown.getBitWidth() == 32 && BKnown.getBitWidth() == 32 &&
8442 "PRMT must have i32 operands");
8443 assert(Known.getBitWidth() == 32 && "PRMT must have i32 result");
8444 KnownBits BitField = BKnown.concat(Lo: AKnown);
8445
8446 APInt SelectorVal = getPRMTSelector(Selector: Selector->getAPIntValue(), Mode);
8447 for (unsigned I : llvm::seq(Size: 4)) {
8448 APInt Sel = SelectorVal.extractBits(numBits: 4, bitPosition: I * 4);
8449 unsigned Idx = Sel.getLoBits(numBits: 3).getZExtValue();
8450 unsigned Sign = Sel.getHiBits(numBits: 1).getZExtValue();
8451 KnownBits Byte = BitField.extractBits(NumBits: 8, BitPosition: Idx * 8);
8452 if (Sign)
8453 Byte = KnownBits::ashr(LHS: Byte, RHS: KnownBits::makeConstant(C: APInt(8, 7)));
8454 Known.insertBits(SubBits: Byte, BitPosition: I * 8);
8455 }
8456}
8457
8458static void computeKnownBitsForLoadV(const SDValue Op, KnownBits &Known) {
8459 MemSDNode *LD = cast<MemSDNode>(Val: Op);
8460
8461 // We can't do anything without knowing the sign bit.
8462 auto ExtType = LD->getConstantOperandVal(Num: LD->getNumOperands() - 1);
8463 if (ExtType == ISD::SEXTLOAD)
8464 return;
8465
8466 // ExtLoading to vector types is weird and may not work well with known bits.
8467 auto DestVT = LD->getValueType(ResNo: 0);
8468 if (DestVT.isVector())
8469 return;
8470
8471 assert(Known.getBitWidth() == DestVT.getSizeInBits());
8472 auto ElementBitWidth = getFromTypeWidthForLoad(Mem: LD);
8473 Known.Zero.setHighBits(Known.getBitWidth() - ElementBitWidth);
8474}
8475
8476void NVPTXTargetLowering::computeKnownBitsForTargetNode(
8477 const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
8478 const SelectionDAG &DAG, unsigned Depth) const {
8479 Known.resetAll();
8480
8481 switch (Op.getOpcode()) {
8482 case NVPTXISD::MUL_WIDE_SIGNED:
8483 case NVPTXISD::MUL_WIDE_UNSIGNED: {
8484 KnownBits LHS = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
8485 KnownBits RHS = DAG.computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
8486 unsigned BitWidth = Known.getBitWidth();
8487 bool IsSigned = Op.getOpcode() == NVPTXISD::MUL_WIDE_SIGNED;
8488 // Forming mul.wide must not hide bits that let subsequent combines remove
8489 // unused parts of its operands, such as the high bytes of narrow loads.
8490 Known = KnownBits::mul(LHS: IsSigned ? LHS.sext(BitWidth) : LHS.zext(BitWidth),
8491 RHS: IsSigned ? RHS.sext(BitWidth) : RHS.zext(BitWidth));
8492 break;
8493 }
8494 case NVPTXISD::CLZ_B64:
8495 case NVPTXISD::POPC_B64: {
8496 KnownBits Input =
8497 DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
8498 unsigned MaxCount = Op.getOpcode() == NVPTXISD::CLZ_B64
8499 ? Input.countMaxLeadingZeros()
8500 : Input.countMaxPopulation();
8501 Known.Zero.setBitsFrom(llvm::bit_width(Value: MaxCount));
8502 break;
8503 }
8504 case NVPTXISD::PRMT:
8505 computeKnownBitsForPRMT(Op, Known, DAG, Depth);
8506 break;
8507 case NVPTXISD::LoadV2:
8508 case NVPTXISD::LoadV4:
8509 case NVPTXISD::LoadV8:
8510 computeKnownBitsForLoadV(Op, Known);
8511 break;
8512 default:
8513 break;
8514 }
8515}
8516
8517static std::pair<APInt, APInt> getPRMTDemandedBits(const APInt &SelectorVal,
8518 const APInt &DemandedBits) {
8519 APInt DemandedLHS = APInt(32, 0);
8520 APInt DemandedRHS = APInt(32, 0);
8521
8522 for (unsigned I : llvm::seq(Size: 4)) {
8523 if (DemandedBits.extractBits(numBits: 8, bitPosition: I * 8).isZero())
8524 continue;
8525
8526 APInt Sel = SelectorVal.extractBits(numBits: 4, bitPosition: I * 4);
8527 unsigned Idx = Sel.getLoBits(numBits: 3).getZExtValue();
8528 unsigned Sign = Sel.getHiBits(numBits: 1).getZExtValue();
8529
8530 APInt &Src = Idx < 4 ? DemandedLHS : DemandedRHS;
8531 unsigned ByteStart = (Idx % 4) * 8;
8532 if (Sign)
8533 Src.setBit(ByteStart + 7);
8534 else
8535 Src.setBits(loBit: ByteStart, hiBit: ByteStart + 8);
8536 }
8537
8538 return {DemandedLHS, DemandedRHS};
8539}
8540
8541// Replace undef with 0 as this is easier for other optimizations such as
8542// known bits.
8543static SDValue canonicalizePRMTInput(SDValue Op, SelectionDAG &DAG) {
8544 if (!Op)
8545 return SDValue();
8546 if (Op.isUndef())
8547 return DAG.getConstant(Val: 0, DL: SDLoc(), VT: MVT::i32);
8548 return Op;
8549}
8550
8551static SDValue simplifyDemandedBitsForPRMT(SDValue PRMT,
8552 const APInt &DemandedBits,
8553 SelectionDAG &DAG,
8554 const TargetLowering &TLI,
8555 unsigned Depth) {
8556 assert(PRMT.getOpcode() == NVPTXISD::PRMT);
8557 SDValue Op0 = PRMT.getOperand(i: 0);
8558 SDValue Op1 = PRMT.getOperand(i: 1);
8559 auto *SelectorConst = dyn_cast<ConstantSDNode>(Val: PRMT.getOperand(i: 2));
8560 if (!SelectorConst)
8561 return SDValue();
8562
8563 unsigned Mode = PRMT.getConstantOperandVal(i: 3);
8564 const APInt Selector = getPRMTSelector(Selector: SelectorConst->getAPIntValue(), Mode);
8565
8566 // Try to simplify the PRMT to one of the inputs if the used bytes are all
8567 // from the same input in the correct order.
8568 const unsigned LeadingBytes = DemandedBits.countLeadingZeros() / 8;
8569 const unsigned SelBits = (4 - LeadingBytes) * 4;
8570 if (Selector.getLoBits(numBits: SelBits) == APInt(32, 0x3210).getLoBits(numBits: SelBits))
8571 return Op0;
8572 if (Selector.getLoBits(numBits: SelBits) == APInt(32, 0x7654).getLoBits(numBits: SelBits))
8573 return Op1;
8574
8575 auto [DemandedLHS, DemandedRHS] = getPRMTDemandedBits(SelectorVal: Selector, DemandedBits);
8576
8577 // Attempt to avoid multi-use ops if we don't need anything from them.
8578 SDValue DemandedOp0 =
8579 TLI.SimplifyMultipleUseDemandedBits(Op: Op0, DemandedBits: DemandedLHS, DAG, Depth: Depth + 1);
8580 SDValue DemandedOp1 =
8581 TLI.SimplifyMultipleUseDemandedBits(Op: Op1, DemandedBits: DemandedRHS, DAG, Depth: Depth + 1);
8582
8583 DemandedOp0 = canonicalizePRMTInput(Op: DemandedOp0, DAG);
8584 DemandedOp1 = canonicalizePRMTInput(Op: DemandedOp1, DAG);
8585 if ((DemandedOp0 && DemandedOp0 != Op0) ||
8586 (DemandedOp1 && DemandedOp1 != Op1)) {
8587 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
8588 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
8589 return getPRMT(A: Op0, B: Op1, Selector: Selector.getZExtValue(), DL: SDLoc(PRMT), DAG);
8590 }
8591
8592 return SDValue();
8593}
8594
8595bool NVPTXTargetLowering::SimplifyDemandedBitsForTargetNode(
8596 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
8597 KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
8598 Known.resetAll();
8599
8600 switch (Op.getOpcode()) {
8601 case NVPTXISD::PRMT:
8602 if (SDValue Result = simplifyDemandedBitsForPRMT(PRMT: Op, DemandedBits, DAG&: TLO.DAG,
8603 TLI: *this, Depth)) {
8604 TLO.CombineTo(O: Op, N: Result);
8605 return true;
8606 }
8607 break;
8608 default:
8609 break;
8610 }
8611
8612 computeKnownBitsForTargetNode(Op, Known, DemandedElts, DAG: TLO.DAG, Depth);
8613 return false;
8614}
8615