1//===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an instruction selector for the NVPTX target.
10//
11//===----------------------------------------------------------------------===//
12
13#include "MCTargetDesc/NVPTXBaseInfo.h"
14#include "NVPTX.h"
15#include "NVPTXISelLowering.h"
16#include "NVPTXSelectionDAGInfo.h"
17#include "NVPTXTargetMachine.h"
18#include "NVPTXUtilities.h"
19#include "llvm/ADT/APInt.h"
20#include "llvm/ADT/MapVector.h"
21#include "llvm/ADT/StringSwitch.h"
22#include "llvm/ADT/Twine.h"
23#include "llvm/Analysis/ValueTracking.h"
24#include "llvm/CodeGen/ISDOpcodes.h"
25#include "llvm/CodeGen/SelectionDAG.h"
26#include "llvm/CodeGen/SelectionDAGISel.h"
27#include "llvm/CodeGen/SelectionDAGNodes.h"
28#include "llvm/IR/Constants.h"
29#include "llvm/IR/DiagnosticInfo.h"
30#include "llvm/IR/InlineAsm.h"
31#include "llvm/IR/Instructions.h"
32#include "llvm/IR/Intrinsics.h"
33#include "llvm/IR/IntrinsicsNVPTX.h"
34#include "llvm/IR/LLVMContext.h"
35#include "llvm/IR/Metadata.h"
36#include "llvm/Support/AtomicOrdering.h"
37#include "llvm/Support/ErrorHandling.h"
38#include "llvm/Support/FormatVariadic.h"
39#include "llvm/Support/KnownFPClass.h"
40#include "llvm/Support/MathExtras.h"
41#include "llvm/TargetParser/AtomicScope.h"
42#include <optional>
43
44using namespace llvm;
45
46#define DEBUG_TYPE "nvptx-isel"
47#define PASS_NAME "NVPTX DAG->DAG Pattern Instruction Selection"
48
49namespace {
50
51struct NVPTXScopes {
52 NVPTXScopes() = default;
53 NVPTXScopes(LLVMContext &C, const Triple &T);
54 NVPTX::Scope operator[](SyncScope::ID ID) const;
55 bool empty() const;
56
57private:
58 SmallMapVector<SyncScope::ID, NVPTX::Scope, 8> Scopes{};
59 LLVMContext *Context = nullptr;
60};
61
62enum class NVPTXMemCacheHintInstruction { Ld, St, Atom };
63
64struct NVPTXMemCacheHintAccess {
65 NVPTXMemCacheHintInstruction Instruction;
66 NVPTX::AddressSpace AddrSpace;
67 unsigned NumElts;
68 unsigned EltWidth;
69 bool IsVolatile;
70};
71
72struct NVPTXMemCacheHintOperands {
73 SDValue EvictionAndPrefetchHint;
74 SDValue CachePolicyReg;
75};
76
77class NVPTXDAGToDAGISel : public SelectionDAGISel {
78 const NVPTXTargetMachine &TM;
79
80 NVPTX::DivPrecisionLevel getDivF32Level(const SDNode *N) const;
81 bool usePrecSqrtF32(const SDNode *N) const;
82 bool useF32FTZ() const;
83 bool useF16FTZ() const;
84 bool allowFMA() const;
85 bool doRsqrtOpt() const;
86 bool doMADWideOpt() const;
87
88 NVPTXScopes Scopes{};
89
90public:
91 NVPTXDAGToDAGISel() = delete;
92
93 explicit NVPTXDAGToDAGISel(NVPTXTargetMachine &tm, CodeGenOptLevel OptLevel);
94
95 bool runOnMachineFunction(MachineFunction &MF) override;
96 const NVPTXSubtarget *Subtarget = nullptr;
97
98 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
99 InlineAsm::ConstraintCode ConstraintID,
100 std::vector<SDValue> &OutOps) override;
101
102private:
103// Include the pieces autogenerated from the target description.
104#include "NVPTXGenDAGISel.inc"
105
106 void Select(SDNode *N) override;
107 bool tryIntrinsicChain(SDNode *N);
108 bool tryIntrinsicVoid(SDNode *N);
109 bool tryLoad(SDNode *N);
110 bool tryLoadVector(SDNode *N);
111 bool tryLDU(SDNode *N);
112 bool tryLDG(MemSDNode *N);
113 bool tryStore(SDNode *N);
114 bool tryStoreVector(SDNode *N);
115 bool tryFence(SDNode *N);
116 bool tryBFE(SDNode *N);
117 bool tryBF16ArithToFMA(SDNode *N);
118 bool SelectSETP_F16X2(SDNode *N);
119 bool SelectSETP_BF16X2(SDNode *N);
120 bool tryUNPACK_VECTOR(SDNode *N);
121 bool tryEXTRACT_VECTOR_ELEMENT(SDNode *N);
122 void PostprocessISelDAG() override;
123 void SelectV2I64toI128(SDNode *N);
124 void SelectI128toV2I64(SDNode *N);
125 void SelectTcgen05Ld(SDNode *N, bool hasOffset = false);
126 void SelectTcgen05St(SDNode *N, bool hasOffset = false);
127 void selectAtomicSwap128(SDNode *N);
128
129 inline SDValue getI32Imm(unsigned Imm, const SDLoc &DL) {
130 return CurDAG->getTargetConstant(Val: Imm, DL, VT: MVT::i32);
131 }
132 NVPTX::Ordering getMemOrder(const MemSDNode *N) const;
133 NVPTX::Scope getAtomicScope(const MemSDNode *N) const;
134
135 bool SelectADDR(SDValue Addr, SDValue &Base, SDValue &Offset);
136 bool SelectFAbs(SDValue N, SDValue &Src);
137 SDValue getPTXCmpMode(const CondCodeSDNode &CondCode);
138 SDValue selectPossiblyImm(SDValue V);
139
140 // Returns the encoded eviction/prefetch hint and cache policy register for a
141 // memory operation. Hints unsupported by the subtarget or address space are
142 // dropped. If L2::cache_hint is active, returns the hint with
143 // L2CacheHintBit set and a register containing the 64-bit cache policy
144 // value. Otherwise returns NOREG for the policy operand.
145 NVPTXMemCacheHintOperands
146 getMemCacheHintOperands(const MemSDNode *N, NVPTXMemCacheHintAccess Access,
147 const SDLoc &DL, bool EmitDiagnostics = true);
148
149 // Returns the Memory Order and Scope that the PTX memory instruction should
150 // use.
151 std::pair<NVPTX::Ordering, NVPTX::Scope>
152 getMemInstOrderingScope(MemSDNode *N);
153
154 NVPTX::Scope getOperationScope(MemSDNode *N, NVPTX::Ordering O) const;
155
156public:
157 static NVPTX::AddressSpace getAddrSpace(const MemSDNode *N);
158};
159
160class NVPTXDAGToDAGISelLegacy : public SelectionDAGISelLegacy {
161public:
162 static char ID;
163 explicit NVPTXDAGToDAGISelLegacy(NVPTXTargetMachine &tm,
164 CodeGenOptLevel OptLevel);
165};
166
167} // end anonymous namespace
168
169/// createNVPTXISelDag - This pass converts a legalized DAG into a
170/// NVPTX-specific DAG, ready for instruction scheduling.
171FunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM,
172 llvm::CodeGenOptLevel OptLevel) {
173 return new NVPTXDAGToDAGISelLegacy(TM, OptLevel);
174}
175
176NVPTXDAGToDAGISelLegacy::NVPTXDAGToDAGISelLegacy(NVPTXTargetMachine &tm,
177 CodeGenOptLevel OptLevel)
178 : SelectionDAGISelLegacy(
179 ID, std::make_unique<NVPTXDAGToDAGISel>(args&: tm, args&: OptLevel)) {}
180
181char NVPTXDAGToDAGISelLegacy::ID = 0;
182
183INITIALIZE_PASS(NVPTXDAGToDAGISelLegacy, DEBUG_TYPE, PASS_NAME, false, false)
184
185NVPTXISelDAGToDAGPass::NVPTXISelDAGToDAGPass(NVPTXTargetMachine &TM,
186 CodeGenOptLevel OptLevel)
187 : SelectionDAGISelPass(std::make_unique<NVPTXDAGToDAGISel>(args&: TM, args&: OptLevel)) {}
188
189NVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm,
190 CodeGenOptLevel OptLevel)
191 : SelectionDAGISel(tm, OptLevel), TM(tm) {}
192
193bool NVPTXDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
194 Subtarget = &MF.getSubtarget<NVPTXSubtarget>();
195 Scopes = NVPTXScopes(MF.getFunction().getContext(),
196 MF.getTarget().getTargetTriple());
197 return SelectionDAGISel::runOnMachineFunction(mf&: MF);
198}
199
200NVPTX::DivPrecisionLevel
201NVPTXDAGToDAGISel::getDivF32Level(const SDNode *N) const {
202 return Subtarget->getTargetLowering()->getDivF32Level(MF: *MF, N: *N);
203}
204
205bool NVPTXDAGToDAGISel::usePrecSqrtF32(const SDNode *N) const {
206 return Subtarget->getTargetLowering()->usePrecSqrtF32(N);
207}
208
209bool NVPTXDAGToDAGISel::useF16FTZ() const {
210 return Subtarget->getTargetLowering()->useF16FTZ(MF: *MF);
211}
212
213bool NVPTXDAGToDAGISel::useF32FTZ() const {
214 return Subtarget->getTargetLowering()->useF32FTZ(MF: *MF);
215}
216
217bool NVPTXDAGToDAGISel::allowFMA() const {
218 const NVPTXTargetLowering *TL = Subtarget->getTargetLowering();
219 return TL->allowFMA(MF&: *MF, OptLevel);
220}
221
222bool NVPTXDAGToDAGISel::doRsqrtOpt() const {
223 return Subtarget->getCLOpts().rsqrt_approx_opt;
224}
225
226bool NVPTXDAGToDAGISel::doMADWideOpt() const {
227 return Subtarget->getCLOpts().mad_wide_opt;
228}
229
230/// Select - Select instructions not customized! Used for
231/// expanded, promoted and normal instructions.
232void NVPTXDAGToDAGISel::Select(SDNode *N) {
233
234 if (N->isMachineOpcode()) {
235 N->setNodeId(-1);
236 return; // Already selected.
237 }
238
239 switch (N->getOpcode()) {
240 case ISD::LOAD:
241 case ISD::ATOMIC_LOAD:
242 case NVPTXISD::ATOMIC_LOAD_B128:
243 case NVPTXISD::MLoad:
244 if (tryLoad(N))
245 return;
246 break;
247 case ISD::STORE:
248 case ISD::ATOMIC_STORE:
249 case NVPTXISD::ATOMIC_STORE_B128:
250 if (tryStore(N))
251 return;
252 break;
253 case ISD::ATOMIC_FENCE:
254 if (tryFence(N))
255 return;
256 break;
257 case NVPTXISD::UNPACK_VECTOR:
258 tryUNPACK_VECTOR(N);
259 return;
260 case ISD::EXTRACT_VECTOR_ELT:
261 if (tryEXTRACT_VECTOR_ELEMENT(N))
262 return;
263 break;
264 case NVPTXISD::SETP_F16X2:
265 SelectSETP_F16X2(N);
266 return;
267 case NVPTXISD::SETP_BF16X2:
268 SelectSETP_BF16X2(N);
269 return;
270 case NVPTXISD::LoadV2:
271 case NVPTXISD::LoadV4:
272 case NVPTXISD::LoadV8:
273 if (tryLoadVector(N))
274 return;
275 break;
276 case NVPTXISD::LDUV2:
277 case NVPTXISD::LDUV4:
278 if (tryLDU(N))
279 return;
280 break;
281 case NVPTXISD::StoreV2:
282 case NVPTXISD::StoreV4:
283 case NVPTXISD::StoreV8:
284 if (tryStoreVector(N))
285 return;
286 break;
287 case ISD::INTRINSIC_W_CHAIN:
288 if (tryIntrinsicChain(N))
289 return;
290 break;
291 case ISD::INTRINSIC_VOID:
292 if (tryIntrinsicVoid(N))
293 return;
294 break;
295 case ISD::AND:
296 case ISD::SRA:
297 case ISD::SRL:
298 // Try to select BFE
299 if (tryBFE(N))
300 return;
301 break;
302 case ISD::CopyToReg: {
303 if (N->getOperand(Num: 1).getValueType() == MVT::i128) {
304 SelectV2I64toI128(N);
305 return;
306 }
307 break;
308 }
309 case ISD::CopyFromReg: {
310 if (N->getOperand(Num: 1).getValueType() == MVT::i128) {
311 SelectI128toV2I64(N);
312 return;
313 }
314 break;
315 }
316 case NVPTXISD::ATOMIC_CMP_SWAP_B128:
317 case NVPTXISD::ATOMIC_SWAP_B128:
318 selectAtomicSwap128(N);
319 return;
320 case ISD::FADD:
321 case ISD::FMUL:
322 case ISD::FSUB:
323 if (tryBF16ArithToFMA(N))
324 return;
325 break;
326 default:
327 break;
328 }
329 SelectCode(N);
330}
331
332#define TCGEN05_LD_OPCODE(SHAPE, NUM) \
333 (enablePack ? NVPTX::TCGEN05_LD_##SHAPE##_##NUM##_PACK \
334 : NVPTX::TCGEN05_LD_##SHAPE##_##NUM)
335
336static unsigned getTcgen05LdOpcode(unsigned IID, bool enablePack) {
337 switch (IID) {
338 case Intrinsic::nvvm_tcgen05_ld_16x64b_x1:
339 return TCGEN05_LD_OPCODE(16x64b, x1);
340 case Intrinsic::nvvm_tcgen05_ld_16x64b_x2:
341 return TCGEN05_LD_OPCODE(16x64b, x2);
342 case Intrinsic::nvvm_tcgen05_ld_16x64b_x4:
343 return TCGEN05_LD_OPCODE(16x64b, x4);
344 case Intrinsic::nvvm_tcgen05_ld_16x64b_x8:
345 return TCGEN05_LD_OPCODE(16x64b, x8);
346 case Intrinsic::nvvm_tcgen05_ld_16x64b_x16:
347 return TCGEN05_LD_OPCODE(16x64b, x16);
348 case Intrinsic::nvvm_tcgen05_ld_16x64b_x32:
349 return TCGEN05_LD_OPCODE(16x64b, x32);
350 case Intrinsic::nvvm_tcgen05_ld_16x64b_x64:
351 return TCGEN05_LD_OPCODE(16x64b, x64);
352 case Intrinsic::nvvm_tcgen05_ld_16x64b_x128:
353 return TCGEN05_LD_OPCODE(16x64b, x128);
354 case Intrinsic::nvvm_tcgen05_ld_16x128b_x1:
355 return TCGEN05_LD_OPCODE(16x128b, x1);
356 case Intrinsic::nvvm_tcgen05_ld_16x128b_x2:
357 return TCGEN05_LD_OPCODE(16x128b, x2);
358 case Intrinsic::nvvm_tcgen05_ld_16x128b_x4:
359 return TCGEN05_LD_OPCODE(16x128b, x4);
360 case Intrinsic::nvvm_tcgen05_ld_16x128b_x8:
361 return TCGEN05_LD_OPCODE(16x128b, x8);
362 case Intrinsic::nvvm_tcgen05_ld_16x128b_x16:
363 return TCGEN05_LD_OPCODE(16x128b, x16);
364 case Intrinsic::nvvm_tcgen05_ld_16x128b_x32:
365 return TCGEN05_LD_OPCODE(16x128b, x32);
366 case Intrinsic::nvvm_tcgen05_ld_16x128b_x64:
367 return TCGEN05_LD_OPCODE(16x128b, x64);
368 case Intrinsic::nvvm_tcgen05_ld_16x256b_x1:
369 return TCGEN05_LD_OPCODE(16x256b, x1);
370 case Intrinsic::nvvm_tcgen05_ld_16x256b_x2:
371 return TCGEN05_LD_OPCODE(16x256b, x2);
372 case Intrinsic::nvvm_tcgen05_ld_16x256b_x4:
373 return TCGEN05_LD_OPCODE(16x256b, x4);
374 case Intrinsic::nvvm_tcgen05_ld_16x256b_x8:
375 return TCGEN05_LD_OPCODE(16x256b, x8);
376 case Intrinsic::nvvm_tcgen05_ld_16x256b_x16:
377 return TCGEN05_LD_OPCODE(16x256b, x16);
378 case Intrinsic::nvvm_tcgen05_ld_16x256b_x32:
379 return TCGEN05_LD_OPCODE(16x256b, x32);
380 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x1:
381 return TCGEN05_LD_OPCODE(16x32bx2, x1);
382 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x2:
383 return TCGEN05_LD_OPCODE(16x32bx2, x2);
384 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x4:
385 return TCGEN05_LD_OPCODE(16x32bx2, x4);
386 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x8:
387 return TCGEN05_LD_OPCODE(16x32bx2, x8);
388 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x16:
389 return TCGEN05_LD_OPCODE(16x32bx2, x16);
390 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x32:
391 return TCGEN05_LD_OPCODE(16x32bx2, x32);
392 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x64:
393 return TCGEN05_LD_OPCODE(16x32bx2, x64);
394 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x128:
395 return TCGEN05_LD_OPCODE(16x32bx2, x128);
396 case Intrinsic::nvvm_tcgen05_ld_32x32b_x1:
397 return TCGEN05_LD_OPCODE(32x32b, x1);
398 case Intrinsic::nvvm_tcgen05_ld_32x32b_x2:
399 return TCGEN05_LD_OPCODE(32x32b, x2);
400 case Intrinsic::nvvm_tcgen05_ld_32x32b_x4:
401 return TCGEN05_LD_OPCODE(32x32b, x4);
402 case Intrinsic::nvvm_tcgen05_ld_32x32b_x8:
403 return TCGEN05_LD_OPCODE(32x32b, x8);
404 case Intrinsic::nvvm_tcgen05_ld_32x32b_x16:
405 return TCGEN05_LD_OPCODE(32x32b, x16);
406 case Intrinsic::nvvm_tcgen05_ld_32x32b_x32:
407 return TCGEN05_LD_OPCODE(32x32b, x32);
408 case Intrinsic::nvvm_tcgen05_ld_32x32b_x64:
409 return TCGEN05_LD_OPCODE(32x32b, x64);
410 case Intrinsic::nvvm_tcgen05_ld_32x32b_x128:
411 return TCGEN05_LD_OPCODE(32x32b, x128);
412 }
413 llvm_unreachable("unhandled tcgen05.ld lowering");
414}
415
416void NVPTXDAGToDAGISel::SelectTcgen05Ld(SDNode *N, bool hasOffset) {
417 if (!Subtarget->hasTcgen05InstSupport())
418 report_fatal_error(
419 reason: "tcgen05.ld is not supported on this architecture variant");
420
421 SDLoc DL(N);
422 unsigned IID = cast<ConstantSDNode>(Val: N->getOperand(Num: 1))->getZExtValue();
423
424 if (hasOffset) {
425 bool enablePack = cast<ConstantSDNode>(Val: N->getOperand(Num: 4))->getZExtValue();
426 auto OffsetNode = CurDAG->getTargetConstant(
427 Val: cast<ConstantSDNode>(Val: N->getOperand(Num: 3))->getZExtValue(), DL, VT: MVT::i32);
428 ReplaceNode(F: N, T: CurDAG->getMachineNode(
429 Opcode: getTcgen05LdOpcode(IID, enablePack), dl: DL, VTs: N->getVTList(),
430 Ops: {N->getOperand(Num: 2), OffsetNode, N->getOperand(Num: 0)}));
431 } else {
432 bool enablePack = cast<ConstantSDNode>(Val: N->getOperand(Num: 3))->getZExtValue();
433 ReplaceNode(F: N, T: CurDAG->getMachineNode(
434 Opcode: getTcgen05LdOpcode(IID, enablePack), dl: DL, VTs: N->getVTList(),
435 Ops: {N->getOperand(Num: 2), N->getOperand(Num: 0)}));
436 }
437}
438
439bool NVPTXDAGToDAGISel::tryIntrinsicChain(SDNode *N) {
440 unsigned IID = N->getConstantOperandVal(Num: 1);
441 switch (IID) {
442 default:
443 return false;
444 case Intrinsic::nvvm_ldu_global_f:
445 case Intrinsic::nvvm_ldu_global_i:
446 case Intrinsic::nvvm_ldu_global_p:
447 return tryLDU(N);
448
449 case Intrinsic::nvvm_tcgen05_ld_16x64b_x1:
450 case Intrinsic::nvvm_tcgen05_ld_16x64b_x2:
451 case Intrinsic::nvvm_tcgen05_ld_16x64b_x4:
452 case Intrinsic::nvvm_tcgen05_ld_16x64b_x8:
453 case Intrinsic::nvvm_tcgen05_ld_16x64b_x16:
454 case Intrinsic::nvvm_tcgen05_ld_16x64b_x32:
455 case Intrinsic::nvvm_tcgen05_ld_16x64b_x64:
456 case Intrinsic::nvvm_tcgen05_ld_16x64b_x128:
457 case Intrinsic::nvvm_tcgen05_ld_16x128b_x1:
458 case Intrinsic::nvvm_tcgen05_ld_16x128b_x2:
459 case Intrinsic::nvvm_tcgen05_ld_16x128b_x4:
460 case Intrinsic::nvvm_tcgen05_ld_16x128b_x16:
461 case Intrinsic::nvvm_tcgen05_ld_16x128b_x32:
462 case Intrinsic::nvvm_tcgen05_ld_16x128b_x64:
463 case Intrinsic::nvvm_tcgen05_ld_16x256b_x1:
464 case Intrinsic::nvvm_tcgen05_ld_16x128b_x8:
465 case Intrinsic::nvvm_tcgen05_ld_16x256b_x2:
466 case Intrinsic::nvvm_tcgen05_ld_16x256b_x4:
467 case Intrinsic::nvvm_tcgen05_ld_16x256b_x8:
468 case Intrinsic::nvvm_tcgen05_ld_16x256b_x16:
469 case Intrinsic::nvvm_tcgen05_ld_16x256b_x32:
470 case Intrinsic::nvvm_tcgen05_ld_32x32b_x1:
471 case Intrinsic::nvvm_tcgen05_ld_32x32b_x2:
472 case Intrinsic::nvvm_tcgen05_ld_32x32b_x4:
473 case Intrinsic::nvvm_tcgen05_ld_32x32b_x8:
474 case Intrinsic::nvvm_tcgen05_ld_32x32b_x16:
475 case Intrinsic::nvvm_tcgen05_ld_32x32b_x32:
476 case Intrinsic::nvvm_tcgen05_ld_32x32b_x64:
477 case Intrinsic::nvvm_tcgen05_ld_32x32b_x128: {
478 SelectTcgen05Ld(N);
479 return true;
480 }
481
482 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x1:
483 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x2:
484 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x4:
485 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x8:
486 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x16:
487 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x32:
488 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x64:
489 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x128: {
490 SelectTcgen05Ld(N, /* hasOffset */ true);
491 return true;
492 }
493 }
494}
495
496// Map ISD:CONDCODE value to appropriate CmpMode expected by
497// NVPTXInstPrinter::printCmpMode()
498SDValue NVPTXDAGToDAGISel::getPTXCmpMode(const CondCodeSDNode &CondCode) {
499 using NVPTX::PTXCmpMode::CmpMode;
500 const unsigned PTXCmpMode = [](ISD::CondCode CC) {
501 switch (CC) {
502 default:
503 llvm_unreachable("Unexpected condition code.");
504 case ISD::SETOEQ:
505 case ISD::SETEQ:
506 return CmpMode::EQ;
507 case ISD::SETOGT:
508 case ISD::SETGT:
509 return CmpMode::GT;
510 case ISD::SETOGE:
511 case ISD::SETGE:
512 return CmpMode::GE;
513 case ISD::SETOLT:
514 case ISD::SETLT:
515 return CmpMode::LT;
516 case ISD::SETOLE:
517 case ISD::SETLE:
518 return CmpMode::LE;
519 case ISD::SETONE:
520 case ISD::SETNE:
521 return CmpMode::NE;
522 case ISD::SETO:
523 return CmpMode::NUM;
524 case ISD::SETUO:
525 return CmpMode::NotANumber;
526 case ISD::SETUEQ:
527 return CmpMode::EQU;
528 case ISD::SETUGT:
529 return CmpMode::GTU;
530 case ISD::SETUGE:
531 return CmpMode::GEU;
532 case ISD::SETULT:
533 return CmpMode::LTU;
534 case ISD::SETULE:
535 return CmpMode::LEU;
536 case ISD::SETUNE:
537 return CmpMode::NEU;
538 }
539 }(CondCode.get());
540 return CurDAG->getTargetConstant(Val: PTXCmpMode, DL: SDLoc(), VT: MVT::i32);
541}
542
543bool NVPTXDAGToDAGISel::SelectSETP_F16X2(SDNode *N) {
544 SDValue PTXCmpMode = getPTXCmpMode(CondCode: *cast<CondCodeSDNode>(Val: N->getOperand(Num: 2)));
545 SDLoc DL(N);
546 SDNode *SetP = CurDAG->getMachineNode(
547 Opcode: NVPTX::SETP_f16x2, dl: DL, VT1: MVT::i1, VT2: MVT::i1,
548 Ops: {N->getOperand(Num: 0), N->getOperand(Num: 1), PTXCmpMode,
549 CurDAG->getTargetConstant(Val: useF16FTZ(), DL, VT: MVT::i1)});
550 ReplaceNode(F: N, T: SetP);
551 return true;
552}
553
554bool NVPTXDAGToDAGISel::SelectSETP_BF16X2(SDNode *N) {
555 SDValue PTXCmpMode = getPTXCmpMode(CondCode: *cast<CondCodeSDNode>(Val: N->getOperand(Num: 2)));
556 SDLoc DL(N);
557 SDNode *SetP =
558 CurDAG->getMachineNode(Opcode: NVPTX::SETP_bf16x2, dl: DL, VT1: MVT::i1, VT2: MVT::i1,
559 Ops: {N->getOperand(Num: 0), N->getOperand(Num: 1), PTXCmpMode});
560 ReplaceNode(F: N, T: SetP);
561 return true;
562}
563
564bool NVPTXDAGToDAGISel::tryUNPACK_VECTOR(SDNode *N) {
565 SDValue Vector = N->getOperand(Num: 0);
566 MVT EltVT = N->getSimpleValueType(ResNo: 0);
567
568 MachineSDNode *N2 =
569 CurDAG->getMachineNode(Opcode: NVPTX::I64toV2I32, dl: SDLoc(N), VT1: EltVT, VT2: EltVT, Ops: Vector);
570
571 ReplaceNode(F: N, T: N2);
572 return true;
573}
574
575// Find all instances of extract_vector_elt that use this v2f16 vector
576// and coalesce them into a scattering move instruction.
577bool NVPTXDAGToDAGISel::tryEXTRACT_VECTOR_ELEMENT(SDNode *N) {
578 SDValue Vector = N->getOperand(Num: 0);
579
580 MVT VT = Vector.getSimpleValueType();
581 if (!(NVPTX::isPackedVectorTy(VT) && VT.getVectorNumElements() == 2))
582 return false;
583
584 unsigned Opcode;
585 if (VT.is32BitVector())
586 Opcode = NVPTX::I32toV2I16;
587 else if (VT.is64BitVector())
588 Opcode = NVPTX::I64toV2I32;
589 else
590 llvm_unreachable("Unhandled packed type");
591
592 // Find and record all uses of this vector that extract element 0 or 1.
593 SmallVector<SDNode *, 4> E0, E1;
594 for (auto *U : Vector.getNode()->users()) {
595 if (U->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
596 continue;
597 if (U->getOperand(Num: 0) != Vector)
598 continue;
599 if (const ConstantSDNode *IdxConst =
600 dyn_cast<ConstantSDNode>(Val: U->getOperand(Num: 1))) {
601 if (IdxConst->getZExtValue() == 0)
602 E0.push_back(Elt: U);
603 else if (IdxConst->getZExtValue() == 1)
604 E1.push_back(Elt: U);
605 else
606 llvm_unreachable("Invalid vector index.");
607 }
608 }
609
610 // There's no point scattering f16x2 if we only ever access one
611 // element of it.
612 if (E0.empty() || E1.empty())
613 return false;
614
615 // Merge (EltTy extractelt(V, 0), EltTy extractelt(V,1))
616 // into EltTy,EltTy Split[EltTy]x2(V)
617 MVT EltVT = VT.getVectorElementType();
618 SDNode *ScatterOp =
619 CurDAG->getMachineNode(Opcode, dl: SDLoc(N), VT1: EltVT, VT2: EltVT, Ops: Vector);
620 for (auto *Node : E0)
621 ReplaceUses(F: SDValue(Node, 0), T: SDValue(ScatterOp, 0));
622 for (auto *Node : E1)
623 ReplaceUses(F: SDValue(Node, 0), T: SDValue(ScatterOp, 1));
624
625 return true;
626}
627
628static std::optional<unsigned> getI16ExtractIndex(const SDNode *N) {
629 if (!N->isMachineOpcode() || N->use_empty())
630 return std::nullopt;
631 switch (N->getMachineOpcode()) {
632 default:
633 return std::nullopt;
634 case NVPTX::CVT_u16_u32:
635 if (N->getConstantOperandVal(Num: 1) != NVPTX::PTXCvtMode::NONE)
636 return std::nullopt;
637 [[fallthrough]];
638 case NVPTX::I32toI16L:
639 case NVPTX::I32toI16L_Sink:
640 return 0;
641 case NVPTX::I32toI16H:
642 case NVPTX::I32toI16H_Sink:
643 return 1;
644 }
645}
646
647// Merge scalar truncations and individual vector extracts after selection, when
648// bitcasts no longer hide that they extract the same 16-bit halves.
649static void mergeI16Unpacks(SelectionDAG &DAG) {
650 auto Position = DAG.allnodes_begin();
651 // Replacing extracts can CSE downstream nodes, including the next node.
652 SelectionDAG::DAGNodeDeletedListener Listener(
653 DAG, [&](SDNode *Deleted, SDNode *) {
654 if (Position == SelectionDAG::allnodes_iterator(Deleted))
655 ++Position;
656 });
657
658 bool Changed = false;
659 while (Position != DAG.allnodes_end()) {
660 SDNode *N = &*Position++;
661 if (!getI16ExtractIndex(N))
662 continue;
663 SDValue Word = N->getOperand(Num: 0);
664 SmallVector<SDValue, 2> Halves[2];
665 for (SDNode *User : Word->users()) {
666 std::optional<unsigned> Index = getI16ExtractIndex(N: User);
667 if (Index && User->getOperand(Num: 0) == Word)
668 Halves[*Index].emplace_back(Args&: User, Args: 0);
669 }
670 if (Halves[0].empty() || Halves[1].empty())
671 continue;
672
673 SDNode *Scatter = DAG.getMachineNode(Opcode: NVPTX::I32toV2I16, dl: SDLoc(N), VT1: MVT::i16,
674 VT2: MVT::i16, Ops: Word);
675 for (unsigned Index : {0u, 1u})
676 for (SDValue V : Halves[Index])
677 DAG.ReplaceAllUsesOfValueWith(From: V, To: SDValue(Scatter, Index));
678 Changed = true;
679 }
680 if (Changed)
681 DAG.RemoveDeadNodes();
682}
683
684void NVPTXDAGToDAGISel::PostprocessISelDAG() { mergeI16Unpacks(DAG&: *CurDAG); }
685
686NVPTX::AddressSpace NVPTXDAGToDAGISel::getAddrSpace(const MemSDNode *N) {
687 auto AS =
688 static_cast<NVPTX::AddressSpace>(N->getMemOperand()->getAddrSpace());
689 switch (AS) {
690 case NVPTX::AddressSpace::Generic:
691 case NVPTX::AddressSpace::Global:
692 case NVPTX::AddressSpace::Shared:
693 case NVPTX::AddressSpace::Const:
694 case NVPTX::AddressSpace::Local:
695 case NVPTX::AddressSpace::SharedCluster:
696 case NVPTX::AddressSpace::EntryParam:
697 case NVPTX::AddressSpace::DeviceParam:
698 return AS;
699 }
700 llvm_unreachable("Unexpected address space");
701}
702
703NVPTX::Ordering NVPTXDAGToDAGISel::getMemOrder(const MemSDNode *N) const {
704 // No "sem" orderings for SM/PTX versions which do not support memory ordering
705 if (!Subtarget->hasMemoryOrdering())
706 return NVPTX::Ordering::NotAtomic;
707 auto Ordering = N->getMergedOrdering();
708 switch (Ordering) {
709 case AtomicOrdering::NotAtomic:
710 return NVPTX::Ordering::NotAtomic;
711 case AtomicOrdering::Unordered:
712 case AtomicOrdering::Monotonic:
713 return NVPTX::Ordering::Relaxed;
714 case AtomicOrdering::Acquire:
715 return NVPTX::Ordering::Acquire;
716 case AtomicOrdering::Release:
717 return NVPTX::Ordering::Release;
718 case AtomicOrdering::AcquireRelease:
719 return NVPTX::Ordering::AcquireRelease;
720 case AtomicOrdering::SequentiallyConsistent:
721 return NVPTX::Ordering::SequentiallyConsistent;
722 }
723 llvm_unreachable("Invalid atomic ordering");
724}
725
726// Clusters contain exactly 1 block on targets without cluster support.
727static NVPTX::Scope resolveScope(NVPTX::Scope S, const NVPTXSubtarget *T) {
728 if (S == NVPTX::Scope::Cluster && !T->hasClusters())
729 return NVPTX::Scope::Block;
730 return S;
731}
732
733NVPTX::Scope NVPTXDAGToDAGISel::getAtomicScope(const MemSDNode *N) const {
734 NVPTX::Scope Scope = resolveScope(S: Scopes[N->getSyncScopeID()], T: Subtarget);
735 if (!Subtarget->hasAtomScope()) {
736 if (Scope == NVPTX::Scope::System)
737 CurDAG->getContext()->diagnose(DI: DiagnosticInfoUnsupported(
738 CurDAG->getMachineFunction().getFunction(),
739 "NVPTX system scope atomics require sm_60 or later",
740 N->getDebugLoc()));
741 return NVPTX::Scope::DefaultDevice;
742 }
743 return Scope;
744}
745
746namespace {
747
748static NVPTX::Ordering getInstructionOrdering(MemSDNode *N,
749 const NVPTXSubtarget *Subtarget) {
750 AtomicOrdering Ordering = N->getSuccessOrdering();
751 auto CodeAddrSpace = NVPTXDAGToDAGISel::getAddrSpace(N);
752
753 bool HasMemoryOrdering = Subtarget->hasMemoryOrdering();
754 bool HasRelaxedMMIO = Subtarget->hasRelaxedMMIO();
755
756 // clang-format off
757
758 // Lowering for Load/Store Operations (note: AcquireRelease Loads or Stores error).
759 // Note: uses of Relaxed in the Atomic column of this table refer
760 // to LLVM AtomicOrdering::Monotonic.
761 //
762 // | Atomic | Volatile | Statespace | PTX sm_60- | PTX sm_70+ |
763 // |---------|----------|--------------------|------------|------------------------------|
764 // | No | No | All | plain | .weak |
765 // | No | Yes | Generic,Shared, | .volatile | .volatile |
766 // | | | Global [0] | | |
767 // | No | Yes | Local (PTX 9.0-) | plain [1] | .weak [1] |
768 // | No | Yes | Local (PTX 9.1+) | .volatile | .volatile |
769 // | No | Yes | Const,Param | plain [1] | .weak [1] |
770 // | Unorder | Yes/No | All | == Relaxed | == Relaxed |
771 // | Relaxed | No | Generic,Shared, | .volatile | <atomic sem> |
772 // | | | Global [0] | | |
773 // | Other | No | Generic,Shared, | Error [2] | <atomic sem> |
774 // | | | Global [0] | | |
775 // | Yes | No | Local,Const,Param | plain [1] | .weak [1] |
776 // | Relaxed | Yes | Generic,Shared [0] | .volatile | .volatile |
777 // | Relaxed | Yes | Global [0] | .volatile | .mmio.relaxed.sys (PTX 8.2+) |
778 // | | | | | or .volatile (PTX 8.1-) |
779 // | Yes | Yes | Local (PTX 9.0-) | plain [1] | .weak [1] |
780 // | Yes | Yes | Local (PTX 9.1+) | .volatile | .volatile |
781 // | Relaxed | Yes | Const,Param | plain [1] | .weak [1] |
782 // | Other | Yes | Generic, Shared, | Error [2] | <atomic sem> [3] |
783 // | | | / Global [0] | | |
784
785 // Lowering of CUDA C++ SequentiallyConsistent Operations and Fences to PTX
786 // by following the ABI proven sound in:
787 // Lustig et al, A Formal Analysis of the NVIDIA PTX Memory Consistency Model, ASPLOS’19.
788 // https://dl.acm.org/doi/pdf/10.1145/3297858.3304043
789 //
790 // | CUDA C++ Atomic Operation or Atomic Fence | PTX Atomic Operation or Fence |
791 // |------------------------------------------------------|-------------------------------|
792 // | cuda::atomic_thread_fence | fence.sc.<scope>; |
793 // | (memory_order_seq_cst, cuda::thread_scope_<scope>) | |
794 // |------------------------------------------------------|-------------------------------|
795 // | cuda::atomic_load | fence.sc.<scope>; |
796 // | (memory_order_seq_cst, cuda::thread_scope_<scope>) | ld.acquire.<scope>; |
797 // |------------------------------------------------------|-------------------------------|
798 // | cuda::atomic_store | fence.sc.<scope>; |
799 // | (memory_order_seq_cst, cuda::thread_scope_<scope>) | st.release.<scope>; |
800 // |------------------------------------------------------|-------------------------------|
801 // | cuda::atomic_fetch_<op> | fence.sc.<scope>; |
802 // | (memory_order_seq_cst, cuda::thread_scope_<scope>) | atom.acq_rel.<scope>; |
803
804 // clang-format on
805
806 // [0]: volatile and atomics are only supported on global or shared
807 // memory locations, accessed via generic/shared/global pointers.
808 // PTX 9.1 adds volatile support on local ld/st.
809 // MMIO is only supported on global memory locations,
810 // accessed via generic/global pointers.
811 // TODO: Implement MMIO access via generic pointer to global.
812 // Currently implemented for global pointers only.
813
814 // [1]: Lowering volatile/atomic operations to non-volatile/non-atomic
815 // PTX instructions fails to preserve their C++ side-effects.
816 //
817 // Example (https://github.com/llvm/llvm-project/issues/62057):
818 //
819 // void example() {
820 // std::atomic<bool> True = true;
821 // while (True.load(std::memory_order_relaxed));
822 // }
823 //
824 // A C++ program that calls "example" is well-defined: the infinite loop
825 // performs an atomic operation. By lowering volatile/atomics to
826 // "weak" memory operations, we are transforming the above into:
827 //
828 // void undefined_behavior() {
829 // bool True = true;
830 // while (True);
831 // }
832 //
833 // which exhibits undefined behavior in both C++ and PTX.
834 //
835 // Calling "example" in CUDA C++ compiled for sm_60- exhibits undefined
836 // behavior due to lack of Independent Forward Progress. Lowering these
837 // to weak memory operations in sm_60- is therefore fine.
838 //
839 // TODO: Where direct volatile or atomic operations are unsupported,
840 // preserve the side-effect using the weak memory instruction and
841 // another instruction, such as a dead dummy volatile load.
842
843 if (CodeAddrSpace == NVPTX::AddressSpace::Local) {
844 // Local memory is private to a thread. Drop atomic ordering but preserve
845 // volatile accesses where supported.
846 return Subtarget->hasLocalVolatile() && N->isVolatile()
847 ? NVPTX::Ordering::Volatile
848 : NVPTX::Ordering::NotAtomic;
849 }
850
851 if (CodeAddrSpace == NVPTX::AddressSpace::Const ||
852 CodeAddrSpace == NVPTX::AddressSpace::EntryParam ||
853 CodeAddrSpace == NVPTX::AddressSpace::DeviceParam) {
854 return NVPTX::Ordering::NotAtomic;
855 }
856
857 // [2]: Atomics with Ordering different than Unordered or Relaxed are not
858 // supported on sm_60 and older; this includes volatile atomics.
859 if (!(Ordering == AtomicOrdering::NotAtomic ||
860 Ordering == AtomicOrdering::Unordered ||
861 Ordering == AtomicOrdering::Monotonic) &&
862 !HasMemoryOrdering) {
863 report_fatal_error(
864 reason: formatv(Fmt: "PTX does not support \"atomic\" for orderings different than"
865 "\"NotAtomic\" or \"Monotonic\" for sm_60 or older, but order "
866 "is: \"{}\".",
867 Vals: toIRString(ao: Ordering)));
868 }
869
870 // [3]: TODO: these should eventually use .mmio<.atomic sem>; for now we drop
871 // the volatile semantics and preserve the atomic ones.
872
873 // Apart from local volatile accesses handled above, PTX atomics and volatile
874 // operations are only available in generic, global, or shared memory.
875 bool AddrSupportsVolatileOrAtomic =
876 (CodeAddrSpace == NVPTX::AddressSpace::Generic ||
877 CodeAddrSpace == NVPTX::AddressSpace::Global ||
878 CodeAddrSpace == NVPTX::AddressSpace::Shared ||
879 CodeAddrSpace == NVPTX::AddressSpace::SharedCluster);
880 if (!AddrSupportsVolatileOrAtomic)
881 return NVPTX::Ordering::NotAtomic;
882
883 bool UseRelaxedMMIO =
884 HasRelaxedMMIO && CodeAddrSpace == NVPTX::AddressSpace::Global;
885
886 switch (Ordering) {
887 case AtomicOrdering::NotAtomic:
888 return N->isVolatile() ? NVPTX::Ordering::Volatile
889 : NVPTX::Ordering::NotAtomic;
890 case AtomicOrdering::Unordered:
891 // We lower unordered in the exact same way as 'monotonic' to respect
892 // LLVM IR atomicity requirements.
893 case AtomicOrdering::Monotonic:
894 if (N->isVolatile())
895 return UseRelaxedMMIO ? NVPTX::Ordering::RelaxedMMIO
896 : NVPTX::Ordering::Volatile;
897 else
898 return HasMemoryOrdering ? NVPTX::Ordering::Relaxed
899 : NVPTX::Ordering::Volatile;
900 // case AtomicOrdering::Consume: // If LLVM ever provides this, lower it to
901 // Acquire.
902 case AtomicOrdering::Acquire:
903 if (!N->readMem())
904 report_fatal_error(
905 reason: formatv(Fmt: "PTX only supports Acquire Ordering on reads: {}",
906 Vals: N->getOperationName()));
907 return NVPTX::Ordering::Acquire;
908 case AtomicOrdering::Release:
909 if (!N->writeMem())
910 report_fatal_error(
911 reason: formatv(Fmt: "PTX only supports Release Ordering on writes: {}",
912 Vals: N->getOperationName()));
913 return NVPTX::Ordering::Release;
914 case AtomicOrdering::AcquireRelease: {
915 report_fatal_error(
916 reason: formatv(Fmt: "NVPTX does not support AcquireRelease Ordering on "
917 "read-modify-write "
918 "yet and PTX does not support it on loads or stores: {}",
919 Vals: N->getOperationName()));
920 }
921 case AtomicOrdering::SequentiallyConsistent: {
922 // LLVM-IR SequentiallyConsistent atomics map to a two-instruction PTX
923 // sequence including a "fence.sc.sco" and the memory instruction with an
924 // Ordering that differs from "sc": acq, rel, or acq_rel, depending on
925 // whether the memory operation is a read, write, or read-modify-write.
926 //
927 // This sets the ordering of the fence to SequentiallyConsistent, and
928 // sets the corresponding ordering for the instruction.
929 llvm_unreachable("seq_cst ordering should not reach this point, "
930 "is emulated in AtomicExpandPass");
931 }
932 }
933 report_fatal_error(
934 reason: formatv(Fmt: "NVPTX backend does not support AtomicOrdering \"{}\" yet.",
935 Vals: toIRString(ao: Ordering)));
936}
937
938} // namespace
939
940NVPTX::Scope NVPTXDAGToDAGISel::getOperationScope(MemSDNode *N,
941 NVPTX::Ordering O) const {
942 switch (O) {
943 case NVPTX::Ordering::NotAtomic:
944 case NVPTX::Ordering::Volatile: // Non-atomic volatile operations
945 // NVPTX uses Thread scope as the scope of non-atomic operations.
946 return NVPTX::Scope::Thread;
947 case NVPTX::Ordering::RelaxedMMIO:
948 // RelaxedMMIO operations are always system scope.
949 // If a RelaxedMMIO order was generated from an atomic volatile operation
950 // with a smaller thread scope, we bump it here to system scope.
951 return NVPTX::Scope::System;
952 case NVPTX::Ordering::Relaxed:
953 case NVPTX::Ordering::Acquire:
954 case NVPTX::Ordering::Release:
955 case NVPTX::Ordering::AcquireRelease:
956 case NVPTX::Ordering::SequentiallyConsistent:
957 auto S = Scopes[N->getSyncScopeID()];
958
959 S = resolveScope(S, T: Subtarget);
960
961 // If operation is volatile, then its scope is system.
962 return N->isVolatile() ? NVPTX::Scope::System : S;
963 }
964 llvm_unreachable("unhandled ordering");
965}
966
967static bool canLowerToLDG(const MemSDNode &N, const NVPTXSubtarget &Subtarget,
968 NVPTX::AddressSpace CodeAddrSpace) {
969 // We use ldg (i.e. ld.global.nc) for invariant loads from the global address
970 // space.
971 return Subtarget.hasLDG() && CodeAddrSpace == NVPTX::AddressSpace::Global &&
972 N.isInvariant();
973}
974
975static unsigned int getFenceOp(NVPTX::Ordering O, NVPTX::Scope S,
976 NVPTXSubtarget const *T) {
977 S = resolveScope(S, T);
978
979 // Fall back to .acq_rel if .acquire, .release is not supported.
980 if (!T->hasSplitAcquireAndReleaseFences() &&
981 (O == NVPTX::Ordering::Acquire || O == NVPTX::Ordering::Release))
982 O = NVPTX::Ordering::AcquireRelease;
983
984 switch (O) {
985 case NVPTX::Ordering::Acquire:
986 switch (S) {
987 case NVPTX::Scope::System:
988 return T->hasMemoryOrdering() ? NVPTX::atomic_thread_fence_acquire_sys
989 : NVPTX::INT_MEMBAR_SYS;
990 case NVPTX::Scope::Block:
991 return T->hasMemoryOrdering() ? NVPTX::atomic_thread_fence_acquire_cta
992 : NVPTX::INT_MEMBAR_CTA;
993 case NVPTX::Scope::Cluster:
994 return NVPTX::atomic_thread_fence_acquire_cluster;
995 case NVPTX::Scope::Device:
996 return T->hasMemoryOrdering() ? NVPTX::atomic_thread_fence_acquire_gpu
997 : NVPTX::INT_MEMBAR_GL;
998 case NVPTX::Scope::Thread:
999 case NVPTX::Scope::DefaultDevice:
1000 report_fatal_error(
1001 reason: formatv(Fmt: "Unsupported scope \"{}\" for acquire/release/acq_rel fence.",
1002 Vals: ScopeToString(S)));
1003 }
1004 break;
1005 case NVPTX::Ordering::Release:
1006 switch (S) {
1007 case NVPTX::Scope::System:
1008 return T->hasMemoryOrdering() ? NVPTX::atomic_thread_fence_release_sys
1009 : NVPTX::INT_MEMBAR_SYS;
1010 case NVPTX::Scope::Block:
1011 return T->hasMemoryOrdering() ? NVPTX::atomic_thread_fence_release_cta
1012 : NVPTX::INT_MEMBAR_CTA;
1013 case NVPTX::Scope::Cluster:
1014 return NVPTX::atomic_thread_fence_release_cluster;
1015 case NVPTX::Scope::Device:
1016 return T->hasMemoryOrdering() ? NVPTX::atomic_thread_fence_release_gpu
1017 : NVPTX::INT_MEMBAR_GL;
1018 case NVPTX::Scope::Thread:
1019 case NVPTX::Scope::DefaultDevice:
1020 report_fatal_error(
1021 reason: formatv(Fmt: "Unsupported scope \"{}\" for acquire/release/acq_rel fence.",
1022 Vals: ScopeToString(S)));
1023 }
1024 break;
1025 case NVPTX::Ordering::AcquireRelease: {
1026 switch (S) {
1027 case NVPTX::Scope::System:
1028 return T->hasMemoryOrdering() ? NVPTX::atomic_thread_fence_acq_rel_sys
1029 : NVPTX::INT_MEMBAR_SYS;
1030 case NVPTX::Scope::Block:
1031 return T->hasMemoryOrdering() ? NVPTX::atomic_thread_fence_acq_rel_cta
1032 : NVPTX::INT_MEMBAR_CTA;
1033 case NVPTX::Scope::Cluster:
1034 return NVPTX::atomic_thread_fence_acq_rel_cluster;
1035 case NVPTX::Scope::Device:
1036 return T->hasMemoryOrdering() ? NVPTX::atomic_thread_fence_acq_rel_gpu
1037 : NVPTX::INT_MEMBAR_GL;
1038 case NVPTX::Scope::Thread:
1039 case NVPTX::Scope::DefaultDevice:
1040 report_fatal_error(
1041 reason: formatv(Fmt: "Unsupported scope \"{}\" for acquire/release/acq_rel fence.",
1042 Vals: ScopeToString(S)));
1043 }
1044 break;
1045 }
1046 case NVPTX::Ordering::SequentiallyConsistent: {
1047 switch (S) {
1048 case NVPTX::Scope::System:
1049 return T->hasMemoryOrdering() ? NVPTX::atomic_thread_fence_seq_cst_sys
1050 : NVPTX::INT_MEMBAR_SYS;
1051 case NVPTX::Scope::Block:
1052 return T->hasMemoryOrdering() ? NVPTX::atomic_thread_fence_seq_cst_cta
1053 : NVPTX::INT_MEMBAR_CTA;
1054 case NVPTX::Scope::Cluster:
1055 return NVPTX::atomic_thread_fence_seq_cst_cluster;
1056 case NVPTX::Scope::Device:
1057 return T->hasMemoryOrdering() ? NVPTX::atomic_thread_fence_seq_cst_gpu
1058 : NVPTX::INT_MEMBAR_GL;
1059 case NVPTX::Scope::Thread:
1060 case NVPTX::Scope::DefaultDevice:
1061 report_fatal_error(reason: formatv(Fmt: "Unsupported scope \"{}\" for seq_cst fence.",
1062 Vals: ScopeToString(S)));
1063 }
1064 break;
1065 }
1066 case NVPTX::Ordering::NotAtomic:
1067 case NVPTX::Ordering::Relaxed:
1068 case NVPTX::Ordering::Volatile:
1069 case NVPTX::Ordering::RelaxedMMIO:
1070 report_fatal_error(
1071 reason: formatv(Fmt: "Unsupported \"{}\" ordering and \"{}\" scope for fence.",
1072 Vals: OrderingToString(Order: O), Vals: ScopeToString(S)));
1073 }
1074 llvm_unreachable("unhandled ordering");
1075}
1076
1077// Returns Memory Order and Scope of a memory instruction.
1078std::pair<NVPTX::Ordering, NVPTX::Scope>
1079NVPTXDAGToDAGISel::getMemInstOrderingScope(MemSDNode *N) {
1080 auto InstructionOrdering = getInstructionOrdering(N, Subtarget);
1081 auto Scope = getOperationScope(N, O: InstructionOrdering);
1082
1083 // b128 loads/stores were introduced in PTX 8.3, but system scope support
1084 // requires PTX 8.4. Check the resolved scope, including volatile promotion.
1085 if (N->getMemoryVT() == MVT::i128 && Scope == NVPTX::Scope::System &&
1086 !Subtarget->hasSysScopeLdStB128())
1087 CurDAG->getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1088 CurDAG->getMachineFunction().getFunction(),
1089 "NVPTX system scope b128 loads and stores require PTX ISA version 8.4 "
1090 "or later",
1091 N->getDebugLoc()));
1092
1093 // Singlethread scope has no inter-thread synchronization requirements, so
1094 // the atomic operation is lowered as plain and the fence is skipped.
1095 // NotAtomic and Volatile operations naturally have Thread scope and must
1096 // preserve their ordering.
1097 if (Scope == NVPTX::Scope::Thread &&
1098 InstructionOrdering != NVPTX::Ordering::NotAtomic &&
1099 InstructionOrdering != NVPTX::Ordering::Volatile)
1100 return {NVPTX::Ordering::NotAtomic, Scope};
1101
1102 return {InstructionOrdering, Scope};
1103}
1104
1105// Helper function template to reduce amount of boilerplate code for
1106// opcode selection.
1107static std::optional<unsigned>
1108pickOpcodeForVT(MVT::SimpleValueType VT, std::optional<unsigned> Opcode_i16,
1109 std::optional<unsigned> Opcode_i32,
1110 std::optional<unsigned> Opcode_i64) {
1111 switch (VT) {
1112 case MVT::f16:
1113 case MVT::i16:
1114 case MVT::bf16:
1115 return Opcode_i16;
1116 case MVT::v2f16:
1117 case MVT::v2bf16:
1118 case MVT::v2i16:
1119 case MVT::v4i8:
1120 case MVT::i32:
1121 case MVT::f32:
1122 return Opcode_i32;
1123 case MVT::v2f32:
1124 case MVT::v2i32:
1125 case MVT::i64:
1126 case MVT::f64:
1127 return Opcode_i64;
1128 default:
1129 return std::nullopt;
1130 }
1131}
1132
1133static inline bool isAddLike(const SDValue V) {
1134 return V.getOpcode() == ISD::ADD ||
1135 (V->getOpcode() == ISD::OR && V->getFlags().hasDisjoint());
1136}
1137
1138static SDValue stripAssertAlign(SDValue N) {
1139 if (N.getOpcode() == ISD::AssertAlign)
1140 N = N.getOperand(i: 0);
1141 return N;
1142}
1143
1144// selectBaseADDR - Match a dag node which will serve as the base address for an
1145// ADDR operand pair.
1146static SDValue selectBaseADDR(SDValue N, SelectionDAG *DAG) {
1147 N = stripAssertAlign(N);
1148 if (const auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: N))
1149 return DAG->getTargetGlobalAddress(GV: GA->getGlobal(), DL: SDLoc(N),
1150 VT: GA->getValueType(ResNo: 0), offset: GA->getOffset(),
1151 TargetFlags: GA->getTargetFlags());
1152 if (const auto *ES = dyn_cast<ExternalSymbolSDNode>(Val&: N))
1153 return DAG->getTargetExternalSymbol(Sym: ES->getSymbol(), VT: ES->getValueType(ResNo: 0),
1154 TargetFlags: ES->getTargetFlags());
1155 if (const auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: N))
1156 return DAG->getTargetFrameIndex(FI: FIN->getIndex(), VT: FIN->getValueType(ResNo: 0));
1157 if (N.getOpcode() == NVPTXISD::Symbol)
1158 return N.getOperand(i: 0);
1159
1160 return N;
1161}
1162
1163static SDValue accumulateOffset(SDValue &Addr, SDLoc DL, SelectionDAG *DAG) {
1164 Addr = stripAssertAlign(N: Addr);
1165 APInt AccumulatedOffset(64u, 0);
1166 while (isAddLike(V: Addr)) {
1167 const auto *CN = dyn_cast<ConstantSDNode>(Val: Addr.getOperand(i: 1));
1168 if (!CN)
1169 break;
1170
1171 const APInt CI = CN->getAPIntValue().sext(width: 64);
1172 if (!(CI + AccumulatedOffset).isSignedIntN(N: 32))
1173 break;
1174
1175 AccumulatedOffset += CI;
1176 Addr = stripAssertAlign(N: Addr->getOperand(Num: 0));
1177 }
1178 return DAG->getSignedTargetConstant(Val: AccumulatedOffset.getSExtValue(), DL,
1179 VT: MVT::i32);
1180}
1181
1182static std::pair<SDValue, SDValue> selectADDR(SDValue Addr, SelectionDAG *DAG) {
1183 SDValue Offset = accumulateOffset(Addr, DL: SDLoc(Addr), DAG);
1184 SDValue Base = selectBaseADDR(N: Addr, DAG);
1185 return {Base, Offset};
1186}
1187
1188// Select a pair of operands which represent a valid PTX address, this could be
1189// one of the following things:
1190// - [var] - Offset is simply set to 0
1191// - [reg] - Offset is simply set to 0
1192// - [reg+immOff]
1193// - [var+immOff]
1194// Note that immOff must fit into a 32-bit signed integer.
1195bool NVPTXDAGToDAGISel::SelectADDR(SDValue Addr, SDValue &Base,
1196 SDValue &Offset) {
1197 std::tie(args&: Base, args&: Offset) = selectADDR(Addr, DAG: CurDAG);
1198 return true;
1199}
1200
1201static void emitInvalidMemCacheHint(LLVMContext &Ctx, const Twine &Msg) {
1202 Ctx.diagnose(DI: DiagnosticInfoGeneric(
1203 Twine("invalid NVPTX !mem.cache_hint metadata: ") + Msg, DS_Warning));
1204}
1205
1206static std::optional<NVPTX::L1Eviction> parseL1Eviction(StringRef Str) {
1207 return StringSwitch<std::optional<NVPTX::L1Eviction>>(Str)
1208 .Case(S: "normal", Value: NVPTX::L1Eviction::Normal)
1209 .Case(S: "unchanged", Value: NVPTX::L1Eviction::Unchanged)
1210 .Case(S: "first", Value: NVPTX::L1Eviction::First)
1211 .Case(S: "last", Value: NVPTX::L1Eviction::Last)
1212 .Case(S: "no_allocate", Value: NVPTX::L1Eviction::NoAllocate)
1213 .Default(Value: std::nullopt);
1214}
1215
1216static std::optional<NVPTX::L2Eviction> parseL2Eviction(StringRef Str) {
1217 return StringSwitch<std::optional<NVPTX::L2Eviction>>(Str)
1218 .Case(S: "normal", Value: NVPTX::L2Eviction::Normal)
1219 .Case(S: "first", Value: NVPTX::L2Eviction::First)
1220 .Case(S: "last", Value: NVPTX::L2Eviction::Last)
1221 .Default(Value: std::nullopt);
1222}
1223
1224static std::optional<NVPTX::L2Prefetch> parseL2Prefetch(StringRef Str) {
1225 return StringSwitch<std::optional<NVPTX::L2Prefetch>>(Str)
1226 .Case(S: "64B", Value: NVPTX::L2Prefetch::Bytes64)
1227 .Case(S: "128B", Value: NVPTX::L2Prefetch::Bytes128)
1228 .Case(S: "256B", Value: NVPTX::L2Prefetch::Bytes256)
1229 .Default(Value: std::nullopt);
1230}
1231
1232template <typename T>
1233static std::optional<T> parseMemCacheHintStringValue(
1234 LLVMContext &Ctx, StringRef Key, const Metadata *Value,
1235 std::optional<T> (*Parse)(StringRef), bool EmitDiagnostics) {
1236 const auto *Val = dyn_cast<MDString>(Val: Value);
1237 if (!Val) {
1238 if (EmitDiagnostics)
1239 emitInvalidMemCacheHint(Ctx,
1240 Msg: Twine("'") + Key + "' expects a string value");
1241 return std::nullopt;
1242 }
1243
1244 StringRef ValStr = Val->getString();
1245 auto Parsed = Parse(ValStr);
1246 if (!Parsed && EmitDiagnostics)
1247 emitInvalidMemCacheHint(Ctx, Msg: Twine("unknown value '") + ValStr + "' for '" +
1248 Key + "'");
1249 return Parsed;
1250}
1251
1252static bool isGlobalOrGeneric(NVPTX::AddressSpace AddrSpace) {
1253 return AddrSpace == NVPTX::AddressSpace::Global ||
1254 AddrSpace == NVPTX::AddressSpace::Generic;
1255}
1256
1257static bool isLdOrSt(NVPTXMemCacheHintAccess Access) {
1258 return Access.Instruction == NVPTXMemCacheHintInstruction::Ld ||
1259 Access.Instruction == NVPTXMemCacheHintInstruction::St;
1260}
1261
1262static bool isL1EvictionSupported(const NVPTXSubtarget &Subtarget,
1263 NVPTX::L1Eviction Eviction,
1264 NVPTXMemCacheHintAccess Access) {
1265 if (Eviction == NVPTX::L1Eviction::Normal)
1266 return true;
1267
1268 return isLdOrSt(Access) && !Access.IsVolatile &&
1269 Subtarget.hasL1EvictionHint();
1270}
1271
1272static bool isL2PrefetchSupported(const NVPTXSubtarget &Subtarget,
1273 NVPTX::L2Prefetch Prefetch,
1274 NVPTXMemCacheHintAccess Access) {
1275 switch (Prefetch) {
1276 case NVPTX::L2Prefetch::None:
1277 return true;
1278 case NVPTX::L2Prefetch::Bytes64:
1279 return Access.Instruction == NVPTXMemCacheHintInstruction::Ld &&
1280 isGlobalOrGeneric(AddrSpace: Access.AddrSpace) && Subtarget.hasL2Prefetch64B();
1281 case NVPTX::L2Prefetch::Bytes128:
1282 return Access.Instruction == NVPTXMemCacheHintInstruction::Ld &&
1283 isGlobalOrGeneric(AddrSpace: Access.AddrSpace) && Subtarget.hasL2Prefetch128B();
1284 case NVPTX::L2Prefetch::Bytes256:
1285 return Access.Instruction == NVPTXMemCacheHintInstruction::Ld &&
1286 isGlobalOrGeneric(AddrSpace: Access.AddrSpace) && Subtarget.hasL2Prefetch256B();
1287 }
1288 llvm_unreachable("Unexpected L2 prefetch hint");
1289}
1290
1291static bool isL2EvictionSupported(const NVPTXSubtarget &Subtarget,
1292 NVPTX::L2Eviction Eviction,
1293 NVPTXMemCacheHintAccess Access) {
1294 if (Eviction == NVPTX::L2Eviction::Normal)
1295 return true;
1296
1297 return isLdOrSt(Access) && !Access.IsVolatile &&
1298 Subtarget.hasL2EvictionHint() && isGlobalOrGeneric(AddrSpace: Access.AddrSpace) &&
1299 ((Access.NumElts == 8 && Access.EltWidth == 32) ||
1300 (Access.NumElts == 4 && Access.EltWidth == 64));
1301}
1302
1303static bool isCachePolicySupported(const NVPTXSubtarget &Subtarget,
1304 NVPTXMemCacheHintAccess Access) {
1305 return !Access.IsVolatile && isGlobalOrGeneric(AddrSpace: Access.AddrSpace) &&
1306 Subtarget.hasL2CacheHint();
1307}
1308
1309NVPTXMemCacheHintOperands NVPTXDAGToDAGISel::getMemCacheHintOperands(
1310 const MemSDNode *N, NVPTXMemCacheHintAccess Access, const SDLoc &DL,
1311 bool EmitDiagnostics) {
1312 LLVMContext &Ctx = *CurDAG->getContext();
1313 const MDNode *Node = N->getMemCacheHint();
1314 SDValue PolicyReg = CurDAG->getRegister(Reg: NVPTX::NoRegister, VT: MVT::i64);
1315 if (!Node)
1316 return {.EvictionAndPrefetchHint: getI32Imm(Imm: 0, DL), .CachePolicyReg: PolicyReg};
1317 if (Node->getNumOperands() == 0) {
1318 if (EmitDiagnostics)
1319 emitInvalidMemCacheHint(Ctx, Msg: "empty hint node");
1320 return {.EvictionAndPrefetchHint: getI32Imm(Imm: 0, DL), .CachePolicyReg: PolicyReg};
1321 }
1322
1323 NVPTX::L1Eviction L1 = NVPTX::L1Eviction::Normal;
1324 NVPTX::L2Eviction L2 = NVPTX::L2Eviction::Normal;
1325 NVPTX::L2Prefetch Prefetch = NVPTX::L2Prefetch::None;
1326 std::optional<uint64_t> CachePolicy;
1327
1328 for (unsigned I = 0; I + 1 < Node->getNumOperands(); I += 2) {
1329 const auto *Key = cast<MDString>(Val: Node->getOperand(I));
1330 StringRef KeyStr = Key->getString();
1331 const Metadata *Value = Node->getOperand(I: I + 1).get();
1332
1333 if (KeyStr == "nvvm.l1_eviction") {
1334 auto ParsedL1 = parseMemCacheHintStringValue(
1335 Ctx, Key: KeyStr, Value, Parse: parseL1Eviction, EmitDiagnostics);
1336 if (ParsedL1 && isL1EvictionSupported(Subtarget: *Subtarget, Eviction: *ParsedL1, Access))
1337 L1 = *ParsedL1;
1338 continue;
1339 }
1340
1341 if (KeyStr == "nvvm.l2_eviction") {
1342 auto ParsedL2 = parseMemCacheHintStringValue(
1343 Ctx, Key: KeyStr, Value, Parse: parseL2Eviction, EmitDiagnostics);
1344 if (ParsedL2 && isL2EvictionSupported(Subtarget: *Subtarget, Eviction: *ParsedL2, Access))
1345 L2 = *ParsedL2;
1346 continue;
1347 }
1348
1349 if (KeyStr == "nvvm.l2_prefetch_size") {
1350 auto ParsedPrefetch = parseMemCacheHintStringValue(
1351 Ctx, Key: KeyStr, Value, Parse: parseL2Prefetch, EmitDiagnostics);
1352 if (ParsedPrefetch &&
1353 isL2PrefetchSupported(Subtarget: *Subtarget, Prefetch: *ParsedPrefetch, Access))
1354 Prefetch = *ParsedPrefetch;
1355 continue;
1356 }
1357
1358 if (KeyStr == "nvvm.l2_cache_hint") {
1359 const auto *ValCI = mdconst::dyn_extract<ConstantInt>(MD&: Value);
1360 if (!ValCI) {
1361 if (EmitDiagnostics)
1362 emitInvalidMemCacheHint(
1363 Ctx, Msg: "'nvvm.l2_cache_hint' expects an integer value");
1364 } else if (isCachePolicySupported(Subtarget: *Subtarget, Access)) {
1365 CachePolicy = ValCI->getZExtValue();
1366 }
1367 continue;
1368 }
1369
1370 if (EmitDiagnostics)
1371 emitInvalidMemCacheHint(Ctx, Msg: Twine("unknown key '") + KeyStr + "'");
1372 }
1373
1374 unsigned EvictionAndPrefetchHint =
1375 NVPTX::encodeEvictionAndPrefetchHint(L1, L2, P: Prefetch);
1376 if (CachePolicy) {
1377 SDValue PolicyConst = CurDAG->getTargetConstant(Val: *CachePolicy, DL, VT: MVT::i64);
1378 PolicyReg = SDValue(
1379 CurDAG->getMachineNode(Opcode: NVPTX::MOV_B64_i, dl: DL, VT: MVT::i64, Op1: PolicyConst), 0);
1380 Bitfield::set<NVPTX::L2CacheHintBit>(Packed&: EvictionAndPrefetchHint, Value: true);
1381 }
1382
1383 return {.EvictionAndPrefetchHint: getI32Imm(Imm: EvictionAndPrefetchHint, DL), .CachePolicyReg: PolicyReg};
1384}
1385
1386bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
1387 MemSDNode *LD = cast<MemSDNode>(Val: N);
1388 assert(LD->readMem() && "Expected load");
1389
1390 // do not support pre/post inc/dec
1391 const LoadSDNode *PlainLoad = dyn_cast<LoadSDNode>(Val: LD);
1392 if (PlainLoad && PlainLoad->isIndexed())
1393 return false;
1394
1395 // Address Space Setting
1396 const auto CodeAddrSpace = getAddrSpace(N: LD);
1397 if (canLowerToLDG(N: *LD, Subtarget: *Subtarget, CodeAddrSpace))
1398 return tryLDG(N: LD);
1399
1400 SDLoc DL(LD);
1401 SDValue Chain = N->getOperand(Num: 0);
1402 const auto [Ordering, Scope] = getMemInstOrderingScope(N: LD);
1403
1404 const unsigned FromTypeWidth = LD->getMemoryVT().getSizeInBits();
1405
1406 // Vector Setting
1407 const unsigned FromType =
1408 (PlainLoad && (PlainLoad->getExtensionType() == ISD::SEXTLOAD))
1409 ? NVPTX::PTXLdStInstCode::Signed
1410 : NVPTX::PTXLdStInstCode::Untyped;
1411
1412 uint32_t UsedBytesMask;
1413 switch (N->getOpcode()) {
1414 case ISD::LOAD:
1415 case ISD::ATOMIC_LOAD:
1416 case NVPTXISD::ATOMIC_LOAD_B128:
1417 UsedBytesMask = UINT32_MAX;
1418 break;
1419 case NVPTXISD::MLoad:
1420 UsedBytesMask = N->getConstantOperandVal(Num: 3);
1421 break;
1422 default:
1423 llvm_unreachable("Unexpected opcode");
1424 }
1425
1426 assert(isPowerOf2_32(FromTypeWidth) && FromTypeWidth >= 8 &&
1427 FromTypeWidth <= 128 && "Invalid width for load");
1428
1429 const auto [Base, Offset] = selectADDR(Addr: N->getOperand(Num: 1), DAG: CurDAG);
1430 const auto [EvictionAndPrefetchHint, PolicyReg] = getMemCacheHintOperands(
1431 N: LD,
1432 Access: {.Instruction: NVPTXMemCacheHintInstruction::Ld, .AddrSpace: CodeAddrSpace,
1433 /*NumElts=*/1, /*EltWidth=*/FromTypeWidth, .IsVolatile: LD->isVolatile()},
1434 DL);
1435
1436 // Create the machine instruction DAG
1437 SDValue Ops[] = {getI32Imm(Imm: Ordering, DL),
1438 getI32Imm(Imm: Scope, DL),
1439 getI32Imm(Imm: CodeAddrSpace, DL),
1440 getI32Imm(Imm: FromType, DL),
1441 getI32Imm(Imm: FromTypeWidth, DL),
1442 getI32Imm(Imm: UsedBytesMask, DL),
1443 Base,
1444 Offset,
1445 EvictionAndPrefetchHint,
1446 PolicyReg,
1447 Chain};
1448
1449 const MVT::SimpleValueType TargetVT = LD->getSimpleValueType(ResNo: 0).SimpleTy;
1450 // The result of an i128 load is a pair of i64s, so TargetVT is i64 and would
1451 // otherwise select LD_i64.
1452 const bool IsB128 = N->getOpcode() == NVPTXISD::ATOMIC_LOAD_B128;
1453 const std::optional<unsigned> Opcode =
1454 IsB128 ? NVPTX::LD_i128
1455 : pickOpcodeForVT(VT: TargetVT, Opcode_i16: NVPTX::LD_i16, Opcode_i32: NVPTX::LD_i32,
1456 Opcode_i64: NVPTX::LD_i64);
1457 if (!Opcode)
1458 return false;
1459
1460 SDVTList VTs =
1461 IsB128 ? CurDAG->getVTList(VT1: MVT::i128, VT2: MVT::Other) : LD->getVTList();
1462 SDNode *NVPTXLD = CurDAG->getMachineNode(Opcode: *Opcode, dl: DL, VTs, Ops);
1463 if (!NVPTXLD)
1464 return false;
1465
1466 MachineMemOperand *MemRef = LD->getMemOperand();
1467 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: NVPTXLD), NewMemRefs: {MemRef});
1468
1469 if (IsB128) {
1470 SDNode *Unpacked = CurDAG->getMachineNode(
1471 Opcode: NVPTX::I128toV2I64, dl: DL, ResultTys: {MVT::i64, MVT::i64}, Ops: SDValue(NVPTXLD, 0));
1472 // Replace results directly: DAGCombiner has already run, and emission
1473 // does not assign registers to MERGE_VALUES results.
1474 SDValue LoadChain(NVPTXLD, 1);
1475 SDValue OldResults[] = {SDValue(LD, 0), SDValue(LD, 1), SDValue(LD, 2)};
1476 SDValue Results[] = {SDValue(Unpacked, 0), SDValue(Unpacked, 1), LoadChain};
1477 ReplaceUses(F: OldResults, T: Results, Num: 3);
1478 CurDAG->RemoveDeadNode(N: LD);
1479 } else {
1480 ReplaceNode(F: LD, T: NVPTXLD);
1481 }
1482 return true;
1483}
1484
1485static unsigned getStoreVectorNumElts(SDNode *N) {
1486 switch (N->getOpcode()) {
1487 case NVPTXISD::StoreV2:
1488 return 2;
1489 case NVPTXISD::StoreV4:
1490 return 4;
1491 case NVPTXISD::StoreV8:
1492 return 8;
1493 default:
1494 llvm_unreachable("Unexpected opcode");
1495 }
1496}
1497
1498bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
1499 MemSDNode *LD = cast<MemSDNode>(Val: N);
1500
1501 // Address Space Setting
1502 const auto CodeAddrSpace = getAddrSpace(N: LD);
1503 if (canLowerToLDG(N: *LD, Subtarget: *Subtarget, CodeAddrSpace))
1504 return tryLDG(N: LD);
1505
1506 const MVT EltVT = LD->getSimpleValueType(ResNo: 0);
1507 SDLoc DL(LD);
1508 SDValue Chain = LD->getChain();
1509 const auto [Ordering, Scope] = getMemInstOrderingScope(N: LD);
1510
1511 // Type Setting: fromType + fromTypeWidth
1512 //
1513 // Sign : ISD::SEXTLOAD
1514 // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
1515 // type is integer
1516 // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
1517 // Read at least 8 bits (predicates are stored as 8-bit values)
1518 // Get the original LoadSDNode::getExtensionType() value
1519 const unsigned ExtensionType = N->getConstantOperandVal(Num: 4);
1520 const unsigned FromType = (ExtensionType == ISD::SEXTLOAD)
1521 ? NVPTX::PTXLdStInstCode::Signed
1522 : NVPTX::PTXLdStInstCode::Untyped;
1523
1524 const unsigned FromTypeWidth = getFromTypeWidthForLoad(Mem: LD);
1525 const uint32_t UsedBytesMask = N->getConstantOperandVal(Num: 3);
1526
1527 assert(!(EltVT.isVector() && ExtensionType != ISD::NON_EXTLOAD));
1528
1529 const auto [EvictionAndPrefetchHint, PolicyReg] =
1530 getMemCacheHintOperands(N: LD,
1531 Access: {.Instruction: NVPTXMemCacheHintInstruction::Ld, .AddrSpace: CodeAddrSpace,
1532 /*NumElts=*/LD->getNumValues() - 1,
1533 /*EltWidth=*/FromTypeWidth, .IsVolatile: LD->isVolatile()},
1534 DL);
1535 const auto [Base, Offset] = selectADDR(Addr: N->getOperand(Num: 1), DAG: CurDAG);
1536 SDValue Ops[] = {getI32Imm(Imm: Ordering, DL),
1537 getI32Imm(Imm: Scope, DL),
1538 getI32Imm(Imm: CodeAddrSpace, DL),
1539 getI32Imm(Imm: FromType, DL),
1540 getI32Imm(Imm: FromTypeWidth, DL),
1541 getI32Imm(Imm: UsedBytesMask, DL),
1542 Base,
1543 Offset,
1544 EvictionAndPrefetchHint,
1545 PolicyReg,
1546 Chain};
1547
1548 std::optional<unsigned> Opcode;
1549 switch (N->getOpcode()) {
1550 default:
1551 llvm_unreachable("Unexpected opcode");
1552 case NVPTXISD::LoadV2:
1553 Opcode = pickOpcodeForVT(VT: EltVT.SimpleTy, Opcode_i16: NVPTX::LDV_i16_v2,
1554 Opcode_i32: NVPTX::LDV_i32_v2, Opcode_i64: NVPTX::LDV_i64_v2);
1555 break;
1556 case NVPTXISD::LoadV4:
1557 Opcode = pickOpcodeForVT(VT: EltVT.SimpleTy, Opcode_i16: NVPTX::LDV_i16_v4,
1558 Opcode_i32: NVPTX::LDV_i32_v4, Opcode_i64: NVPTX::LDV_i64_v4);
1559 break;
1560 case NVPTXISD::LoadV8:
1561 Opcode = pickOpcodeForVT(VT: EltVT.SimpleTy, Opcode_i16: {/* no v8i16 */},
1562 Opcode_i32: NVPTX::LDV_i32_v8, Opcode_i64: {/* no v8i64 */});
1563 break;
1564 }
1565 if (!Opcode)
1566 return false;
1567
1568 SDNode *NVPTXLD = CurDAG->getMachineNode(Opcode: *Opcode, dl: DL, VTs: LD->getVTList(), Ops);
1569
1570 MachineMemOperand *MemRef = LD->getMemOperand();
1571 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: NVPTXLD), NewMemRefs: {MemRef});
1572
1573 ReplaceNode(F: LD, T: NVPTXLD);
1574 return true;
1575}
1576
1577bool NVPTXDAGToDAGISel::tryLDG(MemSDNode *LD) {
1578 SDLoc DL(LD);
1579
1580 unsigned ExtensionType;
1581 uint32_t UsedBytesMask;
1582 if (const auto *Load = dyn_cast<LoadSDNode>(Val: LD)) {
1583 ExtensionType = Load->getExtensionType();
1584 UsedBytesMask = UINT32_MAX;
1585 } else {
1586 ExtensionType = LD->getConstantOperandVal(Num: 4);
1587 UsedBytesMask = LD->getConstantOperandVal(Num: 3);
1588 }
1589 const unsigned FromType = (ExtensionType == ISD::SEXTLOAD)
1590 ? NVPTX::PTXLdStInstCode::Signed
1591 : NVPTX::PTXLdStInstCode::Untyped;
1592
1593 const unsigned FromTypeWidth = getFromTypeWidthForLoad(Mem: LD);
1594
1595 assert(!(LD->getSimpleValueType(0).isVector() &&
1596 ExtensionType != ISD::NON_EXTLOAD));
1597
1598 const auto [Base, Offset] = selectADDR(Addr: LD->getOperand(Num: 1), DAG: CurDAG);
1599 const auto [EvictionAndPrefetchHint, PolicyReg] = getMemCacheHintOperands(
1600 N: LD,
1601 Access: {.Instruction: NVPTXMemCacheHintInstruction::Ld, .AddrSpace: NVPTX::AddressSpace::Global,
1602 .NumElts: LD->getNumValues() - 1, .EltWidth: FromTypeWidth, .IsVolatile: LD->isVolatile()},
1603 DL);
1604 SDValue Ops[] = {getI32Imm(Imm: FromType, DL),
1605 getI32Imm(Imm: FromTypeWidth, DL),
1606 getI32Imm(Imm: UsedBytesMask, DL),
1607 Base,
1608 Offset,
1609 EvictionAndPrefetchHint,
1610 PolicyReg,
1611 LD->getChain()};
1612
1613 const MVT::SimpleValueType TargetVT = LD->getSimpleValueType(ResNo: 0).SimpleTy;
1614 std::optional<unsigned> Opcode;
1615 switch (LD->getOpcode()) {
1616 default:
1617 llvm_unreachable("Unexpected opcode");
1618 case ISD::LOAD:
1619 Opcode = pickOpcodeForVT(VT: TargetVT, Opcode_i16: NVPTX::LD_GLOBAL_NC_i16,
1620 Opcode_i32: NVPTX::LD_GLOBAL_NC_i32, Opcode_i64: NVPTX::LD_GLOBAL_NC_i64);
1621 break;
1622 case NVPTXISD::MLoad:
1623 Opcode = pickOpcodeForVT(VT: TargetVT, Opcode_i16: std::nullopt, Opcode_i32: NVPTX::LD_GLOBAL_NC_i32,
1624 Opcode_i64: NVPTX::LD_GLOBAL_NC_i64);
1625 break;
1626 case NVPTXISD::LoadV2:
1627 Opcode =
1628 pickOpcodeForVT(VT: TargetVT, Opcode_i16: NVPTX::LD_GLOBAL_NC_v2i16,
1629 Opcode_i32: NVPTX::LD_GLOBAL_NC_v2i32, Opcode_i64: NVPTX::LD_GLOBAL_NC_v2i64);
1630 break;
1631 case NVPTXISD::LoadV4:
1632 Opcode =
1633 pickOpcodeForVT(VT: TargetVT, Opcode_i16: NVPTX::LD_GLOBAL_NC_v4i16,
1634 Opcode_i32: NVPTX::LD_GLOBAL_NC_v4i32, Opcode_i64: NVPTX::LD_GLOBAL_NC_v4i64);
1635 break;
1636 case NVPTXISD::LoadV8:
1637 Opcode = pickOpcodeForVT(VT: TargetVT, Opcode_i16: {/* no v8i16 */},
1638 Opcode_i32: NVPTX::LD_GLOBAL_NC_v8i32, Opcode_i64: {/* no v8i64 */});
1639 break;
1640 }
1641 if (!Opcode)
1642 return false;
1643
1644 SDNode *NVPTXLDG = CurDAG->getMachineNode(Opcode: *Opcode, dl: DL, VTs: LD->getVTList(), Ops);
1645
1646 ReplaceNode(F: LD, T: NVPTXLDG);
1647 return true;
1648}
1649
1650bool NVPTXDAGToDAGISel::tryLDU(SDNode *N) {
1651 auto *LD = cast<MemSDNode>(Val: N);
1652
1653 SDLoc DL(N);
1654 const unsigned FromTypeWidth = getFromTypeWidthForLoad(Mem: LD);
1655 const MVT::SimpleValueType TargetVT = LD->getSimpleValueType(ResNo: 0).SimpleTy;
1656
1657 // If this is an LDU intrinsic, the address is the third operand. If its an
1658 // LDU SD node (from custom vector handling), then its the second operand
1659 SDValue Addr =
1660 LD->getOperand(Num: LD->getOpcode() == ISD::INTRINSIC_W_CHAIN ? 2 : 1);
1661
1662 const auto [Base, Offset] = selectADDR(Addr, DAG: CurDAG);
1663 SDValue Ops[] = {getI32Imm(Imm: FromTypeWidth, DL), Base, Offset, LD->getChain()};
1664
1665 std::optional<unsigned> Opcode;
1666 switch (N->getOpcode()) {
1667 default:
1668 llvm_unreachable("Unexpected opcode");
1669 case ISD::INTRINSIC_W_CHAIN:
1670 Opcode = pickOpcodeForVT(VT: TargetVT, Opcode_i16: NVPTX::LDU_GLOBAL_i16,
1671 Opcode_i32: NVPTX::LDU_GLOBAL_i32, Opcode_i64: NVPTX::LDU_GLOBAL_i64);
1672 break;
1673 case NVPTXISD::LDUV2:
1674 Opcode = pickOpcodeForVT(VT: TargetVT, Opcode_i16: NVPTX::LDU_GLOBAL_v2i16,
1675 Opcode_i32: NVPTX::LDU_GLOBAL_v2i32, Opcode_i64: NVPTX::LDU_GLOBAL_v2i64);
1676 break;
1677 case NVPTXISD::LDUV4:
1678 Opcode = pickOpcodeForVT(VT: TargetVT, Opcode_i16: NVPTX::LDU_GLOBAL_v4i16,
1679 Opcode_i32: NVPTX::LDU_GLOBAL_v4i32, Opcode_i64: {/* no v4i64 */});
1680 break;
1681 }
1682 if (!Opcode)
1683 return false;
1684
1685 SDNode *NVPTXLDU = CurDAG->getMachineNode(Opcode: *Opcode, dl: DL, VTs: LD->getVTList(), Ops);
1686
1687 ReplaceNode(F: LD, T: NVPTXLDU);
1688 return true;
1689}
1690
1691bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
1692 MemSDNode *ST = cast<MemSDNode>(Val: N);
1693 assert(ST->writeMem() && "Expected store");
1694 StoreSDNode *PlainStore = dyn_cast<StoreSDNode>(Val: ST);
1695 AtomicSDNode *AtomicStore = dyn_cast<AtomicSDNode>(Val: ST);
1696 // i128 is not a legal type, so ATOMIC_STORE_B128 is neither of the above: it
1697 // carries its value as a pair of 64-bit operands.
1698 const bool IsB128 = N->getOpcode() == NVPTXISD::ATOMIC_STORE_B128;
1699 assert((PlainStore || AtomicStore || IsB128) && "Expected store");
1700
1701 // do not support pre/post inc/dec
1702 if (PlainStore && PlainStore->isIndexed())
1703 return false;
1704
1705 // Address Space Setting
1706 const auto CodeAddrSpace = getAddrSpace(N: ST);
1707
1708 SDLoc DL(ST);
1709 SDValue Chain = ST->getChain();
1710 const auto [Ordering, Scope] = getMemInstOrderingScope(N: ST);
1711
1712 // Vector Setting
1713 const unsigned ToTypeWidth = ST->getMemoryVT().getSizeInBits();
1714
1715 // Create the machine instruction DAG
1716 SDValue Value;
1717 if (IsB128) {
1718 SDNode *Packed =
1719 CurDAG->getMachineNode(Opcode: NVPTX::V2I64toI128, dl: DL, VT: MVT::i128,
1720 Ops: {selectPossiblyImm(V: N->getOperand(Num: 2)),
1721 selectPossiblyImm(V: N->getOperand(Num: 3))});
1722 Value = SDValue(Packed, 0);
1723 } else {
1724 Value = PlainStore ? PlainStore->getValue() : AtomicStore->getVal();
1725 }
1726
1727 assert(isPowerOf2_32(ToTypeWidth) && ToTypeWidth >= 8 && ToTypeWidth <= 128 &&
1728 "Invalid width for store");
1729
1730 const auto [Base, Offset] = selectADDR(Addr: ST->getBasePtr(), DAG: CurDAG);
1731
1732 // Extract eviction/prefetch hint and cache policy register.
1733 const auto [EvictionAndPrefetchHint, PolicyReg] = getMemCacheHintOperands(
1734 N: ST,
1735 Access: {.Instruction: NVPTXMemCacheHintInstruction::St, .AddrSpace: CodeAddrSpace,
1736 /*NumElts=*/1, /*EltWidth=*/ToTypeWidth, .IsVolatile: ST->isVolatile()},
1737 DL);
1738
1739 SDValue Ops[] = {selectPossiblyImm(V: Value),
1740 getI32Imm(Imm: Ordering, DL),
1741 getI32Imm(Imm: Scope, DL),
1742 getI32Imm(Imm: CodeAddrSpace, DL),
1743 getI32Imm(Imm: ToTypeWidth, DL),
1744 Base,
1745 Offset,
1746 EvictionAndPrefetchHint,
1747 PolicyReg,
1748 Chain};
1749
1750 const std::optional<unsigned> Opcode =
1751 IsB128 ? NVPTX::ST_i128
1752 : pickOpcodeForVT(VT: Value.getSimpleValueType().SimpleTy,
1753 Opcode_i16: NVPTX::ST_i16, Opcode_i32: NVPTX::ST_i32, Opcode_i64: NVPTX::ST_i64);
1754 if (!Opcode)
1755 return false;
1756
1757 SDNode *NVPTXST = CurDAG->getMachineNode(Opcode: *Opcode, dl: DL, VT: MVT::Other, Ops);
1758
1759 if (!NVPTXST)
1760 return false;
1761
1762 MachineMemOperand *MemRef = ST->getMemOperand();
1763 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: NVPTXST), NewMemRefs: {MemRef});
1764 ReplaceNode(F: ST, T: NVPTXST);
1765 return true;
1766}
1767
1768bool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) {
1769 MemSDNode *ST = cast<MemSDNode>(Val: N);
1770 const unsigned TotalWidth = ST->getMemoryVT().getSizeInBits();
1771
1772 // Address Space Setting
1773 const auto CodeAddrSpace = getAddrSpace(N: ST);
1774 if (CodeAddrSpace == NVPTX::AddressSpace::Const) {
1775 report_fatal_error(reason: "Cannot store to pointer that points to constant "
1776 "memory space");
1777 }
1778
1779 SDLoc DL(ST);
1780 SDValue Chain = ST->getChain();
1781 const auto [Ordering, Scope] = getMemInstOrderingScope(N: ST);
1782
1783 const unsigned NumElts = getStoreVectorNumElts(N: ST);
1784
1785 SmallVector<SDValue, 16> Ops;
1786 for (auto &V : ST->ops().slice(N: 1, M: NumElts))
1787 Ops.push_back(Elt: selectPossiblyImm(V));
1788 SDValue Addr = N->getOperand(Num: NumElts + 1);
1789 const unsigned ToTypeWidth = TotalWidth / NumElts;
1790
1791 assert(isPowerOf2_32(ToTypeWidth) && ToTypeWidth >= 8 && ToTypeWidth <= 128 &&
1792 TotalWidth <= 256 && "Invalid width for store");
1793
1794 // Extract eviction/prefetch hint and cache policy register.
1795 const auto [EvictionAndPrefetchHint, PolicyReg] = getMemCacheHintOperands(
1796 N: ST,
1797 Access: {.Instruction: NVPTXMemCacheHintInstruction::St, .AddrSpace: CodeAddrSpace,
1798 /*NumElts=*/NumElts, /*EltWidth=*/ToTypeWidth, .IsVolatile: ST->isVolatile()},
1799 DL);
1800
1801 const auto [Base, Offset] = selectADDR(Addr, DAG: CurDAG);
1802 Ops.append(IL: {getI32Imm(Imm: Ordering, DL), getI32Imm(Imm: Scope, DL),
1803 getI32Imm(Imm: CodeAddrSpace, DL), getI32Imm(Imm: ToTypeWidth, DL), Base,
1804 Offset, EvictionAndPrefetchHint, PolicyReg, Chain});
1805
1806 const MVT::SimpleValueType EltVT =
1807 ST->getOperand(Num: 1).getSimpleValueType().SimpleTy;
1808 std::optional<unsigned> Opcode;
1809 switch (ST->getOpcode()) {
1810 default:
1811 return false;
1812 case NVPTXISD::StoreV2:
1813 Opcode = pickOpcodeForVT(VT: EltVT, Opcode_i16: NVPTX::STV_i16_v2, Opcode_i32: NVPTX::STV_i32_v2,
1814 Opcode_i64: NVPTX::STV_i64_v2);
1815 break;
1816 case NVPTXISD::StoreV4:
1817 Opcode = pickOpcodeForVT(VT: EltVT, Opcode_i16: NVPTX::STV_i16_v4, Opcode_i32: NVPTX::STV_i32_v4,
1818 Opcode_i64: NVPTX::STV_i64_v4);
1819 break;
1820 case NVPTXISD::StoreV8:
1821 Opcode = pickOpcodeForVT(VT: EltVT, Opcode_i16: {/* no v8i16 */}, Opcode_i32: NVPTX::STV_i32_v8,
1822 Opcode_i64: {/* no v8i64 */});
1823 break;
1824 }
1825
1826 if (!Opcode)
1827 return false;
1828
1829 SDNode *NVPTXST = CurDAG->getMachineNode(Opcode: *Opcode, dl: DL, VT: MVT::Other, Ops);
1830
1831 MachineMemOperand *MemRef = ST->getMemOperand();
1832 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: NVPTXST), NewMemRefs: {MemRef});
1833
1834 ReplaceNode(F: ST, T: NVPTXST);
1835 return true;
1836}
1837
1838/// SelectBFE - Look for instruction sequences that can be made more efficient
1839/// by using the 'bfe' (bit-field extract) PTX instruction
1840bool NVPTXDAGToDAGISel::tryBFE(SDNode *N) {
1841 // BFE is not a native instruction starting with sm_70. Let ptxas see the
1842 // original operations so it can optimize them for the target architecture.
1843 if (Subtarget->hasFeature(Feature: NVPTX::SM70))
1844 return false;
1845
1846 SDLoc DL(N);
1847 SDValue LHS = N->getOperand(Num: 0);
1848 SDValue RHS = N->getOperand(Num: 1);
1849 SDValue Len;
1850 SDValue Start;
1851 SDValue Val;
1852 bool IsSigned = false;
1853
1854 if (N->getOpcode() == ISD::AND) {
1855 // Canonicalize the operands
1856 // We want 'and %val, %mask'
1857 if (isa<ConstantSDNode>(Val: LHS) && !isa<ConstantSDNode>(Val: RHS)) {
1858 std::swap(a&: LHS, b&: RHS);
1859 }
1860
1861 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(Val&: RHS);
1862 if (!Mask) {
1863 // We need a constant mask on the RHS of the AND
1864 return false;
1865 }
1866
1867 // Extract the mask bits
1868 uint64_t MaskVal = Mask->getZExtValue();
1869 if (!isMask_64(Value: MaskVal)) {
1870 // We *could* handle shifted masks here, but doing so would require an
1871 // 'and' operation to fix up the low-order bits so we would trade
1872 // shr+and for bfe+and, which has the same throughput
1873 return false;
1874 }
1875
1876 // How many bits are in our mask?
1877 int64_t NumBits = countr_one(Value: MaskVal);
1878 Len = CurDAG->getTargetConstant(Val: NumBits, DL, VT: MVT::i32);
1879
1880 if (LHS.getOpcode() == ISD::SRL || LHS.getOpcode() == ISD::SRA) {
1881 // We have a 'srl/and' pair, extract the effective start bit and length
1882 Val = LHS.getNode()->getOperand(Num: 0);
1883 Start = LHS.getNode()->getOperand(Num: 1);
1884 ConstantSDNode *StartConst = dyn_cast<ConstantSDNode>(Val&: Start);
1885 if (StartConst) {
1886 uint64_t StartVal = StartConst->getZExtValue();
1887 // How many "good" bits do we have left? "good" is defined here as bits
1888 // that exist in the original value, not shifted in.
1889 int64_t GoodBits = Val.getValueSizeInBits() - StartVal;
1890 if (NumBits > GoodBits) {
1891 // Do not handle the case where bits have been shifted in. In theory
1892 // we could handle this, but the cost is likely higher than just
1893 // emitting the srl/and pair.
1894 return false;
1895 }
1896 Start = CurDAG->getTargetConstant(Val: StartVal, DL, VT: MVT::i32);
1897 } else {
1898 // Do not handle the case where the shift amount (can be zero if no srl
1899 // was found) is not constant. We could handle this case, but it would
1900 // require run-time logic that would be more expensive than just
1901 // emitting the srl/and pair.
1902 return false;
1903 }
1904 } else {
1905 // Do not handle the case where the LHS of the and is not a shift. While
1906 // it would be trivial to handle this case, it would just transform
1907 // 'and' -> 'bfe', but 'and' has higher-throughput.
1908 return false;
1909 }
1910 } else if (N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) {
1911 if (LHS->getOpcode() == ISD::AND) {
1912 ConstantSDNode *ShiftCnst = dyn_cast<ConstantSDNode>(Val&: RHS);
1913 if (!ShiftCnst) {
1914 // Shift amount must be constant
1915 return false;
1916 }
1917
1918 uint64_t ShiftAmt = ShiftCnst->getZExtValue();
1919
1920 SDValue AndLHS = LHS->getOperand(Num: 0);
1921 SDValue AndRHS = LHS->getOperand(Num: 1);
1922
1923 // Canonicalize the AND to have the mask on the RHS
1924 if (isa<ConstantSDNode>(Val: AndLHS)) {
1925 std::swap(a&: AndLHS, b&: AndRHS);
1926 }
1927
1928 ConstantSDNode *MaskCnst = dyn_cast<ConstantSDNode>(Val&: AndRHS);
1929 if (!MaskCnst) {
1930 // Mask must be constant
1931 return false;
1932 }
1933
1934 uint64_t MaskVal = MaskCnst->getZExtValue();
1935 uint64_t NumZeros;
1936 uint64_t NumBits;
1937 if (isMask_64(Value: MaskVal)) {
1938 NumZeros = 0;
1939 // The number of bits in the result bitfield will be the number of
1940 // trailing ones (the AND) minus the number of bits we shift off
1941 NumBits = llvm::countr_one(Value: MaskVal) - ShiftAmt;
1942 } else if (isShiftedMask_64(Value: MaskVal)) {
1943 NumZeros = llvm::countr_zero(Val: MaskVal);
1944 unsigned NumOnes = llvm::countr_one(Value: MaskVal >> NumZeros);
1945 // The number of bits in the result bitfield will be the number of
1946 // trailing zeros plus the number of set bits in the mask minus the
1947 // number of bits we shift off
1948 NumBits = NumZeros + NumOnes - ShiftAmt;
1949 } else {
1950 // This is not a mask we can handle
1951 return false;
1952 }
1953
1954 if (ShiftAmt < NumZeros) {
1955 // Handling this case would require extra logic that would make this
1956 // transformation non-profitable
1957 return false;
1958 }
1959
1960 Val = AndLHS;
1961 Start = CurDAG->getTargetConstant(Val: ShiftAmt, DL, VT: MVT::i32);
1962 Len = CurDAG->getTargetConstant(Val: NumBits, DL, VT: MVT::i32);
1963
1964 // If pre-shift AND includes the sign bit in the bitfield, we must use
1965 // signed BFE to replicate that bit during bitfield extraction. If the
1966 // sign bit is not part of the mask, unsigned BFE will zero out upper bits
1967 // of the result
1968 if (N->getOpcode() == ISD::SRA)
1969 IsSigned = (ShiftAmt + NumBits) == Val.getValueSizeInBits();
1970 } else if (LHS->getOpcode() == ISD::SHL) {
1971 // Here, we have a pattern like:
1972 //
1973 // (sra (shl val, NN), MM)
1974 // or
1975 // (srl (shl val, NN), MM)
1976 //
1977 // If MM >= NN, we can efficiently optimize this with bfe
1978 Val = LHS->getOperand(Num: 0);
1979
1980 SDValue ShlRHS = LHS->getOperand(Num: 1);
1981 ConstantSDNode *ShlCnst = dyn_cast<ConstantSDNode>(Val&: ShlRHS);
1982 if (!ShlCnst) {
1983 // Shift amount must be constant
1984 return false;
1985 }
1986 uint64_t InnerShiftAmt = ShlCnst->getZExtValue();
1987
1988 SDValue ShrRHS = RHS;
1989 ConstantSDNode *ShrCnst = dyn_cast<ConstantSDNode>(Val&: ShrRHS);
1990 if (!ShrCnst) {
1991 // Shift amount must be constant
1992 return false;
1993 }
1994 uint64_t OuterShiftAmt = ShrCnst->getZExtValue();
1995
1996 // To avoid extra codegen and be profitable, we need Outer >= Inner
1997 if (OuterShiftAmt < InnerShiftAmt) {
1998 return false;
1999 }
2000
2001 // If the outer shift is more than the type size, we have no bitfield to
2002 // extract (since we also check that the inner shift is <= the outer shift
2003 // then this also implies that the inner shift is < the type size)
2004 if (OuterShiftAmt >= Val.getValueSizeInBits()) {
2005 return false;
2006 }
2007
2008 Start = CurDAG->getTargetConstant(Val: OuterShiftAmt - InnerShiftAmt, DL,
2009 VT: MVT::i32);
2010 Len = CurDAG->getTargetConstant(Val: Val.getValueSizeInBits() - OuterShiftAmt,
2011 DL, VT: MVT::i32);
2012
2013 if (N->getOpcode() == ISD::SRA) {
2014 // If we have a arithmetic right shift, we need to use the signed bfe
2015 // variant
2016 IsSigned = true;
2017 }
2018 } else {
2019 // No can do...
2020 return false;
2021 }
2022 } else {
2023 // No can do...
2024 return false;
2025 }
2026 // Only 32-bit BFE has a native SASS implementation.
2027 if (Val.getValueType() != MVT::i32)
2028 return false;
2029
2030 unsigned Opc = IsSigned ? NVPTX::BFE_S32 : NVPTX::BFE_U32;
2031
2032 SDValue Ops[] = {selectPossiblyImm(V: Val), Start, Len};
2033
2034 ReplaceNode(F: N, T: CurDAG->getMachineNode(Opcode: Opc, dl: DL, VTs: N->getVTList(), Ops));
2035 return true;
2036}
2037
2038// Select bf16/bf16v2 FADD, FSUB, FMUL as fma on targets with only fma
2039bool NVPTXDAGToDAGISel::tryBF16ArithToFMA(SDNode *N) {
2040 EVT VT = SDValue(N, 0).getValueType();
2041 if (VT.getScalarType() != MVT::bf16)
2042 return false;
2043
2044 const NVPTXSubtarget *STI = TM.getSubtargetImpl();
2045 if (STI->hasNativeBF16Support(Opcode: N->getOpcode()))
2046 return false;
2047
2048 const bool IsVec = VT.isVector();
2049 assert(!IsVec || VT.getVectorNumElements() == 2);
2050 SDLoc DL(N);
2051 SDValue N0 = N->getOperand(Num: 0);
2052 SDValue N1 = N->getOperand(Num: 1);
2053 SmallVector<SDValue, 3> Operands;
2054 auto GetConstant = [&](float Value) -> SDValue {
2055 // BF16 immediates must be legalized to integer register values
2056 APFloat APF(Value);
2057 bool LosesInfo;
2058 APF.convert(ToSemantics: APFloat::BFloat(), RM: APFloat::rmNearestTiesToEven, losesInfo: &LosesInfo);
2059 assert(!LosesInfo);
2060 if (IsVec) {
2061 auto API = APF.bitcastToAPInt();
2062 API = API.concat(NewLSB: API);
2063 auto Const = CurDAG->getTargetConstant(Val: API, DL, VT: MVT::i32);
2064 return SDValue(CurDAG->getMachineNode(Opcode: NVPTX::MOV_B32_i, dl: DL, VT, Op1: Const),
2065 0);
2066 }
2067 auto Const = CurDAG->getTargetConstantFP(Val: APF, DL, VT);
2068 return SDValue(CurDAG->getMachineNode(Opcode: NVPTX::MOV_BF16_i, dl: DL, VT, Op1: Const), 0);
2069 };
2070
2071 switch (N->getOpcode()) {
2072 case ISD::FADD:
2073 // add(a, b) -> fma(a, 1.0, b)
2074 Operands = {N0, GetConstant(1.0), N1};
2075 break;
2076 case ISD::FSUB:
2077 // sub(a, b) -> fma(b, -1.0, a)
2078 Operands = {N1, GetConstant(-1.0), N0};
2079 break;
2080 case ISD::FMUL:
2081 // mul(a, b) -> fma(a, b, -0.0)
2082 // NOTE: The identity is -0, not 0, because -0 + 0 == 0 for floats
2083 Operands = {N0, N1, GetConstant(-0.0)};
2084 break;
2085 default:
2086 llvm_unreachable("Unexpected opcode");
2087 };
2088
2089 int Opcode = IsVec ? NVPTX::FMA_BF16x2 : NVPTX::FMA_BF16;
2090 MachineSDNode *FMA = CurDAG->getMachineNode(Opcode, dl: DL, VT, Ops: Operands);
2091 ReplaceNode(F: N, T: FMA);
2092 return true;
2093}
2094
2095// The min/max .abs modifier also accepts operands already known to have no
2096// negative values (not even -0). NaN signs are immaterial to these
2097// instructions.
2098bool NVPTXDAGToDAGISel::SelectFAbs(SDValue N, SDValue &Src) {
2099 if (N.getOpcode() == ISD::FABS)
2100 Src = N.getOperand(i: 0);
2101 else if (CurDAG->computeKnownFPClass(Op: N, InterestedClasses: fcNegative).signBitIsZeroOrNaN())
2102 Src = N;
2103 else
2104 return false;
2105 Src = selectPossiblyImm(V: Src);
2106 return true;
2107}
2108
2109SDValue NVPTXDAGToDAGISel::selectPossiblyImm(SDValue V) {
2110 if (V.getOpcode() == ISD::BITCAST)
2111 V = V.getOperand(i: 0);
2112
2113 if (auto *CN = dyn_cast<ConstantSDNode>(Val&: V))
2114 return CurDAG->getTargetConstant(Val: CN->getAPIntValue(), DL: SDLoc(V),
2115 VT: V.getValueType());
2116 if (auto *CN = dyn_cast<ConstantFPSDNode>(Val&: V))
2117 return CurDAG->getTargetConstantFP(Val: CN->getValueAPF(), DL: SDLoc(V),
2118 VT: V.getValueType());
2119 return V;
2120}
2121
2122/// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
2123/// inline asm expressions.
2124bool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand(
2125 const SDValue &Op, InlineAsm::ConstraintCode ConstraintID,
2126 std::vector<SDValue> &OutOps) {
2127 switch (ConstraintID) {
2128 default:
2129 return true;
2130 case InlineAsm::ConstraintCode::m: { // memory
2131 const auto [Base, Offset] = selectADDR(Addr: Op, DAG: CurDAG);
2132 OutOps.push_back(x: Base);
2133 OutOps.push_back(x: Offset);
2134 return false;
2135 }
2136 }
2137 return true;
2138}
2139
2140void NVPTXDAGToDAGISel::SelectV2I64toI128(SDNode *N) {
2141 // Lower a CopyToReg with two 64-bit inputs
2142 // Dst:i128, lo:i64, hi:i64
2143 //
2144 // CopyToReg Dst, lo, hi;
2145 //
2146 // ==>
2147 //
2148 // tmp = V2I64toI128 {lo, hi};
2149 // CopyToReg Dst, tmp;
2150 SDValue Dst = N->getOperand(Num: 1);
2151 SDValue Lo = selectPossiblyImm(V: N->getOperand(Num: 2));
2152 SDValue Hi = selectPossiblyImm(V: N->getOperand(Num: 3));
2153
2154 SDLoc DL(N);
2155 SDNode *Mov =
2156 CurDAG->getMachineNode(Opcode: NVPTX::V2I64toI128, dl: DL, VT: MVT::i128, Ops: {Lo, Hi});
2157
2158 SmallVector<SDValue, 4> NewOps(N->getNumOperands() - 1);
2159 NewOps[0] = N->getOperand(Num: 0);
2160 NewOps[1] = Dst;
2161 NewOps[2] = SDValue(Mov, 0);
2162 if (N->getNumOperands() == 5)
2163 NewOps[3] = N->getOperand(Num: 4);
2164 SDValue NewValue = CurDAG->getNode(Opcode: ISD::CopyToReg, DL, ResultTys: SmallVector<EVT>(N->values()), Ops: NewOps);
2165
2166 ReplaceNode(F: N, T: NewValue.getNode());
2167}
2168
2169void NVPTXDAGToDAGISel::SelectI128toV2I64(SDNode *N) {
2170 // Lower CopyFromReg from a 128-bit regs to two 64-bit regs
2171 // Dst:i128, Src:i128
2172 //
2173 // {lo, hi} = CopyFromReg Src
2174 //
2175 // ==>
2176 //
2177 // {lo, hi} = I128toV2I64 Src
2178 //
2179 SDValue Ch = N->getOperand(Num: 0);
2180 SDValue Src = N->getOperand(Num: 1);
2181 SDValue Glue = N->getOperand(Num: 2);
2182 SDLoc DL(N);
2183
2184 // Add Glue and Ch to the operands and results to avoid break the execution
2185 // order
2186 SDNode *Mov = CurDAG->getMachineNode(
2187 Opcode: NVPTX::I128toV2I64, dl: DL,
2188 ResultTys: {MVT::i64, MVT::i64, Ch.getValueType(), Glue.getValueType()},
2189 Ops: {Src, Ch, Glue});
2190
2191 ReplaceNode(F: N, T: Mov);
2192}
2193
2194bool NVPTXDAGToDAGISel::tryFence(SDNode *N) {
2195 SDLoc DL(N);
2196 assert(N->getOpcode() == ISD::ATOMIC_FENCE);
2197 auto Scope = Scopes[N->getConstantOperandVal(Num: 2)];
2198
2199 // Singlethread fences have no inter-thread synchronization requirements.
2200 // Note: std::atomic_signal_fence lowers to singlethread LLVM IR fences;
2201 // this intentionally drops these before emitting PTX.
2202 if (Scope == NVPTX::Scope::Thread) {
2203 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: N->getOperand(Num: 0));
2204 CurDAG->RemoveDeadNode(N);
2205 return true;
2206 }
2207
2208 unsigned int FenceOp = getFenceOp(
2209 O: NVPTX::Ordering(N->getConstantOperandVal(Num: 1)), S: Scope, T: Subtarget);
2210 SDValue Chain = N->getOperand(Num: 0);
2211 SDNode *FenceNode = CurDAG->getMachineNode(Opcode: FenceOp, dl: DL, VT: MVT::Other, Op1: Chain);
2212 ReplaceNode(F: N, T: FenceNode);
2213 return true;
2214}
2215
2216NVPTXScopes::NVPTXScopes(LLVMContext &C, const Triple &T) : Context(&C) {
2217 auto ScopeID = [&](AtomicScope Scope) {
2218 return C.getOrInsertSyncScopeID(SSN: *getAtomicScopeIRString(T, S: Scope));
2219 };
2220 Scopes[ScopeID(AtomicScope::Single)] = NVPTX::Scope::Thread;
2221 Scopes[ScopeID(AtomicScope::System)] = NVPTX::Scope::System;
2222 Scopes[ScopeID(AtomicScope::Workgroup)] = NVPTX::Scope::Block;
2223 Scopes[ScopeID(AtomicScope::Cluster)] = NVPTX::Scope::Cluster;
2224 Scopes[ScopeID(AtomicScope::Device)] = NVPTX::Scope::Device;
2225}
2226
2227NVPTX::Scope NVPTXScopes::operator[](SyncScope::ID ID) const {
2228 if (Scopes.empty())
2229 llvm_unreachable("NVPTX Scopes must be initialized before calling "
2230 "NVPTXScopes::operator[]");
2231
2232 auto S = Scopes.find(Key: ID);
2233 if (S == Scopes.end()) {
2234 auto scopeName = Context->getSyncScopeName(Id: ID);
2235 assert(scopeName.has_value() && "Scope name must exist.");
2236
2237 // Build list of supported syncscopes programmatically
2238 SmallVector<StringRef> supportedScopes;
2239 for (const auto &Entry : Scopes) {
2240 if (auto name = Context->getSyncScopeName(Id: Entry.first))
2241 supportedScopes.push_back(Elt: name->empty() ? "<empty string>" : *name);
2242 }
2243
2244 reportFatalUsageError(
2245 reason: formatv(Fmt: "NVPTX backend does not support syncscope \"{0}\" (ID={1}).\n"
2246 "Supported syncscopes are: {2}.",
2247 Vals&: scopeName.value(), Vals: int(ID),
2248 Vals: make_range(x: supportedScopes.begin(), y: supportedScopes.end())));
2249 }
2250 return S->second;
2251}
2252
2253bool NVPTXScopes::empty() const { return Scopes.size() == 0; }
2254
2255#define TCGEN05_ST_OPCODE(SHAPE, NUM) \
2256 (enableUnpack ? NVPTX::TCGEN05_ST_##SHAPE##_##NUM##_UNPACK \
2257 : NVPTX::TCGEN05_ST_##SHAPE##_##NUM)
2258
2259static unsigned getTcgen05StOpcode(unsigned IID, bool enableUnpack) {
2260 switch (IID) {
2261 case Intrinsic::nvvm_tcgen05_st_16x64b_x1:
2262 return TCGEN05_ST_OPCODE(16x64b, x1);
2263 case Intrinsic::nvvm_tcgen05_st_16x64b_x2:
2264 return TCGEN05_ST_OPCODE(16x64b, x2);
2265 case Intrinsic::nvvm_tcgen05_st_16x64b_x4:
2266 return TCGEN05_ST_OPCODE(16x64b, x4);
2267 case Intrinsic::nvvm_tcgen05_st_16x64b_x8:
2268 return TCGEN05_ST_OPCODE(16x64b, x8);
2269 case Intrinsic::nvvm_tcgen05_st_16x64b_x16:
2270 return TCGEN05_ST_OPCODE(16x64b, x16);
2271 case Intrinsic::nvvm_tcgen05_st_16x64b_x32:
2272 return TCGEN05_ST_OPCODE(16x64b, x32);
2273 case Intrinsic::nvvm_tcgen05_st_16x64b_x64:
2274 return TCGEN05_ST_OPCODE(16x64b, x64);
2275 case Intrinsic::nvvm_tcgen05_st_16x64b_x128:
2276 return TCGEN05_ST_OPCODE(16x64b, x128);
2277 case Intrinsic::nvvm_tcgen05_st_16x128b_x1:
2278 return TCGEN05_ST_OPCODE(16x128b, x1);
2279 case Intrinsic::nvvm_tcgen05_st_16x128b_x2:
2280 return TCGEN05_ST_OPCODE(16x128b, x2);
2281 case Intrinsic::nvvm_tcgen05_st_16x128b_x4:
2282 return TCGEN05_ST_OPCODE(16x128b, x4);
2283 case Intrinsic::nvvm_tcgen05_st_16x128b_x8:
2284 return TCGEN05_ST_OPCODE(16x128b, x8);
2285 case Intrinsic::nvvm_tcgen05_st_16x128b_x16:
2286 return TCGEN05_ST_OPCODE(16x128b, x16);
2287 case Intrinsic::nvvm_tcgen05_st_16x128b_x32:
2288 return TCGEN05_ST_OPCODE(16x128b, x32);
2289 case Intrinsic::nvvm_tcgen05_st_16x128b_x64:
2290 return TCGEN05_ST_OPCODE(16x128b, x64);
2291 case Intrinsic::nvvm_tcgen05_st_16x256b_x1:
2292 return TCGEN05_ST_OPCODE(16x256b, x1);
2293 case Intrinsic::nvvm_tcgen05_st_16x256b_x2:
2294 return TCGEN05_ST_OPCODE(16x256b, x2);
2295 case Intrinsic::nvvm_tcgen05_st_16x256b_x4:
2296 return TCGEN05_ST_OPCODE(16x256b, x4);
2297 case Intrinsic::nvvm_tcgen05_st_16x256b_x8:
2298 return TCGEN05_ST_OPCODE(16x256b, x8);
2299 case Intrinsic::nvvm_tcgen05_st_16x256b_x16:
2300 return TCGEN05_ST_OPCODE(16x256b, x16);
2301 case Intrinsic::nvvm_tcgen05_st_16x256b_x32:
2302 return TCGEN05_ST_OPCODE(16x256b, x32);
2303 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x1:
2304 return TCGEN05_ST_OPCODE(16x32bx2, x1);
2305 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x2:
2306 return TCGEN05_ST_OPCODE(16x32bx2, x2);
2307 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x4:
2308 return TCGEN05_ST_OPCODE(16x32bx2, x4);
2309 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x8:
2310 return TCGEN05_ST_OPCODE(16x32bx2, x8);
2311 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x16:
2312 return TCGEN05_ST_OPCODE(16x32bx2, x16);
2313 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x32:
2314 return TCGEN05_ST_OPCODE(16x32bx2, x32);
2315 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x64:
2316 return TCGEN05_ST_OPCODE(16x32bx2, x64);
2317 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x128:
2318 return TCGEN05_ST_OPCODE(16x32bx2, x128);
2319 case Intrinsic::nvvm_tcgen05_st_32x32b_x1:
2320 return TCGEN05_ST_OPCODE(32x32b, x1);
2321 case Intrinsic::nvvm_tcgen05_st_32x32b_x2:
2322 return TCGEN05_ST_OPCODE(32x32b, x2);
2323 case Intrinsic::nvvm_tcgen05_st_32x32b_x4:
2324 return TCGEN05_ST_OPCODE(32x32b, x4);
2325 case Intrinsic::nvvm_tcgen05_st_32x32b_x8:
2326 return TCGEN05_ST_OPCODE(32x32b, x8);
2327 case Intrinsic::nvvm_tcgen05_st_32x32b_x16:
2328 return TCGEN05_ST_OPCODE(32x32b, x16);
2329 case Intrinsic::nvvm_tcgen05_st_32x32b_x32:
2330 return TCGEN05_ST_OPCODE(32x32b, x32);
2331 case Intrinsic::nvvm_tcgen05_st_32x32b_x64:
2332 return TCGEN05_ST_OPCODE(32x32b, x64);
2333 case Intrinsic::nvvm_tcgen05_st_32x32b_x128:
2334 return TCGEN05_ST_OPCODE(32x32b, x128);
2335 }
2336 llvm_unreachable("unhandled tcgen05.st lowering");
2337}
2338
2339void NVPTXDAGToDAGISel::SelectTcgen05St(SDNode *N, bool hasOffset) {
2340 if (!Subtarget->hasTcgen05InstSupport())
2341 report_fatal_error(
2342 reason: "tcgen05.st is not supported on this architecture variant");
2343
2344 SDLoc DL(N);
2345 unsigned IID = cast<ConstantSDNode>(Val: N->getOperand(Num: 1))->getZExtValue();
2346
2347 SmallVector<SDValue, 128> Operands = {
2348 N->getOperand(Num: 2) // taddr
2349 };
2350
2351 if (hasOffset)
2352 Operands.push_back(Elt: CurDAG->getTargetConstant(
2353 Val: cast<ConstantSDNode>(Val: N->getOperand(Num: 3))->getZExtValue(), DL,
2354 VT: MVT::i32)); // Offset
2355
2356 for (unsigned I = hasOffset ? 4 : 3; I < (N->getNumOperands() - 1); I++)
2357 Operands.push_back(Elt: N->getOperand(Num: I));
2358
2359 bool enableUnpack =
2360 cast<ConstantSDNode>(Val: N->getOperand(Num: N->getNumOperands() - 1))
2361 ->getZExtValue();
2362
2363 Operands.push_back(Elt: N->getOperand(Num: 0)); // Chain
2364 ReplaceNode(F: N, T: CurDAG->getMachineNode(Opcode: getTcgen05StOpcode(IID, enableUnpack),
2365 dl: DL, VTs: N->getVTList(), Ops: Operands));
2366}
2367
2368bool NVPTXDAGToDAGISel::tryIntrinsicVoid(SDNode *N) {
2369 unsigned IID = N->getConstantOperandVal(Num: 1);
2370 switch (IID) {
2371 default:
2372 return false;
2373 case Intrinsic::nvvm_tcgen05_st_16x64b_x1:
2374 case Intrinsic::nvvm_tcgen05_st_16x64b_x2:
2375 case Intrinsic::nvvm_tcgen05_st_16x64b_x4:
2376 case Intrinsic::nvvm_tcgen05_st_16x64b_x8:
2377 case Intrinsic::nvvm_tcgen05_st_16x64b_x16:
2378 case Intrinsic::nvvm_tcgen05_st_16x64b_x32:
2379 case Intrinsic::nvvm_tcgen05_st_16x64b_x64:
2380 case Intrinsic::nvvm_tcgen05_st_16x64b_x128:
2381 case Intrinsic::nvvm_tcgen05_st_32x32b_x1:
2382 case Intrinsic::nvvm_tcgen05_st_32x32b_x2:
2383 case Intrinsic::nvvm_tcgen05_st_32x32b_x4:
2384 case Intrinsic::nvvm_tcgen05_st_32x32b_x8:
2385 case Intrinsic::nvvm_tcgen05_st_32x32b_x16:
2386 case Intrinsic::nvvm_tcgen05_st_32x32b_x32:
2387 case Intrinsic::nvvm_tcgen05_st_32x32b_x64:
2388 case Intrinsic::nvvm_tcgen05_st_32x32b_x128:
2389 case Intrinsic::nvvm_tcgen05_st_16x128b_x1:
2390 case Intrinsic::nvvm_tcgen05_st_16x128b_x2:
2391 case Intrinsic::nvvm_tcgen05_st_16x128b_x4:
2392 case Intrinsic::nvvm_tcgen05_st_16x128b_x8:
2393 case Intrinsic::nvvm_tcgen05_st_16x128b_x16:
2394 case Intrinsic::nvvm_tcgen05_st_16x128b_x32:
2395 case Intrinsic::nvvm_tcgen05_st_16x128b_x64:
2396 case Intrinsic::nvvm_tcgen05_st_16x256b_x1:
2397 case Intrinsic::nvvm_tcgen05_st_16x256b_x2:
2398 case Intrinsic::nvvm_tcgen05_st_16x256b_x4:
2399 case Intrinsic::nvvm_tcgen05_st_16x256b_x8:
2400 case Intrinsic::nvvm_tcgen05_st_16x256b_x16:
2401 case Intrinsic::nvvm_tcgen05_st_16x256b_x32: {
2402 SelectTcgen05St(N);
2403 return true;
2404 }
2405
2406 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x1:
2407 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x2:
2408 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x4:
2409 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x8:
2410 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x16:
2411 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x32:
2412 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x64:
2413 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x128: {
2414 SelectTcgen05St(N, /* hasOffset */ true);
2415 return true;
2416 }
2417 }
2418}
2419
2420void NVPTXDAGToDAGISel::selectAtomicSwap128(SDNode *N) {
2421 MemSDNode *AN = cast<MemSDNode>(Val: N);
2422 SDLoc dl(N);
2423
2424 const SDValue Chain = N->getOperand(Num: 0);
2425 const auto [Base, Offset] = selectADDR(Addr: N->getOperand(Num: 1), DAG: CurDAG);
2426 SmallVector<SDValue, 10> Ops{Base, Offset};
2427 for (const SDValue &Operand : drop_begin(RangeOrContainer: N->op_values(), N: 2))
2428 Ops.push_back(Elt: selectPossiblyImm(V: Operand));
2429 Ops.append(IL: {getI32Imm(Imm: getMemOrder(N: AN), DL: dl), getI32Imm(Imm: getAtomicScope(N: AN), DL: dl),
2430 getI32Imm(Imm: getAddrSpace(N: AN), DL: dl)});
2431
2432 if (N->getOpcode() == NVPTXISD::ATOMIC_SWAP_B128) {
2433 unsigned EltWidth = AN->getMemoryVT().getFixedSizeInBits();
2434 NVPTXMemCacheHintAccess Access{.Instruction: NVPTXMemCacheHintInstruction::Atom,
2435 .AddrSpace: getAddrSpace(N: AN),
2436 /*NumElts=*/1, .EltWidth: EltWidth, .IsVolatile: AN->isVolatile()};
2437 const auto [EvictionAndPrefetchHint, CachePolicyReg] =
2438 getMemCacheHintOperands(N: AN, Access, DL: dl);
2439 Ops.push_back(Elt: EvictionAndPrefetchHint);
2440 Ops.push_back(Elt: CachePolicyReg);
2441 }
2442
2443 Ops.push_back(Elt: Chain);
2444
2445 assert(N->getOpcode() == NVPTXISD::ATOMIC_CMP_SWAP_B128 ||
2446 N->getOpcode() == NVPTXISD::ATOMIC_SWAP_B128);
2447 unsigned Opcode = N->getOpcode() == NVPTXISD::ATOMIC_SWAP_B128
2448 ? NVPTX::ATOM_EXCH_B128
2449 : NVPTX::ATOM_CAS_B128;
2450
2451 auto *ATOM = CurDAG->getMachineNode(Opcode, dl, VTs: N->getVTList(), Ops);
2452 CurDAG->setNodeMemRefs(N: ATOM, NewMemRefs: AN->getMemOperand());
2453
2454 ReplaceNode(F: N, T: ATOM);
2455}
2456