1//===-- NVPTX.h - Top-level interface for NVPTX representation --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the entry points for global functions defined in
10// the LLVM NVPTX back-end.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIB_TARGET_NVPTX_NVPTX_H
15#define LLVM_LIB_TARGET_NVPTX_NVPTX_H
16
17#include "llvm/ADT/Bitfields.h"
18#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
19#include "llvm/CodeGen/SelectionDAGISel.h"
20#include "llvm/IR/PassManager.h"
21#include "llvm/Pass.h"
22#include "llvm/Support/AtomicOrdering.h"
23#include "llvm/Support/CodeGen.h"
24#include "llvm/Support/NVPTXAddrSpace.h"
25#include "llvm/Target/TargetMachine.h"
26
27namespace llvm {
28class SelectionDAGISelPass;
29class FunctionPass;
30class MachineFunctionPass;
31class NVPTXTargetMachine;
32class PassRegistry;
33
34namespace NVPTXCC {
35enum CondCodes {
36 EQ,
37 NE,
38 LT,
39 LE,
40 GT,
41 GE
42};
43}
44
45FunctionPass *createNVPTXISelDag(NVPTXTargetMachine &TM,
46 llvm::CodeGenOptLevel OptLevel);
47ModulePass *createNVPTXAssignValidGlobalNamesLegacyPass();
48ModulePass *createGenericToNVVMLegacyPass();
49ModulePass *createNVPTXCtorDtorLoweringLegacyPass();
50FunctionPass *createNVPTXAtomicLowerLegacyPass();
51FunctionPass *createNVVMIntrRangePass();
52ModulePass *createNVVMReflectPass(unsigned int SmVersion);
53MachineFunctionPass *createNVPTXPrologEpilogLegacyPass();
54MachineFunctionPass *createNVPTXReplaceImageHandlesLegacyPass();
55FunctionPass *createNVPTXImageOptimizerLegacyPass();
56ModulePass *createNVPTXLowerArgsPass();
57ModulePass *createNVPTXPromoteParamAlignPass();
58FunctionPass *createNVPTXAllocaHoistingLegacyPass();
59FunctionPass *createNVPTXLowerAllocaLegacyPass();
60FunctionPass *createNVPTXLowerUnreachableLegacyPass(bool TrapUnreachable,
61 bool NoTrapAfterNoreturn);
62FunctionPass *createNVPTXLowerAggrCopiesLegacyPass();
63FunctionPass *createNVPTXMarkKernelPtrsGlobalPass();
64FunctionPass *createNVPTXTagInvariantLoadsPass();
65FunctionPass *createNVPTXIRPeepholePass();
66MachineFunctionPass *createNVPTXPeepholeLegacyPass();
67MachineFunctionPass *createNVPTXProxyRegErasureLegacyPass();
68MachineFunctionPass *createNVPTXForwardParamsLegacyPass();
69MachineFunctionPass *createNVPTXAddressFolderLegacyPass();
70
71void initializeNVVMReflectLegacyPassPass(PassRegistry &);
72void initializeGenericToNVVMLegacyPassPass(PassRegistry &);
73void initializeNVPTXAllocaHoistingLegacyPassPass(PassRegistry &);
74void initializeNVPTXAsmPrinterPass(PassRegistry &);
75void initializeNVPTXAssignValidGlobalNamesLegacyPassPass(PassRegistry &);
76void initializeNVPTXAtomicLowerLegacyPassPass(PassRegistry &);
77void initializeNVPTXCtorDtorLoweringLegacyPass(PassRegistry &);
78void initializeNVPTXLowerAggrCopiesLegacyPassPass(PassRegistry &);
79void initializeNVPTXLowerAllocaLegacyPassPass(PassRegistry &);
80void initializeNVPTXLowerUnreachableLegacyPassPass(PassRegistry &);
81void initializeNVPTXLowerArgsLegacyPassPass(PassRegistry &);
82void initializeNVPTXPromoteParamAlignLegacyPassPass(PassRegistry &);
83void initializeNVPTXProxyRegErasureLegacyPassPass(PassRegistry &);
84void initializeNVPTXForwardParamsLegacyPassPass(PassRegistry &);
85void initializeNVPTXAddressFolderLegacyPassPass(PassRegistry &);
86void initializeNVVMIntrRangePass(PassRegistry &);
87void initializeNVVMReflectPass(PassRegistry &);
88void initializeNVPTXAAWrapperPassPass(PassRegistry &);
89void initializeNVPTXExternalAAWrapperPass(PassRegistry &);
90void initializeNVPTXPeepholeLegacyPassPass(PassRegistry &);
91void initializeNVPTXMarkKernelPtrsGlobalLegacyPassPass(PassRegistry &);
92void initializeNVPTXTagInvariantLoadLegacyPassPass(PassRegistry &);
93void initializeNVPTXIRPeepholePass(PassRegistry &);
94void initializeNVPTXPrologEpilogLegacyPassPass(PassRegistry &);
95
96// Module passes
97class GenericToNVVMPass : public RequiredPassInfoMixin<GenericToNVVMPass> {
98public:
99 PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
100};
101
102class NVPTXAssignValidGlobalNamesPass
103 : public RequiredPassInfoMixin<NVPTXAssignValidGlobalNamesPass> {
104public:
105 PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
106};
107
108class NVPTXCtorDtorLoweringPass
109 : public RequiredPassInfoMixin<NVPTXCtorDtorLoweringPass> {
110public:
111 PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
112};
113
114class NVPTXLowerArgsPass : public RequiredPassInfoMixin<NVPTXLowerArgsPass> {
115 TargetMachine &TM;
116
117public:
118 NVPTXLowerArgsPass(TargetMachine &TM) : TM(TM) {}
119 PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
120};
121
122class NVPTXPromoteParamAlignPass
123 : public OptionalPassInfoMixin<NVPTXPromoteParamAlignPass> {
124public:
125 PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
126};
127
128class NVVMReflectPass : public RequiredPassInfoMixin<NVVMReflectPass> {
129 unsigned SmVersion;
130
131public:
132 NVVMReflectPass() : SmVersion(0) {}
133 NVVMReflectPass(unsigned SmVersion) : SmVersion(SmVersion) {}
134 PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
135};
136
137// Function passes
138class NVPTXAllocaHoistingPass
139 : public RequiredPassInfoMixin<NVPTXAllocaHoistingPass> {
140public:
141 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
142};
143
144class NVPTXAtomicLowerPass
145 : public RequiredPassInfoMixin<NVPTXAtomicLowerPass> {
146public:
147 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
148};
149
150class NVPTXCopyByValArgsPass
151 : public OptionalPassInfoMixin<NVPTXCopyByValArgsPass> {
152public:
153 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
154};
155
156class NVPTXImageOptimizerPass
157 : public OptionalPassInfoMixin<NVPTXImageOptimizerPass> {
158public:
159 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
160};
161
162class NVPTXIRPeepholePass : public OptionalPassInfoMixin<NVPTXIRPeepholePass> {
163public:
164 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
165};
166
167class NVPTXLowerAggrCopiesPass
168 : public RequiredPassInfoMixin<NVPTXLowerAggrCopiesPass> {
169public:
170 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
171};
172
173class NVPTXLowerAllocaPass
174 : public RequiredPassInfoMixin<NVPTXLowerAllocaPass> {
175public:
176 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
177};
178
179class NVPTXLowerUnreachablePass
180 : public OptionalPassInfoMixin<NVPTXLowerUnreachablePass> {
181 bool TrapUnreachable;
182 bool NoTrapAfterNoreturn;
183
184public:
185 NVPTXLowerUnreachablePass(bool TrapUnreachable, bool NoTrapAfterNoreturn)
186 : TrapUnreachable(TrapUnreachable),
187 NoTrapAfterNoreturn(NoTrapAfterNoreturn) {}
188 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
189};
190
191class NVPTXMarkKernelPtrsGlobalPass
192 : public OptionalPassInfoMixin<NVPTXMarkKernelPtrsGlobalPass> {
193public:
194 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
195};
196
197class NVPTXTagInvariantLoadsPass
198 : public OptionalPassInfoMixin<NVPTXTagInvariantLoadsPass> {
199public:
200 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
201};
202
203class NVVMIntrRangePass : public OptionalPassInfoMixin<NVVMIntrRangePass> {
204public:
205 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
206};
207
208// Machine function passes
209class NVPTXAddressFolderPass
210 : public OptionalPassInfoMixin<NVPTXAddressFolderPass> {
211public:
212 PreservedAnalyses run(MachineFunction &MF,
213 MachineFunctionAnalysisManager &MFAM);
214};
215
216class NVPTXForwardParamsPass
217 : public RequiredPassInfoMixin<NVPTXForwardParamsPass> {
218public:
219 PreservedAnalyses run(MachineFunction &MF,
220 MachineFunctionAnalysisManager &MFAM);
221};
222
223class NVPTXISelDAGToDAGPass : public SelectionDAGISelPass {
224public:
225 NVPTXISelDAGToDAGPass(NVPTXTargetMachine &TM, CodeGenOptLevel OptLevel);
226};
227
228class NVPTXPeepholePass : public OptionalPassInfoMixin<NVPTXPeepholePass> {
229public:
230 PreservedAnalyses run(MachineFunction &MF,
231 MachineFunctionAnalysisManager &MFAM);
232};
233
234class NVPTXPrologEpilogPass
235 : public RequiredPassInfoMixin<NVPTXPrologEpilogPass> {
236public:
237 PreservedAnalyses run(MachineFunction &MF,
238 MachineFunctionAnalysisManager &MFAM);
239};
240
241class NVPTXProxyRegErasurePass
242 : public RequiredPassInfoMixin<NVPTXProxyRegErasurePass> {
243public:
244 PreservedAnalyses run(MachineFunction &MF,
245 MachineFunctionAnalysisManager &MFAM);
246};
247
248class NVPTXReplaceImageHandlesPass
249 : public RequiredPassInfoMixin<NVPTXReplaceImageHandlesPass> {
250public:
251 PreservedAnalyses run(MachineFunction &MF,
252 MachineFunctionAnalysisManager &MFAM);
253};
254
255namespace NVPTX {
256enum DrvInterface {
257 NVCL,
258 CUDA
259};
260
261// A field inside TSFlags needs a shift and a mask. The usage is
262// always as follows :
263// ((TSFlags & fieldMask) >> fieldShift)
264// The enum keeps the mask, the shift, and all valid values of the
265// field in one place.
266enum VecInstType {
267 VecInstTypeShift = 0,
268 VecInstTypeMask = 0xF,
269
270 VecNOP = 0,
271 VecLoad = 1,
272 VecStore = 2,
273 VecBuild = 3,
274 VecShuffle = 4,
275 VecExtract = 5,
276 VecInsert = 6,
277 VecDest = 7,
278 VecOther = 15
279};
280
281enum SimpleMove {
282 SimpleMoveMask = 0x10,
283 SimpleMoveShift = 4
284};
285enum LoadStore {
286 isLoadMask = 0x20,
287 isLoadShift = 5,
288 isStoreMask = 0x40,
289 isStoreShift = 6
290};
291
292// Extends LLVM AtomicOrdering with PTX Orderings:
293using OrderingUnderlyingType = unsigned int;
294enum Ordering : OrderingUnderlyingType {
295 NotAtomic = (OrderingUnderlyingType)
296 AtomicOrdering::NotAtomic, // PTX calls these: "Weak"
297 // Unordered = 1, // NVPTX maps LLVM Unorderd to Relaxed
298 Relaxed = (OrderingUnderlyingType)AtomicOrdering::Monotonic,
299 // Consume = 3, // Unimplemented in LLVM; NVPTX would map to "Acquire"
300 Acquire = (OrderingUnderlyingType)AtomicOrdering::Acquire,
301 Release = (OrderingUnderlyingType)AtomicOrdering::Release,
302 AcquireRelease = (OrderingUnderlyingType)AtomicOrdering::AcquireRelease,
303 SequentiallyConsistent =
304 (OrderingUnderlyingType)AtomicOrdering::SequentiallyConsistent,
305 Volatile = SequentiallyConsistent + 1,
306 RelaxedMMIO = Volatile + 1,
307};
308
309using ScopeUnderlyingType = unsigned int;
310enum Scope : ScopeUnderlyingType {
311 Thread = 0,
312 Block = 1,
313 Cluster = 2,
314 Device = 3,
315 System = 4,
316 DefaultDevice = 5, // For SM < 70: denotes PTX op implicit/default .gpu scope
317 LASTSCOPE = DefaultDevice
318};
319
320using AddressSpaceUnderlyingType = unsigned int;
321enum AddressSpace : AddressSpaceUnderlyingType {
322 Generic = NVPTXAS::ADDRESS_SPACE_GENERIC,
323 Global = NVPTXAS::ADDRESS_SPACE_GLOBAL,
324 Shared = NVPTXAS::ADDRESS_SPACE_SHARED,
325 Const = NVPTXAS::ADDRESS_SPACE_CONST,
326 Local = NVPTXAS::ADDRESS_SPACE_LOCAL,
327 SharedCluster = NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER,
328 EntryParam = NVPTXAS::ADDRESS_SPACE_ENTRY_PARAM,
329
330 // DeviceParam is not a real address space, as it does not support pointers
331 // and instead can only be referenced by param+offset. For this reason it is
332 // only used in MIR as an instruction modifier and should not be used in LLVM
333 // IR.
334 DeviceParam
335};
336
337// Eviction and prefetch hint enums for !mem.cache_hint metadata. These
338// correspond to PTX L1::evict_*, L2::evict_*, and L2::*B qualifiers.
339
340// L1 Eviction Policy - maps to PTX L1::evict_* qualifiers
341enum class L1Eviction : uint8_t {
342 Normal = 0, // Default behavior (no qualifier)
343 Unchanged = 1, // L1::evict_unchanged
344 First = 2, // L1::evict_first
345 Last = 3, // L1::evict_last
346 NoAllocate = 4, // L1::no_allocate
347};
348
349// L2 Eviction Policy - maps to PTX L2::evict_* qualifiers
350enum class L2Eviction : uint8_t {
351 Normal = 0, // Default behavior (no qualifier)
352 First = 1, // L2::evict_first
353 Last = 2, // L2::evict_last
354};
355
356// L2 Prefetch Size - maps to PTX L2::*B qualifiers
357enum class L2Prefetch : uint8_t {
358 None = 0, // No prefetch hint
359 Bytes64 = 1, // L2::64B
360 Bytes128 = 2, // L2::128B
361 Bytes256 = 3, // L2::256B
362};
363
364// Bitfield layout for encoded eviction/prefetch hints (stored in unsigned):
365// Bits 0-2: L1 Eviction (3 bits, 5 values)
366// Bits 3-4: L2 Eviction (2 bits, 3 values)
367// Bits 5-6: L2 Prefetch (2 bits, 4 values)
368// Bit 7: L2::cache_hint mode flag (set when using CachePolicy)
369// Bits 8-31: Reserved
370//
371// Using llvm::Bitfield for type-safe access with compile-time validation.
372using L1EvictionBits =
373 Bitfield::Element<L1Eviction, 0, 3, L1Eviction::NoAllocate>;
374using L2EvictionBits = Bitfield::Element<L2Eviction, 3, 2, L2Eviction::Last>;
375using L2PrefetchBits =
376 Bitfield::Element<L2Prefetch, 5, 2, L2Prefetch::Bytes256>;
377using L2CacheHintBit = Bitfield::Element<bool, 7, 1>;
378
379inline unsigned encodeEvictionAndPrefetchHint(L1Eviction L1, L2Eviction L2,
380 L2Prefetch P) {
381 unsigned Hint = 0;
382 Bitfield::set<L1EvictionBits>(Packed&: Hint, Value: L1);
383 Bitfield::set<L2EvictionBits>(Packed&: Hint, Value: L2);
384 Bitfield::set<L2PrefetchBits>(Packed&: Hint, Value: P);
385 return Hint;
386}
387
388inline L1Eviction decodeL1Eviction(unsigned Hint) {
389 return Bitfield::get<L1EvictionBits>(Packed: Hint);
390}
391
392inline L2Eviction decodeL2Eviction(unsigned Hint) {
393 return Bitfield::get<L2EvictionBits>(Packed: Hint);
394}
395
396inline L2Prefetch decodeL2Prefetch(unsigned Hint) {
397 return Bitfield::get<L2PrefetchBits>(Packed: Hint);
398}
399
400inline bool isL2CacheHintMode(unsigned Hint) {
401 return Bitfield::get<L2CacheHintBit>(Packed: Hint);
402}
403
404namespace PTXLdStInstCode {
405enum FromType { Unsigned = 0, Signed, Float, Untyped };
406} // namespace PTXLdStInstCode
407
408/// PTXCvtMode - Conversion code enumeration
409namespace PTXCvtMode {
410enum CvtMode {
411 NONE = 0,
412 RNI,
413 RZI,
414 RMI,
415 RPI,
416 RN,
417 RZ,
418 RM,
419 RP,
420 RNA,
421 RS,
422
423 BASE_MASK = 0x0F,
424 FTZ_FLAG = 0x10,
425 SAT_FLAG = 0x20,
426 RELU_FLAG = 0x40,
427 SATFINITE_FLAG = 0x80
428};
429}
430
431/// PTXCmpMode - Comparison mode enumeration
432namespace PTXCmpMode {
433enum CmpMode {
434 EQ = 0,
435 NE,
436 LT,
437 LE,
438 GT,
439 GE,
440 EQU,
441 NEU,
442 LTU,
443 LEU,
444 GTU,
445 GEU,
446 NUM,
447 // NAN is a MACRO
448 NotANumber,
449};
450}
451
452namespace PTXPrmtMode {
453enum PrmtMode {
454 NONE,
455 F4E,
456 B4E,
457 RC8,
458 ECL,
459 ECR,
460 RC16,
461};
462}
463
464enum class DivPrecisionLevel : unsigned {
465 Approx = 0,
466 Full = 1,
467 IEEE754 = 2,
468 IEEE754_NoFTZ = 3,
469};
470
471} // namespace NVPTX
472void initializeNVPTXDAGToDAGISelLegacyPass(PassRegistry &);
473} // namespace llvm
474
475// Defines symbolic names for NVPTX registers. This defines a mapping from
476// register name to register number.
477#define GET_REGINFO_ENUM
478#include "NVPTXGenRegisterInfo.inc"
479
480// Defines symbolic names for NVPTX instructions, MC helper declarations,
481// and named operand helpers generated from UseNamedOperandTable=1.
482#define GET_INSTRINFO_ENUM
483#define GET_INSTRINFO_MC_HELPER_DECLS
484#define GET_INSTRINFO_OPERAND_ENUM
485#include "NVPTXGenInstrInfo.inc"
486
487#define GET_SUBTARGETINFO_ENUM
488#include "NVPTXGenSubtargetInfo.inc"
489
490#endif
491