1//===-- NVPTX.h - Top-level interface for NVPTX representation --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the entry points for global functions defined in
10// the LLVM NVPTX back-end.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIB_TARGET_NVPTX_NVPTX_H
15#define LLVM_LIB_TARGET_NVPTX_NVPTX_H
16
17#include "llvm/ADT/Bitfields.h"
18#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
19#include "llvm/CodeGen/SelectionDAGISel.h"
20#include "llvm/IR/PassManager.h"
21#include "llvm/Pass.h"
22#include "llvm/Support/AtomicOrdering.h"
23#include "llvm/Support/CodeGen.h"
24#include "llvm/Support/NVPTXAddrSpace.h"
25#include "llvm/Target/TargetMachine.h"
26
27namespace llvm {
28class SelectionDAGISelPass;
29class FunctionPass;
30class MachineFunctionPass;
31class NVPTXTargetMachine;
32class PassRegistry;
33
34namespace NVPTXCC {
35enum CondCodes {
36 EQ,
37 NE,
38 LT,
39 LE,
40 GT,
41 GE
42};
43}
44
45FunctionPass *createNVPTXISelDag(NVPTXTargetMachine &TM,
46 llvm::CodeGenOptLevel OptLevel);
47ModulePass *createNVPTXAssignValidGlobalNamesLegacyPass();
48ModulePass *createGenericToNVVMLegacyPass();
49ModulePass *createNVPTXCtorDtorLoweringLegacyPass();
50FunctionPass *createNVPTXAtomicLowerLegacyPass();
51FunctionPass *createNVVMIntrRangePass();
52ModulePass *createNVVMReflectPass(unsigned int SmVersion);
53MachineFunctionPass *createNVPTXPrologEpilogLegacyPass();
54MachineFunctionPass *createNVPTXReplaceImageHandlesLegacyPass();
55FunctionPass *createNVPTXImageOptimizerLegacyPass();
56ModulePass *createNVPTXLowerArgsPass();
57ModulePass *createNVPTXPromoteParamAlignPass();
58FunctionPass *createNVPTXAllocaHoistingLegacyPass();
59FunctionPass *createNVPTXLowerAllocaLegacyPass();
60FunctionPass *createNVPTXLowerUnreachableLegacyPass(bool TrapUnreachable,
61 bool NoTrapAfterNoreturn);
62FunctionPass *createNVPTXLowerAggrCopiesLegacyPass();
63FunctionPass *createNVPTXMarkKernelPtrsGlobalPass();
64FunctionPass *createNVPTXTagInvariantLoadsPass();
65FunctionPass *createNVPTXIRPeepholePass();
66MachineFunctionPass *createNVPTXPeepholeLegacyPass();
67MachineFunctionPass *createNVPTXProxyRegErasureLegacyPass();
68MachineFunctionPass *createNVPTXForwardParamsLegacyPass();
69MachineFunctionPass *createNVPTXAddressFolderLegacyPass();
70
71void initializeNVVMReflectLegacyPassPass(PassRegistry &);
72void initializeGenericToNVVMLegacyPassPass(PassRegistry &);
73void initializeNVPTXAllocaHoistingLegacyPassPass(PassRegistry &);
74void initializeNVPTXAsmPrinterPass(PassRegistry &);
75void initializeNVPTXAssignValidGlobalNamesLegacyPassPass(PassRegistry &);
76void initializeNVPTXAtomicLowerLegacyPassPass(PassRegistry &);
77void initializeNVPTXCtorDtorLoweringLegacyPass(PassRegistry &);
78void initializeNVPTXLowerAggrCopiesLegacyPassPass(PassRegistry &);
79void initializeNVPTXLowerAllocaLegacyPassPass(PassRegistry &);
80void initializeNVPTXLowerUnreachableLegacyPassPass(PassRegistry &);
81void initializeNVPTXLowerArgsLegacyPassPass(PassRegistry &);
82void initializeNVPTXPromoteParamAlignLegacyPassPass(PassRegistry &);
83void initializeNVPTXProxyRegErasureLegacyPassPass(PassRegistry &);
84void initializeNVPTXForwardParamsLegacyPassPass(PassRegistry &);
85void initializeNVPTXAddressFolderLegacyPassPass(PassRegistry &);
86void initializeNVVMIntrRangePass(PassRegistry &);
87void initializeNVPTXAAWrapperPassPass(PassRegistry &);
88void initializeNVPTXExternalAAWrapperPass(PassRegistry &);
89void initializeNVPTXPeepholeLegacyPassPass(PassRegistry &);
90void initializeNVPTXMarkKernelPtrsGlobalLegacyPassPass(PassRegistry &);
91void initializeNVPTXTagInvariantLoadLegacyPassPass(PassRegistry &);
92void initializeNVPTXIRPeepholePass(PassRegistry &);
93void initializeNVPTXPrologEpilogLegacyPassPass(PassRegistry &);
94
95// Module passes
96class GenericToNVVMPass : public RequiredPassInfoMixin<GenericToNVVMPass> {
97public:
98 PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
99};
100
101class NVPTXAssignValidGlobalNamesPass
102 : public RequiredPassInfoMixin<NVPTXAssignValidGlobalNamesPass> {
103public:
104 PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
105};
106
107class NVPTXCtorDtorLoweringPass
108 : public RequiredPassInfoMixin<NVPTXCtorDtorLoweringPass> {
109public:
110 PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
111};
112
113class NVPTXLowerArgsPass : public RequiredPassInfoMixin<NVPTXLowerArgsPass> {
114 TargetMachine &TM;
115
116public:
117 NVPTXLowerArgsPass(TargetMachine &TM) : TM(TM) {}
118 PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
119};
120
121class NVPTXPromoteParamAlignPass
122 : public OptionalPassInfoMixin<NVPTXPromoteParamAlignPass> {
123public:
124 PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
125};
126
127class NVVMReflectPass : public RequiredPassInfoMixin<NVVMReflectPass> {
128 unsigned SmVersion;
129
130public:
131 NVVMReflectPass() : SmVersion(0) {}
132 NVVMReflectPass(unsigned SmVersion) : SmVersion(SmVersion) {}
133 PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
134};
135
136// Function passes
137class NVPTXAllocaHoistingPass
138 : public RequiredPassInfoMixin<NVPTXAllocaHoistingPass> {
139public:
140 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
141};
142
143class NVPTXAtomicLowerPass
144 : public RequiredPassInfoMixin<NVPTXAtomicLowerPass> {
145public:
146 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
147};
148
149class NVPTXCopyByValArgsPass
150 : public OptionalPassInfoMixin<NVPTXCopyByValArgsPass> {
151public:
152 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
153};
154
155class NVPTXImageOptimizerPass
156 : public OptionalPassInfoMixin<NVPTXImageOptimizerPass> {
157public:
158 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
159};
160
161class NVPTXIRPeepholePass : public OptionalPassInfoMixin<NVPTXIRPeepholePass> {
162public:
163 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
164};
165
166class NVPTXLowerAggrCopiesPass
167 : public RequiredPassInfoMixin<NVPTXLowerAggrCopiesPass> {
168public:
169 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
170};
171
172class NVPTXLowerAllocaPass
173 : public RequiredPassInfoMixin<NVPTXLowerAllocaPass> {
174public:
175 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
176};
177
178class NVPTXLowerUnreachablePass
179 : public OptionalPassInfoMixin<NVPTXLowerUnreachablePass> {
180 bool TrapUnreachable;
181 bool NoTrapAfterNoreturn;
182
183public:
184 NVPTXLowerUnreachablePass(bool TrapUnreachable, bool NoTrapAfterNoreturn)
185 : TrapUnreachable(TrapUnreachable),
186 NoTrapAfterNoreturn(NoTrapAfterNoreturn) {}
187 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
188};
189
190class NVPTXMarkKernelPtrsGlobalPass
191 : public OptionalPassInfoMixin<NVPTXMarkKernelPtrsGlobalPass> {
192public:
193 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
194};
195
196class NVPTXTagInvariantLoadsPass
197 : public OptionalPassInfoMixin<NVPTXTagInvariantLoadsPass> {
198public:
199 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
200};
201
202class NVVMIntrRangePass : public OptionalPassInfoMixin<NVVMIntrRangePass> {
203public:
204 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
205};
206
207// Machine function passes
208class NVPTXAddressFolderPass
209 : public OptionalPassInfoMixin<NVPTXAddressFolderPass> {
210public:
211 PreservedAnalyses run(MachineFunction &MF,
212 MachineFunctionAnalysisManager &MFAM);
213};
214
215class NVPTXForwardParamsPass
216 : public RequiredPassInfoMixin<NVPTXForwardParamsPass> {
217public:
218 PreservedAnalyses run(MachineFunction &MF,
219 MachineFunctionAnalysisManager &MFAM);
220};
221
222class NVPTXISelDAGToDAGPass : public SelectionDAGISelPass {
223public:
224 NVPTXISelDAGToDAGPass(NVPTXTargetMachine &TM, CodeGenOptLevel OptLevel);
225};
226
227class NVPTXPeepholePass : public OptionalPassInfoMixin<NVPTXPeepholePass> {
228public:
229 PreservedAnalyses run(MachineFunction &MF,
230 MachineFunctionAnalysisManager &MFAM);
231};
232
233class NVPTXPrologEpilogPass
234 : public RequiredPassInfoMixin<NVPTXPrologEpilogPass> {
235public:
236 PreservedAnalyses run(MachineFunction &MF,
237 MachineFunctionAnalysisManager &MFAM);
238};
239
240class NVPTXProxyRegErasurePass
241 : public RequiredPassInfoMixin<NVPTXProxyRegErasurePass> {
242public:
243 PreservedAnalyses run(MachineFunction &MF,
244 MachineFunctionAnalysisManager &MFAM);
245};
246
247class NVPTXReplaceImageHandlesPass
248 : public RequiredPassInfoMixin<NVPTXReplaceImageHandlesPass> {
249public:
250 PreservedAnalyses run(MachineFunction &MF,
251 MachineFunctionAnalysisManager &MFAM);
252};
253
254namespace NVPTX {
255enum DrvInterface {
256 NVCL,
257 CUDA
258};
259
260// A field inside TSFlags needs a shift and a mask. The usage is
261// always as follows :
262// ((TSFlags & fieldMask) >> fieldShift)
263// The enum keeps the mask, the shift, and all valid values of the
264// field in one place.
265enum VecInstType {
266 VecInstTypeShift = 0,
267 VecInstTypeMask = 0xF,
268
269 VecNOP = 0,
270 VecLoad = 1,
271 VecStore = 2,
272 VecBuild = 3,
273 VecShuffle = 4,
274 VecExtract = 5,
275 VecInsert = 6,
276 VecDest = 7,
277 VecOther = 15
278};
279
280enum SimpleMove {
281 SimpleMoveMask = 0x10,
282 SimpleMoveShift = 4
283};
284enum LoadStore {
285 isLoadMask = 0x20,
286 isLoadShift = 5,
287 isStoreMask = 0x40,
288 isStoreShift = 6
289};
290
291// Extends LLVM AtomicOrdering with PTX Orderings:
292using OrderingUnderlyingType = unsigned int;
293enum Ordering : OrderingUnderlyingType {
294 NotAtomic = (OrderingUnderlyingType)
295 AtomicOrdering::NotAtomic, // PTX calls these: "Weak"
296 // Unordered = 1, // NVPTX maps LLVM Unorderd to Relaxed
297 Relaxed = (OrderingUnderlyingType)AtomicOrdering::Monotonic,
298 // Consume = 3, // Unimplemented in LLVM; NVPTX would map to "Acquire"
299 Acquire = (OrderingUnderlyingType)AtomicOrdering::Acquire,
300 Release = (OrderingUnderlyingType)AtomicOrdering::Release,
301 AcquireRelease = (OrderingUnderlyingType)AtomicOrdering::AcquireRelease,
302 SequentiallyConsistent =
303 (OrderingUnderlyingType)AtomicOrdering::SequentiallyConsistent,
304 Volatile = SequentiallyConsistent + 1,
305 RelaxedMMIO = Volatile + 1,
306};
307
308using ScopeUnderlyingType = unsigned int;
309enum Scope : ScopeUnderlyingType {
310 Thread = 0,
311 Block = 1,
312 Cluster = 2,
313 Device = 3,
314 System = 4,
315 DefaultDevice = 5, // For SM < 70: denotes PTX op implicit/default .gpu scope
316 LASTSCOPE = DefaultDevice
317};
318
319using AddressSpaceUnderlyingType = unsigned int;
320enum AddressSpace : AddressSpaceUnderlyingType {
321 Generic = NVPTXAS::ADDRESS_SPACE_GENERIC,
322 Global = NVPTXAS::ADDRESS_SPACE_GLOBAL,
323 Shared = NVPTXAS::ADDRESS_SPACE_SHARED,
324 Const = NVPTXAS::ADDRESS_SPACE_CONST,
325 Local = NVPTXAS::ADDRESS_SPACE_LOCAL,
326 SharedCluster = NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER,
327 EntryParam = NVPTXAS::ADDRESS_SPACE_ENTRY_PARAM,
328
329 // DeviceParam is not a real address space, as it does not support pointers
330 // and instead can only be referenced by param+offset. For this reason it is
331 // only used in MIR as an instruction modifier and should not be used in LLVM
332 // IR.
333 DeviceParam
334};
335
336// Eviction and prefetch hint enums for !mem.cache_hint metadata. These
337// correspond to PTX L1::evict_*, L2::evict_*, and L2::*B qualifiers.
338
339// L1 Eviction Policy - maps to PTX L1::evict_* qualifiers
340enum class L1Eviction : uint8_t {
341 Normal = 0, // Default behavior (no qualifier)
342 Unchanged = 1, // L1::evict_unchanged
343 First = 2, // L1::evict_first
344 Last = 3, // L1::evict_last
345 NoAllocate = 4, // L1::no_allocate
346};
347
348// L2 Eviction Policy - maps to PTX L2::evict_* qualifiers
349enum class L2Eviction : uint8_t {
350 Normal = 0, // Default behavior (no qualifier)
351 First = 1, // L2::evict_first
352 Last = 2, // L2::evict_last
353};
354
355// L2 Prefetch Size - maps to PTX L2::*B qualifiers
356enum class L2Prefetch : uint8_t {
357 None = 0, // No prefetch hint
358 Bytes64 = 1, // L2::64B
359 Bytes128 = 2, // L2::128B
360 Bytes256 = 3, // L2::256B
361};
362
363// Bitfield layout for encoded eviction/prefetch hints (stored in unsigned):
364// Bits 0-2: L1 Eviction (3 bits, 5 values)
365// Bits 3-4: L2 Eviction (2 bits, 3 values)
366// Bits 5-6: L2 Prefetch (2 bits, 4 values)
367// Bit 7: L2::cache_hint mode flag (set when using CachePolicy)
368// Bits 8-31: Reserved
369//
370// Using llvm::Bitfield for type-safe access with compile-time validation.
371using L1EvictionBits =
372 Bitfield::Element<L1Eviction, 0, 3, L1Eviction::NoAllocate>;
373using L2EvictionBits = Bitfield::Element<L2Eviction, 3, 2, L2Eviction::Last>;
374using L2PrefetchBits =
375 Bitfield::Element<L2Prefetch, 5, 2, L2Prefetch::Bytes256>;
376using L2CacheHintBit = Bitfield::Element<bool, 7, 1>;
377
378inline unsigned encodeEvictionAndPrefetchHint(L1Eviction L1, L2Eviction L2,
379 L2Prefetch P) {
380 unsigned Hint = 0;
381 Bitfield::set<L1EvictionBits>(Packed&: Hint, Value: L1);
382 Bitfield::set<L2EvictionBits>(Packed&: Hint, Value: L2);
383 Bitfield::set<L2PrefetchBits>(Packed&: Hint, Value: P);
384 return Hint;
385}
386
387inline L1Eviction decodeL1Eviction(unsigned Hint) {
388 return Bitfield::get<L1EvictionBits>(Packed: Hint);
389}
390
391inline L2Eviction decodeL2Eviction(unsigned Hint) {
392 return Bitfield::get<L2EvictionBits>(Packed: Hint);
393}
394
395inline L2Prefetch decodeL2Prefetch(unsigned Hint) {
396 return Bitfield::get<L2PrefetchBits>(Packed: Hint);
397}
398
399inline bool isL2CacheHintMode(unsigned Hint) {
400 return Bitfield::get<L2CacheHintBit>(Packed: Hint);
401}
402
403namespace PTXLdStInstCode {
404enum FromType { Unsigned = 0, Signed, Float, Untyped };
405} // namespace PTXLdStInstCode
406
407/// PTXCvtMode - Conversion code enumeration
408namespace PTXCvtMode {
409enum CvtMode {
410 NONE = 0,
411 RNI,
412 RZI,
413 RMI,
414 RPI,
415 RN,
416 RZ,
417 RM,
418 RP,
419 RNA,
420 RS,
421
422 BASE_MASK = 0x0F,
423 FTZ_FLAG = 0x10,
424 SAT_FLAG = 0x20,
425 RELU_FLAG = 0x40,
426 SATFINITE_FLAG = 0x80,
427 PZO_FLAG = 0x100
428};
429}
430
431/// PTXCmpMode - Comparison mode enumeration
432namespace PTXCmpMode {
433enum CmpMode {
434 EQ = 0,
435 NE,
436 LT,
437 LE,
438 GT,
439 GE,
440 EQU,
441 NEU,
442 LTU,
443 LEU,
444 GTU,
445 GEU,
446 NUM,
447 // NAN is a MACRO
448 NotANumber,
449};
450}
451
452namespace PTXPrmtMode {
453enum PrmtMode {
454 NONE,
455 F4E,
456 B4E,
457 RC8,
458 ECL,
459 ECR,
460 RC16,
461};
462}
463
464enum class DivPrecisionLevel : unsigned {
465 Approx = 0,
466 Full = 1,
467 IEEE754 = 2,
468 IEEE754_NoFTZ = 3,
469};
470
471} // namespace NVPTX
472void initializeNVPTXDAGToDAGISelLegacyPass(PassRegistry &);
473} // namespace llvm
474
475// Defines symbolic names for NVPTX registers. This defines a mapping from
476// register name to register number.
477#define GET_REGINFO_ENUM
478#include "NVPTXGenRegisterInfo.inc"
479
480// Defines symbolic names for NVPTX instructions, MC helper declarations,
481// and named operand helpers generated from UseNamedOperandTable=1.
482#define GET_INSTRINFO_ENUM
483#define GET_INSTRINFO_MC_HELPER_DECLS
484#define GET_INSTRINFO_OPERAND_ENUM
485#include "NVPTXGenInstrInfo.inc"
486
487#define GET_SUBTARGETINFO_ENUM
488#include "NVPTXGenSubtargetInfo.inc"
489
490#endif
491