1//===-- NVPTX.h - Top-level interface for NVPTX representation --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the entry points for global functions defined in
10// the LLVM NVPTX back-end.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIB_TARGET_NVPTX_NVPTX_H
15#define LLVM_LIB_TARGET_NVPTX_NVPTX_H
16
17#include "llvm/ADT/Bitfields.h"
18#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
19#include "llvm/CodeGen/SelectionDAGISel.h"
20#include "llvm/IR/PassManager.h"
21#include "llvm/Pass.h"
22#include "llvm/Support/AtomicOrdering.h"
23#include "llvm/Support/CodeGen.h"
24#include "llvm/Support/NVPTXAddrSpace.h"
25#include "llvm/Target/TargetMachine.h"
26
27namespace llvm {
28class SelectionDAGISelPass;
29class FunctionPass;
30class MachineFunctionPass;
31class NVPTXTargetMachine;
32class PassRegistry;
33
34namespace NVPTXCC {
35enum CondCodes {
36 EQ,
37 NE,
38 LT,
39 LE,
40 GT,
41 GE
42};
43}
44
45FunctionPass *createNVPTXISelDag(NVPTXTargetMachine &TM,
46 llvm::CodeGenOptLevel OptLevel);
47ModulePass *createNVPTXAssignValidGlobalNamesLegacyPass();
48ModulePass *createNVPTXCanonicalizeDynamicSharedLegacyPass();
49ModulePass *createGenericToNVVMLegacyPass();
50ModulePass *createNVPTXCtorDtorLoweringLegacyPass();
51FunctionPass *createNVPTXAtomicLowerLegacyPass();
52FunctionPass *createNVVMIntrRangePass();
53ModulePass *createNVVMReflectPass(unsigned int SmVersion);
54MachineFunctionPass *createNVPTXPrologEpilogLegacyPass();
55MachineFunctionPass *createNVPTXReplaceImageHandlesLegacyPass();
56FunctionPass *createNVPTXImageOptimizerLegacyPass();
57ModulePass *createNVPTXLowerArgsPass();
58ModulePass *createNVPTXPromoteParamAlignPass();
59FunctionPass *createNVPTXAllocaHoistingLegacyPass();
60FunctionPass *createNVPTXLowerAllocaLegacyPass();
61FunctionPass *createNVPTXLowerUnreachableLegacyPass(bool TrapUnreachable,
62 bool NoTrapAfterNoreturn);
63FunctionPass *createNVPTXLowerAggrCopiesLegacyPass();
64FunctionPass *createNVPTXMarkKernelPtrsGlobalPass();
65FunctionPass *createNVPTXTagInvariantLoadsPass();
66FunctionPass *createNVPTXIRPeepholePass();
67MachineFunctionPass *createNVPTXPeepholeLegacyPass();
68MachineFunctionPass *createNVPTXProxyRegErasureLegacyPass();
69MachineFunctionPass *createNVPTXForwardParamsLegacyPass();
70MachineFunctionPass *createNVPTXAddressFolderLegacyPass();
71
72void initializeNVVMReflectLegacyPassPass(PassRegistry &);
73void initializeGenericToNVVMLegacyPassPass(PassRegistry &);
74void initializeNVPTXAllocaHoistingLegacyPassPass(PassRegistry &);
75void initializeNVPTXAsmPrinterPass(PassRegistry &);
76void initializeNVPTXAssignValidGlobalNamesLegacyPassPass(PassRegistry &);
77void initializeNVPTXAtomicLowerLegacyPassPass(PassRegistry &);
78void initializeNVPTXCanonicalizeDynamicSharedLegacyPassPass(PassRegistry &);
79void initializeNVPTXCtorDtorLoweringLegacyPass(PassRegistry &);
80void initializeNVPTXLowerAggrCopiesLegacyPassPass(PassRegistry &);
81void initializeNVPTXLowerAllocaLegacyPassPass(PassRegistry &);
82void initializeNVPTXLowerUnreachableLegacyPassPass(PassRegistry &);
83void initializeNVPTXLowerArgsLegacyPassPass(PassRegistry &);
84void initializeNVPTXPromoteParamAlignLegacyPassPass(PassRegistry &);
85void initializeNVPTXProxyRegErasureLegacyPassPass(PassRegistry &);
86void initializeNVPTXForwardParamsLegacyPassPass(PassRegistry &);
87void initializeNVPTXAddressFolderLegacyPassPass(PassRegistry &);
88void initializeNVVMIntrRangePass(PassRegistry &);
89void initializeNVPTXAAWrapperPassPass(PassRegistry &);
90void initializeNVPTXExternalAAWrapperPass(PassRegistry &);
91void initializeNVPTXPeepholeLegacyPassPass(PassRegistry &);
92void initializeNVPTXMarkKernelPtrsGlobalLegacyPassPass(PassRegistry &);
93void initializeNVPTXTagInvariantLoadLegacyPassPass(PassRegistry &);
94void initializeNVPTXIRPeepholePass(PassRegistry &);
95void initializeNVPTXPrologEpilogLegacyPassPass(PassRegistry &);
96
97// Module passes
98class GenericToNVVMPass : public RequiredPassInfoMixin<GenericToNVVMPass> {
99public:
100 PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
101};
102
103class NVPTXAssignValidGlobalNamesPass
104 : public RequiredPassInfoMixin<NVPTXAssignValidGlobalNamesPass> {
105public:
106 PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
107};
108
109class NVPTXCanonicalizeDynamicSharedPass
110 : public RequiredPassInfoMixin<NVPTXCanonicalizeDynamicSharedPass> {
111public:
112 PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
113};
114
115class NVPTXCtorDtorLoweringPass
116 : public RequiredPassInfoMixin<NVPTXCtorDtorLoweringPass> {
117public:
118 PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
119};
120
121class NVPTXLowerArgsPass : public RequiredPassInfoMixin<NVPTXLowerArgsPass> {
122 TargetMachine &TM;
123
124public:
125 NVPTXLowerArgsPass(TargetMachine &TM) : TM(TM) {}
126 PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
127};
128
129class NVPTXPromoteParamAlignPass
130 : public OptionalPassInfoMixin<NVPTXPromoteParamAlignPass> {
131public:
132 PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
133};
134
135class NVVMReflectPass : public RequiredPassInfoMixin<NVVMReflectPass> {
136 unsigned SmVersion;
137
138public:
139 NVVMReflectPass() : SmVersion(0) {}
140 NVVMReflectPass(unsigned SmVersion) : SmVersion(SmVersion) {}
141 PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
142};
143
144// Function passes
145class NVPTXAllocaHoistingPass
146 : public RequiredPassInfoMixin<NVPTXAllocaHoistingPass> {
147public:
148 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
149};
150
151class NVPTXAtomicLowerPass
152 : public RequiredPassInfoMixin<NVPTXAtomicLowerPass> {
153public:
154 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
155};
156
157class NVPTXCopyByValArgsPass
158 : public OptionalPassInfoMixin<NVPTXCopyByValArgsPass> {
159public:
160 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
161};
162
163class NVPTXImageOptimizerPass
164 : public OptionalPassInfoMixin<NVPTXImageOptimizerPass> {
165public:
166 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
167};
168
169class NVPTXIRPeepholePass : public OptionalPassInfoMixin<NVPTXIRPeepholePass> {
170public:
171 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
172};
173
174class NVPTXLowerAggrCopiesPass
175 : public RequiredPassInfoMixin<NVPTXLowerAggrCopiesPass> {
176public:
177 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
178};
179
180class NVPTXLowerAllocaPass
181 : public RequiredPassInfoMixin<NVPTXLowerAllocaPass> {
182public:
183 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
184};
185
186class NVPTXLowerUnreachablePass
187 : public OptionalPassInfoMixin<NVPTXLowerUnreachablePass> {
188 bool TrapUnreachable;
189 bool NoTrapAfterNoreturn;
190
191public:
192 NVPTXLowerUnreachablePass(bool TrapUnreachable, bool NoTrapAfterNoreturn)
193 : TrapUnreachable(TrapUnreachable),
194 NoTrapAfterNoreturn(NoTrapAfterNoreturn) {}
195 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
196};
197
198class NVPTXMarkKernelPtrsGlobalPass
199 : public OptionalPassInfoMixin<NVPTXMarkKernelPtrsGlobalPass> {
200public:
201 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
202};
203
204class NVPTXTagInvariantLoadsPass
205 : public OptionalPassInfoMixin<NVPTXTagInvariantLoadsPass> {
206public:
207 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
208};
209
210class NVVMIntrRangePass : public OptionalPassInfoMixin<NVVMIntrRangePass> {
211public:
212 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
213};
214
215// Machine function passes
216class NVPTXAddressFolderPass
217 : public OptionalPassInfoMixin<NVPTXAddressFolderPass> {
218public:
219 PreservedAnalyses run(MachineFunction &MF,
220 MachineFunctionAnalysisManager &MFAM);
221};
222
223class NVPTXForwardParamsPass
224 : public RequiredPassInfoMixin<NVPTXForwardParamsPass> {
225public:
226 PreservedAnalyses run(MachineFunction &MF,
227 MachineFunctionAnalysisManager &MFAM);
228};
229
230class NVPTXISelDAGToDAGPass : public SelectionDAGISelPass {
231public:
232 NVPTXISelDAGToDAGPass(NVPTXTargetMachine &TM, CodeGenOptLevel OptLevel);
233};
234
235class NVPTXPeepholePass : public OptionalPassInfoMixin<NVPTXPeepholePass> {
236public:
237 PreservedAnalyses run(MachineFunction &MF,
238 MachineFunctionAnalysisManager &MFAM);
239};
240
241class NVPTXPrologEpilogPass
242 : public RequiredPassInfoMixin<NVPTXPrologEpilogPass> {
243public:
244 PreservedAnalyses run(MachineFunction &MF,
245 MachineFunctionAnalysisManager &MFAM);
246};
247
248class NVPTXProxyRegErasurePass
249 : public RequiredPassInfoMixin<NVPTXProxyRegErasurePass> {
250public:
251 PreservedAnalyses run(MachineFunction &MF,
252 MachineFunctionAnalysisManager &MFAM);
253};
254
255class NVPTXReplaceImageHandlesPass
256 : public RequiredPassInfoMixin<NVPTXReplaceImageHandlesPass> {
257public:
258 PreservedAnalyses run(MachineFunction &MF,
259 MachineFunctionAnalysisManager &MFAM);
260};
261
262namespace NVPTX {
263enum DrvInterface {
264 NVCL,
265 CUDA
266};
267
268// A field inside TSFlags needs a shift and a mask. The usage is
269// always as follows :
270// ((TSFlags & fieldMask) >> fieldShift)
271// The enum keeps the mask, the shift, and all valid values of the
272// field in one place.
273enum VecInstType {
274 VecInstTypeShift = 0,
275 VecInstTypeMask = 0xF,
276
277 VecNOP = 0,
278 VecLoad = 1,
279 VecStore = 2,
280 VecBuild = 3,
281 VecShuffle = 4,
282 VecExtract = 5,
283 VecInsert = 6,
284 VecDest = 7,
285 VecOther = 15
286};
287
288enum SimpleMove {
289 SimpleMoveMask = 0x10,
290 SimpleMoveShift = 4
291};
292enum LoadStore {
293 isLoadMask = 0x20,
294 isLoadShift = 5,
295 isStoreMask = 0x40,
296 isStoreShift = 6
297};
298
299// Extends LLVM AtomicOrdering with PTX Orderings:
300using OrderingUnderlyingType = unsigned int;
301enum Ordering : OrderingUnderlyingType {
302 NotAtomic = (OrderingUnderlyingType)
303 AtomicOrdering::NotAtomic, // PTX calls these: "Weak"
304 // Unordered = 1, // NVPTX maps LLVM Unorderd to Relaxed
305 Relaxed = (OrderingUnderlyingType)AtomicOrdering::Monotonic,
306 // Consume = 3, // Unimplemented in LLVM; NVPTX would map to "Acquire"
307 Acquire = (OrderingUnderlyingType)AtomicOrdering::Acquire,
308 Release = (OrderingUnderlyingType)AtomicOrdering::Release,
309 AcquireRelease = (OrderingUnderlyingType)AtomicOrdering::AcquireRelease,
310 SequentiallyConsistent =
311 (OrderingUnderlyingType)AtomicOrdering::SequentiallyConsistent,
312 Volatile = SequentiallyConsistent + 1,
313 RelaxedMMIO = Volatile + 1,
314};
315
316using ScopeUnderlyingType = unsigned int;
317enum Scope : ScopeUnderlyingType {
318 Thread = 0,
319 Block = 1,
320 Cluster = 2,
321 Device = 3,
322 System = 4,
323 DefaultDevice = 5, // For SM < 70: denotes PTX op implicit/default .gpu scope
324 LASTSCOPE = DefaultDevice
325};
326
327using AddressSpaceUnderlyingType = unsigned int;
328enum AddressSpace : AddressSpaceUnderlyingType {
329 Generic = NVPTXAS::ADDRESS_SPACE_GENERIC,
330 Global = NVPTXAS::ADDRESS_SPACE_GLOBAL,
331 Shared = NVPTXAS::ADDRESS_SPACE_SHARED,
332 Const = NVPTXAS::ADDRESS_SPACE_CONST,
333 Local = NVPTXAS::ADDRESS_SPACE_LOCAL,
334 SharedCluster = NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER,
335 EntryParam = NVPTXAS::ADDRESS_SPACE_ENTRY_PARAM,
336
337 // DeviceParam is not a real address space, as it does not support pointers
338 // and instead can only be referenced by param+offset. For this reason it is
339 // only used in MIR as an instruction modifier and should not be used in LLVM
340 // IR.
341 DeviceParam
342};
343
344// Eviction and prefetch hint enums for !mem.cache_hint metadata. These
345// correspond to PTX L1::evict_*, L2::evict_*, and L2::*B qualifiers.
346
347// L1 Eviction Policy - maps to PTX L1::evict_* qualifiers
348enum class L1Eviction : uint8_t {
349 Normal = 0, // Default behavior (no qualifier)
350 Unchanged = 1, // L1::evict_unchanged
351 First = 2, // L1::evict_first
352 Last = 3, // L1::evict_last
353 NoAllocate = 4, // L1::no_allocate
354};
355
356// L2 Eviction Policy - maps to PTX L2::evict_* qualifiers
357enum class L2Eviction : uint8_t {
358 Normal = 0, // Default behavior (no qualifier)
359 First = 1, // L2::evict_first
360 Last = 2, // L2::evict_last
361};
362
363// L2 Prefetch Size - maps to PTX L2::*B qualifiers
364enum class L2Prefetch : uint8_t {
365 None = 0, // No prefetch hint
366 Bytes64 = 1, // L2::64B
367 Bytes128 = 2, // L2::128B
368 Bytes256 = 3, // L2::256B
369};
370
371// Bitfield layout for encoded eviction/prefetch hints (stored in unsigned):
372// Bits 0-2: L1 Eviction (3 bits, 5 values)
373// Bits 3-4: L2 Eviction (2 bits, 3 values)
374// Bits 5-6: L2 Prefetch (2 bits, 4 values)
375// Bit 7: L2::cache_hint mode flag (set when using CachePolicy)
376// Bits 8-31: Reserved
377//
378// Using llvm::Bitfield for type-safe access with compile-time validation.
379using L1EvictionBits =
380 Bitfield::Element<L1Eviction, 0, 3, L1Eviction::NoAllocate>;
381using L2EvictionBits = Bitfield::Element<L2Eviction, 3, 2, L2Eviction::Last>;
382using L2PrefetchBits =
383 Bitfield::Element<L2Prefetch, 5, 2, L2Prefetch::Bytes256>;
384using L2CacheHintBit = Bitfield::Element<bool, 7, 1>;
385
386inline unsigned encodeEvictionAndPrefetchHint(L1Eviction L1, L2Eviction L2,
387 L2Prefetch P) {
388 unsigned Hint = 0;
389 Bitfield::set<L1EvictionBits>(Packed&: Hint, Value: L1);
390 Bitfield::set<L2EvictionBits>(Packed&: Hint, Value: L2);
391 Bitfield::set<L2PrefetchBits>(Packed&: Hint, Value: P);
392 return Hint;
393}
394
395inline L1Eviction decodeL1Eviction(unsigned Hint) {
396 return Bitfield::get<L1EvictionBits>(Packed: Hint);
397}
398
399inline L2Eviction decodeL2Eviction(unsigned Hint) {
400 return Bitfield::get<L2EvictionBits>(Packed: Hint);
401}
402
403inline L2Prefetch decodeL2Prefetch(unsigned Hint) {
404 return Bitfield::get<L2PrefetchBits>(Packed: Hint);
405}
406
407inline bool isL2CacheHintMode(unsigned Hint) {
408 return Bitfield::get<L2CacheHintBit>(Packed: Hint);
409}
410
411namespace PTXLdStInstCode {
412enum FromType { Unsigned = 0, Signed, Float, Untyped };
413} // namespace PTXLdStInstCode
414
415/// PTXCvtMode - Conversion code enumeration
416namespace PTXCvtMode {
417enum CvtMode {
418 NONE = 0,
419 RNI,
420 RZI,
421 RMI,
422 RPI,
423 RN,
424 RZ,
425 RM,
426 RP,
427 RNA,
428 RS,
429
430 BASE_MASK = 0x0F,
431 FTZ_FLAG = 0x10,
432 SAT_FLAG = 0x20,
433 RELU_FLAG = 0x40,
434 SATFINITE_FLAG = 0x80,
435 PZO_FLAG = 0x100
436};
437}
438
439/// PTXCmpMode - Comparison mode enumeration
440namespace PTXCmpMode {
441enum CmpMode {
442 EQ = 0,
443 NE,
444 LT,
445 LE,
446 GT,
447 GE,
448 EQU,
449 NEU,
450 LTU,
451 LEU,
452 GTU,
453 GEU,
454 NUM,
455 // NAN is a MACRO
456 NotANumber,
457};
458}
459
460namespace PTXPrmtMode {
461enum PrmtMode {
462 NONE,
463 F4E,
464 B4E,
465 RC8,
466 ECL,
467 ECR,
468 RC16,
469};
470}
471
472enum class DivPrecisionLevel : unsigned {
473 Approx = 0,
474 Full = 1,
475 IEEE754 = 2,
476 IEEE754_NoFTZ = 3,
477};
478
479} // namespace NVPTX
480void initializeNVPTXDAGToDAGISelLegacyPass(PassRegistry &);
481} // namespace llvm
482
483// Defines symbolic names for NVPTX registers. This defines a mapping from
484// register name to register number.
485#define GET_REGINFO_ENUM
486#include "NVPTXGenRegisterInfo.inc"
487
488// Defines symbolic names for NVPTX instructions, MC helper declarations,
489// and named operand helpers generated from UseNamedOperandTable=1.
490#define GET_INSTRINFO_ENUM
491#define GET_INSTRINFO_MC_HELPER_DECLS
492#define GET_INSTRINFO_OPERAND_ENUM
493#include "NVPTXGenInstrInfo.inc"
494
495#define GET_SUBTARGETINFO_ENUM
496#include "NVPTXGenSubtargetInfo.inc"
497
498#endif
499