1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements functions associated with NVVM Intrinsics.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/IR/NVVMIntrinsicUtils.h"
14#include "llvm/ADT/StringRef.h"
15
16using namespace llvm;
17using namespace nvvm;
18
19void nvvm::printEvictPolicyType(raw_ostream &OS, const Constant *ImmArgVal) {
20 const auto *CI = dyn_cast<ConstantInt>(Val: ImmArgVal);
21 if (!CI ||
22 CI->getZExtValue() > static_cast<uint64_t>(EvictPolicyType::EVICT_LAST)) {
23 OS << "Unsupported evict policy";
24 return;
25 }
26 OS << getEvictPolicyName(Policy: static_cast<EvictPolicyType>(CI->getZExtValue()));
27}
28
29void nvvm::printTMAReductionOp(raw_ostream &OS, const Constant *ImmArgVal) {
30 const auto *CI = dyn_cast<ConstantInt>(Val: ImmArgVal);
31 if (!CI || CI->getZExtValue() > static_cast<uint64_t>(TMAReductionOp::XOR))
32 llvm_unreachable(
33 "printTMAReductionOp called with invalid value for immediate argument");
34
35 OS << getTMATensorReductionOpName(
36 Op: static_cast<TMAReductionOp>(CI->getZExtValue()));
37}
38
39void nvvm::printTMAValidateDataPattern(raw_ostream &OS,
40 const Constant *ImmArgVal) {
41 const auto *CI = dyn_cast<ConstantInt>(Val: ImmArgVal);
42 if (!CI || CI->getZExtValue() > static_cast<uint64_t>(
43 TMAValidateDataPattern::PER_ELEMENT_FF)) {
44 OS << "Unknown validate data pattern";
45 return;
46 }
47
48 OS << getTMAValidateDataPatternName(
49 Pattern: static_cast<TMAValidateDataPattern>(CI->getZExtValue()));
50}
51
52void nvvm::printMBarrierLayout(raw_ostream &OS, const Constant *ImmArgVal) {
53 if (const auto *CI = dyn_cast<ConstantInt>(Val: ImmArgVal)) {
54 switch (static_cast<MBarrierLayout>(CI->getZExtValue())) {
55 case MBarrierLayout::V0:
56 OS << "v0";
57 return;
58 case MBarrierLayout::V1:
59 OS << "v1";
60 return;
61 }
62 }
63}
64
65void nvvm::printMemScope(raw_ostream &OS, const Constant *ImmArgVal) {
66 const auto *CI = dyn_cast<ConstantInt>(Val: ImmArgVal);
67 if (!CI || CI->getZExtValue() > static_cast<uint64_t>(MemScope::SYS)) {
68 OS << "Unknown memory scope";
69 return;
70 }
71
72 OS << getMemScopeName(Scope: static_cast<MemScope>(CI->getZExtValue()));
73}
74
75void nvvm::printTcgen05MMAKind(raw_ostream &OS, const Constant *ImmArgVal) {
76 if (const auto *CI = dyn_cast<ConstantInt>(Val: ImmArgVal)) {
77 uint64_t Val = CI->getZExtValue();
78 switch (static_cast<Tcgen05MMAKind>(Val)) {
79 case Tcgen05MMAKind::F16:
80 OS << "f16";
81 return;
82 case Tcgen05MMAKind::TF32:
83 OS << "tf32";
84 return;
85 case Tcgen05MMAKind::F8F6F4:
86 OS << "f8f6f4";
87 return;
88 case Tcgen05MMAKind::I8:
89 OS << "i8";
90 return;
91 case Tcgen05MMAKind::TI16:
92 OS << "ti16";
93 return;
94 }
95 }
96}
97
98void nvvm::printTcgen05CollectorUsageOp(raw_ostream &OS,
99 const Constant *ImmArgVal) {
100 if (const auto *CI = dyn_cast<ConstantInt>(Val: ImmArgVal)) {
101 uint64_t Val = CI->getZExtValue();
102 switch (static_cast<Tcgen05CollectorUsageOp>(Val)) {
103 case Tcgen05CollectorUsageOp::DISCARD:
104 OS << "discard";
105 return;
106 case Tcgen05CollectorUsageOp::LASTUSE:
107 OS << "lastuse";
108 return;
109 case Tcgen05CollectorUsageOp::FILL:
110 OS << "fill";
111 return;
112 case Tcgen05CollectorUsageOp::USE:
113 OS << "use";
114 return;
115 }
116 }
117}
118
119void nvvm::printTcgen05MMACollectorBBuffer(raw_ostream &OS,
120 const Constant *ImmArgVal) {
121 if (const auto *CI = dyn_cast<ConstantInt>(Val: ImmArgVal)) {
122 uint64_t Val = CI->getZExtValue();
123 switch (static_cast<Tcgen05MMACollectorBBuffer>(Val)) {
124 case Tcgen05MMACollectorBBuffer::B0:
125 OS << "b0";
126 return;
127 case Tcgen05MMACollectorBBuffer::B1:
128 OS << "b1";
129 return;
130 case Tcgen05MMACollectorBBuffer::B2:
131 OS << "b2";
132 return;
133 case Tcgen05MMACollectorBBuffer::B3:
134 OS << "b3";
135 return;
136 }
137 }
138}
139
140void nvvm::printTensormapElemType(raw_ostream &OS, const Constant *ImmArgVal) {
141 static constexpr StringRef TensormapElemTypes[] = {
142 "u8", "u16", "u32", "s32", "u64", "s64",
143 "f16", "f32", "f32.ftz", "f64", "bf16", "tf32",
144 "tf32.ftz", "b4x16", "b4x16_p64", "b6x16_p32"};
145 if (const auto *CI = dyn_cast<ConstantInt>(Val: ImmArgVal)) {
146 uint64_t Val = CI->getZExtValue();
147 if (Val <= static_cast<uint64_t>(nvvm::TensormapElemType::B6x16_p32)) {
148 OS << TensormapElemTypes[Val];
149 return;
150 }
151 }
152}
153
154void nvvm::printTensormapInterleaveLayout(raw_ostream &OS,
155 const Constant *ImmArgVal) {
156 if (const auto *CI = dyn_cast<ConstantInt>(Val: ImmArgVal)) {
157 uint64_t Val = CI->getZExtValue();
158 switch (static_cast<TensormapInterleaveLayout>(Val)) {
159 case TensormapInterleaveLayout::NO_INTERLEAVE:
160 OS << "No interleave";
161 return;
162 case TensormapInterleaveLayout::INTERLEAVE_16B:
163 OS << "16B interleave";
164 return;
165 case TensormapInterleaveLayout::INTERLEAVE_32B:
166 OS << "32B interleave";
167 return;
168 }
169 }
170}
171
172void nvvm::printTensormapSwizzleMode(raw_ostream &OS,
173 const Constant *ImmArgVal) {
174 static constexpr StringRef TensormapSwizzleModes[] = {
175 "No swizzling", "32B swizzling", "64B swizzling", "128B swizzling",
176 "96B swizzling"};
177 if (const auto *CI = dyn_cast<ConstantInt>(Val: ImmArgVal)) {
178 uint64_t Val = CI->getZExtValue();
179 if (Val <= static_cast<uint64_t>(nvvm::TensormapSwizzleMode::SWIZZLE_96B)) {
180 OS << TensormapSwizzleModes[Val];
181 return;
182 }
183 }
184}
185
186void nvvm::printTensormapSwizzleAtomicity(raw_ostream &OS,
187 const Constant *ImmArgVal) {
188 static constexpr StringRef TensormapSwizzleAtomicities[] = {
189 "16B", "32B", "32B + 8B flip", "64B"};
190 if (const auto *CI = dyn_cast<ConstantInt>(Val: ImmArgVal)) {
191 uint64_t Val = CI->getZExtValue();
192 if (Val <= static_cast<uint64_t>(
193 nvvm::TensormapSwizzleAtomicity::SWIZZLE_ATOMICITY_64B)) {
194 OS << TensormapSwizzleAtomicities[Val];
195 return;
196 }
197 }
198}
199
200void nvvm::printFPRoundingMode(raw_ostream &OS, const Constant *ImmArgVal) {
201 if (isa<ConstantInt>(Val: ImmArgVal))
202 OS << nvvm::GetRoundingModeName(RM: nvvm::GetRoundingModeFromImmArg(ImmArgVal));
203}
204
205void nvvm::printTensormapFillMode(raw_ostream &OS, const Constant *ImmArgVal) {
206 if (const auto *CI = dyn_cast<ConstantInt>(Val: ImmArgVal)) {
207 uint64_t Val = CI->getZExtValue();
208 OS << (Val == static_cast<uint64_t>(TensormapFillMode::ZERO_FILL)
209 ? "Zero fill"
210 : "OOB-NaN fill");
211 return;
212 }
213}
214