1//===- NVPTXUtilities.cpp - Utility Functions -----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains miscellaneous utility functions
10//
11//===----------------------------------------------------------------------===//
12
13#include "NVPTXUtilities.h"
14#include "NVPTX.h"
15#include "NVPTXTargetMachine.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/IR/Argument.h"
20#include "llvm/IR/Constants.h"
21#include "llvm/IR/Function.h"
22#include "llvm/IR/GlobalVariable.h"
23#include "llvm/IR/Module.h"
24#include "llvm/Support/Alignment.h"
25#include "llvm/Support/ModRef.h"
26#include "llvm/Support/Mutex.h"
27#include <cstdint>
28#include <cstring>
29#include <map>
30#include <mutex>
31#include <optional>
32#include <string>
33#include <vector>
34
35namespace llvm {
36
37namespace {
38typedef std::map<std::string, std::vector<unsigned>> key_val_pair_t;
39typedef std::map<const GlobalValue *, key_val_pair_t> global_val_annot_t;
40
41struct AnnotationCache {
42 sys::Mutex Lock;
43 std::map<const Module *, global_val_annot_t> Cache;
44};
45
46AnnotationCache &getAnnotationCache() {
47 static AnnotationCache AC;
48 return AC;
49}
50} // anonymous namespace
51
52void clearAnnotationCache(const Module *Mod) {
53 auto &AC = getAnnotationCache();
54 std::lock_guard<sys::Mutex> Guard(AC.Lock);
55 AC.Cache.erase(x: Mod);
56}
57
58static void cacheAnnotationFromMD(const MDNode *MetadataNode,
59 key_val_pair_t &retval) {
60 auto &AC = getAnnotationCache();
61 std::lock_guard<sys::Mutex> Guard(AC.Lock);
62 assert(MetadataNode && "Invalid mdnode for annotation");
63 assert((MetadataNode->getNumOperands() % 2) == 1 &&
64 "Invalid number of operands");
65 // start index = 1, to skip the global variable key
66 // increment = 2, to skip the value for each property-value pairs
67 for (unsigned i = 1, e = MetadataNode->getNumOperands(); i != e; i += 2) {
68 // property
69 const MDString *prop = dyn_cast<MDString>(Val: MetadataNode->getOperand(I: i));
70 assert(prop && "Annotation property not a string");
71 std::string Key = prop->getString().str();
72
73 // value
74 if (ConstantInt *Val = mdconst::dyn_extract<ConstantInt>(
75 MD: MetadataNode->getOperand(I: i + 1))) {
76 retval[Key].push_back(x: Val->getZExtValue());
77 } else {
78 llvm_unreachable("Value operand not a constant int");
79 }
80 }
81}
82
83static void cacheAnnotationFromMD(const Module *m, const GlobalValue *gv) {
84 auto &AC = getAnnotationCache();
85 std::lock_guard<sys::Mutex> Guard(AC.Lock);
86 NamedMDNode *NMD = m->getNamedMetadata(Name: "nvvm.annotations");
87 if (!NMD)
88 return;
89 key_val_pair_t tmp;
90 for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) {
91 const MDNode *elem = NMD->getOperand(i);
92
93 GlobalValue *entity =
94 mdconst::dyn_extract_or_null<GlobalValue>(MD: elem->getOperand(I: 0));
95 // entity may be null due to DCE
96 if (!entity)
97 continue;
98 if (entity != gv)
99 continue;
100
101 // accumulate annotations for entity in tmp
102 cacheAnnotationFromMD(MetadataNode: elem, retval&: tmp);
103 }
104
105 if (tmp.empty()) // no annotations for this gv
106 return;
107
108 AC.Cache[m][gv] = std::move(tmp);
109}
110
111static std::optional<unsigned> findOneNVVMAnnotation(const GlobalValue *gv,
112 const std::string &prop) {
113 auto &AC = getAnnotationCache();
114 std::lock_guard<sys::Mutex> Guard(AC.Lock);
115 const Module *m = gv->getParent();
116 auto ACIt = AC.Cache.find(x: m);
117 if (ACIt == AC.Cache.end())
118 cacheAnnotationFromMD(m, gv);
119 else if (ACIt->second.find(x: gv) == ACIt->second.end())
120 cacheAnnotationFromMD(m, gv);
121 // Look up AC.Cache[m][gv] again because cacheAnnotationFromMD may have
122 // inserted the entry.
123 auto &KVP = AC.Cache[m][gv];
124 auto It = KVP.find(x: prop);
125 if (It == KVP.end())
126 return std::nullopt;
127 return It->second[0];
128}
129
130static bool findAllNVVMAnnotation(const GlobalValue *gv,
131 const std::string &prop,
132 std::vector<unsigned> &retval) {
133 auto &AC = getAnnotationCache();
134 std::lock_guard<sys::Mutex> Guard(AC.Lock);
135 const Module *m = gv->getParent();
136 auto ACIt = AC.Cache.find(x: m);
137 if (ACIt == AC.Cache.end())
138 cacheAnnotationFromMD(m, gv);
139 else if (ACIt->second.find(x: gv) == ACIt->second.end())
140 cacheAnnotationFromMD(m, gv);
141 // Look up AC.Cache[m][gv] again because cacheAnnotationFromMD may have
142 // inserted the entry.
143 auto &KVP = AC.Cache[m][gv];
144 auto It = KVP.find(x: prop);
145 if (It == KVP.end())
146 return false;
147 retval = It->second;
148 return true;
149}
150
151static bool globalHasNVVMAnnotation(const Value &V, const std::string &Prop) {
152 if (const auto *GV = dyn_cast<GlobalValue>(Val: &V))
153 if (const auto Annot = findOneNVVMAnnotation(gv: GV, prop: Prop)) {
154 assert((*Annot == 1) && "Unexpected annotation on a symbol");
155 return true;
156 }
157
158 return false;
159}
160
161static bool argHasNVVMAnnotation(const Value &Val,
162 const std::string &Annotation) {
163 if (const Argument *Arg = dyn_cast<Argument>(Val: &Val)) {
164 const Function *Func = Arg->getParent();
165 std::vector<unsigned> Annot;
166 if (findAllNVVMAnnotation(gv: Func, prop: Annotation, retval&: Annot)) {
167 if (is_contained(Range&: Annot, Element: Arg->getArgNo()))
168 return true;
169 }
170 }
171 return false;
172}
173
174static std::optional<unsigned> getFnAttrParsedInt(const Function &F,
175 StringRef Attr) {
176 return F.hasFnAttribute(Kind: Attr)
177 ? std::optional(F.getFnAttributeAsParsedInteger(Kind: Attr))
178 : std::nullopt;
179}
180
181static SmallVector<unsigned, 3> getFnAttrParsedVector(const Function &F,
182 StringRef Attr) {
183 SmallVector<unsigned, 3> V;
184 auto &Ctx = F.getContext();
185
186 if (F.hasFnAttribute(Kind: Attr)) {
187 // We expect the attribute value to be of the form "x[,y[,z]]", where x, y,
188 // and z are unsigned values.
189 StringRef S = F.getFnAttribute(Kind: Attr).getValueAsString();
190 for (unsigned I = 0; I < 3 && !S.empty(); I++) {
191 auto [First, Rest] = S.split(Separator: ",");
192 unsigned IntVal;
193 if (First.trim().getAsInteger(Radix: 0, Result&: IntVal))
194 Ctx.emitError(ErrorStr: "can't parse integer attribute " + First + " in " + Attr);
195
196 V.push_back(Elt: IntVal);
197 S = Rest;
198 }
199 }
200 return V;
201}
202
203static std::optional<uint64_t> getVectorProduct(ArrayRef<unsigned> V) {
204 if (V.empty())
205 return std::nullopt;
206
207 return std::accumulate(first: V.begin(), last: V.end(), init: 1, binary_op: std::multiplies<uint64_t>{});
208}
209
210bool isParamGridConstant(const Argument &Arg) {
211 assert(isKernelFunction(*Arg.getParent()) &&
212 "only kernel arguments can be grid_constant");
213
214 if (!Arg.hasByValAttr())
215 return false;
216
217 // Lowering an argument as a grid_constant violates the byval semantics (and
218 // the C++ API) by reusing the same memory location for the argument across
219 // multiple threads. If an argument doesn't read memory and its address is not
220 // captured (its address is not compared with any value), then the tweak of
221 // the C++ API and byval semantics is unobservable by the program and we can
222 // lower the arg as a grid_constant.
223 if (Arg.onlyReadsMemory()) {
224 const auto CI = Arg.getAttributes().getCaptureInfo();
225 if (!capturesAddress(CC: CI) && !capturesFullProvenance(CC: CI))
226 return true;
227 }
228
229 // "grid_constant" counts argument indices starting from 1
230 if (Arg.hasAttribute(Kind: "nvvm.grid_constant"))
231 return true;
232
233 return false;
234}
235
236bool isTexture(const Value &V) { return globalHasNVVMAnnotation(V, Prop: "texture"); }
237
238bool isSurface(const Value &V) { return globalHasNVVMAnnotation(V, Prop: "surface"); }
239
240bool isSampler(const Value &V) {
241 const char *AnnotationName = "sampler";
242
243 return globalHasNVVMAnnotation(V, Prop: AnnotationName) ||
244 argHasNVVMAnnotation(Val: V, Annotation: AnnotationName);
245}
246
247bool isImageReadOnly(const Value &V) {
248 return argHasNVVMAnnotation(Val: V, Annotation: "rdoimage");
249}
250
251bool isImageWriteOnly(const Value &V) {
252 return argHasNVVMAnnotation(Val: V, Annotation: "wroimage");
253}
254
255bool isImageReadWrite(const Value &V) {
256 return argHasNVVMAnnotation(Val: V, Annotation: "rdwrimage");
257}
258
259bool isImage(const Value &V) {
260 return isImageReadOnly(V) || isImageWriteOnly(V) || isImageReadWrite(V);
261}
262
263bool isManaged(const Value &V) { return globalHasNVVMAnnotation(V, Prop: "managed"); }
264
265StringRef getTextureName(const Value &V) {
266 assert(V.hasName() && "Found texture variable with no name");
267 return V.getName();
268}
269
270StringRef getSurfaceName(const Value &V) {
271 assert(V.hasName() && "Found surface variable with no name");
272 return V.getName();
273}
274
275StringRef getSamplerName(const Value &V) {
276 assert(V.hasName() && "Found sampler variable with no name");
277 return V.getName();
278}
279
280SmallVector<unsigned, 3> getMaxNTID(const Function &F) {
281 return getFnAttrParsedVector(F, Attr: "nvvm.maxntid");
282}
283
284SmallVector<unsigned, 3> getReqNTID(const Function &F) {
285 return getFnAttrParsedVector(F, Attr: "nvvm.reqntid");
286}
287
288SmallVector<unsigned, 3> getClusterDim(const Function &F) {
289 return getFnAttrParsedVector(F, Attr: "nvvm.cluster_dim");
290}
291
292std::optional<uint64_t> getOverallMaxNTID(const Function &F) {
293 // Note: The semantics here are a bit strange. The PTX ISA states the
294 // following (11.4.2. Performance-Tuning Directives: .maxntid):
295 //
296 // Note that this directive guarantees that the total number of threads does
297 // not exceed the maximum, but does not guarantee that the limit in any
298 // particular dimension is not exceeded.
299 const auto MaxNTID = getMaxNTID(F);
300 return getVectorProduct(V: MaxNTID);
301}
302
303std::optional<uint64_t> getOverallReqNTID(const Function &F) {
304 // Note: The semantics here are a bit strange. See getMaxNTID.
305 const auto ReqNTID = getReqNTID(F);
306 return getVectorProduct(V: ReqNTID);
307}
308
309std::optional<uint64_t> getOverallClusterRank(const Function &F) {
310 // maxclusterrank and cluster_dim are mutually exclusive.
311 if (const auto ClusterRank = getMaxClusterRank(F))
312 return ClusterRank;
313
314 // Note: The semantics here are a bit strange. See getMaxNTID.
315 const auto ClusterDim = getClusterDim(F);
316 return getVectorProduct(V: ClusterDim);
317}
318
319std::optional<unsigned> getMaxClusterRank(const Function &F) {
320 return getFnAttrParsedInt(F, Attr: "nvvm.maxclusterrank");
321}
322
323std::optional<unsigned> getMinCTASm(const Function &F) {
324 return getFnAttrParsedInt(F, Attr: "nvvm.minctasm");
325}
326
327std::optional<unsigned> getMaxNReg(const Function &F) {
328 return getFnAttrParsedInt(F, Attr: "nvvm.maxnreg");
329}
330
331bool hasBlocksAreClusters(const Function &F) {
332 return F.hasFnAttribute(Kind: "nvvm.blocksareclusters");
333}
334
335MaybeAlign getAlign(const CallInst &I, unsigned Index) {
336 // First check the alignstack metadata
337 if (MaybeAlign StackAlign =
338 I.getAttributes().getAttributes(Index).getStackAlignment())
339 return StackAlign;
340
341 // If that is missing, check the legacy nvvm metadata
342 if (MDNode *alignNode = I.getMetadata(Kind: "callalign")) {
343 for (int i = 0, n = alignNode->getNumOperands(); i < n; i++) {
344 if (const ConstantInt *CI =
345 mdconst::dyn_extract<ConstantInt>(MD: alignNode->getOperand(I: i))) {
346 unsigned V = CI->getZExtValue();
347 if ((V >> 16) == Index)
348 return Align(V & 0xFFFF);
349 if ((V >> 16) > Index)
350 return std::nullopt;
351 }
352 }
353 }
354 return std::nullopt;
355}
356
357Function *getMaybeBitcastedCallee(const CallBase *CB) {
358 return dyn_cast<Function>(Val: CB->getCalledOperand()->stripPointerCasts());
359}
360
361bool shouldEmitPTXNoReturn(const Value *V, const TargetMachine &TM) {
362 const auto &ST =
363 *static_cast<const NVPTXTargetMachine &>(TM).getSubtargetImpl();
364 if (!ST.hasNoReturn())
365 return false;
366
367 assert((isa<Function>(V) || isa<CallInst>(V)) &&
368 "Expect either a call instruction or a function");
369
370 if (const CallInst *CallI = dyn_cast<CallInst>(Val: V))
371 return CallI->doesNotReturn() &&
372 CallI->getFunctionType()->getReturnType()->isVoidTy();
373
374 const Function *F = cast<Function>(Val: V);
375 return F->doesNotReturn() &&
376 F->getFunctionType()->getReturnType()->isVoidTy() &&
377 !isKernelFunction(F: *F);
378}
379
380} // namespace llvm
381