1//===- NVVMReflect.cpp - NVVM Emulate conditional compilation -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass replaces occurrences of __nvvm_reflect("foo") and llvm.nvvm.reflect
10// with an integer.
11//
12// We choose the value we use by looking at metadata in the module itself. Note
13// that we intentionally only have one way to choose these values, because other
14// parts of LLVM (particularly, InstCombineCall) rely on being able to predict
15// the values chosen by this pass.
16//
17// If we see an unknown string, we replace its call with 0.
18//
19//===----------------------------------------------------------------------===//
20
21#include "NVPTX.h"
22#include "llvm/ADT/SmallVector.h"
23#include "llvm/Analysis/ConstantFolding.h"
24#include "llvm/IR/Constants.h"
25#include "llvm/IR/DerivedTypes.h"
26#include "llvm/IR/Function.h"
27#include "llvm/IR/InstIterator.h"
28#include "llvm/IR/Instructions.h"
29#include "llvm/IR/Intrinsics.h"
30#include "llvm/IR/IntrinsicsNVPTX.h"
31#include "llvm/IR/Module.h"
32#include "llvm/IR/PassManager.h"
33#include "llvm/IR/Type.h"
34#include "llvm/Pass.h"
35#include "llvm/Support/CommandLine.h"
36#include "llvm/Support/Debug.h"
37#include "llvm/Support/raw_os_ostream.h"
38#include "llvm/Support/raw_ostream.h"
39#include "llvm/Transforms/Scalar.h"
40#include "llvm/Transforms/Utils/BasicBlockUtils.h"
41#include "llvm/Transforms/Utils/Local.h"
42#include <algorithm>
43#include <sstream>
44#include <string>
45#define NVVM_REFLECT_FUNCTION "__nvvm_reflect"
46#define NVVM_REFLECT_OCL_FUNCTION "__nvvm_reflect_ocl"
47
48using namespace llvm;
49
50#define DEBUG_TYPE "nvptx-reflect"
51
52namespace llvm { void initializeNVVMReflectPass(PassRegistry &); }
53
54namespace {
55class NVVMReflect : public FunctionPass {
56public:
57 static char ID;
58 unsigned int SmVersion;
59 NVVMReflect() : NVVMReflect(0) {}
60 explicit NVVMReflect(unsigned int Sm) : FunctionPass(ID), SmVersion(Sm) {
61 initializeNVVMReflectPass(*PassRegistry::getPassRegistry());
62 }
63
64 bool runOnFunction(Function &) override;
65};
66}
67
68FunctionPass *llvm::createNVVMReflectPass(unsigned int SmVersion) {
69 return new NVVMReflect(SmVersion);
70}
71
72static cl::opt<bool>
73NVVMReflectEnabled("nvvm-reflect-enable", cl::init(Val: true), cl::Hidden,
74 cl::desc("NVVM reflection, enabled by default"));
75
76char NVVMReflect::ID = 0;
77INITIALIZE_PASS(NVVMReflect, "nvvm-reflect",
78 "Replace occurrences of __nvvm_reflect() calls with 0/1", false,
79 false)
80
81static bool runNVVMReflect(Function &F, unsigned SmVersion) {
82 if (!NVVMReflectEnabled)
83 return false;
84
85 if (F.getName() == NVVM_REFLECT_FUNCTION ||
86 F.getName() == NVVM_REFLECT_OCL_FUNCTION) {
87 assert(F.isDeclaration() && "_reflect function should not have a body");
88 assert(F.getReturnType()->isIntegerTy() &&
89 "_reflect's return type should be integer");
90 return false;
91 }
92
93 SmallVector<Instruction *, 4> ToRemove;
94 SmallVector<Instruction *, 4> ToSimplify;
95
96 // Go through the calls in this function. Each call to __nvvm_reflect or
97 // llvm.nvvm.reflect should be a CallInst with a ConstantArray argument.
98 // First validate that. If the c-string corresponding to the ConstantArray can
99 // be found successfully, see if it can be found in VarMap. If so, replace the
100 // uses of CallInst with the value found in VarMap. If not, replace the use
101 // with value 0.
102
103 // The IR for __nvvm_reflect calls differs between CUDA versions.
104 //
105 // CUDA 6.5 and earlier uses this sequence:
106 // %ptr = tail call i8* @llvm.nvvm.ptr.constant.to.gen.p0i8.p4i8
107 // (i8 addrspace(4)* getelementptr inbounds
108 // ([8 x i8], [8 x i8] addrspace(4)* @str, i32 0, i32 0))
109 // %reflect = tail call i32 @__nvvm_reflect(i8* %ptr)
110 //
111 // The value returned by Sym->getOperand(0) is a Constant with a
112 // ConstantDataSequential operand which can be converted to string and used
113 // for lookup.
114 //
115 // CUDA 7.0 does it slightly differently:
116 // %reflect = call i32 @__nvvm_reflect(i8* addrspacecast
117 // (i8 addrspace(1)* getelementptr inbounds
118 // ([8 x i8], [8 x i8] addrspace(1)* @str, i32 0, i32 0) to i8*))
119 //
120 // In this case, we get a Constant with a GlobalVariable operand and we need
121 // to dig deeper to find its initializer with the string we'll use for lookup.
122 for (Instruction &I : instructions(F)) {
123 CallInst *Call = dyn_cast<CallInst>(Val: &I);
124 if (!Call)
125 continue;
126 Function *Callee = Call->getCalledFunction();
127 if (!Callee || (Callee->getName() != NVVM_REFLECT_FUNCTION &&
128 Callee->getName() != NVVM_REFLECT_OCL_FUNCTION &&
129 Callee->getIntrinsicID() != Intrinsic::nvvm_reflect))
130 continue;
131
132 // FIXME: Improve error handling here and elsewhere in this pass.
133 assert(Call->getNumOperands() == 2 &&
134 "Wrong number of operands to __nvvm_reflect function");
135
136 // In cuda 6.5 and earlier, we will have an extra constant-to-generic
137 // conversion of the string.
138 const Value *Str = Call->getArgOperand(i: 0);
139 if (const CallInst *ConvCall = dyn_cast<CallInst>(Val: Str)) {
140 // FIXME: Add assertions about ConvCall.
141 Str = ConvCall->getArgOperand(i: 0);
142 }
143 // Pre opaque pointers we have a constant expression wrapping the constant
144 // string.
145 Str = Str->stripPointerCasts();
146 assert(isa<Constant>(Str) &&
147 "Format of __nvvm_reflect function not recognized");
148
149 const Value *Operand = cast<Constant>(Val: Str)->getOperand(i: 0);
150 if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: Operand)) {
151 // For CUDA-7.0 style __nvvm_reflect calls, we need to find the operand's
152 // initializer.
153 assert(GV->hasInitializer() &&
154 "Format of _reflect function not recognized");
155 const Constant *Initializer = GV->getInitializer();
156 Operand = Initializer;
157 }
158
159 assert(isa<ConstantDataSequential>(Operand) &&
160 "Format of _reflect function not recognized");
161 assert(cast<ConstantDataSequential>(Operand)->isCString() &&
162 "Format of _reflect function not recognized");
163
164 StringRef ReflectArg = cast<ConstantDataSequential>(Val: Operand)->getAsString();
165 ReflectArg = ReflectArg.substr(Start: 0, N: ReflectArg.size() - 1);
166 LLVM_DEBUG(dbgs() << "Arg of _reflect : " << ReflectArg << "\n");
167
168 int ReflectVal = 0; // The default value is 0
169 if (ReflectArg == "__CUDA_FTZ") {
170 // Try to pull __CUDA_FTZ from the nvvm-reflect-ftz module flag. Our
171 // choice here must be kept in sync with AutoUpgrade, which uses the same
172 // technique to detect whether ftz is enabled.
173 if (auto *Flag = mdconst::extract_or_null<ConstantInt>(
174 MD: F.getParent()->getModuleFlag(Key: "nvvm-reflect-ftz")))
175 ReflectVal = Flag->getSExtValue();
176 } else if (ReflectArg == "__CUDA_ARCH") {
177 ReflectVal = SmVersion * 10;
178 }
179
180 // If the immediate user is a simple comparison we want to simplify it.
181 for (User *U : Call->users())
182 if (Instruction *I = dyn_cast<Instruction>(Val: U))
183 ToSimplify.push_back(Elt: I);
184
185 Call->replaceAllUsesWith(V: ConstantInt::get(Ty: Call->getType(), V: ReflectVal));
186 ToRemove.push_back(Elt: Call);
187 }
188
189 // The code guarded by __nvvm_reflect may be invalid for the target machine.
190 // Traverse the use-def chain, continually simplifying constant expressions
191 // until we find a terminator that we can then remove.
192 while (!ToSimplify.empty()) {
193 Instruction *I = ToSimplify.pop_back_val();
194 if (Constant *C =
195 ConstantFoldInstruction(I, DL: F.getDataLayout())) {
196 for (User *U : I->users())
197 if (Instruction *I = dyn_cast<Instruction>(Val: U))
198 ToSimplify.push_back(Elt: I);
199
200 I->replaceAllUsesWith(V: C);
201 if (isInstructionTriviallyDead(I)) {
202 ToRemove.push_back(Elt: I);
203 }
204 } else if (I->isTerminator()) {
205 ConstantFoldTerminator(BB: I->getParent());
206 }
207 }
208
209 // Removing via isInstructionTriviallyDead may add duplicates to the ToRemove
210 // array. Filter out the duplicates before starting to erase from parent.
211 std::sort(first: ToRemove.begin(), last: ToRemove.end());
212 auto NewLastIter = llvm::unique(R&: ToRemove);
213 ToRemove.erase(CS: NewLastIter, CE: ToRemove.end());
214
215 for (Instruction *I : ToRemove)
216 I->eraseFromParent();
217
218 return ToRemove.size() > 0;
219}
220
221bool NVVMReflect::runOnFunction(Function &F) {
222 return runNVVMReflect(F, SmVersion);
223}
224
225NVVMReflectPass::NVVMReflectPass() : NVVMReflectPass(0) {}
226
227PreservedAnalyses NVVMReflectPass::run(Function &F,
228 FunctionAnalysisManager &AM) {
229 return runNVVMReflect(F, SmVersion) ? PreservedAnalyses::none()
230 : PreservedAnalyses::all();
231}
232