1//===-- NVPTXReplaceImageHandles.cpp - Replace image handles for Fermi ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// On Fermi, image handles are not supported. To work around this, we traverse
10// the machine code and replace image handles with concrete symbols. For this
11// to work reliably, inlining of all function call must be performed.
12//
13//===----------------------------------------------------------------------===//
14#include "MCTargetDesc/NVPTXBaseInfo.h"
15#include "NVPTX.h"
16#include "NVPTXMachineFunctionInfo.h"
17#include "NVPTXTargetMachine.h"
18#include "llvm/ADT/DenseSet.h"
19#include "llvm/CodeGen/MachineFunction.h"
20#include "llvm/CodeGen/MachineFunctionPass.h"
21#include "llvm/CodeGen/MachineRegisterInfo.h"
22#include "llvm/IR/DiagnosticInfo.h"
23
24using namespace llvm;
25
26namespace {
27class NVPTXReplaceImageHandles {
28 DenseSet<MachineInstr *> InstrsToRemove;
29
30public:
31 bool run(MachineFunction &MF);
32
33private:
34 bool processInstr(MachineInstr &MI);
35 bool replaceImageHandle(MachineOperand &Op, MachineFunction &MF,
36 bool IsTexture = false);
37};
38} // namespace
39
40bool NVPTXReplaceImageHandles::run(MachineFunction &MF) {
41 bool Changed = false;
42 InstrsToRemove.clear();
43
44 for (MachineBasicBlock &MBB : MF)
45 for (MachineInstr &MI : MBB)
46 Changed |= processInstr(MI);
47
48 // Now clean up any handle-access instructions
49 // This is needed in debug mode when code cleanup passes are not executed,
50 // but we need the handle access to be eliminated because they are not
51 // valid instructions when image handles are disabled.
52 for (MachineInstr *MI : InstrsToRemove) {
53 unsigned DefReg = MI->getOperand(i: 0).getReg();
54 // Only these that are not used can be removed.
55 if (MF.getRegInfo().use_nodbg_empty(RegNo: DefReg))
56 MI->eraseFromParent();
57 }
58 return Changed;
59}
60
61bool NVPTXReplaceImageHandles::processInstr(MachineInstr &MI) {
62 MachineFunction &MF = *MI.getParent()->getParent();
63 const MCInstrDesc &MCID = MI.getDesc();
64
65 if (MCID.TSFlags & NVPTXII::IsTexFlag) {
66 // This is a texture fetch, so operand 4 is a texref and operand 5 is
67 // a samplerref
68 MachineOperand &TexHandle = MI.getOperand(i: 4);
69 bool Changed = replaceImageHandle(Op&: TexHandle, MF, /*IsTexture=*/true);
70
71 if (!(MCID.TSFlags & NVPTXII::IsTexModeUnifiedFlag)) {
72 MachineOperand &SampHandle = MI.getOperand(i: 5);
73 Changed |= replaceImageHandle(Op&: SampHandle, MF, /*IsTexture=*/true);
74 }
75
76 return Changed;
77 }
78 if (MCID.TSFlags & NVPTXII::IsSuldMask) {
79 unsigned VecSize =
80 1 << (((MCID.TSFlags & NVPTXII::IsSuldMask) >> NVPTXII::IsSuldShift) -
81 1);
82
83 // For a surface load of vector size N, the Nth operand will be the surfref
84 MachineOperand &SurfHandle = MI.getOperand(i: VecSize);
85
86 return replaceImageHandle(Op&: SurfHandle, MF);
87 }
88 if (MCID.TSFlags & NVPTXII::IsSustFlag) {
89 // This is a surface store, so operand 0 is a surfref
90 MachineOperand &SurfHandle = MI.getOperand(i: 0);
91
92 return replaceImageHandle(Op&: SurfHandle, MF);
93 }
94 if (MCID.TSFlags & NVPTXII::IsSurfTexQueryFlag) {
95 // This is a query, so operand 1 is a surfref/texref
96 MachineOperand &Handle = MI.getOperand(i: 1);
97
98 return replaceImageHandle(Op&: Handle, MF);
99 }
100
101 return false;
102}
103
104bool NVPTXReplaceImageHandles::replaceImageHandle(MachineOperand &Op,
105 MachineFunction &MF,
106 bool IsTexture) {
107 const MachineRegisterInfo &MRI = MF.getRegInfo();
108 NVPTXMachineFunctionInfo *MFI = MF.getInfo<NVPTXMachineFunctionInfo>();
109
110 assert(Op.isReg() && "Handle is not in a reg?");
111
112 // Which instruction defines the handle?
113 MachineInstr &TexHandleDef = *MRI.getVRegDef(Reg: Op.getReg());
114
115 switch (TexHandleDef.getOpcode()) {
116 case NVPTX::LD_i64: {
117 // The handle is a parameter value being loaded, replace with the
118 // parameter symbol
119 const auto &TM = static_cast<const NVPTXTargetMachine &>(MF.getTarget());
120 if (TM.getDrvInterface() == NVPTX::CUDA)
121 // For CUDA, we preserve the param loads coming from function arguments
122 return false;
123
124 assert(TexHandleDef.getOperand(7).isMCSymbol() && "Load is not a symbol!");
125 MCSymbol *Sym = TexHandleDef.getOperand(i: 7).getMCSymbol();
126 InstrsToRemove.insert(V: &TexHandleDef);
127 Op.ChangeToMCSymbol(Sym);
128 MFI->addImageHandleSymbol(Symbol: Sym);
129 return true;
130 }
131 case NVPTX::texsurf_handles: {
132 // The handle is a global variable, replace with the global variable name
133 assert(TexHandleDef.getOperand(1).isGlobal() && "Load is not a global!");
134 const GlobalValue *GV = TexHandleDef.getOperand(i: 1).getGlobal();
135 assert(GV->hasName() && "Global sampler must be named!");
136 InstrsToRemove.insert(V: &TexHandleDef);
137 Op.ChangeToGA(GV, Offset: 0);
138 return true;
139 }
140 case NVPTX::nvvm_move_i64:
141 case TargetOpcode::COPY: {
142 bool Res = replaceImageHandle(Op&: TexHandleDef.getOperand(i: 1), MF, IsTexture);
143 if (Res)
144 InstrsToRemove.insert(V: &TexHandleDef);
145 return Res;
146 }
147 default: {
148 // The handle is computed at runtime (e.g. returned by a call or merged by
149 // a phi), so there is no symbol to replace it with. Leave it in a register
150 // where that is supported.
151 const Function &F = MF.getFunction();
152 const DebugLoc &DL = Op.getParent()->getDebugLoc();
153 if (MF.getSubtarget<NVPTXSubtarget>().getSmVersion() < 30) {
154 // Fermi has no bindless textures or surfaces.
155 F.getContext().diagnose(DI: DiagnosticInfoUnsupported(
156 F, "image handle must be a kernel parameter or a global on sm_2x",
157 DL));
158 } else if (IsTexture &&
159 static_cast<const NVPTXTargetMachine &>(MF.getTarget())
160 .getDrvInterface() == NVPTX::NVCL) {
161 // PTX only allows texture and sampler handles in registers in unified
162 // texture mode.
163 F.getContext().diagnose(DI: DiagnosticInfoUnsupported(
164 F,
165 "texture or sampler handle must be a kernel parameter or a global "
166 "in independent texture mode",
167 DL));
168 }
169 return false;
170 }
171 }
172}
173
174namespace {
175class NVPTXReplaceImageHandlesLegacyPass : public MachineFunctionPass {
176public:
177 static char ID;
178 NVPTXReplaceImageHandlesLegacyPass() : MachineFunctionPass(ID) {}
179
180 bool runOnMachineFunction(MachineFunction &MF) override {
181 return NVPTXReplaceImageHandles().run(MF);
182 }
183
184 StringRef getPassName() const override {
185 return "NVPTX Replace Image Handles";
186 }
187
188 void getAnalysisUsage(AnalysisUsage &AU) const override {
189 AU.setPreservesCFG();
190 MachineFunctionPass::getAnalysisUsage(AU);
191 }
192};
193} // namespace
194
195char NVPTXReplaceImageHandlesLegacyPass::ID = 0;
196
197MachineFunctionPass *llvm::createNVPTXReplaceImageHandlesLegacyPass() {
198 return new NVPTXReplaceImageHandlesLegacyPass();
199}
200
201PreservedAnalyses
202NVPTXReplaceImageHandlesPass::run(MachineFunction &MF,
203 MachineFunctionAnalysisManager &MFAM) {
204 if (!NVPTXReplaceImageHandles().run(MF))
205 return PreservedAnalyses::all();
206 return getMachineFunctionPassPreservedAnalyses().preserveSet<CFGAnalyses>();
207}
208