1//===-- NVPTXReplaceImageHandles.cpp - Replace image handles for Fermi ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// On Fermi, image handles are not supported. To work around this, we traverse
10// the machine code and replace image handles with concrete symbols. For this
11// to work reliably, inlining of all function call must be performed.
12//
13//===----------------------------------------------------------------------===//
14#include "MCTargetDesc/NVPTXBaseInfo.h"
15#include "NVPTX.h"
16#include "NVPTXMachineFunctionInfo.h"
17#include "NVPTXSubtarget.h"
18#include "NVPTXTargetMachine.h"
19#include "llvm/ADT/DenseSet.h"
20#include "llvm/CodeGen/MachineFunction.h"
21#include "llvm/CodeGen/MachineFunctionPass.h"
22#include "llvm/CodeGen/MachineRegisterInfo.h"
23
24using namespace llvm;
25
26namespace {
27class NVPTXReplaceImageHandles {
28 DenseSet<MachineInstr *> InstrsToRemove;
29
30public:
31 bool run(MachineFunction &MF);
32
33private:
34 bool processInstr(MachineInstr &MI);
35 bool replaceImageHandle(MachineOperand &Op, MachineFunction &MF);
36};
37} // namespace
38
39bool NVPTXReplaceImageHandles::run(MachineFunction &MF) {
40 bool Changed = false;
41 InstrsToRemove.clear();
42
43 for (MachineBasicBlock &MBB : MF)
44 for (MachineInstr &MI : MBB)
45 Changed |= processInstr(MI);
46
47 // Now clean up any handle-access instructions
48 // This is needed in debug mode when code cleanup passes are not executed,
49 // but we need the handle access to be eliminated because they are not
50 // valid instructions when image handles are disabled.
51 for (MachineInstr *MI : InstrsToRemove) {
52 unsigned DefReg = MI->getOperand(i: 0).getReg();
53 // Only these that are not used can be removed.
54 if (MF.getRegInfo().use_nodbg_empty(RegNo: DefReg))
55 MI->eraseFromParent();
56 }
57 return Changed;
58}
59
60bool NVPTXReplaceImageHandles::processInstr(MachineInstr &MI) {
61 MachineFunction &MF = *MI.getParent()->getParent();
62 const MCInstrDesc &MCID = MI.getDesc();
63
64 if (MCID.TSFlags & NVPTXII::IsTexFlag) {
65 // This is a texture fetch, so operand 4 is a texref and operand 5 is
66 // a samplerref
67 MachineOperand &TexHandle = MI.getOperand(i: 4);
68 bool Changed = replaceImageHandle(Op&: TexHandle, MF);
69
70 if (!(MCID.TSFlags & NVPTXII::IsTexModeUnifiedFlag)) {
71 MachineOperand &SampHandle = MI.getOperand(i: 5);
72 Changed |= replaceImageHandle(Op&: SampHandle, MF);
73 }
74
75 return Changed;
76 }
77 if (MCID.TSFlags & NVPTXII::IsSuldMask) {
78 unsigned VecSize =
79 1 << (((MCID.TSFlags & NVPTXII::IsSuldMask) >> NVPTXII::IsSuldShift) -
80 1);
81
82 // For a surface load of vector size N, the Nth operand will be the surfref
83 MachineOperand &SurfHandle = MI.getOperand(i: VecSize);
84
85 return replaceImageHandle(Op&: SurfHandle, MF);
86 }
87 if (MCID.TSFlags & NVPTXII::IsSustFlag) {
88 // This is a surface store, so operand 0 is a surfref
89 MachineOperand &SurfHandle = MI.getOperand(i: 0);
90
91 return replaceImageHandle(Op&: SurfHandle, MF);
92 }
93 if (MCID.TSFlags & NVPTXII::IsSurfTexQueryFlag) {
94 // This is a query, so operand 1 is a surfref/texref
95 MachineOperand &Handle = MI.getOperand(i: 1);
96
97 return replaceImageHandle(Op&: Handle, MF);
98 }
99
100 return false;
101}
102
103bool NVPTXReplaceImageHandles::replaceImageHandle(MachineOperand &Op,
104 MachineFunction &MF) {
105 const MachineRegisterInfo &MRI = MF.getRegInfo();
106 NVPTXMachineFunctionInfo *MFI = MF.getInfo<NVPTXMachineFunctionInfo>();
107
108 assert(Op.isReg() && "Handle is not in a reg?");
109
110 // Which instruction defines the handle?
111 MachineInstr &TexHandleDef = *MRI.getVRegDef(Reg: Op.getReg());
112
113 switch (TexHandleDef.getOpcode()) {
114 case NVPTX::LD_i64: {
115 // The handle is a parameter value being loaded, replace with the
116 // parameter symbol
117 const auto &TM = static_cast<const NVPTXTargetMachine &>(MF.getTarget());
118 if (TM.getDrvInterface() == NVPTX::CUDA)
119 // For CUDA, we preserve the param loads coming from function arguments
120 return false;
121
122 assert(TexHandleDef.getOperand(7).isMCSymbol() && "Load is not a symbol!");
123 MCSymbol *Sym = TexHandleDef.getOperand(i: 7).getMCSymbol();
124 InstrsToRemove.insert(V: &TexHandleDef);
125 Op.ChangeToMCSymbol(Sym);
126 MFI->addImageHandleSymbol(Symbol: Sym);
127 return true;
128 }
129 case NVPTX::texsurf_handles: {
130 // The handle is a global variable, replace with the global variable name
131 assert(TexHandleDef.getOperand(1).isGlobal() && "Load is not a global!");
132 const GlobalValue *GV = TexHandleDef.getOperand(i: 1).getGlobal();
133 assert(GV->hasName() && "Global sampler must be named!");
134 InstrsToRemove.insert(V: &TexHandleDef);
135 Op.ChangeToGA(GV, Offset: 0);
136 return true;
137 }
138 case NVPTX::nvvm_move_i64:
139 case TargetOpcode::COPY: {
140 bool Res = replaceImageHandle(Op&: TexHandleDef.getOperand(i: 1), MF);
141 if (Res)
142 InstrsToRemove.insert(V: &TexHandleDef);
143 return Res;
144 }
145 default:
146 llvm_unreachable("Unknown instruction operating on handle");
147 }
148}
149
150namespace {
151class NVPTXReplaceImageHandlesLegacyPass : public MachineFunctionPass {
152public:
153 static char ID;
154 NVPTXReplaceImageHandlesLegacyPass() : MachineFunctionPass(ID) {}
155
156 bool runOnMachineFunction(MachineFunction &MF) override {
157 return NVPTXReplaceImageHandles().run(MF);
158 }
159
160 StringRef getPassName() const override {
161 return "NVPTX Replace Image Handles";
162 }
163
164 void getAnalysisUsage(AnalysisUsage &AU) const override {
165 AU.setPreservesCFG();
166 MachineFunctionPass::getAnalysisUsage(AU);
167 }
168};
169} // namespace
170
171char NVPTXReplaceImageHandlesLegacyPass::ID = 0;
172
173MachineFunctionPass *llvm::createNVPTXReplaceImageHandlesLegacyPass() {
174 return new NVPTXReplaceImageHandlesLegacyPass();
175}
176
177PreservedAnalyses
178NVPTXReplaceImageHandlesPass::run(MachineFunction &MF,
179 MachineFunctionAnalysisManager &MFAM) {
180 if (!NVPTXReplaceImageHandles().run(MF))
181 return PreservedAnalyses::all();
182 return getMachineFunctionPassPreservedAnalyses().preserveSet<CFGAnalyses>();
183}
184