1//- NVPTXForwardParams.cpp - NVPTX Forward Device Params Removing Local Copy -//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// PTX supports 2 methods of accessing device function parameters:
10//
11// - "simple" case: If a parameters is only loaded, and all loads can address
12// the parameter via a constant offset, then the parameter may be loaded via
13// the ".param" address space. This case is not possible if the parameters
14// is stored to or has it's address taken. This method is preferable when
15// possible. Ex:
16//
17// ld.param.u32 %r1, [foo_param_1];
18// ld.param.u32 %r2, [foo_param_1+4];
19//
20// - "move param" case: For more complex cases the address of the param may be
21// placed in a register via a "mov" instruction. This "mov" also implicitly
22// moves the param to the ".local" address space and allows for it to be
23// written to. This essentially defers the responsibilty of the byval copy
24// to the PTX calling convention.
25//
26// mov.b64 %rd1, foo_param_0;
27// st.local.u32 [%rd1], 42;
28// add.u64 %rd3, %rd1, %rd2;
29// ld.local.u32 %r2, [%rd3];
30//
31// In NVPTXLowerArgs and SelectionDAG, we pessimistically assume that all
32// parameters will use the "move param" case and the local address space. This
33// pass is responsible for switching to the "simple" case when possible, as it
34// is more efficient.
35//
36// We do this by simply traversing uses of the param "mov" instructions an
37// trivially checking if they are all loads.
38//
39//===----------------------------------------------------------------------===//
40
41#include "NVPTX.h"
42#include "llvm/ADT/SmallVector.h"
43#include "llvm/CodeGen/MachineFunctionPass.h"
44#include "llvm/CodeGen/MachineInstr.h"
45#include "llvm/CodeGen/MachineOperand.h"
46#include "llvm/CodeGen/MachineRegisterInfo.h"
47#include "llvm/CodeGen/TargetRegisterInfo.h"
48#include "llvm/Support/ErrorHandling.h"
49
50using namespace llvm;
51
52static bool traverseMoveUse(MachineInstr &U, const MachineRegisterInfo &MRI,
53 SmallVectorImpl<MachineInstr *> &RemoveList,
54 SmallVectorImpl<MachineInstr *> &LoadInsts) {
55 switch (U.getOpcode()) {
56 case NVPTX::LD_i16:
57 case NVPTX::LD_i32:
58 case NVPTX::LD_i64:
59 case NVPTX::LDV_i16_v2:
60 case NVPTX::LDV_i16_v4:
61 case NVPTX::LDV_i32_v2:
62 case NVPTX::LDV_i32_v4:
63 case NVPTX::LDV_i64_v2:
64 case NVPTX::LDV_i64_v4: {
65 LoadInsts.push_back(Elt: &U);
66 return true;
67 }
68 case NVPTX::cvta_local_32:
69 case NVPTX::cvta_local_64:
70 case NVPTX::cvta_to_local_32:
71 case NVPTX::cvta_to_local_64: {
72 for (auto &U2 : MRI.use_instructions(Reg: U.operands_begin()->getReg()))
73 if (!traverseMoveUse(U&: U2, MRI, RemoveList, LoadInsts))
74 return false;
75
76 RemoveList.push_back(Elt: &U);
77 return true;
78 }
79 default:
80 return false;
81 }
82}
83
84static bool eliminateMove(MachineInstr &Mov, const MachineRegisterInfo &MRI,
85 SmallVectorImpl<MachineInstr *> &RemoveList) {
86 SmallVector<MachineInstr *, 16> MaybeRemoveList;
87 SmallVector<MachineInstr *, 16> LoadInsts;
88
89 for (auto &U : MRI.use_instructions(Reg: Mov.operands_begin()->getReg()))
90 if (!traverseMoveUse(U, MRI, RemoveList&: MaybeRemoveList, LoadInsts))
91 return false;
92
93 RemoveList.append(RHS: MaybeRemoveList);
94 RemoveList.push_back(Elt: &Mov);
95
96 const MachineOperand *ParamSymbol = Mov.uses().begin();
97 assert(ParamSymbol->isSymbol());
98
99 for (MachineInstr *LI : LoadInsts) {
100 unsigned Opc = LI->getOpcode();
101 int Idx = getNamedOperandIdx(Opcode: Opc, Name: NVPTX::OpName::addr);
102 assert(Idx != -1 && "no addr operand");
103 LI->getOperand(i: Idx).ChangeToES(SymName: ParamSymbol->getSymbolName());
104
105 Idx = getNamedOperandIdx(Opcode: Opc, Name: NVPTX::OpName::addsp);
106 assert(Idx != -1 && "no addsp operand");
107 LI->getOperand(i: Idx).ChangeToImmediate(ImmVal: NVPTX::AddressSpace::DeviceParam);
108 // PTX cache hints and policy are not allowed on ld.param
109 Idx = getNamedOperandIdx(Opcode: Opc, Name: NVPTX::OpName::evictionAndPrefetchHint);
110 assert(Idx != -1 && "no evictionAndPrefetchHint operand");
111 LI->getOperand(i: Idx).ChangeToImmediate(ImmVal: 0);
112
113 Idx = getNamedOperandIdx(Opcode: Opc, Name: NVPTX::OpName::policy);
114 assert(Idx != -1 && "no policy operand");
115 MachineOperand &Policy = LI->getOperand(i: Idx);
116 Register PolicyReg = Policy.getReg();
117 MachineInstr *PolicyDef =
118 PolicyReg.isValid() ? MRI.getVRegDef(Reg: PolicyReg) : nullptr;
119 Policy.ChangeToRegister(Reg: NVPTX::NoRegister, isDef: false);
120 // Remove the policy register's definition if it is now dead.
121 if (PolicyDef && PolicyDef->isDead(MRI))
122 RemoveList.push_back(Elt: PolicyDef);
123 }
124 return true;
125}
126
127static bool forwardDeviceParams(MachineFunction &MF) {
128 const auto &MRI = MF.getRegInfo();
129
130 bool Changed = false;
131 SmallVector<MachineInstr *, 16> RemoveList;
132 for (auto &MI : make_early_inc_range(Range&: *MF.begin()))
133 if (MI.getOpcode() == NVPTX::MOV32_PARAM ||
134 MI.getOpcode() == NVPTX::MOV64_PARAM)
135 Changed |= eliminateMove(Mov&: MI, MRI, RemoveList);
136
137 for (auto *MI : RemoveList)
138 MI->eraseFromParent();
139
140 return Changed;
141}
142
143/// ----------------------------------------------------------------------------
144/// Pass (Manager) Boilerplate
145/// ----------------------------------------------------------------------------
146
147namespace {
148struct NVPTXForwardParamsLegacyPass : public MachineFunctionPass {
149 static char ID;
150 NVPTXForwardParamsLegacyPass() : MachineFunctionPass(ID) {}
151
152 bool runOnMachineFunction(MachineFunction &MF) override {
153 return forwardDeviceParams(MF);
154 }
155
156 void getAnalysisUsage(AnalysisUsage &AU) const override {
157 MachineFunctionPass::getAnalysisUsage(AU);
158 }
159};
160} // namespace
161
162char NVPTXForwardParamsLegacyPass::ID = 0;
163
164INITIALIZE_PASS(NVPTXForwardParamsLegacyPass, "nvptx-forward-params",
165 "NVPTX Forward Params", false, false)
166
167MachineFunctionPass *llvm::createNVPTXForwardParamsLegacyPass() {
168 return new NVPTXForwardParamsLegacyPass();
169}
170
171PreservedAnalyses
172NVPTXForwardParamsPass::run(MachineFunction &MF,
173 MachineFunctionAnalysisManager &MFAM) {
174 if (!forwardDeviceParams(MF))
175 return PreservedAnalyses::all();
176 return getMachineFunctionPassPreservedAnalyses().preserveSet<CFGAnalyses>();
177}
178