| 1 | //===- AMDGPURewriteOutArgumentsPass.cpp - Create struct returns ----------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | /// \file This pass attempts to replace out argument usage with a return of a |
| 10 | /// struct. |
| 11 | /// |
| 12 | /// We can support returning a lot of values directly in registers, but |
| 13 | /// idiomatic C code frequently uses a pointer argument to return a second value |
| 14 | /// rather than returning a struct by value. GPU stack access is also quite |
| 15 | /// painful, so we want to avoid that if possible. Passing a stack object |
| 16 | /// pointer to a function also requires an additional address expansion code |
| 17 | /// sequence to convert the pointer to be relative to the kernel's scratch wave |
| 18 | /// offset register since the callee doesn't know what stack frame the incoming |
| 19 | /// pointer is relative to. |
| 20 | /// |
| 21 | /// The goal is to try rewriting code that looks like this: |
| 22 | /// |
| 23 | /// int foo(int a, int b, int* out) { |
| 24 | /// *out = bar(); |
| 25 | /// return a + b; |
| 26 | /// } |
| 27 | /// |
| 28 | /// into something like this: |
| 29 | /// |
| 30 | /// std::pair<int, int> foo(int a, int b) { |
| 31 | /// return std::pair(a + b, bar()); |
| 32 | /// } |
| 33 | /// |
| 34 | /// Typically the incoming pointer is a simple alloca for a temporary variable |
| 35 | /// to use the API, which if replaced with a struct return will be easily SROA'd |
| 36 | /// out when the stub function we create is inlined |
| 37 | /// |
| 38 | /// This pass introduces the struct return, but leaves the unused pointer |
| 39 | /// arguments and introduces a new stub function calling the struct returning |
| 40 | /// body. DeadArgumentElimination should be run after this to clean these up. |
| 41 | // |
| 42 | //===----------------------------------------------------------------------===// |
| 43 | |
| 44 | #include "AMDGPU.h" |
| 45 | #include "Utils/AMDGPUBaseInfo.h" |
| 46 | #include "llvm/ADT/Statistic.h" |
| 47 | #include "llvm/Analysis/MemorySSA.h" |
| 48 | #include "llvm/Analysis/MemorySSAUpdater.h" |
| 49 | #include "llvm/IR/AttributeMask.h" |
| 50 | #include "llvm/IR/Attributes.h" |
| 51 | #include "llvm/IR/IRBuilder.h" |
| 52 | #include "llvm/IR/Instructions.h" |
| 53 | #include "llvm/InitializePasses.h" |
| 54 | #include "llvm/Pass.h" |
| 55 | #include "llvm/Support/Debug.h" |
| 56 | #include "llvm/Support/raw_ostream.h" |
| 57 | |
| 58 | #define DEBUG_TYPE "amdgpu-rewrite-out-arguments" |
| 59 | |
| 60 | using namespace llvm; |
| 61 | |
| 62 | STATISTIC(NumOutArgumentsReplaced, |
| 63 | "Number out arguments moved to struct return values" ); |
| 64 | STATISTIC(NumOutArgumentFunctionsReplaced, |
| 65 | "Number of functions with out arguments moved to struct return values" ); |
| 66 | |
| 67 | namespace { |
| 68 | |
| 69 | class AMDGPURewriteOutArguments : public FunctionPass { |
| 70 | private: |
| 71 | const AMDGPUOptions *CLOpts = nullptr; |
| 72 | const DataLayout *DL = nullptr; |
| 73 | MemorySSA *MSSA = nullptr; |
| 74 | MemorySSAUpdater *MSSAU = nullptr; |
| 75 | AAResults *AA = nullptr; |
| 76 | |
| 77 | Type *getStoredType(Value &Arg) const; |
| 78 | Type *getOutArgumentType(Argument &Arg) const; |
| 79 | |
| 80 | public: |
| 81 | static char ID; |
| 82 | |
| 83 | AMDGPURewriteOutArguments() : FunctionPass(ID) {} |
| 84 | |
| 85 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
| 86 | AU.addRequired<MemorySSAWrapperPass>(); |
| 87 | AU.addRequired<AAResultsWrapperPass>(); |
| 88 | FunctionPass::getAnalysisUsage(AU); |
| 89 | } |
| 90 | |
| 91 | bool doInitialization(Module &M) override; |
| 92 | bool runOnFunction(Function &F) override; |
| 93 | }; |
| 94 | |
| 95 | } // end anonymous namespace |
| 96 | |
| 97 | INITIALIZE_PASS_BEGIN(AMDGPURewriteOutArguments, DEBUG_TYPE, |
| 98 | "AMDGPU Rewrite Out Arguments" , false, false) |
| 99 | INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass) |
| 100 | INITIALIZE_PASS_END(AMDGPURewriteOutArguments, DEBUG_TYPE, |
| 101 | "AMDGPU Rewrite Out Arguments" , false, false) |
| 102 | |
| 103 | char AMDGPURewriteOutArguments::ID = 0; |
| 104 | |
| 105 | Type *AMDGPURewriteOutArguments::getStoredType(Value &Arg) const { |
| 106 | const int MaxUses = 10; |
| 107 | int UseCount = 0; |
| 108 | |
| 109 | SmallVector<Use *> Worklist(llvm::make_pointer_range(Range: Arg.uses())); |
| 110 | |
| 111 | Type *StoredType = nullptr; |
| 112 | while (!Worklist.empty()) { |
| 113 | Use *U = Worklist.pop_back_val(); |
| 114 | |
| 115 | if (auto *BCI = dyn_cast<BitCastInst>(Val: U->getUser())) { |
| 116 | for (Use &U : BCI->uses()) |
| 117 | Worklist.push_back(Elt: &U); |
| 118 | continue; |
| 119 | } |
| 120 | |
| 121 | if (auto *SI = dyn_cast<StoreInst>(Val: U->getUser())) { |
| 122 | if (UseCount++ > MaxUses) |
| 123 | return nullptr; |
| 124 | |
| 125 | if (!SI->isSimple() || |
| 126 | U->getOperandNo() != StoreInst::getPointerOperandIndex()) |
| 127 | return nullptr; |
| 128 | |
| 129 | if (StoredType && StoredType != SI->getValueOperand()->getType()) |
| 130 | return nullptr; // More than one type. |
| 131 | StoredType = SI->getValueOperand()->getType(); |
| 132 | continue; |
| 133 | } |
| 134 | |
| 135 | // Unsupported user. |
| 136 | return nullptr; |
| 137 | } |
| 138 | |
| 139 | return StoredType; |
| 140 | } |
| 141 | |
| 142 | Type *AMDGPURewriteOutArguments::getOutArgumentType(Argument &Arg) const { |
| 143 | const unsigned MaxOutArgSizeBytes = 4 * CLOpts->max_return_arg_num_regs; |
| 144 | PointerType *ArgTy = dyn_cast<PointerType>(Val: Arg.getType()); |
| 145 | |
| 146 | // TODO: It might be useful for any out arguments, not just privates. |
| 147 | if (!ArgTy || |
| 148 | (ArgTy->getAddressSpace() != DL->getAllocaAddrSpace() && |
| 149 | !CLOpts->any_address_space_out_arguments) || |
| 150 | Arg.hasByValAttr() || Arg.hasStructRetAttr()) { |
| 151 | return nullptr; |
| 152 | } |
| 153 | |
| 154 | Type *StoredType = getStoredType(Arg); |
| 155 | if (!StoredType || DL->getTypeStoreSize(Ty: StoredType) > MaxOutArgSizeBytes) |
| 156 | return nullptr; |
| 157 | |
| 158 | return StoredType; |
| 159 | } |
| 160 | |
| 161 | bool AMDGPURewriteOutArguments::doInitialization(Module &M) { |
| 162 | DL = &M.getDataLayout(); |
| 163 | return false; |
| 164 | } |
| 165 | |
| 166 | static StoreInst *findStoreForOutArgument(BasicBlock *BB, Argument *OutArg, |
| 167 | MemorySSA &MSSA, |
| 168 | BatchAAResults &BAA) { |
| 169 | MemoryLocation ArgLoc = MemoryLocation::getBeforeOrAfter(Ptr: OutArg); |
| 170 | const auto *Accesses = MSSA.getBlockAccesses(BB); |
| 171 | if (!Accesses) |
| 172 | return nullptr; |
| 173 | |
| 174 | for (const MemoryAccess &Access : reverse(C: *Accesses)) { |
| 175 | const auto *UseOrDef = dyn_cast<MemoryUseOrDef>(Val: &Access); |
| 176 | if (!UseOrDef) |
| 177 | continue; |
| 178 | |
| 179 | Instruction *I = UseOrDef->getMemoryInst(); |
| 180 | |
| 181 | // Return the must-alias store to the out argument. |
| 182 | if (auto *Store = dyn_cast<StoreInst>(Val: I)) |
| 183 | if (Store->getPointerOperand() == OutArg) |
| 184 | return Store; |
| 185 | |
| 186 | if (auto *FI = dyn_cast<FenceInst>(Val: I)) |
| 187 | if (FI->getOrdering() == AtomicOrdering::Release) |
| 188 | continue; |
| 189 | |
| 190 | if (auto *LI = dyn_cast<LoadInst>(Val: I)) { |
| 191 | if (LI->isAtomic()) { |
| 192 | // May-alias reads with monotonic ordering are ignored. |
| 193 | if (isStrongerThan(AO: LI->getOrdering(), Other: AtomicOrdering::Monotonic)) |
| 194 | return nullptr; |
| 195 | continue; |
| 196 | } |
| 197 | } |
| 198 | |
| 199 | // Any other memory access that may read or write the location prevents the |
| 200 | // rewrite. |
| 201 | if (isModOrRefSet(MRI: BAA.getModRefInfo(I, OptLoc: ArgLoc))) |
| 202 | return nullptr; |
| 203 | } |
| 204 | |
| 205 | return nullptr; |
| 206 | } |
| 207 | |
| 208 | bool AMDGPURewriteOutArguments::runOnFunction(Function &F) { |
| 209 | if (skipFunction(F)) |
| 210 | return false; |
| 211 | |
| 212 | CLOpts = &F.getContext().getLibraryOptions().get<AMDGPUOptions>(); |
| 213 | const unsigned MaxNumRetRegs = CLOpts->max_return_arg_num_regs; |
| 214 | |
| 215 | // TODO: Could probably handle variadic functions. |
| 216 | if (F.isVarArg() || F.hasStructRetAttr() || |
| 217 | AMDGPU::isEntryFunctionCC(CC: F.getCallingConv())) |
| 218 | return false; |
| 219 | |
| 220 | unsigned ReturnNumRegs = 0; |
| 221 | // Maps an out-argument number to its field index in the return struct. |
| 222 | // Fields are in processing order, which the retry loop below can reorder |
| 223 | // relative to argument order, so the index must be tracked, not recomputed. |
| 224 | SmallDenseMap<unsigned, unsigned, 4> OutArgIndexes; |
| 225 | SmallVector<Type *, 4> ReturnTypes; |
| 226 | Type *RetTy = F.getReturnType(); |
| 227 | if (!RetTy->isVoidTy()) { |
| 228 | ReturnNumRegs = DL->getTypeStoreSize(Ty: RetTy) / 4; |
| 229 | |
| 230 | if (ReturnNumRegs >= MaxNumRetRegs) |
| 231 | return false; |
| 232 | |
| 233 | ReturnTypes.push_back(Elt: RetTy); |
| 234 | } |
| 235 | |
| 236 | SmallVector<std::pair<Argument *, Type *>, 4> OutArgs; |
| 237 | for (Argument &Arg : F.args()) { |
| 238 | if (Type *Ty = getOutArgumentType(Arg)) { |
| 239 | LLVM_DEBUG(dbgs() << "Found possible out argument " << Arg |
| 240 | << " in function " << F.getName() << '\n'); |
| 241 | OutArgs.push_back(Elt: {&Arg, Ty}); |
| 242 | } |
| 243 | } |
| 244 | |
| 245 | if (OutArgs.empty()) |
| 246 | return false; |
| 247 | |
| 248 | using ReplacementVec = SmallVector<std::pair<Argument *, Value *>, 4>; |
| 249 | |
| 250 | DenseMap<ReturnInst *, ReplacementVec> Replacements; |
| 251 | |
| 252 | SmallVector<ReturnInst *, 4> Returns; |
| 253 | for (BasicBlock &BB : F) { |
| 254 | if (ReturnInst *RI = dyn_cast<ReturnInst>(Val: &BB.back())) |
| 255 | Returns.push_back(Elt: RI); |
| 256 | } |
| 257 | |
| 258 | if (Returns.empty()) |
| 259 | return false; |
| 260 | |
| 261 | AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); |
| 262 | MSSA = &getAnalysis<MemorySSAWrapperPass>().getMSSA(); |
| 263 | |
| 264 | BatchAAResults BatchAA(*AA); |
| 265 | MemorySSAUpdater MSSAUpdater(MSSA); |
| 266 | MSSAU = &MSSAUpdater; |
| 267 | |
| 268 | bool Changing; |
| 269 | |
| 270 | do { |
| 271 | Changing = false; |
| 272 | |
| 273 | // Keep retrying if we are able to successfully eliminate an argument. This |
| 274 | // helps with cases with multiple arguments which may alias, such as in a |
| 275 | // sincos implementation. With 2 stores to may-aliasing arguments, MDA |
| 276 | // returns the second store for the first argument too; the identity guard |
| 277 | // below rejects it, and a later iteration folds the first argument once the |
| 278 | // second store has been removed. |
| 279 | for (const auto &Pair : OutArgs) { |
| 280 | bool ThisReplaceable = true; |
| 281 | SmallVector<std::pair<ReturnInst *, StoreInst *>, 4> ReplaceableStores; |
| 282 | |
| 283 | Argument *OutArg = Pair.first; |
| 284 | Type *ArgTy = Pair.second; |
| 285 | |
| 286 | // Skip this argument if converting it will push us over the register |
| 287 | // count to return limit. |
| 288 | |
| 289 | // TODO: This is an approximation. When legalized this could be more. We |
| 290 | // can ask TLI for exactly how many. |
| 291 | unsigned ArgNumRegs = DL->getTypeStoreSize(Ty: ArgTy) / 4; |
| 292 | if (ArgNumRegs + ReturnNumRegs > MaxNumRetRegs) |
| 293 | continue; |
| 294 | |
| 295 | // An argument is convertible only if all exit blocks are able to replace |
| 296 | // it. |
| 297 | for (ReturnInst *RI : Returns) { |
| 298 | BasicBlock *BB = RI->getParent(); |
| 299 | |
| 300 | StoreInst *SI = findStoreForOutArgument(BB, OutArg, MSSA&: *MSSA, BAA&: BatchAA); |
| 301 | if (SI) { |
| 302 | LLVM_DEBUG(dbgs() << "Found out argument store: " << *SI << '\n'); |
| 303 | ReplaceableStores.emplace_back(Args&: RI, Args&: SI); |
| 304 | } else { |
| 305 | ThisReplaceable = false; |
| 306 | break; |
| 307 | } |
| 308 | } |
| 309 | |
| 310 | if (!ThisReplaceable) |
| 311 | continue; // Try the next argument candidate. |
| 312 | |
| 313 | for (std::pair<ReturnInst *, StoreInst *> Store : ReplaceableStores) { |
| 314 | Value *ReplVal = Store.second->getValueOperand(); |
| 315 | |
| 316 | auto &ValVec = Replacements[Store.first]; |
| 317 | if (llvm::is_contained(Range: llvm::make_first_range(c&: ValVec), Element: OutArg)) { |
| 318 | LLVM_DEBUG(dbgs() |
| 319 | << "Saw multiple out arg stores" << *OutArg << '\n'); |
| 320 | // It is possible to see stores to the same argument multiple times, |
| 321 | // but we expect these would have been optimized out already. |
| 322 | ThisReplaceable = false; |
| 323 | break; |
| 324 | } |
| 325 | |
| 326 | ValVec.emplace_back(Args&: OutArg, Args&: ReplVal); |
| 327 | MSSAU->removeMemoryAccess(I: Store.second); |
| 328 | Store.second->eraseFromParent(); |
| 329 | } |
| 330 | |
| 331 | if (ThisReplaceable) { |
| 332 | OutArgIndexes.insert(KV: {OutArg->getArgNo(), ReturnTypes.size()}); |
| 333 | ReturnTypes.push_back(Elt: ArgTy); |
| 334 | ++NumOutArgumentsReplaced; |
| 335 | Changing = true; |
| 336 | } |
| 337 | } |
| 338 | } while (Changing); |
| 339 | |
| 340 | if (Replacements.empty()) |
| 341 | return false; |
| 342 | |
| 343 | LLVMContext &Ctx = F.getContext(); |
| 344 | StructType *NewRetTy = StructType::create(Context&: Ctx, Elements: ReturnTypes, Name: F.getName()); |
| 345 | |
| 346 | FunctionType *NewFuncTy = FunctionType::get(Result: NewRetTy, |
| 347 | Params: F.getFunctionType()->params(), |
| 348 | isVarArg: F.isVarArg()); |
| 349 | |
| 350 | LLVM_DEBUG(dbgs() << "Computed new return type: " << *NewRetTy << '\n'); |
| 351 | |
| 352 | Function *NewFunc = Function::Create(Ty: NewFuncTy, Linkage: Function::PrivateLinkage, |
| 353 | N: F.getName() + ".body" ); |
| 354 | F.getParent()->getFunctionList().insert(where: F.getIterator(), New: NewFunc); |
| 355 | NewFunc->copyAttributesFrom(Src: &F); |
| 356 | NewFunc->setComdat(F.getComdat()); |
| 357 | |
| 358 | // We want to preserve the function and param attributes, but need to strip |
| 359 | // off any return attributes, e.g. zeroext doesn't make sense with a struct. |
| 360 | NewFunc->stealArgumentListFrom(Src&: F); |
| 361 | |
| 362 | NewFunc->removeRetAttrs(Attrs: AttributeFuncs::typeIncompatible( |
| 363 | Ty: NewRetTy, AS: NewFunc->getAttributes().getRetAttrs())); |
| 364 | // TODO: How to preserve metadata? |
| 365 | |
| 366 | // Move the body of the function into the new rewritten function, and replace |
| 367 | // this function with a stub. |
| 368 | NewFunc->splice(ToIt: NewFunc->begin(), FromF: &F); |
| 369 | |
| 370 | for (auto &Replacement : Replacements) { |
| 371 | ReturnInst *RI = Replacement.first; |
| 372 | IRBuilder<> B(RI); |
| 373 | B.SetCurrentDebugLocation(RI->getDebugLoc()); |
| 374 | |
| 375 | Value *NewRetVal = PoisonValue::get(T: NewRetTy); |
| 376 | |
| 377 | Value *RetVal = RI->getReturnValue(); |
| 378 | if (RetVal) |
| 379 | NewRetVal = B.CreateInsertValue(Agg: NewRetVal, Val: RetVal, Idxs: 0); |
| 380 | |
| 381 | // Use OutArgIndexes so body and stub agree on the field for each argument. |
| 382 | for (std::pair<Argument *, Value *> ReturnPoint : Replacement.second) { |
| 383 | unsigned FieldIdx = OutArgIndexes.lookup(Val: ReturnPoint.first->getArgNo()); |
| 384 | NewRetVal = B.CreateInsertValue(Agg: NewRetVal, Val: ReturnPoint.second, Idxs: FieldIdx); |
| 385 | } |
| 386 | |
| 387 | if (RetVal) |
| 388 | RI->setOperand(i_nocapture: 0, Val_nocapture: NewRetVal); |
| 389 | else { |
| 390 | B.CreateRet(V: NewRetVal); |
| 391 | RI->eraseFromParent(); |
| 392 | } |
| 393 | } |
| 394 | |
| 395 | SmallVector<Value *, 16> StubCallArgs; |
| 396 | for (Argument &Arg : F.args()) { |
| 397 | if (OutArgIndexes.count(Val: Arg.getArgNo())) { |
| 398 | // It's easier to preserve the type of the argument list. We rely on |
| 399 | // DeadArgumentElimination to take care of these. |
| 400 | StubCallArgs.push_back(Elt: PoisonValue::get(T: Arg.getType())); |
| 401 | } else { |
| 402 | StubCallArgs.push_back(Elt: &Arg); |
| 403 | } |
| 404 | } |
| 405 | |
| 406 | BasicBlock *StubBB = BasicBlock::Create(Context&: Ctx, Name: "" , Parent: &F); |
| 407 | IRBuilder<> B(StubBB); |
| 408 | CallInst *StubCall = B.CreateCall(Callee: NewFunc, Args: StubCallArgs); |
| 409 | |
| 410 | for (Argument &Arg : F.args()) { |
| 411 | auto It = OutArgIndexes.find(Val: Arg.getArgNo()); |
| 412 | if (It == OutArgIndexes.end()) |
| 413 | continue; |
| 414 | |
| 415 | unsigned FieldIdx = It->second; |
| 416 | Type *EltTy = NewRetTy->getElementType(N: FieldIdx); |
| 417 | const auto Align = |
| 418 | DL->getValueOrABITypeAlignment(Alignment: Arg.getParamAlign(), Ty: EltTy); |
| 419 | |
| 420 | Value *Val = B.CreateExtractValue(Agg: StubCall, Idxs: FieldIdx); |
| 421 | B.CreateAlignedStore(Val, Ptr: &Arg, Align); |
| 422 | } |
| 423 | |
| 424 | if (!RetTy->isVoidTy()) { |
| 425 | B.CreateRet(V: B.CreateExtractValue(Agg: StubCall, Idxs: 0)); |
| 426 | } else { |
| 427 | B.CreateRetVoid(); |
| 428 | } |
| 429 | |
| 430 | // The function is now a stub we want to inline. |
| 431 | F.addFnAttr(Kind: Attribute::AlwaysInline); |
| 432 | |
| 433 | ++NumOutArgumentFunctionsReplaced; |
| 434 | return true; |
| 435 | } |
| 436 | |