| 1 | //===- ACCRoutineToGPUFunc.cpp - Move ACC routines to GPU module ----------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // The OpenACC `routine` directive defines functions that may be invoked from |
| 10 | // device code. Those functions need to be available in the device compilation |
| 11 | // unit. This pass moves materialized acc routines into the GPU module as |
| 12 | // gpu.func operations so they can be compiled for the device. |
| 13 | // |
| 14 | // Overview: |
| 15 | // --------- |
| 16 | // For each acc.routine that is not bound by name, the corresponding |
| 17 | // specialized function (created by ACCRoutineLowering) or the original |
| 18 | // host function (in case of seq) is cloned into theGPU module as a gpu.func. |
| 19 | // Callees referenced from those routines are processed: device-valid callees |
| 20 | // (runtime, intrinsics, other acc routines) are added to the GPU module as |
| 21 | // declarations or full clones as needed. Bind-name routines are not moved; |
| 22 | // their acc.routine ops are erased. After cloning, the host copies of |
| 23 | // specialized device functions and nohost routines are removed. |
| 24 | // |
| 25 | // Approach: |
| 26 | // ---------------- |
| 27 | // 1. Collect materialized routines (acc.routine without bind(name)); record |
| 28 | // bind-name routines for erasure. Emit remarks for materialized routines. |
| 29 | // |
| 30 | // 2. Process calls: walk each materialized function; for each call, if the |
| 31 | // callee is already in the GPU module or is an acc routine (or specialized |
| 32 | // acc routine), skip; otherwise require OpenACCSupport::isValidSymbolUse. |
| 33 | // Valid callees are added to the clone set (as declaration or full clone). |
| 34 | // |
| 35 | // 3. Clone into GPU module: each function in the clone set is turned into a |
| 36 | // gpu.func (body cloned or declaration only). acc.specialized_routine is |
| 37 | // preserved and symbol uses are updated so the routine name is unchanged. |
| 38 | // |
| 39 | // 4. Cleanup: erase from the host module the specialized device function |
| 40 | // bodies and any nohost routine (host copy removed after move to device). |
| 41 | // |
| 42 | // Example: |
| 43 | // -------- |
| 44 | // Before (after ACCRoutineLowering): |
| 45 | // acc.routine @r_seq func(@foo) seq |
| 46 | // func.func @foo() attributes {acc.specialized_routine = ...} { ... } |
| 47 | // |
| 48 | // After: |
| 49 | // acc.routine @r_seq func(@foo) seq |
| 50 | // gpu.module @acc_gpu_module { |
| 51 | // gpu.func @foo() attributes {acc.specialized_routine = ...} { ... } |
| 52 | // } |
| 53 | // (host @foo erased) |
| 54 | // |
| 55 | // Requirements: |
| 56 | // ------------- |
| 57 | // - Must run after `ACCRoutineLowering` pass which ensures variants for all |
| 58 | // levels of parallelism are created. |
| 59 | // - Uses OpenACCSupport: getOrCreateGPUModule, isValidSymbolUse, emitRemark, |
| 60 | // emitNYI. If no custom implementation is registered, the default is used. |
| 61 | // |
| 62 | //===----------------------------------------------------------------------===// |
| 63 | |
| 64 | #include "mlir/Dialect/OpenACC/Extensions/AllExtensions.h" |
| 65 | #include "mlir/Dialect/OpenACC/Transforms/Passes.h" |
| 66 | |
| 67 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 68 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 69 | #include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h" |
| 70 | #include "mlir/Dialect/OpenACC/OpenACC.h" |
| 71 | #include "mlir/IR/AttrTypeSubElements.h" |
| 72 | #include "mlir/IR/BuiltinAttributes.h" |
| 73 | #include "mlir/IR/IRMapping.h" |
| 74 | #include "mlir/IR/SymbolTable.h" |
| 75 | #include "mlir/Interfaces/CallInterfaces.h" |
| 76 | #include "llvm/ADT/DenseMap.h" |
| 77 | #include "llvm/ADT/SetVector.h" |
| 78 | #include <string> |
| 79 | |
| 80 | namespace mlir { |
| 81 | namespace acc { |
| 82 | #define GEN_PASS_DEF_ACCROUTINETOGPUFUNC |
| 83 | #include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc" |
| 84 | } // namespace acc |
| 85 | } // namespace mlir |
| 86 | |
| 87 | #define DEBUG_TYPE "acc-routine-to-gpu-func" |
| 88 | |
| 89 | using namespace mlir; |
| 90 | using namespace mlir::acc; |
| 91 | |
| 92 | namespace { |
| 93 | /// Create a gpu.func from a func.func by cloning the body. |
| 94 | static gpu::GPUFuncOp createGPUFuncFromFunc(OpBuilder &builder, |
| 95 | func::FuncOp sourceFunc) { |
| 96 | Location loc = sourceFunc.getLoc(); |
| 97 | StringRef name = sourceFunc.getName(); |
| 98 | FunctionType type = sourceFunc.getFunctionType(); |
| 99 | // Do not copy any attributes from the source; specialized_routine is set |
| 100 | // later when applicable. |
| 101 | gpu::GPUFuncOp gpuFunc = |
| 102 | gpu::GPUFuncOp::create(builder, location: loc, name, type, |
| 103 | /*workgroupAttributions=*/TypeRange(), |
| 104 | /*privateAttributions=*/TypeRange(), /*attrs=*/{}); |
| 105 | |
| 106 | Region &sourceBody = sourceFunc.getBody(); |
| 107 | Region &deviceBody = gpuFunc.getBody(); |
| 108 | Block &deviceEntryBlock = deviceBody.front(); |
| 109 | |
| 110 | // Map source block arguments to the GPU func's entry block arguments (which |
| 111 | // GPUFuncOp::create already created). |
| 112 | IRMapping mapping; |
| 113 | Block &sourceEntryBlock = sourceBody.front(); |
| 114 | for (auto [srcArg, destArg] : llvm::zip(t: sourceEntryBlock.getArguments(), |
| 115 | u: deviceEntryBlock.getArguments())) |
| 116 | mapping.map(from: srcArg, to: destArg); |
| 117 | |
| 118 | sourceBody.cloneInto(dest: &deviceBody, mapper&: mapping); |
| 119 | |
| 120 | // Replace func.return with gpu.return in the cloned blocks. |
| 121 | gpuFunc.walk(callback: [](func::ReturnOp op) { |
| 122 | OpBuilder replacer(op); |
| 123 | gpu::ReturnOp gpuReturn = gpu::ReturnOp::create(builder&: replacer, location: op.getLoc()); |
| 124 | gpuReturn->setOperands(op.getOperands()); |
| 125 | op.erase(); |
| 126 | }); |
| 127 | |
| 128 | // Splice the cloned entry block's operations into the GPU func's entry block |
| 129 | // (cloneInto created a separate block for the cloned content), then remove |
| 130 | // the now-empty cloned block. |
| 131 | Block *clonedSourceEntry = mapping.lookup(from: &sourceEntryBlock); |
| 132 | deviceEntryBlock.getOperations().splice( |
| 133 | where: deviceEntryBlock.getOperations().end(), |
| 134 | L2&: clonedSourceEntry->getOperations()); |
| 135 | clonedSourceEntry->erase(); |
| 136 | |
| 137 | return gpuFunc; |
| 138 | } |
| 139 | |
| 140 | using CloneCandidate = std::pair<func::FuncOp, RoutineOp>; |
| 141 | |
| 142 | /// Collect materialized and bind routines; fill candidate func names and |
| 143 | /// materialized routine set. Emit remarks for materialized routines. |
| 144 | static void collectRoutineCandidates( |
| 145 | ModuleOp mod, SymbolTable &symTab, acc::DeviceType deviceType, |
| 146 | OpenACCSupport &accSupport, |
| 147 | llvm::SmallSetVector<llvm::StringRef, 4> &funcsToCloneCandidates, |
| 148 | llvm::SmallSetVector<RoutineOp, 4> &materializedAccRoutines, |
| 149 | llvm::SmallSetVector<RoutineOp, 4> &bindAccRoutines) { |
| 150 | auto isParallelRoutine = [deviceType](RoutineOp routineOp) { |
| 151 | return routineOp.hasGang(deviceType) || routineOp.hasGang() || |
| 152 | routineOp.hasWorker(deviceType) || routineOp.hasWorker() || |
| 153 | routineOp.hasVector(deviceType) || routineOp.hasVector() || |
| 154 | routineOp.getGangDimValue(deviceType) || routineOp.getGangDimValue(); |
| 155 | }; |
| 156 | |
| 157 | mod.walk(callback: [&](RoutineOp op) { |
| 158 | if (op.getEffectiveBindName(targetDeviceType: deviceType)) { |
| 159 | bindAccRoutines.insert(X: op); |
| 160 | return; |
| 161 | } |
| 162 | func::FuncOp callee = |
| 163 | symTab.lookup<func::FuncOp>(name: op.getFuncName().getLeafReference()); |
| 164 | accSupport.emitRemark( |
| 165 | op: callee ? callee.getOperation() : op.getOperation(), |
| 166 | messageFn: [&op, &isParallelRoutine]() { |
| 167 | std::string msg = "Generating" ; |
| 168 | if (op.getImplicitAttr()) |
| 169 | msg += " implicit" ; |
| 170 | msg += " acc routine" ; |
| 171 | if (!isParallelRoutine(op)) |
| 172 | msg += " seq" ; |
| 173 | return msg; |
| 174 | }, |
| 175 | DEBUG_TYPE); |
| 176 | funcsToCloneCandidates.insert(X: op.getFuncName().getLeafReference()); |
| 177 | materializedAccRoutines.insert(X: op); |
| 178 | }); |
| 179 | } |
| 180 | |
| 181 | /// Process calls in ACC routines: add valid callees to funcsToClone (for |
| 182 | /// declaration or clone). Returns failure() if any call is unsupported. |
| 183 | static LogicalResult processCallsInRoutines( |
| 184 | SymbolTable &symTab, SymbolTable &gpuSymTab, OpenACCSupport &accSupport, |
| 185 | const llvm::SmallSetVector<llvm::StringRef, 4> &funcsToCloneCandidates, |
| 186 | const llvm::SmallSetVector<RoutineOp, 4> &materializedAccRoutines, |
| 187 | llvm::SmallSetVector<CloneCandidate, 4> &funcsToClone) { |
| 188 | LogicalResult callCheckResult = success(); |
| 189 | auto processCalls = [&](CallOpInterface callOp) { |
| 190 | if (!callOp.getCallableForCallee()) |
| 191 | return; |
| 192 | auto calleeSymbolRef = |
| 193 | dyn_cast<SymbolRefAttr>(Val: callOp.getCallableForCallee()); |
| 194 | if (!calleeSymbolRef) |
| 195 | return; |
| 196 | |
| 197 | auto callee = |
| 198 | symTab.lookup<func::FuncOp>(name: calleeSymbolRef.getLeafReference()); |
| 199 | if (!callee) |
| 200 | return; |
| 201 | |
| 202 | if (gpuSymTab.lookup(name: callee.getName())) |
| 203 | return; |
| 204 | if (isAccRoutine(op: callee) || isSpecializedAccRoutine(op: callee)) |
| 205 | return; |
| 206 | |
| 207 | if (!accSupport.isValidSymbolUse(user: callOp.getOperation(), symbol: calleeSymbolRef)) { |
| 208 | accSupport.emitNYI(loc: callOp->getLoc(), message: "Unsupported call in acc routine" ); |
| 209 | callCheckResult = failure(); |
| 210 | return; |
| 211 | } |
| 212 | funcsToClone.insert(X: {callee, RoutineOp{}}); |
| 213 | }; |
| 214 | |
| 215 | for (auto [funcName, accRoutine] : |
| 216 | llvm::zip(t: funcsToCloneCandidates, u: materializedAccRoutines)) { |
| 217 | func::FuncOp func = symTab.lookup<func::FuncOp>(name: funcName); |
| 218 | if (!func) |
| 219 | continue; |
| 220 | if (!gpuSymTab.lookup(name: funcName)) |
| 221 | funcsToClone.insert(X: {func, accRoutine}); |
| 222 | func.walk(callback: [&](CallOpInterface callOp) { processCalls(callOp); }); |
| 223 | if (failed(Result: callCheckResult)) |
| 224 | return failure(); |
| 225 | } |
| 226 | return success(); |
| 227 | } |
| 228 | |
| 229 | /// Rewrite symbol uses of uniqued specialized names back to the original |
| 230 | /// routine names. One walk of the module body instead of |
| 231 | /// replaceAllSymbolUses per routine (which walks all of the IR each time). |
| 232 | static void |
| 233 | rewriteSpecializedSymbolUses(ModuleOp mod, |
| 234 | const DenseMap<StringAttr, StringAttr> &renames) { |
| 235 | if (renames.empty()) |
| 236 | return; |
| 237 | |
| 238 | AttrTypeReplacer replacer; |
| 239 | replacer.addReplacement( |
| 240 | callback: [&](SymbolRefAttr attr) -> std::pair<Attribute, WalkResult> { |
| 241 | auto it = renames.find(Val: attr.getRootReference()); |
| 242 | if (it == renames.end()) |
| 243 | return {attr, WalkResult::skip()}; |
| 244 | if (isa<FlatSymbolRefAttr>(Val: attr)) |
| 245 | return {FlatSymbolRefAttr::get(value: it->second), WalkResult::skip()}; |
| 246 | return {SymbolRefAttr::get(rootReference: it->second, nestedReferences: attr.getNestedReferences()), |
| 247 | WalkResult::skip()}; |
| 248 | }); |
| 249 | |
| 250 | // Match SymbolTable::replaceAllSymbolUses: walk the module body and do |
| 251 | // not enter nested symbol tables (e.g. gpu.module). |
| 252 | for (Region ®ion : mod->getRegions()) { |
| 253 | region.walk(callback: [&](Operation *op) { |
| 254 | if (op->hasTrait<OpTrait::SymbolTable>()) |
| 255 | return WalkResult::skip(); |
| 256 | replacer.replaceElementsIn(op); |
| 257 | return WalkResult::advance(); |
| 258 | }); |
| 259 | } |
| 260 | } |
| 261 | |
| 262 | /// Clone each function in funcsToClone into the GPU module (declaration or |
| 263 | /// full body). Fix up symbol names and specialized_routine attr for ACC |
| 264 | /// routines. |
| 265 | static LogicalResult cloneFuncsToGPUModule( |
| 266 | ModuleOp mod, SymbolTable &gpuSymTab, |
| 267 | const llvm::SmallSetVector<CloneCandidate, 4> &funcsToClone) { |
| 268 | OpBuilder builder(mod.getContext()); |
| 269 | |
| 270 | // ACCRoutineLowering inserts the device copy next to the host function, so |
| 271 | // the symbol table uniqued its name while acc.specialized_routine still |
| 272 | // holds the original one. Collect all of those renames and apply them in a |
| 273 | // single walk below. |
| 274 | DenseMap<StringAttr, StringAttr> specializedRenames; |
| 275 | for (CloneCandidate candidate : funcsToClone) { |
| 276 | func::FuncOp srcFunc = candidate.first; |
| 277 | if (srcFunc.isDeclaration()) |
| 278 | continue; |
| 279 | if (auto specAttr = |
| 280 | srcFunc->getDiscardableAttrOfType<SpecializedRoutineAttr>( |
| 281 | name: getSpecializedRoutineAttrName())) { |
| 282 | StringAttr destName = specAttr.getFuncName(); |
| 283 | if (srcFunc.getNameAttr() != destName) |
| 284 | specializedRenames.try_emplace(Key: srcFunc.getNameAttr(), Args&: destName); |
| 285 | } |
| 286 | } |
| 287 | rewriteSpecializedSymbolUses(mod, renames: specializedRenames); |
| 288 | |
| 289 | for (CloneCandidate candidate : funcsToClone) { |
| 290 | func::FuncOp srcFunc = candidate.first; |
| 291 | |
| 292 | if (srcFunc.isDeclaration()) { |
| 293 | Operation *cloned = srcFunc->clone(); |
| 294 | gpuSymTab.insert(symbol: cloned); |
| 295 | continue; |
| 296 | } |
| 297 | |
| 298 | gpu::GPUFuncOp deviceFuncOp = createGPUFuncFromFunc(builder, sourceFunc: srcFunc); |
| 299 | |
| 300 | if (auto specAttr = |
| 301 | srcFunc->getDiscardableAttrOfType<SpecializedRoutineAttr>( |
| 302 | name: getSpecializedRoutineAttrName())) { |
| 303 | deviceFuncOp.setName(specAttr.getFuncName()); |
| 304 | deviceFuncOp->setDiscardableAttr(name: getSpecializedRoutineAttrName(), |
| 305 | value: specAttr); |
| 306 | } |
| 307 | |
| 308 | gpuSymTab.insert(symbol: deviceFuncOp); |
| 309 | } |
| 310 | return success(); |
| 311 | } |
| 312 | |
| 313 | /// Remove specialized device copies and nohost routines from the host module. |
| 314 | static void |
| 315 | cleanupHostModule(const llvm::SmallSetVector<CloneCandidate, 4> &funcsToClone) { |
| 316 | for (CloneCandidate candidate : funcsToClone) { |
| 317 | func::FuncOp funcCandidate = candidate.first; |
| 318 | RoutineOp routineCandidate = candidate.second; |
| 319 | if ((routineCandidate && routineCandidate.getNohost()) || |
| 320 | acc::isSpecializedAccRoutine(op: funcCandidate)) |
| 321 | funcCandidate.erase(); |
| 322 | } |
| 323 | } |
| 324 | |
| 325 | class ACCRoutineToGPUFunc |
| 326 | : public acc::impl::ACCRoutineToGPUFuncBase<ACCRoutineToGPUFunc> { |
| 327 | public: |
| 328 | using acc::impl::ACCRoutineToGPUFuncBase< |
| 329 | ACCRoutineToGPUFunc>::ACCRoutineToGPUFuncBase; |
| 330 | |
| 331 | void getDependentDialects(DialectRegistry ®istry) const override { |
| 332 | acc::impl::ACCRoutineToGPUFuncBase< |
| 333 | ACCRoutineToGPUFunc>::getDependentDialects(registry); |
| 334 | acc::registerAllExtensions(registry); |
| 335 | } |
| 336 | |
| 337 | void runOnOperation() override { |
| 338 | ModuleOp mod = getOperation(); |
| 339 | if (mod.getOps<RoutineOp>().empty()) { |
| 340 | LLVM_DEBUG(llvm::dbgs() |
| 341 | << "Skipping ACCRoutineToGPUFunc - no acc.routine ops\n" ); |
| 342 | return; |
| 343 | } |
| 344 | |
| 345 | OpenACCSupport &accSupport = getAnalysis<OpenACCSupport>(); |
| 346 | std::optional<gpu::GPUModuleOp> gpuModOpt = |
| 347 | accSupport.getOrCreateGPUModule(mod); |
| 348 | if (!gpuModOpt) { |
| 349 | accSupport.emitNYI(loc: mod.getLoc(), message: "Failed to create GPU module" ); |
| 350 | return signalPassFailure(); |
| 351 | } |
| 352 | gpu::GPUModuleOp gpuMod = *gpuModOpt; |
| 353 | |
| 354 | SymbolTable symTab(mod); |
| 355 | SymbolTable gpuSymTab(gpuMod); |
| 356 | |
| 357 | llvm::SmallSetVector<llvm::StringRef, 4> funcsToCloneCandidates; |
| 358 | llvm::SmallSetVector<RoutineOp, 4> materializedAccRoutines; |
| 359 | llvm::SmallSetVector<RoutineOp, 4> bindAccRoutines; |
| 360 | |
| 361 | collectRoutineCandidates(mod, symTab, deviceType: this->deviceType, accSupport, |
| 362 | funcsToCloneCandidates, materializedAccRoutines, |
| 363 | bindAccRoutines); |
| 364 | |
| 365 | llvm::SmallSetVector<CloneCandidate, 4> funcsToClone; |
| 366 | if (failed(Result: processCallsInRoutines(symTab, gpuSymTab, accSupport, |
| 367 | funcsToCloneCandidates, |
| 368 | materializedAccRoutines, funcsToClone))) |
| 369 | return signalPassFailure(); |
| 370 | |
| 371 | if (failed(Result: cloneFuncsToGPUModule(mod, gpuSymTab, funcsToClone))) |
| 372 | return signalPassFailure(); |
| 373 | |
| 374 | cleanupHostModule(funcsToClone); |
| 375 | } |
| 376 | }; |
| 377 | } // namespace |
| 378 | |