| 1 | //===- ACCBindRoutine.cpp - OpenACC bind routine transform ---------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // The OpenACC `routine` directive may specify a `bind(name)` clause to |
| 10 | // associate the routine with a different symbol for device code. This pass |
| 11 | // finds calls inside offload regions that target such routines and rewrites the |
| 12 | // callee to the bound symbol. |
| 13 | // |
| 14 | // Overview: |
| 15 | // --------- |
| 16 | // For the current function, walk call operations inside offload regions, or |
| 17 | // gpu.func). If the callee is a function with an acc routine that has |
| 18 | // bind(name), replace the call to use the bound symbol. |
| 19 | // |
| 20 | // Requirements: |
| 21 | // ------------- |
| 22 | // - OffloadRegionOpInterface: the pass walks operations implementing this |
| 23 | // interface to discover offload regions (e.g. acc.compute_region) and |
| 24 | // rewrites calls inside their getOffloadRegion(). |
| 25 | // - CallOpInterface with working setCalleeFromCallable: call operations |
| 26 | // must implement CallOpInterface and setCalleeFromCallable so the pass |
| 27 | // can rewrite the callee to the symbol without invalidating the call. |
| 28 | // |
| 29 | //===----------------------------------------------------------------------===// |
| 30 | |
| 31 | #include "mlir/Dialect/OpenACC/Extensions/AllExtensions.h" |
| 32 | #include "mlir/Dialect/OpenACC/Transforms/Passes.h" |
| 33 | |
| 34 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 35 | #include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h" |
| 36 | #include "mlir/Dialect/OpenACC/OpenACC.h" |
| 37 | #include "mlir/IR/BuiltinAttributes.h" |
| 38 | #include "mlir/IR/SymbolTable.h" |
| 39 | #include "mlir/Interfaces/CallInterfaces.h" |
| 40 | #include "llvm/Support/Debug.h" |
| 41 | |
| 42 | namespace mlir { |
| 43 | namespace acc { |
| 44 | #define GEN_PASS_DEF_ACCBINDROUTINE |
| 45 | #define GEN_PASS_DEF_ACCMATERIALIZEROUTINEBINDTARGETS |
| 46 | #include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc" |
| 47 | } // namespace acc |
| 48 | } // namespace mlir |
| 49 | |
| 50 | #define DEBUG_TYPE "acc-bind-routine" |
| 51 | |
| 52 | using namespace mlir; |
| 53 | using namespace mlir::acc; |
| 54 | |
| 55 | namespace { |
| 56 | static RoutineOp getFirstAccRoutineOp(FunctionOpInterface funcOp, |
| 57 | const SymbolTable &fallbackSymTab) { |
| 58 | SymbolRefAttr routineRef; |
| 59 | if (isSpecializedAccRoutine(op: funcOp)) { |
| 60 | auto attr = funcOp->getDiscardableAttrOfType<SpecializedRoutineAttr>( |
| 61 | name: getSpecializedRoutineAttrName()); |
| 62 | routineRef = attr.getRoutine(); |
| 63 | } else { |
| 64 | auto routineInfo = funcOp->getDiscardableAttrOfType<RoutineInfoAttr>( |
| 65 | name: getRoutineInfoAttrName()); |
| 66 | assert(routineInfo && "expected acc.routine_info for acc routine function" ); |
| 67 | auto accRoutines = routineInfo.getAccRoutines(); |
| 68 | assert(!accRoutines.empty() && "expected at least one acc routine" ); |
| 69 | routineRef = accRoutines[0]; |
| 70 | } |
| 71 | |
| 72 | RoutineOp routine = |
| 73 | SymbolTable::lookupNearestSymbolFrom<RoutineOp>(from: funcOp, symbol: routineRef); |
| 74 | if (routine) |
| 75 | return routine; |
| 76 | return fallbackSymTab.lookup<RoutineOp>(name: routineRef.getLeafReference()); |
| 77 | } |
| 78 | |
| 79 | class ACCMaterializeRoutineBindTargets |
| 80 | : public acc::impl::ACCMaterializeRoutineBindTargetsBase< |
| 81 | ACCMaterializeRoutineBindTargets> { |
| 82 | using Base = acc::impl::ACCMaterializeRoutineBindTargetsBase< |
| 83 | ACCMaterializeRoutineBindTargets>; |
| 84 | |
| 85 | public: |
| 86 | using Base::Base; |
| 87 | |
| 88 | void runOnOperation() override { |
| 89 | Operation *symbolTableOp = getOperation(); |
| 90 | if (!symbolTableOp->hasTrait<OpTrait::SymbolTable>()) { |
| 91 | symbolTableOp->emitError( |
| 92 | message: "root operation must have the SymbolTable trait" ); |
| 93 | signalPassFailure(); |
| 94 | return; |
| 95 | } |
| 96 | |
| 97 | ModuleOp mod = dyn_cast<ModuleOp>(Val: symbolTableOp); |
| 98 | if (!mod) |
| 99 | mod = symbolTableOp->getParentOfType<ModuleOp>(); |
| 100 | if (!mod) { |
| 101 | symbolTableOp->emitError( |
| 102 | message: "root operation must be a builtin.module or nested in one" ); |
| 103 | signalPassFailure(); |
| 104 | return; |
| 105 | } |
| 106 | |
| 107 | SymbolTable outerSymTab(mod); |
| 108 | SymbolTable insertSymTab(symbolTableOp); |
| 109 | Block *insertBlock = &symbolTableOp->getRegion(index: 0).front(); |
| 110 | |
| 111 | symbolTableOp->walk<WalkOrder::PreOrder>(callback: [&](Operation *op) { |
| 112 | if (op != symbolTableOp && op->hasTrait<OpTrait::SymbolTable>()) |
| 113 | return WalkResult::skip(); |
| 114 | |
| 115 | auto callOp = dyn_cast<CallOpInterface>(Val: op); |
| 116 | if (!callOp || !callOp.getCallableForCallee()) |
| 117 | return WalkResult::advance(); |
| 118 | if (!callOp->getParentOfType<OffloadRegionOpInterface>() && |
| 119 | !callOp->getParentOfType<gpu::GPUFuncOp>()) |
| 120 | return WalkResult::advance(); |
| 121 | |
| 122 | auto calleeSymbolRef = |
| 123 | dyn_cast<SymbolRefAttr>(Val: callOp.getCallableForCallee()); |
| 124 | if (!calleeSymbolRef) |
| 125 | return WalkResult::advance(); |
| 126 | FunctionOpInterface callee = |
| 127 | SymbolTable::lookupNearestSymbolFrom<FunctionOpInterface>( |
| 128 | from: callOp, symbol: calleeSymbolRef); |
| 129 | if (!callee) |
| 130 | callee = outerSymTab.lookup<FunctionOpInterface>( |
| 131 | name: calleeSymbolRef.getLeafReference()); |
| 132 | if (!callee || !(isAccRoutine(op: callee) || isSpecializedAccRoutine(op: callee))) |
| 133 | return WalkResult::advance(); |
| 134 | |
| 135 | if (auto routineInfo = callee->getDiscardableAttrOfType<RoutineInfoAttr>( |
| 136 | name: getRoutineInfoAttrName())) |
| 137 | if (routineInfo.getAccRoutines().size() > 1) |
| 138 | return WalkResult::advance(); |
| 139 | |
| 140 | RoutineOp routine = getFirstAccRoutineOp(funcOp: callee, fallbackSymTab: outerSymTab); |
| 141 | if (!routine) |
| 142 | return WalkResult::advance(); |
| 143 | |
| 144 | auto bindNameOpt = routine.getEffectiveBindName(targetDeviceType: this->deviceType); |
| 145 | if (!bindNameOpt || !std::holds_alternative<StringAttr>(v: *bindNameOpt)) |
| 146 | return WalkResult::advance(); |
| 147 | |
| 148 | StringRef bindName = std::get<StringAttr>(v&: *bindNameOpt).getValue(); |
| 149 | if (insertSymTab.lookup(name: bindName)) |
| 150 | return WalkResult::advance(); |
| 151 | |
| 152 | OpBuilder builder(mod.getContext()); |
| 153 | builder.setInsertionPointToEnd(insertBlock); |
| 154 | auto funcType = cast<FunctionType>(Val: callee.getFunctionType()); |
| 155 | func::FuncOp bindFunc = |
| 156 | func::FuncOp::create(builder, location: callee.getLoc(), name: bindName, type: funcType); |
| 157 | bindFunc.setPrivate(); |
| 158 | insertSymTab.insert(symbol: bindFunc); |
| 159 | return WalkResult::advance(); |
| 160 | }); |
| 161 | } |
| 162 | }; |
| 163 | |
| 164 | class ACCBindRoutine : public acc::impl::ACCBindRoutineBase<ACCBindRoutine> { |
| 165 | public: |
| 166 | using acc::impl::ACCBindRoutineBase<ACCBindRoutine>::ACCBindRoutineBase; |
| 167 | |
| 168 | void getDependentDialects(DialectRegistry ®istry) const override { |
| 169 | acc::impl::ACCBindRoutineBase<ACCBindRoutine>::getDependentDialects( |
| 170 | registry); |
| 171 | acc::registerAllExtensions(registry); |
| 172 | } |
| 173 | |
| 174 | void runOnOperation() override { |
| 175 | FunctionOpInterface func = getOperation(); |
| 176 | ModuleOp module = func->getParentOfType<ModuleOp>(); |
| 177 | if (!module) |
| 178 | return; |
| 179 | |
| 180 | auto cachedAnalysis = |
| 181 | getCachedParentAnalysis<OpenACCSupport>(parent: func->getParentOp()); |
| 182 | OpenACCSupport &accSupport = |
| 183 | cachedAnalysis ? cachedAnalysis->get() : getAnalysis<OpenACCSupport>(); |
| 184 | SymbolTable outerSymTab(module); |
| 185 | bool failed = false; |
| 186 | |
| 187 | func.walk(callback: [&](CallOpInterface callOp) { |
| 188 | if (!callOp.getCallableForCallee()) |
| 189 | return; |
| 190 | if (!callOp->getParentOfType<OffloadRegionOpInterface>() && |
| 191 | !callOp->getParentOfType<gpu::GPUFuncOp>()) |
| 192 | return; |
| 193 | SymbolRefAttr calleeSymbolRef = |
| 194 | dyn_cast<SymbolRefAttr>(Val: callOp.getCallableForCallee()); |
| 195 | if (!calleeSymbolRef) |
| 196 | return; |
| 197 | FunctionOpInterface callee = |
| 198 | SymbolTable::lookupNearestSymbolFrom<FunctionOpInterface>( |
| 199 | from: callOp, symbol: calleeSymbolRef); |
| 200 | if (!callee) |
| 201 | callee = outerSymTab.lookup<FunctionOpInterface>( |
| 202 | name: calleeSymbolRef.getLeafReference()); |
| 203 | if (!callee) |
| 204 | return; |
| 205 | |
| 206 | if (!(isAccRoutine(op: callee) || isSpecializedAccRoutine(op: callee))) |
| 207 | return; |
| 208 | |
| 209 | if (auto routineInfo = callee->getDiscardableAttrOfType<RoutineInfoAttr>( |
| 210 | name: getRoutineInfoAttrName())) { |
| 211 | if (routineInfo.getAccRoutines().size() > 1) { |
| 212 | (void)accSupport.emitNYI(loc: callOp.getLoc(), message: "multiple `acc routine`s" ); |
| 213 | failed = true; |
| 214 | return; |
| 215 | } |
| 216 | } |
| 217 | |
| 218 | RoutineOp routine = getFirstAccRoutineOp(funcOp: callee, fallbackSymTab: outerSymTab); |
| 219 | if (!routine) |
| 220 | return; |
| 221 | |
| 222 | auto bindNameOpt = routine.getEffectiveBindName(targetDeviceType: this->deviceType); |
| 223 | if (!bindNameOpt) |
| 224 | return; |
| 225 | SymbolRefAttr calleeRef; |
| 226 | if (auto *symRef = std::get_if<SymbolRefAttr>(ptr: &*bindNameOpt)) { |
| 227 | calleeRef = *symRef; |
| 228 | } else { |
| 229 | StringRef bindName = std::get<StringAttr>(v&: *bindNameOpt).getValue(); |
| 230 | auto gpuMod = func->getParentOfType<gpu::GPUModuleOp>(); |
| 231 | Operation *symbolTableOp = |
| 232 | gpuMod ? gpuMod.getOperation() : module.getOperation(); |
| 233 | SymbolTable symbolTable(symbolTableOp); |
| 234 | if (!symbolTable.lookup(name: bindName)) { |
| 235 | callOp.emitError( |
| 236 | message: "string-bound ACC routine target was not materialized" ); |
| 237 | failed = true; |
| 238 | return; |
| 239 | } |
| 240 | calleeRef = FlatSymbolRefAttr::get(ctx: callOp.getContext(), value: bindName); |
| 241 | } |
| 242 | callOp.setCalleeFromCallable(calleeRef); |
| 243 | }); |
| 244 | |
| 245 | if (failed) |
| 246 | signalPassFailure(); |
| 247 | } |
| 248 | }; |
| 249 | } // namespace |
| 250 | |