| 1 | //===- ACCRoutineLowering.cpp - Wrap ACC routines in compute_region -------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This pass handles `acc routine` directive by creating specialized |
| 10 | // functions with appropriate parallelism information that can be used for |
| 11 | // eventual creation of device function. |
| 12 | // |
| 13 | // Overview: |
| 14 | // --------- |
| 15 | // For each acc.routine that is not bound by name, the pass creates a new |
| 16 | // function (the "device" copy) whose body is a single acc.compute_region |
| 17 | // containing a clone of the original (host) function body. Parallelism is |
| 18 | // expressed by one acc.par_width derived from the routine's clauses (seq, |
| 19 | // vector, worker, gang). The device copy created is simply a staging |
| 20 | // place for eventual move to device module level function. |
| 21 | // |
| 22 | // Transformations: |
| 23 | // ---------------- |
| 24 | // 1. Device function: Same signature as the host; attributes copied except |
| 25 | // acc.routine_info. The acc.specialized_routine attribute is set with the |
| 26 | // routine symbol, par level, and original function name. |
| 27 | // |
| 28 | // 2. Body: One acc.par_width, one acc.compute_region that clones the host |
| 29 | // body. Multi-block host bodies are wrapped in scf.execute_region inside |
| 30 | // the compute_region. |
| 31 | // |
| 32 | // 3. Finalization: acc.routine's func_name is updated to the device function. |
| 33 | // |
| 34 | //===----------------------------------------------------------------------===// |
| 35 | |
| 36 | #include "mlir/Dialect/OpenACC/Extensions/AllExtensions.h" |
| 37 | #include "mlir/Dialect/OpenACC/Transforms/Passes.h" |
| 38 | |
| 39 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 40 | #include "mlir/Dialect/OpenACC/OpenACC.h" |
| 41 | #include "mlir/Dialect/OpenACC/OpenACCParMapping.h" |
| 42 | #include "mlir/Dialect/OpenACC/OpenACCUtilsCG.h" |
| 43 | #include "mlir/IR/BuiltinAttributes.h" |
| 44 | #include "mlir/IR/IRMapping.h" |
| 45 | #include "mlir/IR/PatternMatch.h" |
| 46 | #include "mlir/IR/SymbolTable.h" |
| 47 | #include "mlir/IR/Value.h" |
| 48 | |
| 49 | namespace mlir { |
| 50 | namespace acc { |
| 51 | #define GEN_PASS_DEF_ACCROUTINELOWERING |
| 52 | #include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc" |
| 53 | } // namespace acc |
| 54 | } // namespace mlir |
| 55 | |
| 56 | #define DEBUG_TYPE "acc-routine-lowering" |
| 57 | |
| 58 | using namespace mlir; |
| 59 | using namespace mlir::acc; |
| 60 | |
| 61 | namespace { |
| 62 | /// Compute the ParLevel from an acc.routine op for specialization. |
| 63 | static ParLevel computeParLevel(RoutineOp routineOp, DeviceType deviceType) { |
| 64 | auto gangDim = routineOp.getGangDimValue(deviceType); |
| 65 | if (!gangDim) |
| 66 | gangDim = routineOp.getGangDimValue(); |
| 67 | if (gangDim) { |
| 68 | switch (*gangDim) { |
| 69 | case 1: |
| 70 | return ParLevel::gang_dim1; |
| 71 | case 2: |
| 72 | return ParLevel::gang_dim2; |
| 73 | case 3: |
| 74 | return ParLevel::gang_dim3; |
| 75 | default: |
| 76 | break; |
| 77 | } |
| 78 | } |
| 79 | if (routineOp.hasGang(deviceType) || routineOp.hasGang()) |
| 80 | return ParLevel::gang_dim1; |
| 81 | if (routineOp.hasWorker(deviceType) || routineOp.hasWorker()) |
| 82 | return ParLevel::worker; |
| 83 | if (routineOp.hasVector(deviceType) || routineOp.hasVector()) |
| 84 | return ParLevel::vector; |
| 85 | return ParLevel::seq; |
| 86 | } |
| 87 | |
| 88 | /// Collect return operands from the function (first block with func.return). |
| 89 | static void getReturnValues(func::FuncOp func, SmallVectorImpl<Value> &result) { |
| 90 | result.clear(); |
| 91 | for (Block &block : func.getBody().getBlocks()) { |
| 92 | if (auto returnOp = dyn_cast<func::ReturnOp>(Val: block.getTerminator())) { |
| 93 | result.assign(in_start: returnOp.operand_begin(), in_end: returnOp.operand_end()); |
| 94 | break; |
| 95 | } |
| 96 | } |
| 97 | } |
| 98 | |
| 99 | /// Create the device function with the same signature as the host, set |
| 100 | /// specialized_routine, and add a single block with the same block arguments. |
| 101 | static func::FuncOp createFunctionForDeviceStaging(func::FuncOp hostFunc, |
| 102 | RoutineOp routineOp, |
| 103 | ParLevel parLevel, |
| 104 | MLIRContext *ctx, |
| 105 | IRRewriter &rewriter) { |
| 106 | Location loc = hostFunc.getLoc(); |
| 107 | FunctionType funcType = hostFunc.getFunctionType(); |
| 108 | func::FuncOp deviceFunc = |
| 109 | func::FuncOp::create(builder&: rewriter, location: loc, name: hostFunc.getName(), type: funcType); |
| 110 | deviceFunc->copyProperties(rhs: hostFunc->getPropertiesStorage()); |
| 111 | deviceFunc->setDiscardableAttrs(hostFunc->getDiscardableAttrDictionary()); |
| 112 | deviceFunc->removeDiscardableAttr(name: getRoutineInfoAttrName()); |
| 113 | deviceFunc->setDiscardableAttr( |
| 114 | name: getSpecializedRoutineAttrName(), |
| 115 | value: SpecializedRoutineAttr::get( |
| 116 | context: ctx, routine: SymbolRefAttr::get(ctx, value: routineOp.getSymName()), |
| 117 | level: ParLevelAttr::get(context: ctx, value: parLevel), |
| 118 | funcName: StringAttr::get(context: ctx, bytes: hostFunc.getName()))); |
| 119 | |
| 120 | Block *sourceBlock = &hostFunc.getBody().front(); |
| 121 | Block *newBlock = rewriter.createBlock(parent: &deviceFunc.getRegion()); |
| 122 | for (BlockArgument arg : sourceBlock->getArguments()) |
| 123 | newBlock->addArgument(type: arg.getType(), loc: hostFunc.getLoc()); |
| 124 | |
| 125 | return deviceFunc; |
| 126 | } |
| 127 | |
| 128 | /// Fill the device function body: one acc.par_width, one acc.compute_region |
| 129 | /// (cloning the host body with inputArgsToMap), then func.return. |
| 130 | static LogicalResult |
| 131 | buildRoutineBody(func::FuncOp deviceFunc, func::FuncOp hostFunc, |
| 132 | ArrayRef<Value> funcReturnVals, ParLevel parLevel, |
| 133 | DefaultACCToGPUMappingPolicy &policy, IRRewriter &rewriter) { |
| 134 | Block *newBlock = &deviceFunc.getBody().front(); |
| 135 | Block *sourceBlock = &hostFunc.getBody().front(); |
| 136 | Location loc = hostFunc.getLoc(); |
| 137 | MLIRContext *ctx = rewriter.getContext(); |
| 138 | |
| 139 | rewriter.setInsertionPointToStart(newBlock); |
| 140 | GPUParallelDimAttr parDim = policy.map(ctx, level: parLevel); |
| 141 | Value parWidthVal = ParWidthOp::create(builder&: rewriter, location: loc, launchArg: Value(), par_dim: parDim); |
| 142 | SmallVector<Value, 4> inputArgs(newBlock->getArguments().begin(), |
| 143 | newBlock->getArguments().end()); |
| 144 | |
| 145 | // Normally the region passed to buildComputeRegion is something in the |
| 146 | // current function. Here we pass the body of the original (host) function as |
| 147 | // an optimization to avoid cloning twice (once for a staged device copy and |
| 148 | // again when creating the compute region). Since we clone only once, we must |
| 149 | // also provide the original function's arguments so the mapping is correct |
| 150 | // when cloning the body. |
| 151 | ValueRange sourceArgsToMap = sourceBlock->getArguments(); |
| 152 | |
| 153 | IRMapping mapping; |
| 154 | rewriter.setInsertionPointAfter(parWidthVal.getDefiningOp()); |
| 155 | ComputeRegionOp computeRegion = buildComputeRegion( |
| 156 | loc, launchArgs: {parWidthVal}, inputArgs, origin: RoutineOp::getOperationName(), |
| 157 | regionToClone&: hostFunc.getBody(), rewriter, mapping, |
| 158 | /*output=*/funcReturnVals, /*kernelFuncName=*/{}, |
| 159 | /*kernelModuleName=*/{}, /*stream=*/{}, inputArgsToMap: sourceArgsToMap); |
| 160 | if (!computeRegion) |
| 161 | return failure(); |
| 162 | |
| 163 | rewriter.setInsertionPointAfter(computeRegion); |
| 164 | if (funcReturnVals.empty()) |
| 165 | func::ReturnOp::create(builder&: rewriter, location: loc); |
| 166 | else |
| 167 | func::ReturnOp::create(builder&: rewriter, location: loc, operands: computeRegion.getResults()); |
| 168 | |
| 169 | return success(); |
| 170 | } |
| 171 | |
| 172 | /// Update acc.routine refs |
| 173 | static void finalizeRoutines( |
| 174 | SmallVectorImpl<std::pair<func::FuncOp, RoutineOp>> &accRoutineInfo, |
| 175 | MLIRContext *ctx) { |
| 176 | for (auto &[deviceFunc, routineOp] : accRoutineInfo) { |
| 177 | routineOp.setFuncNameAttr(SymbolRefAttr::get(ctx, value: deviceFunc.getName())); |
| 178 | routineOp->moveBefore(existingOp: deviceFunc); |
| 179 | } |
| 180 | } |
| 181 | |
| 182 | class ACCRoutineLowering |
| 183 | : public acc::impl::ACCRoutineLoweringBase<ACCRoutineLowering> { |
| 184 | public: |
| 185 | using ACCRoutineLoweringBase::ACCRoutineLoweringBase; |
| 186 | |
| 187 | void getDependentDialects(DialectRegistry ®istry) const override { |
| 188 | ACCRoutineLoweringBase::getDependentDialects(registry); |
| 189 | acc::registerAllExtensions(registry); |
| 190 | } |
| 191 | |
| 192 | void runOnOperation() override { |
| 193 | ModuleOp mod = getOperation(); |
| 194 | if (mod.getOps<RoutineOp>().empty()) { |
| 195 | LLVM_DEBUG(llvm::dbgs() |
| 196 | << "Skipping ACCRoutineLowering - no acc.routine ops\n" ); |
| 197 | return; |
| 198 | } |
| 199 | |
| 200 | SymbolTable symTab(mod); |
| 201 | MLIRContext *ctx = mod.getContext(); |
| 202 | IRRewriter rewriter(ctx); |
| 203 | DefaultACCToGPUMappingPolicy policy; |
| 204 | |
| 205 | // Pair: device function, routine operation |
| 206 | SmallVector<std::pair<func::FuncOp, RoutineOp>, 4> accRoutineInfo; |
| 207 | |
| 208 | for (RoutineOp routineOp : mod.getOps<RoutineOp>()) { |
| 209 | if (routineOp.getEffectiveBindName(targetDeviceType: deviceType)) |
| 210 | continue; |
| 211 | |
| 212 | func::FuncOp hostFunc = symTab.lookup<func::FuncOp>( |
| 213 | name: routineOp.getFuncName().getLeafReference()); |
| 214 | if (!hostFunc) { |
| 215 | routineOp.emitError(message: "acc routine function not found in symbol table" ); |
| 216 | return signalPassFailure(); |
| 217 | } |
| 218 | if (hostFunc.isExternal()) |
| 219 | continue; |
| 220 | |
| 221 | SmallVector<Value, 4> funcReturnVals; |
| 222 | getReturnValues(func: hostFunc, result&: funcReturnVals); |
| 223 | |
| 224 | OpBuilder::InsertionGuard guard(rewriter); |
| 225 | ParLevel parLevel = computeParLevel(routineOp, deviceType); |
| 226 | func::FuncOp deviceFunc = createFunctionForDeviceStaging( |
| 227 | hostFunc, routineOp, parLevel, ctx, rewriter); |
| 228 | if (failed(Result: buildRoutineBody(deviceFunc, hostFunc, funcReturnVals, |
| 229 | parLevel, policy, rewriter))) |
| 230 | return signalPassFailure(); |
| 231 | |
| 232 | accRoutineInfo.push_back(Elt: {deviceFunc, routineOp}); |
| 233 | symTab.insert(symbol: deviceFunc); |
| 234 | } |
| 235 | |
| 236 | finalizeRoutines(accRoutineInfo, ctx); |
| 237 | } |
| 238 | }; |
| 239 | } // namespace |
| 240 | |