| 1 | //===- ACCSpecializeForDevice.cpp -----------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This pass strips OpenACC constructs that are invalid or unnecessary inside |
| 10 | // device code (specialized acc routines or compute construct regions). |
| 11 | // |
| 12 | // Overview: |
| 13 | // --------- |
| 14 | // In a specialized acc routine or compute construct, many OpenACC operations |
| 15 | // do not make sense because they are host-side constructs. This pass removes |
| 16 | // or transforms these operations appropriately: |
| 17 | // |
| 18 | // - Data operations that manage device memory from host perspective |
| 19 | // - Compute constructs that launch kernels (we're already on device) |
| 20 | // - Runtime operations like init/shutdown/set/wait |
| 21 | // |
| 22 | // Transformations: |
| 23 | // ---------------- |
| 24 | // The pass applies the following transformations: |
| 25 | // |
| 26 | // 1. Data Entry Ops (replaced with var operand): |
| 27 | // acc.attach, acc.copyin, acc.create, acc.declare_device_resident, |
| 28 | // acc.declare_link, acc.deviceptr, acc.get_deviceptr, acc.nocreate, |
| 29 | // acc.present, acc.update_device, acc.use_device |
| 30 | // |
| 31 | // 2. Data Exit Ops (erased): |
| 32 | // acc.copyout, acc.delete, acc.detach, acc.update_host |
| 33 | // |
| 34 | // 3. Structured Data/Compute Constructs (region inlined): |
| 35 | // acc.data, acc.host_data, acc.kernel_environment, acc.parallel, |
| 36 | // acc.serial, acc.kernels |
| 37 | // |
| 38 | // 4. Unstructured Data Ops (erased): |
| 39 | // acc.enter_data, acc.exit_data, acc.update, acc.declare_enter, |
| 40 | // acc.declare_exit |
| 41 | // |
| 42 | // 5. Runtime Ops (erased): |
| 43 | // acc.init, acc.shutdown, acc.set, acc.wait |
| 44 | // |
| 45 | // 6. acc.on_device (folded): |
| 46 | // acc.on_device with constant device type is folded to a boolean constant. |
| 47 | // With a device type only known at runtime it is replaced by comparisons |
| 48 | // against the device types being specialized for. |
| 49 | // |
| 50 | // Scope of Application: |
| 51 | // --------------------- |
| 52 | // - For functions with `acc.specialized_routine` attribute: patterns are |
| 53 | // applied to the entire function body. |
| 54 | // - For non-specialized functions: patterns are applied only to ACC |
| 55 | // operations INSIDE compute constructs (parallel, serial, kernels), |
| 56 | // not to the compute constructs themselves or their data operands. |
| 57 | // |
| 58 | // Note: acc.cache, acc.private, acc.reduction, acc.firstprivate are NOT |
| 59 | // transformed by this pass as they are valid in device code. |
| 60 | // |
| 61 | //===----------------------------------------------------------------------===// |
| 62 | |
| 63 | #include "mlir/Dialect/OpenACC/Extensions/AllExtensions.h" |
| 64 | #include "mlir/Dialect/OpenACC/Transforms/Passes.h" |
| 65 | |
| 66 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 67 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 68 | #include "mlir/Dialect/OpenACC/OpenACC.h" |
| 69 | #include "mlir/Dialect/OpenACC/Transforms/ACCSpecializePatterns.h" |
| 70 | #include "mlir/Dialect/SCF/IR/SCF.h" |
| 71 | #include "mlir/IR/MLIRContext.h" |
| 72 | #include "mlir/IR/PatternMatch.h" |
| 73 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 74 | |
| 75 | namespace mlir { |
| 76 | namespace acc { |
| 77 | #define GEN_PASS_DEF_ACCSPECIALIZEFORDEVICE |
| 78 | #include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc" |
| 79 | } // namespace acc |
| 80 | } // namespace mlir |
| 81 | |
| 82 | using namespace mlir; |
| 83 | using namespace mlir::acc; |
| 84 | |
| 85 | namespace { |
| 86 | /// Fold acc.on_device in device code. A constant device type folds to a |
| 87 | /// boolean constant, any other device type to a comparison against the device |
| 88 | /// types being specialized for. |
| 89 | class FoldAccOnDeviceOpConversion : public OpRewritePattern<acc::OnDeviceOp> { |
| 90 | using OpRewritePattern<acc::OnDeviceOp>::OpRewritePattern; |
| 91 | |
| 92 | const TypesForDevice &theDeviceTypes; |
| 93 | |
| 94 | public: |
| 95 | FoldAccOnDeviceOpConversion(MLIRContext *context, |
| 96 | const TypesForDevice &theDeviceTypes) |
| 97 | : OpRewritePattern<acc::OnDeviceOp>(context), |
| 98 | theDeviceTypes(theDeviceTypes) {} |
| 99 | |
| 100 | LogicalResult matchAndRewrite(acc::OnDeviceOp op, |
| 101 | PatternRewriter &rewriter) const override { |
| 102 | Value deviceType = op.getDeviceType(); |
| 103 | APInt constVal; |
| 104 | if (matchPattern(value: deviceType, pattern: m_ConstantInt(bind_value: &constVal))) { |
| 105 | bool result = theDeviceTypes.contains(key: constVal.getSExtValue()); |
| 106 | rewriter.replaceOpWithNewOp<arith::ConstantOp>( |
| 107 | op, args: rewriter.getI1Type(), args: rewriter.getBoolAttr(value: result)); |
| 108 | return success(); |
| 109 | } |
| 110 | |
| 111 | // The device type is only known at runtime, inline the definition of |
| 112 | // `on_device`. |
| 113 | Location loc = op.getLoc(); |
| 114 | Value result; |
| 115 | for (int64_t type : theDeviceTypes) { |
| 116 | Value typeValue = arith::ConstantOp::create( |
| 117 | builder&: rewriter, location: loc, result: deviceType.getType(), |
| 118 | value: rewriter.getIntegerAttr(type: deviceType.getType(), value: type)); |
| 119 | Value isType = arith::CmpIOp::create( |
| 120 | builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::eq, lhs: deviceType, rhs: typeValue); |
| 121 | if (result) |
| 122 | result = arith::OrIOp::create(builder&: rewriter, location: loc, lhs: result, rhs: isType); |
| 123 | else |
| 124 | result = isType; |
| 125 | } |
| 126 | assert(result && "At least one device type expected." ); |
| 127 | rewriter.replaceOp(op, newValues: result); |
| 128 | return success(); |
| 129 | } |
| 130 | }; |
| 131 | |
| 132 | class ACCSpecializeForDevice |
| 133 | : public acc::impl::ACCSpecializeForDeviceBase<ACCSpecializeForDevice> { |
| 134 | public: |
| 135 | using ACCSpecializeForDeviceBase< |
| 136 | ACCSpecializeForDevice>::ACCSpecializeForDeviceBase; |
| 137 | |
| 138 | void getDependentDialects(DialectRegistry ®istry) const override { |
| 139 | ACCSpecializeForDeviceBase<ACCSpecializeForDevice>::getDependentDialects( |
| 140 | registry); |
| 141 | acc::registerAllExtensions(registry); |
| 142 | } |
| 143 | |
| 144 | void runOnOperation() override { |
| 145 | func::FuncOp func = getOperation(); |
| 146 | |
| 147 | TypesForDevice types{theDeviceTypes.begin(), theDeviceTypes.end()}; |
| 148 | |
| 149 | RewritePatternSet patterns(&getContext()); |
| 150 | acc::populateACCSpecializeForDevicePatterns(patterns, theDeviceTypes: types); |
| 151 | GreedyRewriteConfig config; |
| 152 | config.setUseTopDownTraversal(true); |
| 153 | |
| 154 | if (acc::isSpecializedAccRoutine(op: func)) { |
| 155 | // For specialized acc routines, apply patterns to the entire function |
| 156 | (void)applyPatternsGreedily(op: func, patterns: std::move(patterns), config); |
| 157 | } else { |
| 158 | // For non-specialized functions, apply patterns only to ACC operations |
| 159 | // inside compute constructs (not to the compute constructs themselves). |
| 160 | // Use ExistingOps strictness so the greedy driver does not expand the |
| 161 | // worklist to parent ops, which would accidentally unwrap the compute |
| 162 | // construct (e.g. after inlining acc routines with their own data |
| 163 | // regions). |
| 164 | config.setStrictness(GreedyRewriteStrictness::ExistingOps); |
| 165 | SmallVector<Operation *> opsToTransform; |
| 166 | func.walk(callback: [&](Operation *op) { |
| 167 | if (isa<ACC_COMPUTE_CONSTRUCT_OPS>(Val: op)) { |
| 168 | // Walk inside the compute construct and collect ACC ops |
| 169 | op->walk(callback: [&](Operation *innerOp) { |
| 170 | // Skip the compute construct itself |
| 171 | if (innerOp == op) |
| 172 | return; |
| 173 | if (isa<acc::OpenACCDialect>(Val: innerOp->getDialect())) |
| 174 | opsToTransform.push_back(Elt: innerOp); |
| 175 | }); |
| 176 | } |
| 177 | }); |
| 178 | if (!opsToTransform.empty()) |
| 179 | (void)applyOpPatternsGreedily(ops: opsToTransform, patterns: std::move(patterns), |
| 180 | config); |
| 181 | } |
| 182 | } |
| 183 | }; |
| 184 | } // namespace |
| 185 | |
| 186 | //===----------------------------------------------------------------------===// |
| 187 | // Pattern population functions |
| 188 | //===----------------------------------------------------------------------===// |
| 189 | |
| 190 | void mlir::acc::populateACCSpecializeForDevicePatterns( |
| 191 | RewritePatternSet &patterns, const TypesForDevice &theDeviceTypes) { |
| 192 | MLIRContext *context = patterns.getContext(); |
| 193 | |
| 194 | // Declare patterns - erase declare_enter and its associated declare_exit |
| 195 | patterns.insert<ACCDeclareEnterOpConversion>(arg&: context); |
| 196 | |
| 197 | // Data entry ops - replaced with their var operand |
| 198 | // Note: acc.cache, acc.private, acc.reduction, acc.firstprivate are NOT |
| 199 | // included here - they are valid in device code |
| 200 | patterns.insert<ACCOpReplaceWithVarConversion<acc::AttachOp>, |
| 201 | ACCOpReplaceWithVarConversion<acc::CopyinOp>, |
| 202 | ACCOpReplaceWithVarConversion<acc::CreateOp>, |
| 203 | ACCOpReplaceWithVarConversion<acc::DeclareDeviceResidentOp>, |
| 204 | ACCOpReplaceWithVarConversion<acc::DeclareLinkOp>, |
| 205 | ACCOpReplaceWithVarConversion<acc::DevicePtrOp>, |
| 206 | ACCOpReplaceWithVarConversion<acc::GetDevicePtrOp>, |
| 207 | ACCOpReplaceWithVarConversion<acc::NoCreateOp>, |
| 208 | ACCOpReplaceWithVarConversion<acc::PresentOp>, |
| 209 | ACCOpReplaceWithVarConversion<acc::UpdateDeviceOp>, |
| 210 | ACCOpReplaceWithVarConversion<acc::UseDeviceOp>>(arg&: context); |
| 211 | |
| 212 | // Data exit ops - simply erased (no results) |
| 213 | patterns.insert<ACCOpEraseConversion<acc::CopyoutOp>, |
| 214 | ACCOpEraseConversion<acc::DeleteOp>, |
| 215 | ACCOpEraseConversion<acc::DetachOp>, |
| 216 | ACCOpEraseConversion<acc::UpdateHostOp>>(arg&: context); |
| 217 | |
| 218 | // Structured data constructs - unwrap their regions |
| 219 | patterns.insert<ACCRegionUnwrapConversion<acc::DataOp>, |
| 220 | ACCRegionUnwrapConversion<acc::HostDataOp>, |
| 221 | ACCRegionUnwrapConversion<acc::KernelEnvironmentOp>>(arg&: context); |
| 222 | |
| 223 | // Compute constructs - unwrap their regions |
| 224 | patterns.insert<ACCRegionUnwrapConversion<acc::ParallelOp>, |
| 225 | ACCRegionUnwrapConversion<acc::SerialOp>, |
| 226 | ACCRegionUnwrapConversion<acc::KernelsOp>>(arg&: context); |
| 227 | |
| 228 | // Unstructured data operations - erase them |
| 229 | patterns.insert<ACCOpEraseConversion<acc::EnterDataOp>, |
| 230 | ACCOpEraseConversion<acc::ExitDataOp>, |
| 231 | ACCOpEraseConversion<acc::UpdateOp>>(arg&: context); |
| 232 | |
| 233 | // Runtime operations - erase them |
| 234 | patterns.insert< |
| 235 | ACCOpEraseConversion<acc::InitOp>, ACCOpEraseConversion<acc::ShutdownOp>, |
| 236 | ACCOpEraseConversion<acc::SetOp>, ACCOpEraseConversion<acc::WaitOp>>( |
| 237 | arg&: context); |
| 238 | |
| 239 | // Fold acc.on_device calls so dead-code elimination can remove host-only |
| 240 | // code paths in device code. |
| 241 | patterns.insert<FoldAccOnDeviceOpConversion>(arg&: context, args: theDeviceTypes); |
| 242 | } |
| 243 | |