| 1 | //===- ROCDLAttachTarget.cpp - Attach an ROCDL target ---------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements the `GpuROCDLAttachTarget` pass, attaching |
| 10 | // `#rocdl.target` attributes to GPU modules. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "mlir/Dialect/GPU/Transforms/ROCDL/Passes.h" |
| 15 | |
| 16 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 17 | #include "mlir/Dialect/LLVMIR/ROCDLDialect.h" |
| 18 | #include "mlir/Dialect/LLVMIR/ROCDLTargetInfo.h" |
| 19 | #include "mlir/IR/Builders.h" |
| 20 | #include "mlir/Pass/Pass.h" |
| 21 | #include "llvm/Support/Regex.h" |
| 22 | |
| 23 | namespace mlir { |
| 24 | #define GEN_PASS_DEF_GPUROCDLATTACHTARGET |
| 25 | #include "mlir/Dialect/GPU/Transforms/ROCDL/Passes.h.inc" |
| 26 | } // namespace mlir |
| 27 | |
| 28 | using namespace mlir; |
| 29 | using namespace mlir::ROCDL; |
| 30 | |
| 31 | namespace { |
| 32 | struct ROCDLAttachTarget |
| 33 | : public impl::GpuROCDLAttachTargetBase<ROCDLAttachTarget> { |
| 34 | using Base::Base; |
| 35 | |
| 36 | DictionaryAttr getFlags(OpBuilder &builder, bool isWave64) const; |
| 37 | |
| 38 | void runOnOperation() override; |
| 39 | |
| 40 | void getDependentDialects(DialectRegistry ®istry) const override { |
| 41 | registry.insert<ROCDL::ROCDLDialect>(); |
| 42 | } |
| 43 | }; |
| 44 | } // namespace |
| 45 | |
| 46 | DictionaryAttr ROCDLAttachTarget::getFlags(OpBuilder &builder, |
| 47 | bool isWave64) const { |
| 48 | UnitAttr unitAttr = builder.getUnitAttr(); |
| 49 | SmallVector<NamedAttribute, 6> flags; |
| 50 | auto addFlag = [&](StringRef flag) { |
| 51 | flags.push_back(Elt: builder.getNamedAttr(name: flag, val: unitAttr)); |
| 52 | }; |
| 53 | if (!isWave64) |
| 54 | addFlag("no_wave64" ); |
| 55 | if (fastFlag) |
| 56 | addFlag("fast" ); |
| 57 | if (dazFlag) |
| 58 | addFlag("daz" ); |
| 59 | if (finiteOnlyFlag) |
| 60 | addFlag("finite_only" ); |
| 61 | if (unsafeMathFlag) |
| 62 | addFlag("unsafe_math" ); |
| 63 | if (!correctSqrtFlag) |
| 64 | addFlag("unsafe_sqrt" ); |
| 65 | if (!flags.empty()) |
| 66 | return builder.getDictionaryAttr(value: flags); |
| 67 | return nullptr; |
| 68 | } |
| 69 | |
| 70 | void ROCDLAttachTarget::runOnOperation() { |
| 71 | OpBuilder builder(&getContext()); |
| 72 | |
| 73 | // #rocdl.target feeds the TargetMachine, whose -mcpu is a bare processor |
| 74 | // name. Split an `arch` into the pieces the attribute wants, mirroring |
| 75 | // Clang. |
| 76 | std::string resolvedTriple = triple; |
| 77 | std::string resolvedChip = chip; |
| 78 | std::string resolvedFeatures = features; |
| 79 | // `wavesize` is the only wavefront-size control. Without an `arch` there is |
| 80 | // no target to ask, so leaving it at 0 keeps the Wave64 that `#rocdl.target` |
| 81 | // has always assumed. |
| 82 | bool resolvedWave64 = waveSize != 32; |
| 83 | // Set when `arch` was given and used so its xnack/sramecc modifiers can |
| 84 | // become module flags. |
| 85 | std::optional<ROCDL::TargetInfo> targetInfo; |
| 86 | if (!arch.empty()) { |
| 87 | std::optional<llvm::AMDGPU::TargetID> id = |
| 88 | ROCDL::TargetInfo::parseTargetID(arch); |
| 89 | if (!id) { |
| 90 | emitError(loc: UnknownLoc::get(context: &getContext())) |
| 91 | << "'" << arch << "' is not a valid AMDGPU architecture" ; |
| 92 | return signalPassFailure(); |
| 93 | } |
| 94 | // #rocdl.target requires a chip, so a triple that names no GPU (the legacy |
| 95 | // subarch-less "amdgcn-amd-amdhsa") cannot be attached. |
| 96 | if (id->getGPUKind() == llvm::AMDGPU::GK_NONE) { |
| 97 | emitError(loc: UnknownLoc::get(context: &getContext())) |
| 98 | << "'" << arch |
| 99 | << "' names no GPU; a chip is required to attach a " |
| 100 | "target" ; |
| 101 | return signalPassFailure(); |
| 102 | } |
| 103 | |
| 104 | llvm::Triple parsed(id->getTargetTripleString()); |
| 105 | |
| 106 | StringRef archName = parsed.getArchName(); |
| 107 | llvm::Triple::SubArchType subArch = |
| 108 | llvm::AMDGPU::getSubArch(AK: id->getGPUKind()); |
| 109 | if (StringRef subArchName = llvm::AMDGPU::getSubArchName(SubArch: subArch); |
| 110 | !subArchName.empty()) |
| 111 | archName = subArchName; |
| 112 | |
| 113 | // Drop the "unknown" environment part of triples since a lot of the |
| 114 | // toolchain expects a 3-component form. |
| 115 | resolvedTriple = |
| 116 | (parsed.getEnvironment() == llvm::Triple::UnknownEnvironment |
| 117 | ? llvm::Triple(archName, parsed.getVendorName(), |
| 118 | parsed.getOSName()) |
| 119 | : llvm::Triple(archName, parsed.getVendorName(), |
| 120 | parsed.getOSName(), parsed.getEnvironmentName())) |
| 121 | .str(); |
| 122 | resolvedChip = llvm::AMDGPU::getArchNameAMDGCN(AK: id->getGPUKind()).str(); |
| 123 | |
| 124 | // Take the wavefront size from the target, since `wavesize`'s Wave64 |
| 125 | // fallback is wrong on targets like gfx10. |
| 126 | FailureOr<ROCDL::TargetInfo> info = |
| 127 | ROCDL::TargetInfo::get(arch, waveSize, emitError: [&] { |
| 128 | return emitError(loc: UnknownLoc::get(context: &getContext())); |
| 129 | }); |
| 130 | if (failed(Result: info)) |
| 131 | return signalPassFailure(); |
| 132 | targetInfo = *info; |
| 133 | resolvedWave64 = info->getWavefrontSize() == 64; |
| 134 | |
| 135 | // Record the wavefrontsize option into the features set so that the device |
| 136 | // libraries and codegen agree with each other in cases where both |
| 137 | // wavefontsize32 and wavefrontsize64 are permitted options. |
| 138 | if (info->supportsBothWavefrontSizes()) { |
| 139 | if (!resolvedFeatures.empty()) |
| 140 | resolvedFeatures += "," ; |
| 141 | resolvedFeatures += |
| 142 | resolvedWave64 ? "+wavefrontsize64" : "+wavefrontsize32" ; |
| 143 | } |
| 144 | } else if (waveSize != 0 && waveSize != 32 && waveSize != 64) { |
| 145 | emitError(loc: UnknownLoc::get(context: &getContext())) |
| 146 | << "wavefront size must be 32 or 64, got " << waveSize; |
| 147 | return signalPassFailure(); |
| 148 | } |
| 149 | |
| 150 | ArrayRef<std::string> libs(linkLibs); |
| 151 | SmallVector<StringRef> filesToLink(libs); |
| 152 | auto target = builder.getAttr<ROCDLTargetAttr>( |
| 153 | args&: optLevel, args&: resolvedTriple, args&: resolvedChip, args&: resolvedFeatures, args&: abiVersion, |
| 154 | args: getFlags(builder, isWave64: resolvedWave64), |
| 155 | args: filesToLink.empty() ? nullptr : builder.getStrArrayAttr(values: filesToLink)); |
| 156 | llvm::Regex matcher(moduleMatcher); |
| 157 | for (Region ®ion : getOperation()->getRegions()) |
| 158 | for (Block &block : region.getBlocks()) |
| 159 | for (auto module : block.getOps<gpu::GPUModuleOp>()) { |
| 160 | // Check if the name of the module matches. |
| 161 | if (!moduleMatcher.empty() && !matcher.match(String: module.getName())) |
| 162 | continue; |
| 163 | // Create the target array. |
| 164 | SmallVector<Attribute> targets; |
| 165 | if (std::optional<ArrayAttr> attrs = module.getTargets()) |
| 166 | targets.append(in_start: attrs->getValue().begin(), in_end: attrs->getValue().end()); |
| 167 | targets.push_back(Elt: target); |
| 168 | // Remove any duplicate targets. |
| 169 | targets.erase(CS: llvm::unique(R&: targets), CE: targets.end()); |
| 170 | // Update the target attribute array. |
| 171 | module.setTargetsAttr(builder.getArrayAttr(value: targets)); |
| 172 | if (targetInfo) |
| 173 | targetInfo->migrateArchFeaturesToModuleFlags(op: module); |
| 174 | } |
| 175 | } |
| 176 | |