| 1 | //===- Target.cpp - MLIR LLVM ROCDL target compilation ----------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This files defines ROCDL target related functions including registration |
| 10 | // calls for the `#rocdl.target` compilation attribute. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "mlir/Target/LLVM/ROCDL/Target.h" |
| 15 | |
| 16 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 17 | #include "mlir/Dialect/LLVMIR/ROCDLDialect.h" |
| 18 | #include "mlir/Support/FileUtilities.h" |
| 19 | #include "mlir/Target/LLVM/ROCDL/Utils.h" |
| 20 | #include "mlir/Target/LLVMIR/Export.h" |
| 21 | |
| 22 | #include "llvm/Config/Targets.h" |
| 23 | #include "llvm/IR/Constants.h" |
| 24 | #include "llvm/MC/MCAsmBackend.h" |
| 25 | #include "llvm/MC/MCAsmInfo.h" |
| 26 | #include "llvm/MC/MCCodeEmitter.h" |
| 27 | #include "llvm/MC/MCContext.h" |
| 28 | #include "llvm/MC/MCInstrInfo.h" |
| 29 | #include "llvm/MC/MCObjectFileInfo.h" |
| 30 | #include "llvm/MC/MCObjectWriter.h" |
| 31 | #include "llvm/MC/MCParser/MCTargetAsmParser.h" |
| 32 | #include "llvm/MC/MCRegisterInfo.h" |
| 33 | #include "llvm/MC/MCStreamer.h" |
| 34 | #include "llvm/MC/MCSubtargetInfo.h" |
| 35 | #include "llvm/MC/TargetRegistry.h" |
| 36 | #include "llvm/Support/FileSystem.h" |
| 37 | #include "llvm/Support/FileUtilities.h" |
| 38 | #include "llvm/Support/Path.h" |
| 39 | #include "llvm/Support/Program.h" |
| 40 | #include "llvm/Support/SourceMgr.h" |
| 41 | #include "llvm/Support/TargetSelect.h" |
| 42 | #include "llvm/TargetParser/AMDGPUTargetParser.h" |
| 43 | |
| 44 | #include <cstdlib> |
| 45 | #include <optional> |
| 46 | |
| 47 | using namespace mlir; |
| 48 | using namespace mlir::ROCDL; |
| 49 | |
| 50 | #ifndef __DEFAULT_ROCM_PATH__ |
| 51 | #define __DEFAULT_ROCM_PATH__ "" |
| 52 | #endif |
| 53 | |
| 54 | namespace { |
| 55 | // Implementation of the `TargetAttrInterface` model. |
| 56 | class ROCDLTargetAttrImpl |
| 57 | : public gpu::TargetAttrInterface::FallbackModel<ROCDLTargetAttrImpl> { |
| 58 | public: |
| 59 | std::optional<mlir::gpu::SerializedObject> |
| 60 | serializeToObject(Attribute attribute, Operation *module, |
| 61 | const gpu::TargetOptions &options) const; |
| 62 | |
| 63 | Attribute createObject(Attribute attribute, Operation *module, |
| 64 | const mlir::gpu::SerializedObject &object, |
| 65 | const gpu::TargetOptions &options) const; |
| 66 | }; |
| 67 | } // namespace |
| 68 | |
| 69 | // Register the ROCDL dialect, the ROCDL translation and the target interface. |
| 70 | void mlir::ROCDL::registerROCDLTargetInterfaceExternalModels( |
| 71 | DialectRegistry ®istry) { |
| 72 | registry.addExtension(extensionFn: +[](MLIRContext *ctx, ROCDL::ROCDLDialect *dialect) { |
| 73 | ROCDLTargetAttr::attachInterface<ROCDLTargetAttrImpl>(context&: *ctx); |
| 74 | }); |
| 75 | } |
| 76 | |
| 77 | void mlir::ROCDL::registerROCDLTargetInterfaceExternalModels( |
| 78 | MLIRContext &context) { |
| 79 | DialectRegistry registry; |
| 80 | registerROCDLTargetInterfaceExternalModels(registry); |
| 81 | context.appendDialectRegistry(registry); |
| 82 | } |
| 83 | |
| 84 | // Search for the ROCM path. |
| 85 | StringRef mlir::ROCDL::getROCMPath() { |
| 86 | if (const char *var = std::getenv(name: "ROCM_PATH" )) |
| 87 | return var; |
| 88 | if (const char *var = std::getenv(name: "ROCM_ROOT" )) |
| 89 | return var; |
| 90 | if (const char *var = std::getenv(name: "ROCM_HOME" )) |
| 91 | return var; |
| 92 | return __DEFAULT_ROCM_PATH__; |
| 93 | } |
| 94 | |
| 95 | SerializeGPUModuleBase::SerializeGPUModuleBase( |
| 96 | Operation &module, ROCDLTargetAttr target, |
| 97 | const gpu::TargetOptions &targetOptions) |
| 98 | : ModuleToObject(module, target.getTriple(), target.getChip(), |
| 99 | target.getFeatures(), target.getO(), {}, {}, {}, {}, |
| 100 | targetOptions.getLibraryOptions()), |
| 101 | target(target), toolkitPath(targetOptions.getToolkitPath()), |
| 102 | librariesToLink(targetOptions.getLibrariesToLink()) { |
| 103 | |
| 104 | // If `targetOptions` has an empty toolkitPath use `getROCMPath` |
| 105 | if (toolkitPath.empty()) |
| 106 | toolkitPath = getROCMPath(); |
| 107 | |
| 108 | // Append the files in the target attribute. |
| 109 | if (target.getLink()) |
| 110 | librariesToLink.append(in_start: target.getLink().begin(), in_end: target.getLink().end()); |
| 111 | } |
| 112 | |
| 113 | void SerializeGPUModuleBase::init() { |
| 114 | static llvm::once_flag initializeBackendOnce; |
| 115 | llvm::call_once(flag&: initializeBackendOnce, F: []() { |
| 116 | // If the `AMDGPU` LLVM target was built, initialize it. |
| 117 | #if LLVM_HAS_AMDGPU_TARGET |
| 118 | LLVMInitializeAMDGPUTarget(); |
| 119 | LLVMInitializeAMDGPUTargetInfo(); |
| 120 | LLVMInitializeAMDGPUTargetMC(); |
| 121 | LLVMInitializeAMDGPUAsmParser(); |
| 122 | LLVMInitializeAMDGPUAsmPrinter(); |
| 123 | #endif |
| 124 | }); |
| 125 | } |
| 126 | |
| 127 | ROCDLTargetAttr SerializeGPUModuleBase::getTarget() const { return target; } |
| 128 | |
| 129 | StringRef SerializeGPUModuleBase::getToolkitPath() const { return toolkitPath; } |
| 130 | |
| 131 | ArrayRef<Attribute> SerializeGPUModuleBase::getLibrariesToLink() const { |
| 132 | return librariesToLink; |
| 133 | } |
| 134 | |
| 135 | LogicalResult SerializeGPUModuleBase::appendStandardLibs(AMDGCNLibraries libs) { |
| 136 | if (libs == AMDGCNLibraries::None) |
| 137 | return success(); |
| 138 | StringRef pathRef = getToolkitPath(); |
| 139 | |
| 140 | // Get the path for the device libraries |
| 141 | SmallString<256> path; |
| 142 | path.insert(I: path.begin(), From: pathRef.begin(), To: pathRef.end()); |
| 143 | llvm::sys::path::append(path, a: "amdgcn" , b: "bitcode" ); |
| 144 | pathRef = StringRef(path.data(), path.size()); |
| 145 | |
| 146 | // Fail if the path is invalid. |
| 147 | if (!llvm::sys::fs::is_directory(Path: pathRef)) { |
| 148 | getOperation().emitError() << "ROCm amdgcn bitcode path: " << pathRef |
| 149 | << " does not exist or is not a directory" ; |
| 150 | return failure(); |
| 151 | } |
| 152 | |
| 153 | // Helper function for adding a library. |
| 154 | auto addLib = [&](const Twine &lib) -> bool { |
| 155 | auto baseSize = path.size(); |
| 156 | llvm::sys::path::append(path, a: lib); |
| 157 | StringRef pathRef(path.data(), path.size()); |
| 158 | if (!llvm::sys::fs::is_regular_file(Path: pathRef)) { |
| 159 | getOperation().emitRemark() << "bitcode library path: " << pathRef |
| 160 | << " does not exist or is not a file" ; |
| 161 | return true; |
| 162 | } |
| 163 | librariesToLink.push_back(Elt: StringAttr::get(context: target.getContext(), bytes: pathRef)); |
| 164 | path.truncate(N: baseSize); |
| 165 | return false; |
| 166 | }; |
| 167 | |
| 168 | // Add ROCm device libraries. Fail if any of the libraries is not found, ie. |
| 169 | // if any of the `addLib` failed. |
| 170 | if ((any(Val: libs & AMDGCNLibraries::Ocml) && addLib("ocml.bc" )) || |
| 171 | (any(Val: libs & AMDGCNLibraries::Ockl) && addLib("ockl.bc" )) || |
| 172 | (any(Val: libs & AMDGCNLibraries::Hip) && addLib("hip.bc" )) || |
| 173 | (any(Val: libs & AMDGCNLibraries::OpenCL) && addLib("opencl.bc" ))) |
| 174 | return failure(); |
| 175 | return success(); |
| 176 | } |
| 177 | |
| 178 | std::optional<SmallVector<std::unique_ptr<llvm::Module>>> |
| 179 | SerializeGPUModuleBase::loadBitcodeFiles(llvm::Module &module) { |
| 180 | // Return if there are no libs to load. |
| 181 | if (deviceLibs == AMDGCNLibraries::None && librariesToLink.empty()) |
| 182 | return SmallVector<std::unique_ptr<llvm::Module>>(); |
| 183 | if (failed(Result: appendStandardLibs(libs: deviceLibs))) |
| 184 | return std::nullopt; |
| 185 | SmallVector<std::unique_ptr<llvm::Module>> bcFiles; |
| 186 | if (failed(Result: loadBitcodeFilesFromList(context&: module.getContext(), librariesToLink, |
| 187 | llvmModules&: bcFiles, failureOnError: true))) |
| 188 | return std::nullopt; |
| 189 | return std::move(bcFiles); |
| 190 | } |
| 191 | |
| 192 | LogicalResult SerializeGPUModuleBase::handleBitcodeFile(llvm::Module &module) { |
| 193 | // Some ROCM builds don't strip this like they should |
| 194 | if (auto *openclVersion = module.getNamedMetadata(Name: "opencl.ocl.version" )) |
| 195 | module.eraseNamedMetadata(NMD: openclVersion); |
| 196 | // Stop spamming us with clang version numbers |
| 197 | if (auto *ident = module.getNamedMetadata(Name: "llvm.ident" )) |
| 198 | module.eraseNamedMetadata(NMD: ident); |
| 199 | // Override the libModules datalayout and target triple with the compiler's |
| 200 | // data layout should there be a discrepency. |
| 201 | setDataLayoutAndTriple(module); |
| 202 | return success(); |
| 203 | } |
| 204 | |
| 205 | void SerializeGPUModuleBase::handleModulePreLink(llvm::Module &module) { |
| 206 | // If all libraries are not set, traverse the module to determine which |
| 207 | // libraries are required. |
| 208 | if (deviceLibs != AMDGCNLibraries::All) { |
| 209 | for (llvm::Function &f : module.functions()) { |
| 210 | if (f.hasExternalLinkage() && f.hasName() && !f.hasExactDefinition()) { |
| 211 | StringRef funcName = f.getName(); |
| 212 | if ("printf" == funcName) |
| 213 | deviceLibs |= AMDGCNLibraries::OpenCL | AMDGCNLibraries::Ockl | |
| 214 | AMDGCNLibraries::Ocml; |
| 215 | if (funcName.starts_with(Prefix: "__ockl_" )) |
| 216 | deviceLibs |= AMDGCNLibraries::Ockl; |
| 217 | if (funcName.starts_with(Prefix: "__ocml_" )) |
| 218 | deviceLibs |= AMDGCNLibraries::Ocml; |
| 219 | if (funcName == "__atomic_work_item_fence" ) |
| 220 | deviceLibs |= AMDGCNLibraries::Hip; |
| 221 | } |
| 222 | } |
| 223 | } |
| 224 | addControlVariables(module, libs: deviceLibs, wave64: target.hasWave64(), daz: target.hasDaz(), |
| 225 | finiteOnly: target.hasFiniteOnly(), unsafeMath: target.hasUnsafeMath(), |
| 226 | fastMath: target.hasFastMath(), correctSqrt: target.hasCorrectSqrt(), |
| 227 | abiVer: target.getAbi()); |
| 228 | } |
| 229 | |
| 230 | void SerializeGPUModuleBase::addControlVariables( |
| 231 | llvm::Module &module, AMDGCNLibraries libs, bool wave64, bool daz, |
| 232 | bool finiteOnly, bool unsafeMath, bool fastMath, bool correctSqrt, |
| 233 | StringRef abiVer) { |
| 234 | // Helper function for adding control variables. |
| 235 | auto addControlVariable = [&module](StringRef name, uint32_t value, |
| 236 | uint32_t bitwidth) { |
| 237 | if (module.getNamedGlobal(Name: name)) |
| 238 | return; |
| 239 | llvm::IntegerType *type = |
| 240 | llvm::IntegerType::getIntNTy(C&: module.getContext(), N: bitwidth); |
| 241 | llvm::GlobalVariable *controlVariable = new llvm::GlobalVariable( |
| 242 | module, /*isConstant=*/type, true, |
| 243 | llvm::GlobalValue::LinkageTypes::LinkOnceODRLinkage, |
| 244 | llvm::ConstantInt::get(Ty: type, V: value), name, /*before=*/nullptr, |
| 245 | /*threadLocalMode=*/llvm::GlobalValue::ThreadLocalMode::NotThreadLocal, |
| 246 | /*addressSpace=*/4); |
| 247 | controlVariable->setVisibility( |
| 248 | llvm::GlobalValue::VisibilityTypes::ProtectedVisibility); |
| 249 | controlVariable->setAlignment(llvm::MaybeAlign(bitwidth / 8)); |
| 250 | controlVariable->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Local); |
| 251 | }; |
| 252 | |
| 253 | // Note that COV6 requires ROCm 6.3+. |
| 254 | int abi = 600; |
| 255 | abiVer.getAsInteger(Radix: 0, Result&: abi); |
| 256 | module.addModuleFlag(Behavior: llvm::Module::Error, Key: "amdhsa_code_object_version" , Val: abi); |
| 257 | // Return if no device libraries are required. |
| 258 | if (libs == AMDGCNLibraries::None) |
| 259 | return; |
| 260 | // Add ocml related control variables. |
| 261 | if (any(Val: libs & AMDGCNLibraries::Ocml)) { |
| 262 | addControlVariable("__oclc_finite_only_opt" , finiteOnly || fastMath, 8); |
| 263 | addControlVariable("__oclc_daz_opt" , daz || fastMath, 8); |
| 264 | addControlVariable("__oclc_correctly_rounded_sqrt32" , |
| 265 | correctSqrt && !fastMath, 8); |
| 266 | addControlVariable("__oclc_unsafe_math_opt" , unsafeMath || fastMath, 8); |
| 267 | } |
| 268 | // Add ocml or ockl related control variables. |
| 269 | if (any(Val: libs & (AMDGCNLibraries::Ocml | AMDGCNLibraries::Ockl))) { |
| 270 | addControlVariable("__oclc_wavefrontsize64" , wave64, 8); |
| 271 | // Get the ISA version. |
| 272 | llvm::AMDGPU::IsaVersion isaVersion = llvm::AMDGPU::getIsaVersion(GPU: chip); |
| 273 | // Add the ISA control variable. |
| 274 | addControlVariable("__oclc_ISA_version" , |
| 275 | isaVersion.Minor + 100 * isaVersion.Stepping + |
| 276 | 1000 * isaVersion.Major, |
| 277 | 32); |
| 278 | addControlVariable("__oclc_ABI_version" , abi, 32); |
| 279 | } |
| 280 | } |
| 281 | |
| 282 | FailureOr<SmallVector<char, 0>> |
| 283 | mlir::ROCDL::assembleIsa(StringRef isa, StringRef targetTriple, StringRef chip, |
| 284 | StringRef features, |
| 285 | function_ref<InFlightDiagnostic()> emitError) { |
| 286 | SmallVector<char, 0> result; |
| 287 | llvm::raw_svector_ostream os(result); |
| 288 | |
| 289 | llvm::Triple triple(llvm::Triple::normalize(Str: targetTriple)); |
| 290 | std::string error; |
| 291 | const llvm::Target *target = |
| 292 | llvm::TargetRegistry::lookupTarget(TheTriple: triple, Error&: error); |
| 293 | if (!target) |
| 294 | return emitError() << "failed to lookup target: " << error; |
| 295 | |
| 296 | llvm::SourceMgr srcMgr; |
| 297 | // Copy buffer to ensure it's null terminated. |
| 298 | srcMgr.AddNewSourceBuffer(F: llvm::MemoryBuffer::getMemBufferCopy(InputData: isa), IncludeLoc: SMLoc()); |
| 299 | |
| 300 | const llvm::MCTargetOptions mcOptions; |
| 301 | std::unique_ptr<llvm::MCRegisterInfo> mri(target->createMCRegInfo(TT: triple)); |
| 302 | std::unique_ptr<llvm::MCAsmInfo> mai( |
| 303 | target->createMCAsmInfo(MRI: *mri, TheTriple: triple, Options: mcOptions)); |
| 304 | std::unique_ptr<llvm::MCSubtargetInfo> sti( |
| 305 | target->createMCSubtargetInfo(TheTriple: triple, CPU: chip, Features: features)); |
| 306 | |
| 307 | llvm::MCContext ctx(triple, *mai, *mri, *sti, &srcMgr); |
| 308 | std::unique_ptr<llvm::MCObjectFileInfo> mofi(target->createMCObjectFileInfo( |
| 309 | Ctx&: ctx, /*PIC=*/false, /*LargeCodeModel=*/false)); |
| 310 | ctx.setObjectFileInfo(mofi.get()); |
| 311 | |
| 312 | SmallString<128> cwd; |
| 313 | if (!llvm::sys::fs::current_path(result&: cwd)) |
| 314 | ctx.setCompilationDir(cwd); |
| 315 | |
| 316 | std::unique_ptr<llvm::MCStreamer> mcStreamer; |
| 317 | std::unique_ptr<llvm::MCInstrInfo> mcii(target->createMCInstrInfo()); |
| 318 | |
| 319 | llvm::MCCodeEmitter *ce = target->createMCCodeEmitter(II: *mcii, Ctx&: ctx); |
| 320 | llvm::MCAsmBackend *mab = target->createMCAsmBackend(STI: *sti, MRI: *mri, Options: mcOptions); |
| 321 | mcStreamer.reset(p: target->createMCObjectStreamer( |
| 322 | T: triple, Ctx&: ctx, TAB: std::unique_ptr<llvm::MCAsmBackend>(mab), |
| 323 | OW: mab->createObjectWriter(OS&: os), Emitter: std::unique_ptr<llvm::MCCodeEmitter>(ce), |
| 324 | STI: *sti)); |
| 325 | |
| 326 | std::unique_ptr<llvm::MCAsmParser> parser( |
| 327 | createMCAsmParser(srcMgr, ctx, *mcStreamer, *mai)); |
| 328 | std::unique_ptr<llvm::MCTargetAsmParser> tap( |
| 329 | target->createMCAsmParser(STI: *sti, Parser&: *parser, MII: *mcii)); |
| 330 | |
| 331 | if (!tap) |
| 332 | return emitError() << "assembler initialization error" ; |
| 333 | |
| 334 | parser->setTargetParser(*tap); |
| 335 | parser->Run(NoInitialTextSection: false); |
| 336 | return std::move(result); |
| 337 | } |
| 338 | |
| 339 | FailureOr<SmallVector<char, 0>> |
| 340 | mlir::ROCDL::linkObjectCode(ArrayRef<char> objectCode, StringRef lldPath, |
| 341 | function_ref<InFlightDiagnostic()> emitError) { |
| 342 | // Save the ISA binary to a temp file. |
| 343 | int tempIsaBinaryFd = -1; |
| 344 | SmallString<128> tempIsaBinaryFilename; |
| 345 | if (llvm::sys::fs::createTemporaryFile(Prefix: "kernel%%" , Suffix: "o" , ResultFD&: tempIsaBinaryFd, |
| 346 | ResultPath&: tempIsaBinaryFilename)) |
| 347 | return emitError() |
| 348 | << "failed to create a temporary file for dumping the ISA binary" ; |
| 349 | |
| 350 | llvm::FileRemover cleanupIsaBinary(tempIsaBinaryFilename); |
| 351 | { |
| 352 | llvm::raw_fd_ostream tempIsaBinaryOs(tempIsaBinaryFd, true); |
| 353 | tempIsaBinaryOs << StringRef(objectCode.data(), objectCode.size()); |
| 354 | tempIsaBinaryOs.flush(); |
| 355 | } |
| 356 | |
| 357 | // Create a temp file for HSA code object. |
| 358 | SmallString<128> tempHsacoFilename; |
| 359 | if (llvm::sys::fs::createTemporaryFile(Prefix: "kernel" , Suffix: "hsaco" , ResultPath&: tempHsacoFilename)) |
| 360 | return emitError() |
| 361 | << "failed to create a temporary file for the HSA code object" ; |
| 362 | |
| 363 | llvm::FileRemover cleanupHsaco(tempHsacoFilename); |
| 364 | |
| 365 | int lldResult = llvm::sys::ExecuteAndWait( |
| 366 | Program: lldPath, |
| 367 | Args: {"ld.lld" , "-shared" , tempIsaBinaryFilename, "-o" , tempHsacoFilename}); |
| 368 | if (lldResult != 0) |
| 369 | return emitError() << "lld invocation failed" ; |
| 370 | |
| 371 | // Load the HSA code object. |
| 372 | auto hsacoFile = |
| 373 | llvm::MemoryBuffer::getFile(Filename: tempHsacoFilename, /*IsText=*/false); |
| 374 | if (!hsacoFile) |
| 375 | return emitError() |
| 376 | << "failed to read the HSA code object from the temp file" ; |
| 377 | |
| 378 | StringRef buffer = (*hsacoFile)->getBuffer(); |
| 379 | |
| 380 | return SmallVector<char, 0>(buffer.begin(), buffer.end()); |
| 381 | } |
| 382 | |
| 383 | FailureOr<SmallVector<char, 0>> |
| 384 | SerializeGPUModuleBase::compileToBinary(StringRef serializedISA) { |
| 385 | auto errCallback = [&]() { return getOperation().emitError(); }; |
| 386 | // Assemble the ISA. |
| 387 | FailureOr<SmallVector<char, 0>> isaBinary = ROCDL::assembleIsa( |
| 388 | isa: serializedISA, targetTriple: this->triple, chip: this->chip, features: this->features, emitError: errCallback); |
| 389 | |
| 390 | if (failed(Result: isaBinary)) |
| 391 | return failure(); |
| 392 | |
| 393 | // Link the object code. |
| 394 | llvm::SmallString<128> lldPath(toolkitPath); |
| 395 | llvm::sys::path::append(path&: lldPath, a: "llvm" , b: "bin" , c: "ld.lld" ); |
| 396 | FailureOr<SmallVector<char, 0>> linkedCode = |
| 397 | ROCDL::linkObjectCode(objectCode: *isaBinary, lldPath, emitError: errCallback); |
| 398 | if (failed(Result: linkedCode)) |
| 399 | return failure(); |
| 400 | |
| 401 | return linkedCode; |
| 402 | } |
| 403 | |
| 404 | FailureOr<SmallVector<char, 0>> SerializeGPUModuleBase::moduleToObjectImpl( |
| 405 | const gpu::TargetOptions &targetOptions, llvm::Module &llvmModule) { |
| 406 | // Return LLVM IR if the compilation target is offload. |
| 407 | #define DEBUG_TYPE "serialize-to-llvm" |
| 408 | LLVM_DEBUG({ |
| 409 | llvm::dbgs() << "LLVM IR for module: " |
| 410 | << cast<gpu::GPUModuleOp>(getOperation()).getNameAttr() << "\n" |
| 411 | << llvmModule << "\n" ; |
| 412 | }); |
| 413 | #undef DEBUG_TYPE |
| 414 | if (targetOptions.getCompilationTarget() == gpu::CompilationTarget::Offload) |
| 415 | return SerializeGPUModuleBase::moduleToObject(llvmModule); |
| 416 | |
| 417 | FailureOr<llvm::TargetMachine *> targetMachine = getOrCreateTargetMachine(); |
| 418 | if (failed(Result: targetMachine)) |
| 419 | return getOperation().emitError() |
| 420 | << "target Machine unavailable for triple " << triple |
| 421 | << ", can't compile with LLVM" ; |
| 422 | |
| 423 | // Translate the Module to ISA. |
| 424 | FailureOr<SmallString<0>> serializedISA = |
| 425 | translateModuleToISA(llvmModule, targetMachine&: **targetMachine, |
| 426 | emitError: [&]() { return getOperation().emitError(); }); |
| 427 | if (failed(Result: serializedISA)) |
| 428 | return getOperation().emitError() << "failed translating the module to ISA" ; |
| 429 | |
| 430 | #define DEBUG_TYPE "serialize-to-isa" |
| 431 | LLVM_DEBUG({ |
| 432 | llvm::dbgs() << "ISA for module: " |
| 433 | << cast<gpu::GPUModuleOp>(getOperation()).getNameAttr() << "\n" |
| 434 | << *serializedISA << "\n" ; |
| 435 | }); |
| 436 | #undef DEBUG_TYPE |
| 437 | // Return ISA assembly code if the compilation target is assembly. |
| 438 | if (targetOptions.getCompilationTarget() == gpu::CompilationTarget::Assembly) |
| 439 | return SmallVector<char, 0>(serializedISA->begin(), serializedISA->end()); |
| 440 | |
| 441 | // Compiling to binary requires a valid ROCm path, fail if it's not found. |
| 442 | if (getToolkitPath().empty()) |
| 443 | return getOperation().emitError() |
| 444 | << "invalid ROCm path, please set a valid path" ; |
| 445 | |
| 446 | // Compile to binary. |
| 447 | return compileToBinary(serializedISA: *serializedISA); |
| 448 | } |
| 449 | |
| 450 | #if LLVM_HAS_AMDGPU_TARGET |
| 451 | namespace { |
| 452 | class AMDGPUSerializer : public SerializeGPUModuleBase { |
| 453 | public: |
| 454 | AMDGPUSerializer(Operation &module, ROCDLTargetAttr target, |
| 455 | const gpu::TargetOptions &targetOptions); |
| 456 | |
| 457 | FailureOr<SmallVector<char, 0>> |
| 458 | moduleToObject(llvm::Module &llvmModule) override; |
| 459 | |
| 460 | private: |
| 461 | // Target options. |
| 462 | gpu::TargetOptions targetOptions; |
| 463 | }; |
| 464 | } // namespace |
| 465 | |
| 466 | AMDGPUSerializer::AMDGPUSerializer(Operation &module, ROCDLTargetAttr target, |
| 467 | const gpu::TargetOptions &targetOptions) |
| 468 | : SerializeGPUModuleBase(module, target, targetOptions), |
| 469 | targetOptions(targetOptions) {} |
| 470 | |
| 471 | FailureOr<SmallVector<char, 0>> |
| 472 | AMDGPUSerializer::moduleToObject(llvm::Module &llvmModule) { |
| 473 | return moduleToObjectImpl(targetOptions, llvmModule); |
| 474 | } |
| 475 | #endif // LLVM_HAS_AMDGPU_TARGET |
| 476 | |
| 477 | std::optional<mlir::gpu::SerializedObject> |
| 478 | ROCDLTargetAttrImpl::serializeToObject( |
| 479 | Attribute attribute, Operation *module, |
| 480 | const gpu::TargetOptions &options) const { |
| 481 | assert(module && "The module must be non null." ); |
| 482 | if (!module) |
| 483 | return std::nullopt; |
| 484 | if (!mlir::isa<gpu::GPUModuleOp>(Val: module)) { |
| 485 | module->emitError(message: "module must be a GPU module" ); |
| 486 | return std::nullopt; |
| 487 | } |
| 488 | #if LLVM_HAS_AMDGPU_TARGET |
| 489 | AMDGPUSerializer serializer(*module, cast<ROCDLTargetAttr>(Val&: attribute), |
| 490 | options); |
| 491 | serializer.init(); |
| 492 | std::optional<SmallVector<char, 0>> binary = serializer.run(); |
| 493 | if (!binary) |
| 494 | return std::nullopt; |
| 495 | return gpu::SerializedObject{std::move(*binary)}; |
| 496 | #else |
| 497 | module->emitError("the `AMDGPU` target was not built. Please enable it when " |
| 498 | "building LLVM" ); |
| 499 | return std::nullopt; |
| 500 | #endif // LLVM_HAS_AMDGPU_TARGET |
| 501 | } |
| 502 | |
| 503 | Attribute |
| 504 | ROCDLTargetAttrImpl::createObject(Attribute attribute, Operation *module, |
| 505 | const mlir::gpu::SerializedObject &object, |
| 506 | const gpu::TargetOptions &options) const { |
| 507 | gpu::CompilationTarget format = options.getCompilationTarget(); |
| 508 | // If format is `fatbin` transform it to binary as `fatbin` is not yet |
| 509 | // supported. |
| 510 | gpu::KernelTableAttr kernels; |
| 511 | if (format > gpu::CompilationTarget::Binary) { |
| 512 | format = gpu::CompilationTarget::Binary; |
| 513 | kernels = ROCDL::getKernelMetadata(gpuModule: module, elfData: object.getObject()); |
| 514 | } |
| 515 | DictionaryAttr properties{}; |
| 516 | Builder builder(attribute.getContext()); |
| 517 | StringAttr objectStr = builder.getStringAttr( |
| 518 | bytes: StringRef(object.getObject().data(), object.getObject().size())); |
| 519 | return builder.getAttr<gpu::ObjectAttr>(args&: attribute, args&: format, args&: objectStr, |
| 520 | args&: properties, args&: kernels); |
| 521 | } |
| 522 | |