| 1 | //===- Target.cpp - MLIR LLVM XeVM target compilation -----------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This files defines XeVM target related functions including registration |
| 10 | // calls for the `#xevm.target` compilation attribute. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "mlir/Target/LLVM/XeVM/Target.h" |
| 15 | #include "mlir/Dialect/GPU/IR/CompilationInterfaces.h" |
| 16 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 17 | #include "mlir/Dialect/LLVMIR/XeVMDialect.h" |
| 18 | #include "mlir/IR/BuiltinAttributeInterfaces.h" |
| 19 | #include "mlir/IR/BuiltinDialect.h" |
| 20 | #include "mlir/IR/BuiltinTypes.h" |
| 21 | #include "mlir/IR/DialectResourceBlobManager.h" |
| 22 | #include "mlir/Target/LLVM/XeVM/Utils.h" |
| 23 | #include "mlir/Target/LLVMIR/Dialect/GPU/GPUToLLVMIRTranslation.h" |
| 24 | #include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.h" |
| 25 | #include "mlir/Target/LLVMIR/Dialect/XeVM/XeVMToLLVMIRTranslation.h" |
| 26 | #include "mlir/Target/LLVMIR/Export.h" |
| 27 | #include "llvm/ADT/ScopeExit.h" |
| 28 | #include "llvm/Bitcode/BitcodeWriter.h" |
| 29 | #include "llvm/Config/Targets.h" |
| 30 | #include "llvm/IR/LegacyPassManager.h" |
| 31 | #include "llvm/Support/FileSystem.h" |
| 32 | #include "llvm/Support/FileUtilities.h" |
| 33 | #include "llvm/Support/FormatVariadic.h" |
| 34 | #include "llvm/Support/MemoryBuffer.h" |
| 35 | #include "llvm/Support/Path.h" |
| 36 | #include "llvm/Support/Program.h" |
| 37 | #include "llvm/Support/TargetSelect.h" |
| 38 | #include "llvm/Support/raw_ostream.h" |
| 39 | #include "llvm/Target/TargetMachine.h" |
| 40 | |
| 41 | #include <cstdint> |
| 42 | #include <cstdlib> |
| 43 | |
| 44 | using namespace mlir; |
| 45 | using namespace mlir::xevm; |
| 46 | |
| 47 | #if MLIR_XEVM_OCLOC_LIB_AVAILABLE |
| 48 | // Intel compute runtime includes libocloc in the distribution, but |
| 49 | // <ocloc_api.h> isn't included. Hence forward declarations for the libocloc |
| 50 | // shared-library APIs is needed. These replace the inclusion of <ocloc_api.h> |
| 51 | // so that the header is not a build-time requirement; the symbols are resolved |
| 52 | // at link/load time via the ocloc shared library. |
| 53 | extern "C" { |
| 54 | |
| 55 | // Return code indicating successful ocloc compilation. |
| 56 | // Matches the OCLOC_SUCCESS enumerator in the real header (value 0). |
| 57 | enum OclocErrorCode : int { OCLOC_SUCCESS = 0 }; |
| 58 | |
| 59 | // Drives an in-process ocloc compilation. |
| 60 | /// Invokes ocloc API using C interface. Supported commands match |
| 61 | /// the functionality of ocloc executable (check ocloc's "help" |
| 62 | /// for reference : shared/offline_compiler/source/ocloc_api.cpp). |
| 63 | /// |
| 64 | /// numArgs and argv params represent the command line. |
| 65 | /// Remaining params represent I/O. |
| 66 | /// Output params should be freed using oclocFreeOutput when |
| 67 | /// no longer needed. |
| 68 | /// List and names of outputs match outputs of ocloc executable. |
| 69 | /// |
| 70 | /// \param numArgs is the number of arguments to pass to ocloc |
| 71 | /// |
| 72 | /// \param argv is an array of arguments to be passed to ocloc |
| 73 | /// |
| 74 | /// \param numSources is the number of in-memory representations |
| 75 | /// of source files to be passed to ocloc |
| 76 | /// |
| 77 | /// \param dataSources is an array of in-memory representations |
| 78 | /// of source files to be passed to ocloc |
| 79 | /// |
| 80 | /// \param lenSources is an array of sizes of in-memory representations |
| 81 | /// of source files passed to ocloc as dataSources |
| 82 | /// |
| 83 | /// \param nameSources is an array of names of in-memory representations |
| 84 | /// of source files passed to ocloc as dataSources |
| 85 | /// |
| 86 | /// \param numInputHeaders is the number of in-memory representations |
| 87 | /// of header files to be passed to ocloc |
| 88 | /// |
| 89 | /// \param dataInputHeaders is an array of in-memory representations |
| 90 | /// of header files to be passed to ocloc |
| 91 | /// |
| 92 | /// \param lenInputHeaders is an array of sizes of in-memory representations |
| 93 | /// of header files passed to ocloc as dataInputHeaders |
| 94 | /// |
| 95 | /// \param nameInputHeaders is an array of names of in-memory representations |
| 96 | /// of header files passed to ocloc as dataInputHeaders |
| 97 | /// |
| 98 | /// \param numOutputs returns the number of outputs |
| 99 | /// |
| 100 | /// \param dataOutputs returns an array of in-memory representations |
| 101 | /// of output files |
| 102 | /// |
| 103 | /// \param lenOutputs returns an array of sizes of in-memory representations |
| 104 | /// of output files |
| 105 | /// |
| 106 | /// \param nameOutputs returns an array of names of in-memory representations |
| 107 | /// of output files. Special name stdout.log describes output that contains |
| 108 | /// messages generated by ocloc (e.g. compiler errors/warnings) |
| 109 | /// |
| 110 | /// \returns 0 on success. Returns non-0 in case of failure. |
| 111 | |
| 112 | int oclocInvoke(unsigned numArgs, const char **argv, unsigned numSources, |
| 113 | const uint8_t **dataSources, const uint64_t *lenSources, |
| 114 | const char **nameSources, unsigned numHeaders, |
| 115 | const uint8_t **dataHeaders, const uint64_t *lenHeaders, |
| 116 | const char **nameHeaders, unsigned *numOutputs, |
| 117 | uint8_t ***dataOutputs, uint64_t **lenOutputs, |
| 118 | char ***nameOutputs); |
| 119 | |
| 120 | /// Frees results of oclocInvoke |
| 121 | /// |
| 122 | /// \param numOutputs is number of outputs as returned by oclocInvoke |
| 123 | /// |
| 124 | /// \param dataOutputs is array of outputs as returned by oclocInvoke |
| 125 | /// |
| 126 | /// \param lenOutputs is array of sizes of outputs as returned by oclocInvoke |
| 127 | /// |
| 128 | /// \param nameOutputs is array of names of outputs as returned by oclocInvoke |
| 129 | /// |
| 130 | /// \returns 0 on success. Returns non-0 in case of failure. |
| 131 | int oclocFreeOutput(unsigned *numOutputs, uint8_t ***dataOutputs, |
| 132 | uint64_t **lenOutputs, char ***nameOutputs); |
| 133 | |
| 134 | } // extern "C" |
| 135 | #endif // MLIR_XEVM_OCLOC_LIB_AVAILABLE |
| 136 | |
| 137 | namespace { |
| 138 | // XeVM implementation of the gpu:TargetAttrInterface. |
| 139 | class XeVMTargetAttrImpl |
| 140 | : public gpu::TargetAttrInterface::FallbackModel<XeVMTargetAttrImpl> { |
| 141 | public: |
| 142 | std::optional<mlir::gpu::SerializedObject> |
| 143 | serializeToObject(Attribute attribute, Operation *module, |
| 144 | const gpu::TargetOptions &options) const; |
| 145 | |
| 146 | Attribute createObject(Attribute attribute, Operation *module, |
| 147 | const mlir::gpu::SerializedObject &object, |
| 148 | const gpu::TargetOptions &options) const; |
| 149 | }; |
| 150 | } // namespace |
| 151 | |
| 152 | void mlir::xevm::registerXeVMTargetInterfaceExternalModels( |
| 153 | DialectRegistry ®istry) { |
| 154 | registry.addExtension(extensionFn: +[](MLIRContext *ctx, XeVMDialect *dialect) { |
| 155 | XeVMTargetAttr::attachInterface<XeVMTargetAttrImpl>(context&: *ctx); |
| 156 | }); |
| 157 | } |
| 158 | |
| 159 | void mlir::xevm::registerXeVMTargetInterfaceExternalModels( |
| 160 | MLIRContext &context) { |
| 161 | DialectRegistry registry; |
| 162 | registerXeVMTargetInterfaceExternalModels(registry); |
| 163 | context.appendDialectRegistry(registry); |
| 164 | } |
| 165 | |
| 166 | SerializeGPUModuleBase::SerializeGPUModuleBase( |
| 167 | Operation &module, XeVMTargetAttr xeTarget, |
| 168 | const gpu::TargetOptions &targetOptions) |
| 169 | : ModuleToObject(module, xeTarget.getTriple(), "" , {}, xeTarget.getO(), {}, |
| 170 | {}, {}, {}, targetOptions.getLibraryOptions()), |
| 171 | xeTarget(xeTarget), librariesToLink(targetOptions.getLibrariesToLink()), |
| 172 | targetOptions(targetOptions) { |
| 173 | if (xeTarget.getLinkFiles()) |
| 174 | librariesToLink.append(in_start: xeTarget.getLinkFiles().begin(), |
| 175 | in_end: xeTarget.getLinkFiles().end()); |
| 176 | } |
| 177 | |
| 178 | XeVMTargetAttr SerializeGPUModuleBase::getTarget() const { return xeTarget; } |
| 179 | |
| 180 | std::optional<SmallVector<std::unique_ptr<llvm::Module>>> |
| 181 | SerializeGPUModuleBase::loadBitcodeFiles(llvm::Module &module) { |
| 182 | if (librariesToLink.empty()) |
| 183 | return SmallVector<std::unique_ptr<llvm::Module>>(); |
| 184 | SmallVector<std::unique_ptr<llvm::Module>> bcFiles; |
| 185 | if (failed(Result: loadBitcodeFilesFromList(context&: module.getContext(), librariesToLink, |
| 186 | llvmModules&: bcFiles))) |
| 187 | return std::nullopt; |
| 188 | return std::move(bcFiles); |
| 189 | } |
| 190 | |
| 191 | gpu::GPUModuleOp SerializeGPUModuleBase::getGPUModuleOp() { |
| 192 | return dyn_cast<gpu::GPUModuleOp>(Val: &SerializeGPUModuleBase::getOperation()); |
| 193 | } |
| 194 | |
| 195 | // ---------------------------------------------------------------------------- |
| 196 | // compile via the ocloc shared-library API (in-process, no temp files). Only |
| 197 | // compiled when the library is available at build time. |
| 198 | // ---------------------------------------------------------------------------- |
| 199 | #if MLIR_XEVM_OCLOC_LIB_AVAILABLE |
| 200 | FailureOr<SmallVector<char, 0>> |
| 201 | SerializeGPUModuleBase::compileToBinaryViaLibocloc(StringRef asmStr, |
| 202 | StringRef inputFormat) { |
| 203 | Location loc = getGPUModuleOp().getLoc(); |
| 204 | std::string asmFname = llvm::formatv( |
| 205 | "mlir-{0}-{1}-{2}.asm" , getGPUModuleOp().getNameAttr().getValue(), |
| 206 | getTarget().getTriple(), getTarget().getChip()); |
| 207 | |
| 208 | // Build command-line options. |
| 209 | std::pair<llvm::BumpPtrAllocator, SmallVector<const char *>> cmdOpts = |
| 210 | targetOptions.tokenizeCmdOptions(); |
| 211 | // Example: --gpu-module-to-binary="opts='opt1 opt2'" |
| 212 | const std::string cmdOptsStr = "\"" + llvm::join(cmdOpts.second, " " ) + "\"" ; |
| 213 | std::vector<std::string> oclocArgs = {"ocloc" , |
| 214 | "compile" , |
| 215 | "-file" , |
| 216 | asmFname, |
| 217 | inputFormat.str(), |
| 218 | "-device" , |
| 219 | getTarget().getChip().str(), |
| 220 | "-options" , |
| 221 | cmdOptsStr}; |
| 222 | |
| 223 | // Dump tool invocation commands. |
| 224 | #define DEBUG_TYPE "serialize-to-binary" |
| 225 | LLVM_DEBUG({ |
| 226 | llvm::dbgs() << "libocloc invocation for module: " |
| 227 | << getGPUModuleOp().getNameAttr() << "\n" ; |
| 228 | llvm::interleave(oclocArgs, llvm::dbgs(), " " ); |
| 229 | llvm::dbgs() << "\n" ; |
| 230 | }); |
| 231 | #undef DEBUG_TYPE |
| 232 | |
| 233 | // Build a plain argv array expected by oclocInvoke. |
| 234 | std::vector<const char *> argv; |
| 235 | for (const auto &str : oclocArgs) |
| 236 | argv.push_back(str.c_str()); |
| 237 | |
| 238 | // Wire up in-memory source file. |
| 239 | const uint8_t *dataSources[1] = { |
| 240 | reinterpret_cast<const uint8_t *>(asmStr.data())}; |
| 241 | const uint64_t lenSources[1] = {asmStr.size()}; |
| 242 | const char *nameSources[1] = {asmFname.c_str()}; |
| 243 | |
| 244 | uint32_t outputsNum = 0; |
| 245 | uint8_t **outputs = nullptr; |
| 246 | uint64_t *outputLengths = nullptr; |
| 247 | char **outputNames = nullptr; |
| 248 | // Ensure ocloc output buffers are always freed on exit. |
| 249 | auto freeOutputs = llvm::scope_exit([&]() { |
| 250 | oclocFreeOutput(&outputsNum, &outputs, &outputLengths, &outputNames); |
| 251 | }); |
| 252 | |
| 253 | int err = oclocInvoke(static_cast<uint32_t>(argv.size()), argv.data(), |
| 254 | /*numSources=*/1, dataSources, lenSources, nameSources, |
| 255 | /*numHeaders=*/0, nullptr, nullptr, nullptr, |
| 256 | &outputsNum, &outputs, &outputLengths, &outputNames); |
| 257 | |
| 258 | if (err != OCLOC_SUCCESS) { |
| 259 | emitError(loc) << "libocloc: `oclocInvoke` failed, error code: " << err; |
| 260 | // Emit any compiler log that ocloc produced. |
| 261 | for (uint32_t i = 0; i < outputsNum; ++i) { |
| 262 | if (llvm::StringRef(outputNames[i]).ends_with(".log" )) |
| 263 | emitError(loc) << "Compiler log:\n" |
| 264 | << llvm::StringRef(reinterpret_cast<char *>(outputs[i]), |
| 265 | outputLengths[i]); |
| 266 | } |
| 267 | return failure(); |
| 268 | } |
| 269 | |
| 270 | // Find and return the .bin output. |
| 271 | for (uint32_t i = 0; i < outputsNum; ++i) { |
| 272 | if (llvm::StringRef(outputNames[i]).ends_with(".bin" )) { |
| 273 | char *begin = reinterpret_cast<char *>(outputs[i]); |
| 274 | return SmallVector<char, 0>(begin, begin + outputLengths[i]); |
| 275 | } |
| 276 | } |
| 277 | return emitError(loc) << "`oclocInvoke` did not produce `.bin` output" ; |
| 278 | } |
| 279 | #endif // MLIR_XEVM_OCLOC_LIB_AVAILABLE |
| 280 | |
| 281 | // ---------------------------------------------------------------------------- |
| 282 | // Compile by spawning the `ocloc` command-line tool as a process, |
| 283 | // communicating through temporary files. Acts as a fallback when the shared |
| 284 | // library is not available. |
| 285 | // ---------------------------------------------------------------------------- |
| 286 | FailureOr<SmallVector<char, 0>> |
| 287 | SerializeGPUModuleBase::compileToBinaryViaOclocTool(StringRef asmStr, |
| 288 | StringRef inputFormat) { |
| 289 | using TmpFile = std::pair<llvm::SmallString<128>, llvm::FileRemover>; |
| 290 | |
| 291 | // Locate the `ocloc` executable on PATH. |
| 292 | std::optional<std::string> oclocPath = findTool(tool: "ocloc" ); |
| 293 | if (!oclocPath) { |
| 294 | emitError(loc: getGPUModuleOp().getLoc()) << "Could not find `ocloc` on PATH" ; |
| 295 | return failure(); |
| 296 | } |
| 297 | |
| 298 | Location loc = getGPUModuleOp().getLoc(); |
| 299 | std::string basename = llvm::formatv( |
| 300 | Fmt: "mlir-{0}-{1}-{2}" , Vals: getGPUModuleOp().getNameAttr().getValue(), |
| 301 | Vals: getTarget().getTriple(), Vals: getTarget().getChip()); |
| 302 | |
| 303 | // Helper: create a named temporary file, returning path + auto-remover. |
| 304 | auto createTemp = [&](StringRef name, |
| 305 | StringRef suffix) -> std::optional<TmpFile> { |
| 306 | llvm::SmallString<128> path; |
| 307 | if (auto ec = llvm::sys::fs::createTemporaryFile(Prefix: name, Suffix: suffix, ResultPath&: path)) { |
| 308 | emitError(loc) << "Couldn't create temp file `" << path |
| 309 | << "`: " << ec.message(); |
| 310 | return std::nullopt; |
| 311 | } |
| 312 | return TmpFile(path, llvm::FileRemover(path.c_str())); |
| 313 | }; |
| 314 | |
| 315 | std::optional<TmpFile> asmFile = createTemp(basename, "asm" ); |
| 316 | std::optional<TmpFile> binFile = createTemp(basename, "" ); |
| 317 | std::optional<TmpFile> logFile = createTemp(basename, "log" ); |
| 318 | if (!asmFile || !binFile || !logFile) |
| 319 | return failure(); |
| 320 | |
| 321 | // Write the assembly source to a temp file. |
| 322 | { |
| 323 | std::error_code ec; |
| 324 | llvm::raw_fd_ostream asmStream(asmFile->first, ec); |
| 325 | if (ec) { |
| 326 | emitError(loc) << "Couldn't open `" << asmFile->first |
| 327 | << "`: " << ec.message(); |
| 328 | return failure(); |
| 329 | } |
| 330 | asmStream << asmStr; |
| 331 | if (asmStream.has_error()) { |
| 332 | emitError(loc) << "Error writing assembly to `" << asmFile->first << "`" ; |
| 333 | return failure(); |
| 334 | } |
| 335 | asmStream.flush(); |
| 336 | } |
| 337 | |
| 338 | // Build command-line options. |
| 339 | std::pair<llvm::BumpPtrAllocator, SmallVector<const char *>> cmdOpts = |
| 340 | targetOptions.tokenizeCmdOptions(); |
| 341 | const std::string cmdOptsStr = "\"" + llvm::join(R&: cmdOpts.second, Separator: " " ) + "\"" ; |
| 342 | |
| 343 | SmallVector<StringRef, 12> oclocArgs( |
| 344 | {"ocloc" , "compile" , "-file" , asmFile->first, inputFormat, "-device" , |
| 345 | getTarget().getChip(), "-output" , binFile->first, "-output_no_suffix" , |
| 346 | "-options" , cmdOptsStr}); |
| 347 | |
| 348 | // Dump tool invocation commands. |
| 349 | #define DEBUG_TYPE "serialize-to-binary" |
| 350 | LLVM_DEBUG({ |
| 351 | llvm::dbgs() << "Tool invocation for module: " |
| 352 | << getGPUModuleOp().getNameAttr() << "\n" ; |
| 353 | llvm::interleave(oclocArgs, llvm::dbgs(), " " ); |
| 354 | llvm::dbgs() << "\n" ; |
| 355 | }); |
| 356 | #undef DEBUG_TYPE |
| 357 | |
| 358 | // Redirect stdout/stderr to the log temp file. |
| 359 | std::optional<StringRef> redirects[] = {std::nullopt, logFile->first, |
| 360 | logFile->first}; |
| 361 | |
| 362 | std::string errorMsg; |
| 363 | if (llvm::sys::ExecuteAndWait(Program: *oclocPath, Args: oclocArgs, Env: std::nullopt, Redirects: redirects, |
| 364 | SecondsToWait: 0, MemoryLimit: 0, ErrMsg: &errorMsg)) { |
| 365 | // Prefer a structured error message; otherwise dump the log file. |
| 366 | if (!errorMsg.empty()) { |
| 367 | emitError(loc) << "`ocloc` invocation failed: " << errorMsg; |
| 368 | } else if (auto log = llvm::MemoryBuffer::getFile(Filename: logFile->first)) { |
| 369 | emitError(loc) << "`ocloc` invocation failed. Log:\n" |
| 370 | << (*log)->getBuffer(); |
| 371 | } else { |
| 372 | emitError(loc) << "`ocloc` invocation failed (no log available)" ; |
| 373 | } |
| 374 | return failure(); |
| 375 | } |
| 376 | |
| 377 | // Read back the binary output (ocloc appends ".bin" to the base name). |
| 378 | binFile->first.append(RHS: ".bin" ); |
| 379 | auto binaryBuffer = llvm::MemoryBuffer::getFile(Filename: binFile->first); |
| 380 | if (!binaryBuffer) { |
| 381 | emitError(loc) << "Couldn't open binary output `" << binFile->first |
| 382 | << "`: " << binaryBuffer.getError().message(); |
| 383 | return failure(); |
| 384 | } |
| 385 | StringRef bin = (*binaryBuffer)->getBuffer(); |
| 386 | return SmallVector<char, 0>(bin.begin(), bin.end()); |
| 387 | } |
| 388 | |
| 389 | // ---------------------------------------------------------------------------- |
| 390 | // Public entry-point: prefer the in-process library path; fall back to the |
| 391 | // external tool when the library is not available. |
| 392 | // ---------------------------------------------------------------------------- |
| 393 | FailureOr<SmallVector<char, 0>> |
| 394 | SerializeGPUModuleBase::compileToBinary(StringRef asmStr, |
| 395 | StringRef inputFormat) { |
| 396 | #if MLIR_XEVM_OCLOC_LIB_AVAILABLE |
| 397 | return compileToBinaryViaLibocloc(asmStr, inputFormat); |
| 398 | #else |
| 399 | return compileToBinaryViaOclocTool(asmStr, inputFormat); |
| 400 | #endif |
| 401 | } |
| 402 | |
| 403 | std::optional<std::string> SerializeGPUModuleBase::findTool(StringRef tool) { |
| 404 | // 1. Check the toolkit path given in the command line. |
| 405 | StringRef pathRef = targetOptions.getToolkitPath(); |
| 406 | SmallVector<char, 256> path; |
| 407 | if (!pathRef.empty()) { |
| 408 | path.insert(I: path.begin(), From: pathRef.begin(), To: pathRef.end()); |
| 409 | llvm::sys::path::append(path, a: "bin" , b: tool); |
| 410 | if (llvm::sys::fs::can_execute(Path: path)) |
| 411 | return StringRef(path.data(), path.size()).str(); |
| 412 | } |
| 413 | // 2. Check PATH. |
| 414 | if (llvm::ErrorOr<std::string> toolPath = llvm::sys::findProgramByName(Name: tool)) |
| 415 | return *toolPath; |
| 416 | |
| 417 | getGPUModuleOp().emitError() |
| 418 | << "Couldn't find the `" << tool |
| 419 | << "` binary. Please specify the toolkit " |
| 420 | "path via GpuModuleToBinaryPass or add the compiler to $PATH`." ; |
| 421 | return std::nullopt; |
| 422 | } |
| 423 | namespace { |
| 424 | class SPIRVSerializer : public SerializeGPUModuleBase { |
| 425 | public: |
| 426 | SPIRVSerializer(Operation &module, XeVMTargetAttr xeTarget, |
| 427 | const gpu::TargetOptions &targetOptions) |
| 428 | : SerializeGPUModuleBase(module, xeTarget, targetOptions) {} |
| 429 | |
| 430 | static void init(); |
| 431 | |
| 432 | /// Serializes the LLVM module to an object format, depending on the |
| 433 | /// compilation target selected in target options. |
| 434 | FailureOr<SmallVector<char, 0>> |
| 435 | moduleToObject(llvm::Module &llvmModule) override; |
| 436 | |
| 437 | /// Runs the serialization pipeline, returning `std::nullopt` on error. |
| 438 | std::optional<SmallVector<char, 0>> run() override; |
| 439 | |
| 440 | private: |
| 441 | /// Translates the LLVM module to SPIR-V binary using LLVM's |
| 442 | /// SPIR-V target. |
| 443 | std::optional<std::string> |
| 444 | translateToSPIRVBinary(llvm::Module &llvmModule, |
| 445 | llvm::TargetMachine &targetMachine); |
| 446 | }; |
| 447 | } // namespace |
| 448 | |
| 449 | void SPIRVSerializer::init() { |
| 450 | static llvm::once_flag initializeBackendOnce; |
| 451 | llvm::call_once(flag&: initializeBackendOnce, F: []() { |
| 452 | #if LLVM_HAS_SPIRV_TARGET |
| 453 | LLVMInitializeSPIRVTarget(); |
| 454 | LLVMInitializeSPIRVTargetInfo(); |
| 455 | LLVMInitializeSPIRVTargetMC(); |
| 456 | LLVMInitializeSPIRVAsmPrinter(); |
| 457 | #endif |
| 458 | }); |
| 459 | } |
| 460 | |
| 461 | #if LLVM_HAS_SPIRV_TARGET |
| 462 | // Enable default set of SPIR-V extensions. |
| 463 | // Supported extensions vary based on chip. See, |
| 464 | // https://github.com/intel/intel-graphics-compiler/blob/master/ |
| 465 | // documentation/igc/supported-spirv-extensions.md |
| 466 | // for details. |
| 467 | static const std::vector<std::string> getDefaultSPIRVExtensions() { |
| 468 | return { |
| 469 | "SPV_KHR_bfloat16" , |
| 470 | "SPV_EXT_relaxed_printf_string_address_space" , |
| 471 | "SPV_INTEL_cache_controls" , |
| 472 | "SPV_INTEL_variable_length_array" , |
| 473 | "SPV_EXT_bfloat16_arithmetic" , |
| 474 | }; |
| 475 | } |
| 476 | |
| 477 | namespace llvm { |
| 478 | class Module; |
| 479 | |
| 480 | extern "C" bool |
| 481 | SPIRVTranslateModule(Module *M, std::string &SpirvObj, std::string &ErrMsg, |
| 482 | const std::vector<std::string> &AllowExtNames, |
| 483 | const std::vector<std::string> &Opts); |
| 484 | } // namespace llvm |
| 485 | #endif |
| 486 | |
| 487 | // There is 1 way to finalize IL to native code: IGC |
| 488 | // There are 2 ways to access IGC: AOT (ocloc) and JIT (L0 runtime). |
| 489 | // - L0 runtime consumes IL and is external to MLIR codebase (rt wrappers). |
| 490 | // - `ocloc` tool can be "queried" from within MLIR. |
| 491 | FailureOr<SmallVector<char, 0>> |
| 492 | SPIRVSerializer::moduleToObject(llvm::Module &llvmModule) { |
| 493 | #define DEBUG_TYPE "serialize-to-llvm" |
| 494 | LLVM_DEBUG({ |
| 495 | llvm::dbgs() << "LLVM IR for module: " << getGPUModuleOp().getNameAttr() |
| 496 | << "\n" ; |
| 497 | llvm::dbgs() << llvmModule << "\n" ; |
| 498 | llvm::dbgs().flush(); |
| 499 | }); |
| 500 | #undef DEBUG_TYPE |
| 501 | |
| 502 | // Return LLVM IR if the compilation target is `offload`. |
| 503 | if (targetOptions.getCompilationTarget() == gpu::CompilationTarget::Offload) |
| 504 | return SerializeGPUModuleBase::moduleToObject(llvmModule); |
| 505 | |
| 506 | #if !LLVM_HAS_SPIRV_TARGET |
| 507 | return getGPUModuleOp()->emitError( |
| 508 | "The `SPIRV` target was not built. Please enable " |
| 509 | "it when building LLVM." ); |
| 510 | #else |
| 511 | std::string serializedSPIRVBinary; |
| 512 | std::string ErrMsg; |
| 513 | std::vector<std::string> Opts; |
| 514 | Opts.push_back(x: triple.str()); |
| 515 | Opts.push_back(x: std::to_string(val: optLevel)); |
| 516 | |
| 517 | // Translate the LLVM module to SPIR-V binary using LLVM's SPIR-V Backend API. |
| 518 | bool success = |
| 519 | SPIRVTranslateModule(M: &llvmModule, SpirvObj&: serializedSPIRVBinary, ErrMsg, |
| 520 | AllowExtNames: getDefaultSPIRVExtensions(), Opts); |
| 521 | |
| 522 | if (!success) |
| 523 | return getGPUModuleOp().emitError() |
| 524 | << "Failed translating the module to Binary." |
| 525 | << "Error message: " << ErrMsg; |
| 526 | |
| 527 | if (serializedSPIRVBinary.size() % 4) |
| 528 | return getGPUModuleOp().emitError() |
| 529 | << "SPIRV code size must be a multiple of 4." ; |
| 530 | |
| 531 | StringRef spirvBin(serializedSPIRVBinary.c_str(), |
| 532 | serializedSPIRVBinary.size()); |
| 533 | |
| 534 | // Return SPIRV binary if the compilation target is `assembly`. Optimization |
| 535 | // and SPIR-V extensions are enabled for SPIR-V binary output in both paths |
| 536 | // (assembly and binary) as of now. SPIR-V binary |
| 537 | // is generated directly using the SPIR-V backends `SPIRVTranslateModule` API. |
| 538 | if (targetOptions.getCompilationTarget() == |
| 539 | gpu::CompilationTarget::Assembly) { |
| 540 | #define DEBUG_TYPE "serialize-to-isa" |
| 541 | LLVM_DEBUG({ |
| 542 | llvm::dbgs() << "SPIR-V for module: " << getGPUModuleOp().getNameAttr() |
| 543 | << "\n" ; |
| 544 | llvm::dbgs() << serializedSPIRVBinary << "\n" ; |
| 545 | llvm::dbgs().flush(); |
| 546 | }); |
| 547 | #undef DEBUG_TYPE |
| 548 | return SmallVector<char, 0>(spirvBin.begin(), spirvBin.end()); |
| 549 | } |
| 550 | |
| 551 | // Return native binary. Compile the SPIR-V binary to native binary for Intel |
| 552 | // GPUs using `ocloc` compiler (Intel's OpenCL Offline Compiler). |
| 553 | |
| 554 | return compileToBinary(asmStr: spirvBin, inputFormat: "-spirv_input" ); |
| 555 | #endif // LLVM_HAS_SPIRV_TARGET |
| 556 | } |
| 557 | |
| 558 | std::optional<SmallVector<char, 0>> SPIRVSerializer::run() { |
| 559 | // Translate the module to LLVM IR. |
| 560 | llvm::LLVMContext llvmContext; |
| 561 | std::unique_ptr<llvm::Module> llvmModule = translateToLLVMIR(llvmContext); |
| 562 | if (!llvmModule) { |
| 563 | getOperation().emitError() << "Failed creating the llvm::Module." ; |
| 564 | return std::nullopt; |
| 565 | } |
| 566 | setDataLayoutAndTriple(*llvmModule); |
| 567 | |
| 568 | if (initialLlvmIRCallback) |
| 569 | initialLlvmIRCallback(*llvmModule); |
| 570 | |
| 571 | // Link bitcode files. |
| 572 | handleModulePreLink(module&: *llvmModule); |
| 573 | { |
| 574 | auto libs = loadBitcodeFiles(module&: *llvmModule); |
| 575 | if (!libs) |
| 576 | return std::nullopt; |
| 577 | if (!libs->empty()) |
| 578 | if (failed(Result: linkFiles(module&: *llvmModule, libs: std::move(*libs)))) |
| 579 | return std::nullopt; |
| 580 | handleModulePostLink(module&: *llvmModule); |
| 581 | } |
| 582 | |
| 583 | if (linkedLlvmIRCallback) |
| 584 | linkedLlvmIRCallback(*llvmModule); |
| 585 | |
| 586 | // Return the serialized object. |
| 587 | return moduleToObject(llvmModule&: *llvmModule); |
| 588 | } |
| 589 | |
| 590 | std::optional<std::string> |
| 591 | SPIRVSerializer::translateToSPIRVBinary(llvm::Module &llvmModule, |
| 592 | llvm::TargetMachine &targetMachine) { |
| 593 | std::string targetISA; |
| 594 | llvm::raw_string_ostream stream(targetISA); |
| 595 | |
| 596 | { // Drop pstream after this to prevent the ISA from being stuck buffering |
| 597 | llvm::buffer_ostream pstream(stream); |
| 598 | llvm::legacy::PassManager codegenPasses; |
| 599 | if (targetMachine.addPassesToEmitFile(codegenPasses, pstream, nullptr, |
| 600 | llvm::CodeGenFileType::ObjectFile)) |
| 601 | return std::nullopt; |
| 602 | |
| 603 | codegenPasses.run(M&: llvmModule); |
| 604 | } |
| 605 | return targetISA; |
| 606 | } |
| 607 | |
| 608 | std::optional<mlir::gpu::SerializedObject> |
| 609 | XeVMTargetAttrImpl::serializeToObject(Attribute attribute, Operation *module, |
| 610 | const gpu::TargetOptions &options) const { |
| 611 | if (!module) |
| 612 | return std::nullopt; |
| 613 | auto gpuMod = dyn_cast<gpu::GPUModuleOp>(Val: module); |
| 614 | if (!gpuMod) { |
| 615 | module->emitError(message: "expected to be a gpu.module op" ); |
| 616 | return std::nullopt; |
| 617 | } |
| 618 | auto xeTarget = cast<XeVMTargetAttr>(Val&: attribute); |
| 619 | if (xeTarget.getTriple().starts_with(Prefix: "spirv" )) { |
| 620 | gpuMod.walk(callback: [&](LLVM::LLVMFuncOp funcOp) { |
| 621 | if (funcOp->hasDiscardableAttr(name: gpu::GPUDialect::getKernelFuncAttrName())) |
| 622 | funcOp.setIntelReqdSubGroupSize(16); |
| 623 | }); |
| 624 | |
| 625 | SPIRVSerializer serializer(*module, cast<XeVMTargetAttr>(Val&: attribute), |
| 626 | options); |
| 627 | serializer.init(); |
| 628 | |
| 629 | #if !LLVM_HAS_SPIRV_TARGET |
| 630 | module->emitError("Cannot run `TargetRegistry::lookupTarget()` for SPIRV " |
| 631 | "without having the target built." ); |
| 632 | #endif |
| 633 | |
| 634 | std::optional<SmallVector<char, 0>> binary = serializer.run(); |
| 635 | if (!binary) |
| 636 | return std::nullopt; |
| 637 | return gpu::SerializedObject{std::move(*binary)}; |
| 638 | } |
| 639 | module->emitError(message: "Unsupported XeVM target triple: " ) << xeTarget.getTriple(); |
| 640 | return std::nullopt; |
| 641 | } |
| 642 | |
| 643 | Attribute |
| 644 | XeVMTargetAttrImpl::createObject(Attribute attribute, Operation *module, |
| 645 | const mlir::gpu::SerializedObject &object, |
| 646 | const gpu::TargetOptions &options) const { |
| 647 | Builder builder(attribute.getContext()); |
| 648 | gpu::CompilationTarget format = options.getCompilationTarget(); |
| 649 | auto xeTarget = cast<XeVMTargetAttr>(Val&: attribute); |
| 650 | SmallVector<NamedAttribute, 2> properties; |
| 651 | if (format == gpu::CompilationTarget::Assembly) |
| 652 | properties.push_back( |
| 653 | Elt: builder.getNamedAttr(name: "O" , val: builder.getI32IntegerAttr(value: xeTarget.getO()))); |
| 654 | |
| 655 | DictionaryAttr objectProps; |
| 656 | if (!properties.empty()) |
| 657 | objectProps = builder.getDictionaryAttr(value: properties); |
| 658 | |
| 659 | return builder.getAttr<gpu::ObjectAttr>( |
| 660 | args&: attribute, args&: format, |
| 661 | args: builder.getStringAttr( |
| 662 | bytes: StringRef(object.getObject().data(), object.getObject().size())), |
| 663 | args&: objectProps, /*kernels=*/args: nullptr); |
| 664 | } |
| 665 | |