1//===- AMDGPUOpenCLEnqueuedBlockLowering.cpp - Lower enqueued block -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// \file
10// This post-linking pass replaces the function pointer of enqueued
11// block kernel with a global variable (runtime handle) and adds
12// "runtime-handle" attribute to the enqueued block kernel.
13//
14// In LLVM CodeGen the runtime-handle metadata will be translated to
15// RuntimeHandle metadata in code object. Runtime allocates a global buffer
16// for each kernel with RuntimeHandle metadata and saves the kernel address
17// required for the AQL packet into the buffer. __enqueue_kernel function
18// in device library knows that the invoke function pointer in the block
19// literal is actually runtime handle and loads the kernel address from it
20// and put it into AQL packet for dispatching.
21//
22// This cannot be done in FE since FE cannot create a unique global variable
23// with external linkage across LLVM modules. The global variable with internal
24// linkage does not work since optimization passes will try to replace loads
25// of the global variable with its initialization value.
26//
27// It also identifies the kernels directly or indirectly enqueues kernels
28// and adds "calls-enqueue-kernel" function attribute to them, which will
29// be used to determine whether to emit runtime metadata for the kernel
30// enqueue related hidden kernel arguments.
31//
32//===----------------------------------------------------------------------===//
33
34#include "AMDGPU.h"
35#include "llvm/ADT/DenseSet.h"
36#include "llvm/ADT/SmallString.h"
37#include "llvm/IR/Constants.h"
38#include "llvm/IR/Instructions.h"
39#include "llvm/IR/Mangler.h"
40#include "llvm/IR/Module.h"
41#include "llvm/Pass.h"
42#include "llvm/Support/Debug.h"
43
44#define DEBUG_TYPE "amdgpu-lower-enqueued-block"
45
46using namespace llvm;
47
48namespace {
49
50/// Lower enqueued blocks.
51class AMDGPUOpenCLEnqueuedBlockLowering : public ModulePass {
52public:
53 static char ID;
54
55 explicit AMDGPUOpenCLEnqueuedBlockLowering() : ModulePass(ID) {}
56
57private:
58 bool runOnModule(Module &M) override;
59};
60
61} // end anonymous namespace
62
63char AMDGPUOpenCLEnqueuedBlockLowering::ID = 0;
64
65char &llvm::AMDGPUOpenCLEnqueuedBlockLoweringID =
66 AMDGPUOpenCLEnqueuedBlockLowering::ID;
67
68INITIALIZE_PASS(AMDGPUOpenCLEnqueuedBlockLowering, DEBUG_TYPE,
69 "Lower OpenCL enqueued blocks", false, false)
70
71ModulePass* llvm::createAMDGPUOpenCLEnqueuedBlockLoweringPass() {
72 return new AMDGPUOpenCLEnqueuedBlockLowering();
73}
74
75bool AMDGPUOpenCLEnqueuedBlockLowering::runOnModule(Module &M) {
76 DenseSet<Function *> Callers;
77 auto &C = M.getContext();
78 bool Changed = false;
79
80 // ptr kernel_object, i32 private_segment_size, i32 group_segment_size
81 StructType *HandleTy = nullptr;
82
83 for (auto &F : M.functions()) {
84 if (F.hasFnAttribute(Kind: "enqueued-block")) {
85 if (!F.hasName()) {
86 SmallString<64> Name;
87 Mangler::getNameWithPrefix(OutName&: Name, GVName: "__amdgpu_enqueued_kernel",
88 DL: M.getDataLayout());
89 F.setName(Name);
90 }
91 LLVM_DEBUG(dbgs() << "found enqueued kernel: " << F.getName() << '\n');
92 auto RuntimeHandle = (F.getName() + ".runtime_handle").str();
93 if (!HandleTy) {
94 Type *Int32 = Type::getInt32Ty(C);
95 HandleTy =
96 StructType::create(Context&: C, Elements: {PointerType::getUnqual(C), Int32, Int32},
97 Name: "block.runtime.handle.t");
98 }
99
100 auto *GV = new GlobalVariable(
101 M, HandleTy,
102 /*isConstant=*/true, GlobalValue::ExternalLinkage,
103 /*Initializer=*/Constant::getNullValue(Ty: HandleTy), RuntimeHandle,
104 /*InsertBefore=*/nullptr, GlobalValue::NotThreadLocal,
105 AMDGPUAS::GLOBAL_ADDRESS,
106 /*isExternallyInitialized=*/true);
107 LLVM_DEBUG(dbgs() << "runtime handle created: " << *GV << '\n');
108
109 F.replaceAllUsesWith(V: ConstantExpr::getAddrSpaceCast(C: GV, Ty: F.getType()));
110 F.addFnAttr(Kind: "runtime-handle", Val: RuntimeHandle);
111 F.setLinkage(GlobalValue::ExternalLinkage);
112 Changed = true;
113 }
114 }
115
116 return Changed;
117}
118