1//===- ROCDLToLLVMIRTranslation.cpp - Translate ROCDL to LLVM IR ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a translation between the MLIR ROCDL dialect and
10// LLVM IR.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Target/LLVMIR/Dialect/ROCDL/ROCDLToLLVMIRTranslation.h"
15#include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
16#include "mlir/IR/BuiltinAttributes.h"
17#include "mlir/IR/Operation.h"
18#include "mlir/Target/LLVMIR/ModuleTranslation.h"
19
20#include "llvm/IR/IRBuilder.h"
21#include "llvm/IR/IntrinsicsAMDGPU.h"
22#include "llvm/IR/LLVMContext.h"
23#include "llvm/Support/raw_ostream.h"
24#include <cstdint>
25
26using namespace mlir;
27using namespace mlir::LLVM;
28using mlir::LLVM::detail::createIntrinsicCall;
29
30namespace {
31/// Implementation of the dialect interface that converts operations belonging
32/// to the ROCDL dialect to LLVM IR.
33class ROCDLDialectLLVMIRTranslationInterface
34 : public LLVMTranslationDialectInterface {
35public:
36 using LLVMTranslationDialectInterface::LLVMTranslationDialectInterface;
37
38 /// Translates the given operation to LLVM IR using the provided IR builder
39 /// and saving the state in `moduleTranslation`.
40 LogicalResult
41 convertOperation(Operation *op, llvm::IRBuilderBase &builder,
42 LLVM::ModuleTranslation &moduleTranslation) const final {
43 Operation &opInst = *op;
44#include "mlir/Dialect/LLVMIR/ROCDLConversions.inc"
45
46 return failure();
47 }
48
49 /// Attaches module-level metadata for functions marked as kernels.
50 LogicalResult
51 amendOperation(Operation *op, ArrayRef<llvm::Instruction *> instructions,
52 NamedAttribute attribute,
53 LLVM::ModuleTranslation &moduleTranslation) const final {
54 auto *dialect = dyn_cast<ROCDL::ROCDLDialect>(Val: attribute.getNameDialect());
55 llvm::LLVMContext &llvmContext = moduleTranslation.getLLVMContext();
56 if (dialect->getKernelAttrHelper().getName() == attribute.getName()) {
57 auto func = dyn_cast<LLVM::LLVMFuncOp>(Val: op);
58 if (!func)
59 return op->emitOpError(message: Twine(attribute.getName()) +
60 " is only supported on `llvm.func` operations");
61 ;
62
63 // For GPU kernels,
64 // 1. Insert AMDGPU_KERNEL calling convention.
65 // 2. Insert amdgpu-flat-work-group-size(1, 256) attribute unless the user
66 // has overriden this value - 256 is the default in clang
67 llvm::Function *llvmFunc =
68 moduleTranslation.lookupFunction(name: func.getName());
69 llvmFunc->setCallingConv(llvm::CallingConv::AMDGPU_KERNEL);
70 if (!llvmFunc->hasFnAttribute(Kind: "amdgpu-flat-work-group-size")) {
71 llvmFunc->addFnAttr(Kind: "amdgpu-flat-work-group-size", Val: "1,256");
72 }
73
74 // MLIR's GPU kernel APIs all assume and produce uniformly-sized
75 // workgroups, so the lowering of the `rocdl.kernel` marker encodes this
76 // assumption. This assumption may be overridden by setting
77 // `rocdl.uniform_work_group_size` on a given function.
78 if (!llvmFunc->hasFnAttribute(Kind: "uniform-work-group-size"))
79 llvmFunc->addFnAttr(Kind: "uniform-work-group-size");
80 }
81 // Override flat-work-group-size
82 // TODO: update clients to rocdl.flat_work_group_size instead,
83 // then remove this half of the branch
84 if (dialect->getMaxFlatWorkGroupSizeAttrHelper().getName() ==
85 attribute.getName()) {
86 auto func = dyn_cast<LLVM::LLVMFuncOp>(Val: op);
87 if (!func)
88 return op->emitOpError(message: Twine(attribute.getName()) +
89 " is only supported on `llvm.func` operations");
90 auto value = dyn_cast<IntegerAttr>(Val: attribute.getValue());
91 if (!value)
92 return op->emitOpError(message: Twine(attribute.getName()) +
93 " must be an integer");
94
95 llvm::Function *llvmFunc =
96 moduleTranslation.lookupFunction(name: func.getName());
97 llvm::SmallString<8> llvmAttrValue;
98 llvm::raw_svector_ostream attrValueStream(llvmAttrValue);
99 attrValueStream << "1," << value.getInt();
100 llvmFunc->addFnAttr(Kind: "amdgpu-flat-work-group-size", Val: llvmAttrValue);
101 }
102 if (dialect->getWavesPerEuAttrHelper().getName() == attribute.getName()) {
103 auto func = dyn_cast<LLVM::LLVMFuncOp>(Val: op);
104 if (!func)
105 return op->emitOpError(message: Twine(attribute.getName()) +
106 " is only supported on `llvm.func` operations");
107
108 llvm::Function *llvmFunc =
109 moduleTranslation.lookupFunction(name: func.getName());
110 llvm::SmallString<8> llvmAttrValue;
111 if (auto strValue = dyn_cast<StringAttr>(Val: attribute.getValue())) {
112 llvmAttrValue.append(RHS: strValue.getValue());
113 } else if (auto intValue = dyn_cast<IntegerAttr>(Val: attribute.getValue())) {
114 llvm::raw_svector_ostream attrValueStream(llvmAttrValue);
115 attrValueStream << intValue.getInt();
116 } else {
117 return op->emitOpError(message: Twine(attribute.getName()) +
118 " must be a string or an integer");
119 }
120 llvmFunc->addFnAttr(Kind: "amdgpu-waves-per-eu", Val: llvmAttrValue);
121 }
122 if (dialect->getFlatWorkGroupSizeAttrHelper().getName() ==
123 attribute.getName()) {
124 auto func = dyn_cast<LLVM::LLVMFuncOp>(Val: op);
125 if (!func)
126 return op->emitOpError(message: Twine(attribute.getName()) +
127 " is only supported on `llvm.func` operations");
128 auto value = dyn_cast<StringAttr>(Val: attribute.getValue());
129 if (!value)
130 return op->emitOpError(message: Twine(attribute.getName()) +
131 " must be a string");
132
133 llvm::Function *llvmFunc =
134 moduleTranslation.lookupFunction(name: func.getName());
135 llvm::SmallString<8> llvmAttrValue;
136 llvmAttrValue.append(RHS: value.getValue());
137 llvmFunc->addFnAttr(Kind: "amdgpu-flat-work-group-size", Val: llvmAttrValue);
138 }
139 if (ROCDL::ROCDLDialect::getUniformWorkGroupSizeAttrName() ==
140 attribute.getName()) {
141 auto func = dyn_cast<LLVM::LLVMFuncOp>(Val: op);
142 if (!func)
143 return op->emitOpError(message: Twine(attribute.getName()) +
144 " is only supported on `llvm.func` operations");
145 auto value = dyn_cast<BoolAttr>(Val: attribute.getValue());
146 if (!value)
147 return op->emitOpError(message: Twine(attribute.getName()) +
148 " must be a boolean");
149 llvm::Function *llvmFunc =
150 moduleTranslation.lookupFunction(name: func.getName());
151 if (value.getValue())
152 llvmFunc->addFnAttr(Kind: "uniform-work-group-size");
153 else
154 llvmFunc->removeFnAttr(Kind: "uniform-work-group-size");
155 }
156
157 bool isXnack =
158 dialect->getXnackAttrHelper().getName() == attribute.getName();
159 bool isSramecc =
160 dialect->getSrameccAttrHelper().getName() == attribute.getName();
161 if (isXnack || isSramecc) {
162 auto value = dyn_cast<BoolAttr>(Val: attribute.getValue());
163 if (!value)
164 return op->emitOpError(message: Twine(attribute.getName()) +
165 " must be a boolean");
166 StringRef key = isXnack
167 ? ROCDL::ROCDLDialect::getModuleFlagKeyXnackName()
168 : ROCDL::ROCDLDialect::getModuleFlagKeySramEccName();
169 moduleTranslation.getLLVMModule()->addModuleFlag(
170 Behavior: llvm::Module::Error, Key: key,
171 Val: llvm::ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: llvmContext),
172 V: value.getValue()));
173 }
174 if (dialect->getUnsafeFpAtomicsAttrHelper().getName() ==
175 attribute.getName()) {
176 auto func = dyn_cast<LLVM::LLVMFuncOp>(Val: op);
177 if (!func)
178 return op->emitOpError(message: Twine(attribute.getName()) +
179 " is only supported on `llvm.func` operations");
180 auto value = dyn_cast<BoolAttr>(Val: attribute.getValue());
181 if (!value)
182 return op->emitOpError(message: Twine(attribute.getName()) +
183 " must be a boolean");
184 llvm::Function *llvmFunc =
185 moduleTranslation.lookupFunction(name: func.getName());
186 llvmFunc->addFnAttr(Kind: "amdgpu-unsafe-fp-atomics",
187 Val: value.getValue() ? "true" : "false");
188 }
189 // Set reqd_work_group_size metadata
190 if (dialect->getReqdWorkGroupSizeAttrHelper().getName() ==
191 attribute.getName()) {
192 auto func = dyn_cast<LLVM::LLVMFuncOp>(Val: op);
193 if (!func)
194 return op->emitOpError(message: Twine(attribute.getName()) +
195 " is only supported on `llvm.func` operations");
196 auto value = dyn_cast<DenseI32ArrayAttr>(Val: attribute.getValue());
197 if (!value)
198 return op->emitOpError(message: Twine(attribute.getName()) +
199 " must be a dense i32 array attribute");
200 if (value.asArrayRef().size() != 3)
201 return op->emitOpError(message: Twine(attribute.getName()) +
202 " must contain exactly three values");
203
204 uint64_t FlatWorkGroupSize = 1;
205 SmallVector<llvm::Metadata *, 3> metadata;
206 llvm::Type *i32 = llvm::IntegerType::get(C&: llvmContext, NumBits: 32);
207 for (int32_t i : value.asArrayRef()) {
208 FlatWorkGroupSize *= static_cast<uint32_t>(i);
209 llvm::Constant *constant = llvm::ConstantInt::get(Ty: i32, V: i);
210 metadata.push_back(Elt: llvm::ConstantAsMetadata::get(C: constant));
211 }
212 llvm::Function *llvmFunc =
213 moduleTranslation.lookupFunction(name: func.getName());
214 llvm::SmallString<16> expectedFlatWorkGroupSize;
215 llvm::raw_svector_ostream attrValueStream(expectedFlatWorkGroupSize);
216 attrValueStream << FlatWorkGroupSize << "," << FlatWorkGroupSize;
217
218 StringRef flatAttrName =
219 dialect->getFlatWorkGroupSizeAttrHelper().getName();
220 if (auto flatAttr = dyn_cast_if_present<StringAttr>(
221 Val: op->getDiscardableAttr(name: flatAttrName))) {
222 if (flatAttr.getValue() != expectedFlatWorkGroupSize)
223 return op->emitOpError(message: Twine(flatAttrName) +
224 " must match rocdl.reqd_work_group_size");
225 }
226
227 StringRef maxFlatAttrName =
228 dialect->getMaxFlatWorkGroupSizeAttrHelper().getName();
229 if (auto maxFlatAttr = dyn_cast_if_present<IntegerAttr>(
230 Val: op->getDiscardableAttr(name: maxFlatAttrName))) {
231 llvm::SmallString<16> expectedMaxFlatWorkGroupSize;
232 llvm::raw_svector_ostream maxAttrValueStream(
233 expectedMaxFlatWorkGroupSize);
234 maxAttrValueStream << "1," << maxFlatAttr.getInt();
235 if (expectedMaxFlatWorkGroupSize != expectedFlatWorkGroupSize)
236 return op->emitOpError(message: Twine(maxFlatAttrName) +
237 " must match rocdl.reqd_work_group_size");
238 }
239
240 llvmFunc->addFnAttr(Kind: "amdgpu-flat-work-group-size",
241 Val: expectedFlatWorkGroupSize);
242 llvm::MDNode *node = llvm::MDNode::get(Context&: llvmContext, MDs: metadata);
243 llvmFunc->setMetadata(Kind: "reqd_work_group_size", Node: node);
244 }
245
246 // Atomic and nontemporal metadata
247 if (dialect->getLastUseAttrHelper().getName() == attribute.getName()) {
248 for (llvm::Instruction *i : instructions)
249 i->setMetadata(Kind: "amdgpu.last.use", Node: llvm::MDNode::get(Context&: llvmContext, MDs: {}));
250 }
251 if (dialect->getNoRemoteMemoryAttrHelper().getName() ==
252 attribute.getName()) {
253 for (llvm::Instruction *i : instructions)
254 i->setMetadata(Kind: "amdgpu.no.remote.memory",
255 Node: llvm::MDNode::get(Context&: llvmContext, MDs: {}));
256 }
257 if (dialect->getNoFineGrainedMemoryAttrHelper().getName() ==
258 attribute.getName()) {
259 for (llvm::Instruction *i : instructions)
260 i->setMetadata(Kind: "amdgpu.no.fine.grained.memory",
261 Node: llvm::MDNode::get(Context&: llvmContext, MDs: {}));
262 }
263 if (dialect->getIgnoreDenormalModeAttrHelper().getName() ==
264 attribute.getName()) {
265 op->emitWarning(message: "rocdl.ignore_denormal_mode is deprecated; use "
266 "ignore_denormal_mode on llvm.atomicrmw instead");
267 for (llvm::Instruction *i : instructions)
268 i->setMetadata(KindID: llvm::LLVMContext::MD_atomic_ignore_denormal_mode,
269 Node: llvm::MDNode::get(Context&: llvmContext, MDs: {}));
270 }
271
272 return success();
273 }
274};
275} // namespace
276
277void mlir::registerROCDLDialectTranslation(DialectRegistry &registry) {
278 registry.insert<ROCDL::ROCDLDialect>();
279 registry.addExtension(extensionFn: +[](MLIRContext *ctx, ROCDL::ROCDLDialect *dialect) {
280 dialect->addInterfaces<ROCDLDialectLLVMIRTranslationInterface>();
281 });
282}
283
284void mlir::registerROCDLDialectTranslation(MLIRContext &context) {
285 DialectRegistry registry;
286 registerROCDLDialectTranslation(registry);
287 context.appendDialectRegistry(registry);
288}
289