1//===- GPUBase.cpp - GPU types and attributes -----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements GPU types, attributes, and enums.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Dialect/GPU/IR/GPUBase.h"
14
15#include "mlir/IR/Builders.h"
16#include "mlir/IR/DialectImplementation.h"
17#include "llvm/ADT/TypeSwitch.h"
18#include "llvm/Support/CommandLine.h"
19#include "llvm/Support/StringSaver.h"
20
21using namespace mlir;
22using namespace mlir::gpu;
23
24//===----------------------------------------------------------------------===//
25// GPU Device Mapping Attributes
26//===----------------------------------------------------------------------===//
27
28int64_t GPUBlockMappingAttr::getMappingId() const {
29 return static_cast<int64_t>(getBlock());
30}
31
32bool GPUBlockMappingAttr::isLinearMapping() const {
33 return getMappingId() >= static_cast<int64_t>(MappingId::LinearDim0);
34}
35
36int64_t GPUBlockMappingAttr::getRelativeIndex() const {
37 return isLinearMapping()
38 ? getMappingId() - static_cast<int64_t>(MappingId::LinearDim0)
39 : getMappingId();
40}
41
42int64_t GPUWarpgroupMappingAttr::getMappingId() const {
43 return static_cast<int64_t>(getWarpgroup());
44}
45
46bool GPUWarpgroupMappingAttr::isLinearMapping() const {
47 return getMappingId() >= static_cast<int64_t>(MappingId::LinearDim0);
48}
49
50int64_t GPUWarpgroupMappingAttr::getRelativeIndex() const {
51 return isLinearMapping()
52 ? getMappingId() - static_cast<int64_t>(MappingId::LinearDim0)
53 : getMappingId();
54}
55
56int64_t GPUWarpMappingAttr::getMappingId() const {
57 return static_cast<int64_t>(getWarp());
58}
59
60bool GPUWarpMappingAttr::isLinearMapping() const {
61 return getMappingId() >= static_cast<int64_t>(MappingId::LinearDim0);
62}
63
64int64_t GPUWarpMappingAttr::getRelativeIndex() const {
65 return isLinearMapping()
66 ? getMappingId() - static_cast<int64_t>(MappingId::LinearDim0)
67 : getMappingId();
68}
69
70int64_t GPUThreadMappingAttr::getMappingId() const {
71 return static_cast<int64_t>(getThread());
72}
73
74bool GPUThreadMappingAttr::isLinearMapping() const {
75 return getMappingId() >= static_cast<int64_t>(MappingId::LinearDim0);
76}
77
78int64_t GPUThreadMappingAttr::getRelativeIndex() const {
79 return isLinearMapping()
80 ? getMappingId() - static_cast<int64_t>(MappingId::LinearDim0)
81 : getMappingId();
82}
83
84int64_t GPULaneMappingAttr::getMappingId() const {
85 return static_cast<int64_t>(getLane());
86}
87
88bool GPULaneMappingAttr::isLinearMapping() const {
89 return getMappingId() >= static_cast<int64_t>(MappingId::LinearDim0);
90}
91
92int64_t GPULaneMappingAttr::getRelativeIndex() const {
93 return isLinearMapping()
94 ? getMappingId() - static_cast<int64_t>(MappingId::LinearDim0)
95 : getMappingId();
96}
97
98int64_t GPUMemorySpaceMappingAttr::getMappingId() const {
99 return static_cast<int64_t>(getAddressSpace());
100}
101
102bool GPUMemorySpaceMappingAttr::isLinearMapping() const {
103 llvm_unreachable("GPUMemorySpaceMappingAttr does not support linear mapping");
104}
105
106int64_t GPUMemorySpaceMappingAttr::getRelativeIndex() const {
107 llvm_unreachable("GPUMemorySpaceMappingAttr does not support relative index");
108}
109
110//===----------------------------------------------------------------------===//
111// MMAMatrixType
112//===----------------------------------------------------------------------===//
113
114MMAMatrixType MMAMatrixType::get(ArrayRef<int64_t> shape, Type elementType,
115 StringRef operand) {
116 return Base::get(ctx: elementType.getContext(), args&: shape, args&: elementType, args&: operand);
117}
118
119MMAMatrixType
120MMAMatrixType::getChecked(function_ref<InFlightDiagnostic()> emitError,
121 ArrayRef<int64_t> shape, Type elementType,
122 StringRef operand) {
123 return Base::getChecked(emitErrorFn: emitError, ctx: elementType.getContext(), args: shape,
124 args: elementType, args: operand);
125}
126
127unsigned MMAMatrixType::getNumDims() const { return getImpl()->numDims; }
128
129ArrayRef<int64_t> MMAMatrixType::getShape() const {
130 return getImpl()->getShape();
131}
132
133Type MMAMatrixType::getElementType() const { return getImpl()->elementType; }
134
135StringRef MMAMatrixType::getOperand() const { return getImpl()->getOperand(); }
136
137bool MMAMatrixType::isValidElementType(Type elementType) {
138 return elementType.isF16() || elementType.isF32() || elementType.isF64() ||
139 elementType.isUnsignedInteger(width: 8) || elementType.isSignedInteger(width: 8) ||
140 elementType.isInteger(width: 32);
141}
142
143LogicalResult
144MMAMatrixType::verifyInvariants(function_ref<InFlightDiagnostic()> emitError,
145 ArrayRef<int64_t> shape, Type elementType,
146 StringRef operand) {
147 if (operand != "AOp" && operand != "BOp" && operand != "COp")
148 return emitError() << "operand expected to be one of AOp, BOp or COp";
149
150 if (shape.size() != 2)
151 return emitError() << "MMAMatrixType must have exactly two dimensions";
152
153 if (!MMAMatrixType::isValidElementType(elementType))
154 return emitError()
155 << "MMAMatrixType elements must be SI8, UI8, I32, F16, F32, or F64";
156
157 return success();
158}
159
160//===----------------------------------------------------------------------===//
161// GPUDialect
162//===----------------------------------------------------------------------===//
163
164bool GPUDialect::isWorkgroupMemoryAddressSpace(Attribute memorySpace) {
165 if (!memorySpace)
166 return false;
167 if (auto gpuAttr = llvm::dyn_cast<gpu::AddressSpaceAttr>(Val&: memorySpace))
168 return gpuAttr.getValue() == getWorkgroupAddressSpace();
169 return false;
170}
171
172bool GPUDialect::hasWorkgroupMemoryAddressSpace(MemRefType type) {
173 Attribute memorySpace = type.getMemorySpace();
174 return isWorkgroupMemoryAddressSpace(memorySpace);
175}
176
177bool GPUDialect::isConstantMemoryAddressSpace(Attribute memorySpace) {
178 if (!memorySpace)
179 return false;
180 if (auto gpuAttr = llvm::dyn_cast<gpu::AddressSpaceAttr>(Val&: memorySpace))
181 return gpuAttr.getValue() == getConstantAddressSpace();
182 return false;
183}
184
185bool GPUDialect::hasConstantMemoryAddressSpace(MemRefType type) {
186 Attribute memorySpace = type.getMemorySpace();
187 return isConstantMemoryAddressSpace(memorySpace);
188}
189
190//===----------------------------------------------------------------------===//
191// GPU object attribute
192//===----------------------------------------------------------------------===//
193
194LogicalResult ObjectAttr::verify(function_ref<InFlightDiagnostic()> emitError,
195 Attribute target, CompilationTarget format,
196 StringAttr object, DictionaryAttr properties,
197 KernelTableAttr kernels) {
198 if (!target)
199 return emitError() << "the target attribute cannot be null";
200 if (target.hasPromiseOrImplementsInterface<TargetAttrInterface>())
201 return success();
202 return emitError() << "the target attribute must implement or promise the "
203 "`gpu::TargetAttrInterface`";
204}
205
206namespace {
207ParseResult parseObject(AsmParser &odsParser, CompilationTarget &format,
208 StringAttr &object) {
209 std::optional<CompilationTarget> formatResult;
210 StringRef enumKeyword;
211 auto loc = odsParser.getCurrentLocation();
212 if (failed(Result: odsParser.parseOptionalKeyword(keyword: &enumKeyword)))
213 formatResult = CompilationTarget::Fatbin;
214 if (!formatResult &&
215 (formatResult =
216 gpu::symbolizeEnum<gpu::CompilationTarget>(str: enumKeyword)) &&
217 odsParser.parseEqual())
218 return odsParser.emitError(loc, message: "expected an equal sign");
219 if (!formatResult)
220 return odsParser.emitError(loc, message: "expected keyword for GPU object format");
221 FailureOr<StringAttr> objectResult =
222 FieldParser<StringAttr>::parse(parser&: odsParser);
223 if (failed(Result: objectResult))
224 return odsParser.emitError(loc: odsParser.getCurrentLocation(),
225 message: "failed to parse GPU_ObjectAttr parameter "
226 "'object' which is to be a `StringAttr`");
227 format = *formatResult;
228 object = *objectResult;
229 return success();
230}
231
232void printObject(AsmPrinter &odsParser, CompilationTarget format,
233 StringAttr object) {
234 if (format != CompilationTarget::Fatbin)
235 odsParser << stringifyEnum(enumValue: format) << " = ";
236 odsParser << object;
237}
238} // namespace
239
240//===----------------------------------------------------------------------===//
241// GPU select object attribute
242//===----------------------------------------------------------------------===//
243
244LogicalResult
245gpu::SelectObjectAttr::verify(function_ref<InFlightDiagnostic()> emitError,
246 Attribute target) {
247 // Check `target`, it can be null, an integer attr or a GPU Target attribute.
248 if (target) {
249 if (auto intAttr = mlir::dyn_cast<IntegerAttr>(Val&: target)) {
250 if (intAttr.getInt() < 0) {
251 return emitError() << "the object index must be positive";
252 }
253 } else if (!target.hasPromiseOrImplementsInterface<TargetAttrInterface>()) {
254 return emitError()
255 << "the target attribute must be a GPU Target attribute";
256 }
257 }
258 return success();
259}
260
261//===----------------------------------------------------------------------===//
262// GPU KernelMetadataAttr
263//===----------------------------------------------------------------------===//
264
265KernelMetadataAttr KernelMetadataAttr::get(FunctionOpInterface kernel,
266 DictionaryAttr metadata) {
267 assert(kernel && "invalid kernel");
268 return get(name: kernel.getNameAttr(), functionType: kernel.getFunctionType(),
269 argAttrs: kernel.getAllArgAttrs(), metadata);
270}
271
272KernelMetadataAttr
273KernelMetadataAttr::getChecked(function_ref<InFlightDiagnostic()> emitError,
274 FunctionOpInterface kernel,
275 DictionaryAttr metadata) {
276 assert(kernel && "invalid kernel");
277 return getChecked(emitError, name: kernel.getNameAttr(), functionType: kernel.getFunctionType(),
278 argAttrs: kernel.getAllArgAttrs(), metadata);
279}
280
281KernelMetadataAttr
282KernelMetadataAttr::appendMetadata(ArrayRef<NamedAttribute> attrs) const {
283 if (attrs.empty())
284 return *this;
285 NamedAttrList attrList;
286 if (DictionaryAttr dict = getMetadata())
287 attrList.append(newAttributes&: dict);
288 attrList.append(newAttributes&: attrs);
289 return KernelMetadataAttr::get(name: getName(), functionType: getFunctionType(), argAttrs: getArgAttrs(),
290 metadata: attrList.getDictionary(context: getContext()));
291}
292
293LogicalResult
294KernelMetadataAttr::verify(function_ref<InFlightDiagnostic()> emitError,
295 StringAttr name, Type functionType,
296 ArrayAttr argAttrs, DictionaryAttr metadata) {
297 if (name.empty())
298 return emitError() << "the kernel name can't be empty";
299 if (argAttrs) {
300 if (llvm::any_of(Range&: argAttrs, P: [](Attribute attr) {
301 return !llvm::isa<DictionaryAttr>(Val: attr);
302 }))
303 return emitError()
304 << "all attributes in the array must be a dictionary attribute";
305 }
306 return success();
307}
308
309//===----------------------------------------------------------------------===//
310// GPU KernelTableAttr
311//===----------------------------------------------------------------------===//
312
313KernelTableAttr KernelTableAttr::get(MLIRContext *context,
314 ArrayRef<KernelMetadataAttr> kernels,
315 bool isSorted) {
316 // Note that `is_sorted` is always only invoked once even with assertions ON.
317 assert((!isSorted || llvm::is_sorted(kernels)) &&
318 "expected a sorted kernel array");
319 // Immediately return the attribute if the array is sorted.
320 if (isSorted || llvm::is_sorted(Range&: kernels))
321 return Base::get(ctx: context, args&: kernels);
322 // Sort the array.
323 SmallVector<KernelMetadataAttr> kernelsTmp(kernels);
324 llvm::array_pod_sort(Start: kernelsTmp.begin(), End: kernelsTmp.end());
325 return Base::get(ctx: context, args&: kernelsTmp);
326}
327
328KernelTableAttr KernelTableAttr::getChecked(
329 function_ref<InFlightDiagnostic()> emitError, MLIRContext *context,
330 ArrayRef<KernelMetadataAttr> kernels, bool isSorted) {
331 // Note that `is_sorted` is always only invoked once even with assertions ON.
332 assert((!isSorted || llvm::is_sorted(kernels)) &&
333 "expected a sorted kernel array");
334 // Immediately return the attribute if the array is sorted.
335 if (isSorted || llvm::is_sorted(Range&: kernels))
336 return Base::getChecked(emitErrorFn: emitError, ctx&: context, args: kernels);
337 // Sort the array.
338 SmallVector<KernelMetadataAttr> kernelsTmp(kernels);
339 llvm::array_pod_sort(Start: kernelsTmp.begin(), End: kernelsTmp.end());
340 return Base::getChecked(emitErrorFn: emitError, ctx&: context, args: kernelsTmp);
341}
342
343LogicalResult
344KernelTableAttr::verify(function_ref<InFlightDiagnostic()> emitError,
345 ArrayRef<KernelMetadataAttr> kernels) {
346 if (kernels.size() < 2)
347 return success();
348 // Check that the kernels are uniquely named.
349 if (std::adjacent_find(first: kernels.begin(), last: kernels.end(),
350 binary_pred: [](KernelMetadataAttr l, KernelMetadataAttr r) {
351 return l.getName() == r.getName();
352 }) != kernels.end()) {
353 return emitError() << "expected all kernels to be uniquely named";
354 }
355 return success();
356}
357
358KernelMetadataAttr KernelTableAttr::lookup(StringRef key) const {
359 auto [iterator, found] = impl::findAttrSorted(first: begin(), last: end(), name: key);
360 return found ? *iterator : KernelMetadataAttr();
361}
362
363KernelMetadataAttr KernelTableAttr::lookup(StringAttr key) const {
364 auto [iterator, found] = impl::findAttrSorted(first: begin(), last: end(), name: key);
365 return found ? *iterator : KernelMetadataAttr();
366}
367
368//===----------------------------------------------------------------------===//
369// GPU target options
370//===----------------------------------------------------------------------===//
371
372TargetOptions::TargetOptions(
373 StringRef toolkitPath, ArrayRef<Attribute> librariesToLink,
374 StringRef cmdOptions, StringRef elfSection,
375 CompilationTarget compilationTarget,
376 function_ref<SymbolTable *()> getSymbolTableCallback,
377 function_ref<void(llvm::Module &)> initialLlvmIRCallback,
378 function_ref<void(llvm::Module &)> linkedLlvmIRCallback,
379 function_ref<void(llvm::Module &)> optimizedLlvmIRCallback,
380 function_ref<void(StringRef)> isaCallback)
381 : TargetOptions(TypeID::get<TargetOptions>(), toolkitPath, librariesToLink,
382 cmdOptions, elfSection, compilationTarget,
383 getSymbolTableCallback, initialLlvmIRCallback,
384 linkedLlvmIRCallback, optimizedLlvmIRCallback,
385 isaCallback) {}
386
387TargetOptions::TargetOptions(
388 TypeID typeID, StringRef toolkitPath, ArrayRef<Attribute> librariesToLink,
389 StringRef cmdOptions, StringRef elfSection,
390 CompilationTarget compilationTarget,
391 function_ref<SymbolTable *()> getSymbolTableCallback,
392 function_ref<void(llvm::Module &)> initialLlvmIRCallback,
393 function_ref<void(llvm::Module &)> linkedLlvmIRCallback,
394 function_ref<void(llvm::Module &)> optimizedLlvmIRCallback,
395 function_ref<void(StringRef)> isaCallback)
396 : toolkitPath(toolkitPath.str()), librariesToLink(librariesToLink),
397 cmdOptions(cmdOptions.str()), elfSection(elfSection.str()),
398 compilationTarget(compilationTarget),
399 getSymbolTableCallback(getSymbolTableCallback),
400 initialLlvmIRCallback(initialLlvmIRCallback),
401 linkedLlvmIRCallback(linkedLlvmIRCallback),
402 optimizedLlvmIRCallback(optimizedLlvmIRCallback),
403 isaCallback(isaCallback), typeID(typeID) {}
404
405TypeID TargetOptions::getTypeID() const { return typeID; }
406
407StringRef TargetOptions::getToolkitPath() const { return toolkitPath; }
408
409ArrayRef<Attribute> TargetOptions::getLibrariesToLink() const {
410 return librariesToLink;
411}
412
413StringRef TargetOptions::getCmdOptions() const { return cmdOptions; }
414
415StringRef TargetOptions::getELFSection() const { return elfSection; }
416
417SymbolTable *TargetOptions::getSymbolTable() const {
418 return getSymbolTableCallback ? getSymbolTableCallback() : nullptr;
419}
420
421function_ref<void(llvm::Module &)>
422TargetOptions::getInitialLlvmIRCallback() const {
423 return initialLlvmIRCallback;
424}
425
426function_ref<void(llvm::Module &)>
427TargetOptions::getLinkedLlvmIRCallback() const {
428 return linkedLlvmIRCallback;
429}
430
431function_ref<void(llvm::Module &)>
432TargetOptions::getOptimizedLlvmIRCallback() const {
433 return optimizedLlvmIRCallback;
434}
435
436function_ref<void(StringRef)> TargetOptions::getISACallback() const {
437 return isaCallback;
438}
439
440CompilationTarget TargetOptions::getCompilationTarget() const {
441 return compilationTarget;
442}
443
444CompilationTarget TargetOptions::getDefaultCompilationTarget() {
445 return CompilationTarget::Fatbin;
446}
447
448std::pair<llvm::BumpPtrAllocator, SmallVector<const char *>>
449TargetOptions::tokenizeCmdOptions(const std::string &cmdOptions) {
450 std::pair<llvm::BumpPtrAllocator, SmallVector<const char *>> options;
451 llvm::StringSaver stringSaver(options.first);
452 StringRef opts = cmdOptions;
453 // For a correct tokenization of the command line options `opts` must be
454 // unquoted, otherwise the tokenization function returns a single string: the
455 // unquoted `cmdOptions` -which is not the desired behavior.
456 // Remove any quotes if they are at the beginning and end of the string:
457 if (!opts.empty() && opts.front() == '"' && opts.back() == '"')
458 opts.consume_front(Prefix: "\""), opts.consume_back(Suffix: "\"");
459 if (!opts.empty() && opts.front() == '\'' && opts.back() == '\'')
460 opts.consume_front(Prefix: "'"), opts.consume_back(Suffix: "'");
461#ifdef _WIN32
462 llvm::cl::TokenizeWindowsCommandLine(opts, stringSaver, options.second,
463 /*MarkEOLs=*/false);
464#else
465 llvm::cl::TokenizeGNUCommandLine(Source: opts, Saver&: stringSaver, NewArgv&: options.second,
466 /*MarkEOLs=*/false);
467#endif // _WIN32
468 return options;
469}
470
471std::pair<llvm::BumpPtrAllocator, SmallVector<const char *>>
472TargetOptions::tokenizeCmdOptions() const {
473 return tokenizeCmdOptions(cmdOptions);
474}
475
476std::pair<llvm::BumpPtrAllocator, SmallVector<const char *>>
477TargetOptions::tokenizeAndRemoveSuffixCmdOptions(llvm::StringRef startsWith) {
478 size_t startPos = cmdOptions.find(svt: startsWith);
479 if (startPos == std::string::npos)
480 return {llvm::BumpPtrAllocator(), SmallVector<const char *>()};
481
482 auto tokenized =
483 tokenizeCmdOptions(cmdOptions: cmdOptions.substr(pos: startPos + startsWith.size()));
484 cmdOptions.resize(n: startPos);
485 return tokenized;
486}
487
488MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::gpu::TargetOptions)
489
490#include "mlir/Dialect/GPU/IR/GPUOpsEnums.cpp.inc"
491
492#define GET_ATTRDEF_CLASSES
493#include "mlir/Dialect/GPU/IR/GPUOpsAttributes.cpp.inc"
494
495namespace {
496template <typename... Attrs>
497void registerAttributeStorageImpl(MLIRContext *context) {
498 (mlir::detail::AttributeUniquer::registerAttribute<Attrs>(context), ...);
499}
500} // namespace
501
502void mlir::gpu::detail::registerAttributeStorage(MLIRContext *context) {
503 registerAttributeStorageImpl<
504#define GET_ATTRDEF_LIST
505#include "mlir/Dialect/GPU/IR/GPUOpsAttributes.cpp.inc"
506 >(context);
507}
508
509#include "mlir/Dialect/GPU/IR/CompilationAttrInterfaces.cpp.inc"
510