1//===- DataLayoutInterfaces.cpp - Data Layout Interface Implementation ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Interfaces/DataLayoutInterfaces.h"
10#include "mlir/IR/BuiltinDialect.h"
11#include "mlir/IR/BuiltinOps.h"
12#include "mlir/IR/BuiltinTypes.h"
13#include "mlir/IR/Operation.h"
14
15#include "llvm/ADT/TypeSwitch.h"
16#include "llvm/Support/MathExtras.h"
17
18using namespace mlir;
19
20//===----------------------------------------------------------------------===//
21// Default implementations
22//===----------------------------------------------------------------------===//
23
24/// Reports that the given type is missing the data layout information and
25/// exits.
26[[noreturn]] static void reportMissingDataLayout(Type type) {
27 std::string message;
28 llvm::raw_string_ostream os(message);
29 os << "neither the scoping op nor the type class provide data layout "
30 "information for "
31 << type;
32 llvm::report_fatal_error(reason: Twine(message));
33}
34
35/// Returns the bitwidth of the index type if specified in the param list.
36/// Assumes 64-bit index otherwise.
37static uint64_t getIndexBitwidth(DataLayoutEntryListRef params) {
38 if (params.empty())
39 return 64;
40 auto attr = cast<IntegerAttr>(Val: params.front().getValue());
41 return attr.getValue().getZExtValue();
42}
43
44llvm::TypeSize
45mlir::detail::getDefaultTypeSize(Type type, const DataLayout &dataLayout,
46 ArrayRef<DataLayoutEntryInterface> params) {
47 llvm::TypeSize bits = getDefaultTypeSizeInBits(type, dataLayout, params);
48 return divideCeil(numerator: bits, denominator: 8);
49}
50
51llvm::TypeSize
52mlir::detail::getDefaultTypeSizeInBits(Type type, const DataLayout &dataLayout,
53 DataLayoutEntryListRef params) {
54 if (type.isIntOrFloat())
55 return llvm::TypeSize::getFixed(ExactSize: type.getIntOrFloatBitWidth());
56
57 if (auto ctype = dyn_cast<ComplexType>(Val&: type)) {
58 Type et = ctype.getElementType();
59 uint64_t innerAlignment =
60 getDefaultPreferredAlignment(type: et, dataLayout, params) * 8;
61 llvm::TypeSize innerSize = getDefaultTypeSizeInBits(type: et, dataLayout, params);
62
63 // Include padding required to align the imaginary value in the complex
64 // type.
65 return llvm::alignTo(Size: innerSize, Align: innerAlignment) + innerSize;
66 }
67
68 // Index is an integer of some bitwidth.
69 if (isa<IndexType>(Val: type))
70 return dataLayout.getTypeSizeInBits(
71 t: IntegerType::get(context: type.getContext(), width: getIndexBitwidth(params)));
72
73 // Sizes of vector types are rounded up to those of types with closest
74 // power-of-two number of elements in the innermost dimension.
75 // TODO: make this extensible.
76 if (auto vecType = dyn_cast<VectorType>(Val&: type)) {
77 uint64_t baseSize = vecType.getNumElements() / vecType.getShape().back() *
78 llvm::PowerOf2Ceil(A: vecType.getShape().back()) *
79 dataLayout.getTypeSizeInBits(t: vecType.getElementType());
80 return llvm::TypeSize::get(Quantity: baseSize, Scalable: vecType.isScalable());
81 }
82
83 if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(Val&: type))
84 return typeInterface.getTypeSizeInBits(dataLayout, params);
85
86 reportMissingDataLayout(type);
87}
88
89static DataLayoutEntryInterface
90findEntryForIntegerType(IntegerType intType,
91 ArrayRef<DataLayoutEntryInterface> params) {
92 assert(!params.empty() && "expected non-empty parameter list");
93 std::map<unsigned, DataLayoutEntryInterface> sortedParams;
94 for (DataLayoutEntryInterface entry : params) {
95 sortedParams.insert(x: std::make_pair(
96 x: cast<Type>(Val: entry.getKey()).getIntOrFloatBitWidth(), y&: entry));
97 }
98 auto iter = sortedParams.lower_bound(x: intType.getWidth());
99 if (iter == sortedParams.end())
100 iter = std::prev(x: iter);
101
102 return iter->second;
103}
104
105constexpr const static uint64_t kDefaultBitsInByte = 8u;
106
107static uint64_t extractABIAlignment(DataLayoutEntryInterface entry) {
108 auto values =
109 cast<DenseIntElementsAttr>(Val: entry.getValue()).getValues<uint64_t>();
110 return static_cast<uint64_t>(*values.begin()) / kDefaultBitsInByte;
111}
112
113static uint64_t
114getIntegerTypeABIAlignment(IntegerType intType,
115 ArrayRef<DataLayoutEntryInterface> params) {
116 constexpr uint64_t kDefaultSmallIntAlignment = 4u;
117 constexpr unsigned kSmallIntSize = 64;
118 if (params.empty()) {
119 return intType.getWidth() < kSmallIntSize
120 ? llvm::PowerOf2Ceil(
121 A: llvm::divideCeil(Numerator: intType.getWidth(), Denominator: kDefaultBitsInByte))
122 : kDefaultSmallIntAlignment;
123 }
124
125 return extractABIAlignment(entry: findEntryForIntegerType(intType, params));
126}
127
128static uint64_t
129getFloatTypeABIAlignment(FloatType fltType, const DataLayout &dataLayout,
130 ArrayRef<DataLayoutEntryInterface> params) {
131 assert(params.size() <= 1 && "at most one data layout entry is expected for "
132 "the singleton floating-point type");
133 if (params.empty())
134 return llvm::PowerOf2Ceil(A: dataLayout.getTypeSize(t: fltType).getFixedValue());
135 return extractABIAlignment(entry: params[0]);
136}
137
138uint64_t mlir::detail::getDefaultABIAlignment(
139 Type type, const DataLayout &dataLayout,
140 ArrayRef<DataLayoutEntryInterface> params) {
141 // Natural alignment is the closest power-of-two number above. For scalable
142 // vectors, aligning them to the same as the base vector is sufficient.
143 if (isa<VectorType>(Val: type))
144 return llvm::PowerOf2Ceil(A: dataLayout.getTypeSize(t: type).getKnownMinValue());
145
146 if (auto fltType = dyn_cast<FloatType>(Val&: type))
147 return getFloatTypeABIAlignment(fltType, dataLayout, params);
148
149 // Index is an integer of some bitwidth.
150 if (isa<IndexType>(Val: type))
151 return dataLayout.getTypeABIAlignment(
152 t: IntegerType::get(context: type.getContext(), width: getIndexBitwidth(params)));
153
154 if (auto intType = dyn_cast<IntegerType>(Val&: type))
155 return getIntegerTypeABIAlignment(intType, params);
156
157 if (auto ctype = dyn_cast<ComplexType>(Val&: type))
158 return getDefaultABIAlignment(type: ctype.getElementType(), dataLayout, params);
159
160 if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(Val&: type))
161 return typeInterface.getABIAlignment(dataLayout, params);
162
163 reportMissingDataLayout(type);
164}
165
166static uint64_t extractPreferredAlignment(DataLayoutEntryInterface entry) {
167 auto values =
168 cast<DenseIntElementsAttr>(Val: entry.getValue()).getValues<uint64_t>();
169 return *std::next(x: values.begin(), n: values.size() - 1) / kDefaultBitsInByte;
170}
171
172static uint64_t
173getIntegerTypePreferredAlignment(IntegerType intType,
174 const DataLayout &dataLayout,
175 ArrayRef<DataLayoutEntryInterface> params) {
176 if (params.empty())
177 return llvm::PowerOf2Ceil(A: dataLayout.getTypeSize(t: intType).getFixedValue());
178
179 return extractPreferredAlignment(entry: findEntryForIntegerType(intType, params));
180}
181
182static uint64_t
183getFloatTypePreferredAlignment(FloatType fltType, const DataLayout &dataLayout,
184 ArrayRef<DataLayoutEntryInterface> params) {
185 assert(params.size() <= 1 && "at most one data layout entry is expected for "
186 "the singleton floating-point type");
187 if (params.empty())
188 return dataLayout.getTypeABIAlignment(t: fltType);
189 return extractPreferredAlignment(entry: params[0]);
190}
191
192uint64_t mlir::detail::getDefaultPreferredAlignment(
193 Type type, const DataLayout &dataLayout,
194 ArrayRef<DataLayoutEntryInterface> params) {
195 // Preferred alignment is same as natural for floats and vectors.
196 if (isa<VectorType>(Val: type))
197 return dataLayout.getTypeABIAlignment(t: type);
198
199 if (auto fltType = dyn_cast<FloatType>(Val&: type))
200 return getFloatTypePreferredAlignment(fltType, dataLayout, params);
201
202 // Preferred alignment is the closest power-of-two number above for integers
203 // (ABI alignment may be smaller).
204 if (auto intType = dyn_cast<IntegerType>(Val&: type))
205 return getIntegerTypePreferredAlignment(intType, dataLayout, params);
206
207 if (isa<IndexType>(Val: type)) {
208 return dataLayout.getTypePreferredAlignment(
209 t: IntegerType::get(context: type.getContext(), width: getIndexBitwidth(params)));
210 }
211
212 if (auto ctype = dyn_cast<ComplexType>(Val&: type))
213 return getDefaultPreferredAlignment(type: ctype.getElementType(), dataLayout,
214 params);
215
216 if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(Val&: type))
217 return typeInterface.getPreferredAlignment(dataLayout, params);
218
219 reportMissingDataLayout(type);
220}
221
222std::optional<uint64_t> mlir::detail::getDefaultIndexBitwidth(
223 Type type, const DataLayout &dataLayout,
224 ArrayRef<DataLayoutEntryInterface> params) {
225 if (isa<IndexType>(Val: type))
226 return getIndexBitwidth(params);
227
228 if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(Val&: type))
229 if (std::optional<uint64_t> indexBitwidth =
230 typeInterface.getIndexBitwidth(dataLayout, params))
231 return *indexBitwidth;
232
233 // Return std::nullopt for all other types, which are assumed to be non
234 // pointer-like types.
235 return std::nullopt;
236}
237
238// Returns the endianness if specified in the given entry. If the entry is empty
239// the default endianness represented by an empty attribute is returned.
240Attribute mlir::detail::getDefaultEndianness(DataLayoutEntryInterface entry) {
241 if (entry == DataLayoutEntryInterface())
242 return Attribute();
243
244 return entry.getValue();
245}
246
247// Returns the default memory space if specified in the given entry. If the
248// entry is empty the default memory space represented by an empty attribute is
249// returned.
250Attribute mlir::detail::getDefaultMemorySpace(DataLayoutEntryInterface entry) {
251 if (!entry)
252 return Attribute();
253
254 return entry.getValue();
255}
256
257// Returns the memory space used for alloca operations if specified in the
258// given entry. If the entry is empty the default memory space represented by
259// an empty attribute is returned.
260Attribute
261mlir::detail::getDefaultAllocaMemorySpace(DataLayoutEntryInterface entry) {
262 if (entry == DataLayoutEntryInterface()) {
263 return Attribute();
264 }
265
266 return entry.getValue();
267}
268
269// Returns the mangling mode if specified in the given entry.
270// If the entry is empty, an empty attribute is returned.
271Attribute mlir::detail::getDefaultManglingMode(DataLayoutEntryInterface entry) {
272 if (entry == DataLayoutEntryInterface())
273 return Attribute();
274
275 return entry.getValue();
276}
277
278// Returns the memory space used for the program memory space. if
279// specified in the given entry. If the entry is empty the default
280// memory space represented by an empty attribute is returned.
281Attribute
282mlir::detail::getDefaultProgramMemorySpace(DataLayoutEntryInterface entry) {
283 if (entry == DataLayoutEntryInterface()) {
284 return Attribute();
285 }
286
287 return entry.getValue();
288}
289
290// Returns the memory space used for global the global memory space. if
291// specified in the given entry. If the entry is empty the default memory
292// space represented by an empty attribute is returned.
293Attribute
294mlir::detail::getDefaultGlobalMemorySpace(DataLayoutEntryInterface entry) {
295 if (entry == DataLayoutEntryInterface()) {
296 return Attribute();
297 }
298
299 return entry.getValue();
300}
301
302// Returns the stack alignment if specified in the given entry. If the entry is
303// empty the default alignment zero is returned.
304uint64_t
305mlir::detail::getDefaultStackAlignment(DataLayoutEntryInterface entry) {
306 if (entry == DataLayoutEntryInterface())
307 return 0;
308
309 auto value = cast<IntegerAttr>(Val: entry.getValue());
310 return value.getValue().getZExtValue();
311}
312
313// Returns the function pointer alignment if specified in the given entry. If
314// the entry is empty the default alignment zero is returned.
315Attribute mlir::detail::getDefaultFunctionPointerAlignment(
316 DataLayoutEntryInterface entry) {
317 if (entry == DataLayoutEntryInterface())
318 return Attribute();
319 return entry.getValue();
320}
321
322// Returns the legal int widths if specified in the given entry. If the entry is
323// empty the default legal int widths represented by an empty attribute is
324// returned.
325Attribute
326mlir::detail::getDefaultLegalIntWidths(DataLayoutEntryInterface entry) {
327 if (entry == DataLayoutEntryInterface())
328 return Attribute();
329 return entry.getValue();
330}
331
332std::optional<Attribute>
333mlir::detail::getDevicePropertyValue(DataLayoutEntryInterface entry) {
334 if (entry == DataLayoutEntryInterface())
335 return std::nullopt;
336
337 return entry.getValue();
338}
339
340DataLayoutEntryList
341mlir::detail::filterEntriesForType(DataLayoutEntryListRef entries,
342 TypeID typeID) {
343 return llvm::filter_to_vector<4>(
344 C&: entries, Pred: [typeID](DataLayoutEntryInterface entry) {
345 auto type = llvm::dyn_cast_if_present<Type>(Val: entry.getKey());
346 return type && type.getTypeID() == typeID;
347 });
348}
349
350DataLayoutEntryInterface
351mlir::detail::filterEntryForIdentifier(DataLayoutEntryListRef entries,
352 StringAttr id) {
353 const auto *it = llvm::find_if(Range&: entries, P: [id](DataLayoutEntryInterface entry) {
354 if (auto attr = dyn_cast<StringAttr>(Val: entry.getKey()))
355 return attr == id;
356 return false;
357 });
358 return it == entries.end() ? DataLayoutEntryInterface() : *it;
359}
360
361static DataLayoutSpecInterface getSpec(Operation *operation) {
362 return llvm::TypeSwitch<Operation *, DataLayoutSpecInterface>(operation)
363 .Case<ModuleOp, DataLayoutOpInterface>(
364 caseFn: [&](auto op) { return op.getDataLayoutSpec(); })
365 .DefaultUnreachable(message: "expected an op with data layout spec");
366}
367
368static TargetSystemSpecInterface getTargetSystemSpec(Operation *operation) {
369 if (operation) {
370 ModuleOp moduleOp = dyn_cast<ModuleOp>(Val: operation);
371 if (!moduleOp)
372 moduleOp = operation->getParentOfType<ModuleOp>();
373 return moduleOp.getTargetSystemSpec();
374 }
375 return TargetSystemSpecInterface();
376}
377
378/// Populates `opsWithLayout` with the list of proper ancestors of `leaf` that
379/// are either modules or implement the `DataLayoutOpInterface`.
380static void
381collectParentLayouts(Operation *leaf,
382 SmallVectorImpl<DataLayoutSpecInterface> &specs,
383 SmallVectorImpl<Location> *opLocations = nullptr) {
384 if (!leaf)
385 return;
386
387 for (Operation *parent = leaf->getParentOp(); parent != nullptr;
388 parent = parent->getParentOp()) {
389 llvm::TypeSwitch<Operation *>(parent)
390 .Case(caseFn: [&](ModuleOp op) {
391 // Skip top-level module op unless it has a layout. Top-level module
392 // without layout is most likely the one implicitly added by the
393 // parser and it doesn't have location. Top-level null specification
394 // would have had the same effect as not having a specification at all
395 // (using type defaults).
396 if (!op->getParentOp() && !op.getDataLayoutSpec())
397 return;
398 specs.push_back(Elt: op.getDataLayoutSpec());
399 if (opLocations)
400 opLocations->push_back(Elt: op.getLoc());
401 })
402 .Case(caseFn: [&](DataLayoutOpInterface op) {
403 specs.push_back(Elt: op.getDataLayoutSpec());
404 if (opLocations)
405 opLocations->push_back(Elt: op.getLoc());
406 });
407 }
408}
409
410/// Returns a layout spec that is a combination of the layout specs attached
411/// to the given operation and all its ancestors.
412static DataLayoutSpecInterface getCombinedDataLayout(Operation *leaf) {
413 if (!leaf)
414 return {};
415
416 assert((isa<ModuleOp, DataLayoutOpInterface>(leaf)) &&
417 "expected an op with data layout spec");
418
419 SmallVector<DataLayoutSpecInterface> specs;
420 collectParentLayouts(leaf, specs);
421
422 // Fast track if there are no ancestors.
423 if (specs.empty())
424 return getSpec(operation: leaf);
425
426 // Create the list of non-null specs (null/missing specs can be safely
427 // ignored) from the outermost to the innermost.
428 auto nonNullSpecs = llvm::filter_to_vector<2>(
429 C: llvm::reverse(C&: specs),
430 Pred: [](DataLayoutSpecInterface iface) { return iface != nullptr; });
431
432 // Combine the specs using the innermost as anchor.
433 if (DataLayoutSpecInterface current = getSpec(operation: leaf))
434 return current.combineWith(specs: nonNullSpecs);
435 if (nonNullSpecs.empty())
436 return {};
437 return nonNullSpecs.back().combineWith(
438 specs: llvm::ArrayRef(nonNullSpecs).drop_back());
439}
440
441LogicalResult mlir::detail::verifyDataLayoutOp(Operation *op) {
442 DataLayoutSpecInterface spec = getSpec(operation: op);
443 // The layout specification may be missing and it's fine.
444 if (!spec)
445 return success();
446
447 if (failed(Result: spec.verifySpec(loc: op->getLoc())))
448 return failure();
449 if (!getCombinedDataLayout(leaf: op)) {
450 InFlightDiagnostic diag =
451 op->emitError()
452 << "data layout does not combine with layouts of enclosing ops";
453 SmallVector<DataLayoutSpecInterface> specs;
454 SmallVector<Location> opLocations;
455 collectParentLayouts(leaf: op, specs, opLocations: &opLocations);
456 for (Location loc : opLocations)
457 diag.attachNote(noteLoc: loc) << "enclosing op with data layout";
458 return diag;
459 }
460 return success();
461}
462
463llvm::TypeSize mlir::detail::divideCeil(llvm::TypeSize numerator,
464 uint64_t denominator) {
465 uint64_t divided =
466 llvm::divideCeil(Numerator: numerator.getKnownMinValue(), Denominator: denominator);
467 return llvm::TypeSize::get(Quantity: divided, Scalable: numerator.isScalable());
468}
469
470//===----------------------------------------------------------------------===//
471// DataLayout
472//===----------------------------------------------------------------------===//
473
474template <typename OpTy>
475void checkMissingLayout(DataLayoutSpecInterface originalLayout, OpTy op) {
476 if (!originalLayout) {
477 assert((!op || !op.getDataLayoutSpec()) &&
478 "could not compute layout information for an op (failed to "
479 "combine attributes?)");
480 }
481}
482
483mlir::DataLayout::DataLayout() : DataLayout(ModuleOp()) {}
484
485mlir::DataLayout::DataLayout(DataLayoutOpInterface op)
486 : originalLayout(getCombinedDataLayout(leaf: op)),
487 originalTargetSystemDesc(getTargetSystemSpec(operation: op)), scope(op),
488 allocaMemorySpace(std::nullopt), programMemorySpace(std::nullopt),
489 globalMemorySpace(std::nullopt), stackAlignment(std::nullopt) {
490#if LLVM_ENABLE_ABI_BREAKING_CHECKS
491 checkMissingLayout(originalLayout, op);
492 collectParentLayouts(op, layoutStack);
493#endif
494}
495
496mlir::DataLayout::DataLayout(ModuleOp op)
497 : originalLayout(getCombinedDataLayout(leaf: op)),
498 originalTargetSystemDesc(getTargetSystemSpec(operation: op)), scope(op),
499 allocaMemorySpace(std::nullopt), programMemorySpace(std::nullopt),
500 globalMemorySpace(std::nullopt), stackAlignment(std::nullopt) {
501#if LLVM_ENABLE_ABI_BREAKING_CHECKS
502 checkMissingLayout(originalLayout, op);
503 collectParentLayouts(op, layoutStack);
504#endif
505}
506
507mlir::DataLayout mlir::DataLayout::closest(Operation *op) {
508 // Search the closest parent either being a module operation or implementing
509 // the data layout interface.
510 while (op) {
511 if (auto module = dyn_cast<ModuleOp>(Val: op))
512 return DataLayout(module);
513 if (auto iface = dyn_cast<DataLayoutOpInterface>(Val: op))
514 return DataLayout(iface);
515 op = op->getParentOp();
516 }
517 return DataLayout();
518}
519
520void mlir::DataLayout::checkValid() const {
521#if LLVM_ENABLE_ABI_BREAKING_CHECKS
522 SmallVector<DataLayoutSpecInterface> specs;
523 collectParentLayouts(scope, specs);
524 assert(specs.size() == layoutStack.size() &&
525 "data layout object used, but no longer valid due to the change in "
526 "number of nested layouts");
527 for (auto pair : llvm::zip(specs, layoutStack)) {
528 Attribute newLayout = std::get<0>(pair);
529 Attribute origLayout = std::get<1>(pair);
530 assert(newLayout == origLayout &&
531 "data layout object used, but no longer valid "
532 "due to the change in layout attributes");
533 }
534#endif
535 assert(((!scope && !this->originalLayout) ||
536 (scope && this->originalLayout == getCombinedDataLayout(scope))) &&
537 "data layout object used, but no longer valid due to the change in "
538 "layout spec");
539}
540
541/// Looks up the value for the given type key in the given cache. If there is no
542/// such value in the cache, compute it using the given callback and put it in
543/// the cache before returning.
544template <typename T>
545static T cachedLookup(Type t, DenseMap<Type, T> &cache,
546 function_ref<T(Type)> compute) {
547 auto it = cache.find(t);
548 if (it != cache.end())
549 return it->second;
550
551 auto result = cache.try_emplace(t, compute(t));
552 return result.first->second;
553}
554
555llvm::TypeSize mlir::DataLayout::getTypeSize(Type t) const {
556 checkValid();
557 return cachedLookup<llvm::TypeSize>(t, cache&: sizes, compute: [&](Type ty) {
558 DataLayoutEntryList list;
559 if (originalLayout)
560 list = originalLayout.getSpecForType(type: ty.getTypeID());
561 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(Val: scope))
562 return iface.getTypeSize(type: ty, dataLayout: *this, params: list);
563 return detail::getDefaultTypeSize(type: ty, dataLayout: *this, params: list);
564 });
565}
566
567llvm::TypeSize mlir::DataLayout::getTypeSizeInBits(Type t) const {
568 checkValid();
569 return cachedLookup<llvm::TypeSize>(t, cache&: bitsizes, compute: [&](Type ty) {
570 DataLayoutEntryList list;
571 if (originalLayout)
572 list = originalLayout.getSpecForType(type: ty.getTypeID());
573 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(Val: scope))
574 return iface.getTypeSizeInBits(type: ty, dataLayout: *this, params: list);
575 return detail::getDefaultTypeSizeInBits(type: ty, dataLayout: *this, params: list);
576 });
577}
578
579llvm::TypeSize mlir::DataLayout::getTypeAllocSize(Type t) const {
580 return llvm::alignTo(Size: getTypeSize(t), Align: getTypeABIAlignment(t));
581}
582
583uint64_t mlir::DataLayout::getTypeABIAlignment(Type t) const {
584 checkValid();
585 return cachedLookup<uint64_t>(t, cache&: abiAlignments, compute: [&](Type ty) {
586 DataLayoutEntryList list;
587 if (originalLayout)
588 list = originalLayout.getSpecForType(type: ty.getTypeID());
589 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(Val: scope))
590 return iface.getTypeABIAlignment(type: ty, dataLayout: *this, params: list);
591 return detail::getDefaultABIAlignment(type: ty, dataLayout: *this, params: list);
592 });
593}
594
595uint64_t mlir::DataLayout::getTypePreferredAlignment(Type t) const {
596 checkValid();
597 return cachedLookup<uint64_t>(t, cache&: preferredAlignments, compute: [&](Type ty) {
598 DataLayoutEntryList list;
599 if (originalLayout)
600 list = originalLayout.getSpecForType(type: ty.getTypeID());
601 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(Val: scope))
602 return iface.getTypePreferredAlignment(type: ty, dataLayout: *this, params: list);
603 return detail::getDefaultPreferredAlignment(type: ty, dataLayout: *this, params: list);
604 });
605}
606
607std::optional<uint64_t> mlir::DataLayout::getTypeIndexBitwidth(Type t) const {
608 checkValid();
609 return cachedLookup<std::optional<uint64_t>>(t, cache&: indexBitwidths, compute: [&](Type ty) {
610 DataLayoutEntryList list;
611 if (originalLayout)
612 list = originalLayout.getSpecForType(type: ty.getTypeID());
613 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(Val: scope))
614 return iface.getIndexBitwidth(type: ty, dataLayout: *this, params: list);
615 return detail::getDefaultIndexBitwidth(type: ty, dataLayout: *this, params: list);
616 });
617}
618
619mlir::Attribute mlir::DataLayout::getEndianness() const {
620 checkValid();
621 if (endianness)
622 return *endianness;
623 DataLayoutEntryInterface entry;
624 if (originalLayout)
625 entry = originalLayout.getSpecForIdentifier(
626 identifier: originalLayout.getEndiannessIdentifier(context: originalLayout.getContext()));
627
628 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(Val: scope))
629 endianness = iface.getEndianness(entry);
630 else
631 endianness = detail::getDefaultEndianness(entry);
632 return *endianness;
633}
634
635mlir::Attribute mlir::DataLayout::getDefaultMemorySpace() const {
636 checkValid();
637 if (defaultMemorySpace)
638 return *defaultMemorySpace;
639 DataLayoutEntryInterface entry;
640 if (originalLayout)
641 entry = originalLayout.getSpecForIdentifier(
642 identifier: originalLayout.getDefaultMemorySpaceIdentifier(
643 context: originalLayout.getContext()));
644 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(Val: scope))
645 defaultMemorySpace = iface.getDefaultMemorySpace(entry);
646 else
647 defaultMemorySpace = detail::getDefaultMemorySpace(entry);
648 return *defaultMemorySpace;
649}
650
651mlir::Attribute mlir::DataLayout::getAllocaMemorySpace() const {
652 checkValid();
653 if (allocaMemorySpace)
654 return *allocaMemorySpace;
655 DataLayoutEntryInterface entry;
656 if (originalLayout)
657 entry = originalLayout.getSpecForIdentifier(
658 identifier: originalLayout.getAllocaMemorySpaceIdentifier(
659 context: originalLayout.getContext()));
660 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(Val: scope))
661 allocaMemorySpace = iface.getAllocaMemorySpace(entry);
662 else
663 allocaMemorySpace = detail::getDefaultAllocaMemorySpace(entry);
664 return *allocaMemorySpace;
665}
666
667mlir::Attribute mlir::DataLayout::getManglingMode() const {
668 checkValid();
669 if (manglingMode)
670 return *manglingMode;
671 DataLayoutEntryInterface entry;
672 if (originalLayout)
673 entry = originalLayout.getSpecForIdentifier(
674 identifier: originalLayout.getManglingModeIdentifier(context: originalLayout.getContext()));
675
676 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(Val: scope))
677 manglingMode = iface.getManglingMode(entry);
678 else
679 manglingMode = detail::getDefaultManglingMode(entry);
680 return *manglingMode;
681}
682
683mlir::Attribute mlir::DataLayout::getProgramMemorySpace() const {
684 checkValid();
685 if (programMemorySpace)
686 return *programMemorySpace;
687 DataLayoutEntryInterface entry;
688 if (originalLayout)
689 entry = originalLayout.getSpecForIdentifier(
690 identifier: originalLayout.getProgramMemorySpaceIdentifier(
691 context: originalLayout.getContext()));
692 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(Val: scope))
693 programMemorySpace = iface.getProgramMemorySpace(entry);
694 else
695 programMemorySpace = detail::getDefaultProgramMemorySpace(entry);
696 return *programMemorySpace;
697}
698
699mlir::Attribute mlir::DataLayout::getGlobalMemorySpace() const {
700 checkValid();
701 if (globalMemorySpace)
702 return *globalMemorySpace;
703 DataLayoutEntryInterface entry;
704 if (originalLayout)
705 entry = originalLayout.getSpecForIdentifier(
706 identifier: originalLayout.getGlobalMemorySpaceIdentifier(
707 context: originalLayout.getContext()));
708 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(Val: scope))
709 globalMemorySpace = iface.getGlobalMemorySpace(entry);
710 else
711 globalMemorySpace = detail::getDefaultGlobalMemorySpace(entry);
712 return *globalMemorySpace;
713}
714
715uint64_t mlir::DataLayout::getStackAlignment() const {
716 checkValid();
717 if (stackAlignment)
718 return *stackAlignment;
719 DataLayoutEntryInterface entry;
720 if (originalLayout)
721 entry = originalLayout.getSpecForIdentifier(
722 identifier: originalLayout.getStackAlignmentIdentifier(
723 context: originalLayout.getContext()));
724 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(Val: scope))
725 stackAlignment = iface.getStackAlignment(entry);
726 else
727 stackAlignment = detail::getDefaultStackAlignment(entry);
728 return *stackAlignment;
729}
730
731Attribute mlir::DataLayout::getFunctionPointerAlignment() const {
732 checkValid();
733 if (functionPointerAlignment)
734 return *functionPointerAlignment;
735 DataLayoutEntryInterface entry;
736 if (originalLayout)
737 entry = originalLayout.getSpecForIdentifier(
738 identifier: originalLayout.getFunctionPointerAlignmentIdentifier(
739 context: originalLayout.getContext()));
740 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(Val: scope))
741 functionPointerAlignment = iface.getFunctionPointerAlignment(entry);
742 else
743 functionPointerAlignment =
744 detail::getDefaultFunctionPointerAlignment(entry);
745 return *functionPointerAlignment;
746}
747
748Attribute mlir::DataLayout::getLegalIntWidths() const {
749 checkValid();
750 if (legalIntWidths)
751 return *legalIntWidths;
752 DataLayoutEntryInterface entry;
753 if (originalLayout)
754 entry = originalLayout.getSpecForIdentifier(
755 identifier: originalLayout.getLegalIntWidthsIdentifier(
756 context: originalLayout.getContext()));
757 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(Val: scope))
758 legalIntWidths = iface.getLegalIntWidths(entry);
759 else
760 legalIntWidths = detail::getDefaultLegalIntWidths(entry);
761 return *legalIntWidths;
762}
763
764std::optional<Attribute> mlir::DataLayout::getDevicePropertyValue(
765 TargetSystemSpecInterface::DeviceID deviceID,
766 StringAttr propertyName) const {
767 checkValid();
768 DataLayoutEntryInterface entry;
769 if (originalTargetSystemDesc) {
770 if (std::optional<TargetDeviceSpecInterface> device =
771 originalTargetSystemDesc.getDeviceSpecForDeviceID(deviceID))
772 entry = device->getSpecForIdentifier(identifier: propertyName);
773 }
774 // Currently I am not caching the results because we do not return
775 // default values of these properties. Instead if the property is
776 // missing, we return std::nullopt so that the users can resort to
777 // the default value however they want.
778 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(Val: scope))
779 return iface.getDevicePropertyValue(entry);
780 else
781 return detail::getDevicePropertyValue(entry);
782}
783
784//===----------------------------------------------------------------------===//
785// DataLayoutSpecInterface
786//===----------------------------------------------------------------------===//
787
788void DataLayoutSpecInterface::bucketEntriesByType(
789 llvm::MapVector<TypeID, DataLayoutEntryList> &types,
790 llvm::MapVector<StringAttr, DataLayoutEntryInterface> &ids) {
791 for (DataLayoutEntryInterface entry : getEntries()) {
792 if (auto type = llvm::dyn_cast_if_present<Type>(Val: entry.getKey()))
793 types[type.getTypeID()].push_back(Elt: entry);
794 else
795 ids[llvm::cast<StringAttr>(Val: entry.getKey())] = entry;
796 }
797}
798
799LogicalResult mlir::detail::verifyDataLayoutSpec(DataLayoutSpecInterface spec,
800 Location loc) {
801 // First, verify individual entries.
802 for (DataLayoutEntryInterface entry : spec.getEntries())
803 if (failed(Result: entry.verifyEntry(loc)))
804 return failure();
805
806 // Second, dispatch verifications of entry groups to types or dialects they
807 // are associated with.
808 llvm::MapVector<TypeID, DataLayoutEntryList> types;
809 llvm::MapVector<StringAttr, DataLayoutEntryInterface> ids;
810 spec.bucketEntriesByType(types, ids);
811
812 for (const auto &kvp : types) {
813 auto sampleType = cast<Type>(Val: kvp.second.front().getKey());
814 if (isa<IndexType>(Val: sampleType)) {
815 assert(kvp.second.size() == 1 &&
816 "expected one data layout entry for non-parametric 'index' type");
817 if (!isa<IntegerAttr>(Val: kvp.second.front().getValue()))
818 return emitError(loc)
819 << "expected integer attribute in the data layout entry for "
820 << sampleType;
821 continue;
822 }
823
824 if (sampleType.isIntOrFloat()) {
825 for (DataLayoutEntryInterface entry : kvp.second) {
826 auto value = dyn_cast<DenseIntElementsAttr>(Val: entry.getValue());
827 if (!value || !value.getElementType().isSignlessInteger(width: 64)) {
828 emitError(loc) << "expected a dense i64 elements attribute in the "
829 "data layout entry "
830 << entry;
831 return failure();
832 }
833
834 auto elements = llvm::to_vector<2>(Range: value.getValues<uint64_t>());
835 unsigned numElements = elements.size();
836 if (numElements < 1 || numElements > 2) {
837 emitError(loc) << "expected 1 or 2 elements in the data layout entry "
838 << entry;
839 return failure();
840 }
841
842 uint64_t abi = elements[0];
843 uint64_t preferred = numElements == 2 ? elements[1] : abi;
844 if (preferred < abi) {
845 emitError(loc)
846 << "preferred alignment is expected to be greater than or equal "
847 "to the abi alignment in data layout entry "
848 << entry;
849 return failure();
850 }
851 }
852 continue;
853 }
854
855 if (isa<BuiltinDialect>(Val: &sampleType.getDialect()))
856 return emitError(loc) << "unexpected data layout for a built-in type";
857
858 auto dlType = dyn_cast<DataLayoutTypeInterface>(Val&: sampleType);
859 if (!dlType)
860 return emitError(loc)
861 << "data layout specified for a type that does not support it";
862 if (failed(Result: dlType.verifyEntries(entries: kvp.second, loc)))
863 return failure();
864 }
865
866 for (const auto &kvp : ids) {
867 StringAttr identifier = cast<StringAttr>(Val: kvp.second.getKey());
868 Dialect *dialect = identifier.getReferencedDialect();
869
870 // Ignore attributes that belong to an unknown dialect, the dialect may
871 // actually implement the relevant interface but we don't know about that.
872 if (!dialect)
873 continue;
874
875 const auto *iface = dyn_cast<DataLayoutDialectInterface>(Val: dialect);
876 if (!iface) {
877 return emitError(loc)
878 << "the '" << dialect->getNamespace()
879 << "' dialect does not support identifier data layout entries";
880 }
881 if (failed(Result: iface->verifyEntry(entry: kvp.second, loc)))
882 return failure();
883 }
884
885 return success();
886}
887
888LogicalResult
889mlir::detail::verifyTargetSystemSpec(TargetSystemSpecInterface spec,
890 Location loc) {
891 DenseMap<StringAttr, DataLayoutEntryInterface> deviceDescKeys;
892 DenseSet<TargetSystemSpecInterface::DeviceID> deviceIDs;
893 for (const auto &entry : spec.getEntries()) {
894 auto targetDeviceSpec =
895 dyn_cast<TargetDeviceSpecInterface>(Val: entry.getValue());
896
897 if (!targetDeviceSpec)
898 return failure();
899
900 // First, verify individual target device desc specs.
901 if (failed(Result: targetDeviceSpec.verifyEntry(loc)))
902 return failure();
903
904 // Check that device IDs are unique across all entries.
905 auto deviceID =
906 llvm::dyn_cast<TargetSystemSpecInterface::DeviceID>(Val: entry.getKey());
907 if (!deviceID)
908 return failure();
909
910 if (!deviceIDs.insert(V: deviceID).second) {
911 return failure();
912 }
913
914 // collect all the keys used by all the target device specs.
915 for (DataLayoutEntryInterface entry : targetDeviceSpec.getEntries()) {
916 if (auto type = llvm::dyn_cast_if_present<Type>(Val: entry.getKey())) {
917 // targetDeviceSpec does not support Type as a key.
918 return failure();
919 } else {
920 deviceDescKeys[cast<StringAttr>(Val: entry.getKey())] = entry;
921 }
922 }
923 }
924
925 for (const auto &[keyName, keyVal] : deviceDescKeys) {
926 Dialect *dialect = keyName.getReferencedDialect();
927
928 // Ignore attributes that belong to an unknown dialect, the dialect may
929 // actually implement the relevant interface but we don't know about that.
930 if (!dialect)
931 return failure();
932
933 const auto *iface = dyn_cast<DataLayoutDialectInterface>(Val: dialect);
934 if (!iface) {
935 return emitError(loc)
936 << "the '" << dialect->getNamespace()
937 << "' dialect does not support identifier data layout entries";
938 }
939 if (failed(Result: iface->verifyEntry(entry: keyVal, loc)))
940 return failure();
941 }
942
943 return success();
944}
945
946#include "mlir/Interfaces/DataLayoutAttrInterface.cpp.inc"
947#include "mlir/Interfaces/DataLayoutOpInterface.cpp.inc"
948#include "mlir/Interfaces/DataLayoutTypeInterface.cpp.inc"
949