| 1 | //===- BuiltinAttributes.cpp - MLIR Builtin Attribute Classes -------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/IR/BuiltinAttributes.h" |
| 10 | #include "AttributeDetail.h" |
| 11 | #include "mlir/IR/AffineMap.h" |
| 12 | #include "mlir/IR/BuiltinDialect.h" |
| 13 | #include "mlir/IR/BuiltinTypeInterfaces.h" |
| 14 | #include "mlir/IR/Dialect.h" |
| 15 | #include "mlir/IR/DialectResourceBlobManager.h" |
| 16 | #include "mlir/IR/IntegerSet.h" |
| 17 | #include "mlir/IR/OpImplementation.h" |
| 18 | #include "mlir/IR/Operation.h" |
| 19 | #include "mlir/IR/SymbolTable.h" |
| 20 | #include "mlir/IR/Types.h" |
| 21 | #include "llvm/ADT/APSInt.h" |
| 22 | #include "llvm/Support/Alignment.h" |
| 23 | #include "llvm/Support/Debug.h" |
| 24 | #include "llvm/Support/DebugLog.h" |
| 25 | #include "llvm/Support/Endian.h" |
| 26 | #include <optional> |
| 27 | |
| 28 | #define DEBUG_TYPE "builtinattributes" |
| 29 | |
| 30 | using namespace mlir; |
| 31 | using namespace mlir::detail; |
| 32 | |
| 33 | //===----------------------------------------------------------------------===// |
| 34 | /// Tablegen Attribute Definitions |
| 35 | //===----------------------------------------------------------------------===// |
| 36 | |
| 37 | #define GET_ATTRDEF_CLASSES |
| 38 | #include "mlir/IR/BuiltinAttributes.cpp.inc" |
| 39 | |
| 40 | //===----------------------------------------------------------------------===// |
| 41 | // BuiltinDialect |
| 42 | //===----------------------------------------------------------------------===// |
| 43 | |
| 44 | void BuiltinDialect::registerAttributes() { |
| 45 | addAttributes< |
| 46 | #define GET_ATTRDEF_LIST |
| 47 | #include "mlir/IR/BuiltinAttributes.cpp.inc" |
| 48 | >(); |
| 49 | addAttributes<DistinctAttr>(); |
| 50 | } |
| 51 | |
| 52 | //===----------------------------------------------------------------------===// |
| 53 | // DictionaryAttr |
| 54 | //===----------------------------------------------------------------------===// |
| 55 | |
| 56 | /// Helper function that does either an in place sort or sorts from source array |
| 57 | /// into destination. If inPlace then storage is both the source and the |
| 58 | /// destination, else value is the source and storage destination. Returns |
| 59 | /// whether source was sorted. |
| 60 | template <bool inPlace> |
| 61 | static bool dictionaryAttrSort(ArrayRef<NamedAttribute> value, |
| 62 | SmallVectorImpl<NamedAttribute> &storage) { |
| 63 | // Specialize for the common case. |
| 64 | switch (value.size()) { |
| 65 | case 0: |
| 66 | // Zero already sorted. |
| 67 | if (!inPlace) |
| 68 | storage.clear(); |
| 69 | break; |
| 70 | case 1: |
| 71 | // One already sorted but may need to be copied. |
| 72 | if (!inPlace) |
| 73 | storage.assign(IL: {value[0]}); |
| 74 | break; |
| 75 | case 2: { |
| 76 | bool isSorted = value[0] < value[1]; |
| 77 | if (inPlace) { |
| 78 | if (!isSorted) |
| 79 | std::swap(a&: storage[0], b&: storage[1]); |
| 80 | } else if (isSorted) { |
| 81 | storage.assign(IL: {value[0], value[1]}); |
| 82 | } else { |
| 83 | storage.assign(IL: {value[1], value[0]}); |
| 84 | } |
| 85 | return !isSorted; |
| 86 | } |
| 87 | default: |
| 88 | if (!inPlace) |
| 89 | storage.assign(in_start: value.begin(), in_end: value.end()); |
| 90 | // Check to see they are sorted already. |
| 91 | bool isSorted = llvm::is_sorted(Range&: value); |
| 92 | // If not, do a general sort. |
| 93 | if (!isSorted) |
| 94 | llvm::array_pod_sort(Start: storage.begin(), End: storage.end()); |
| 95 | return !isSorted; |
| 96 | } |
| 97 | return false; |
| 98 | } |
| 99 | |
| 100 | /// Returns an entry with a duplicate name from the given sorted array of named |
| 101 | /// attributes. Returns std::nullopt if all elements have unique names. |
| 102 | static std::optional<NamedAttribute> |
| 103 | findDuplicateElement(ArrayRef<NamedAttribute> value) { |
| 104 | const std::optional<NamedAttribute> none{std::nullopt}; |
| 105 | if (value.size() < 2) |
| 106 | return none; |
| 107 | |
| 108 | if (value.size() == 2) |
| 109 | return value[0].getName() == value[1].getName() ? value[0] : none; |
| 110 | |
| 111 | const auto *it = std::adjacent_find(first: value.begin(), last: value.end(), |
| 112 | binary_pred: [](NamedAttribute l, NamedAttribute r) { |
| 113 | return l.getName() == r.getName(); |
| 114 | }); |
| 115 | return it != value.end() ? *it : none; |
| 116 | } |
| 117 | |
| 118 | bool DictionaryAttr::sort(ArrayRef<NamedAttribute> value, |
| 119 | SmallVectorImpl<NamedAttribute> &storage) { |
| 120 | bool isSorted = dictionaryAttrSort</*inPlace=*/false>(value, storage); |
| 121 | assert(!findDuplicateElement(storage) && |
| 122 | "DictionaryAttr element names must be unique" ); |
| 123 | return isSorted; |
| 124 | } |
| 125 | |
| 126 | bool DictionaryAttr::sortInPlace(SmallVectorImpl<NamedAttribute> &array) { |
| 127 | bool isSorted = dictionaryAttrSort</*inPlace=*/true>(value: array, storage&: array); |
| 128 | assert(!findDuplicateElement(array) && |
| 129 | "DictionaryAttr element names must be unique" ); |
| 130 | return isSorted; |
| 131 | } |
| 132 | |
| 133 | std::optional<NamedAttribute> |
| 134 | DictionaryAttr::findDuplicate(SmallVectorImpl<NamedAttribute> &array, |
| 135 | bool isSorted) { |
| 136 | if (!isSorted) |
| 137 | dictionaryAttrSort</*inPlace=*/true>(value: array, storage&: array); |
| 138 | return findDuplicateElement(value: array); |
| 139 | } |
| 140 | |
| 141 | DictionaryAttr DictionaryAttr::get(MLIRContext *context, |
| 142 | ArrayRef<NamedAttribute> value) { |
| 143 | if (value.empty()) |
| 144 | return DictionaryAttr::getEmpty(context); |
| 145 | |
| 146 | // We need to sort the element list to canonicalize it. |
| 147 | SmallVector<NamedAttribute, 8> storage; |
| 148 | if (dictionaryAttrSort</*inPlace=*/false>(value, storage)) |
| 149 | value = storage; |
| 150 | assert(!findDuplicateElement(value) && |
| 151 | "DictionaryAttr element names must be unique" ); |
| 152 | return Base::get(ctx: context, args&: value); |
| 153 | } |
| 154 | /// Construct a dictionary with an array of values that is known to already be |
| 155 | /// sorted by name and uniqued. |
| 156 | DictionaryAttr DictionaryAttr::getWithSorted(MLIRContext *context, |
| 157 | ArrayRef<NamedAttribute> value) { |
| 158 | if (value.empty()) |
| 159 | return DictionaryAttr::getEmpty(context); |
| 160 | // Ensure that the attribute elements are unique and sorted. |
| 161 | assert(llvm::is_sorted( |
| 162 | value, [](NamedAttribute l, NamedAttribute r) { return l < r; }) && |
| 163 | "expected attribute values to be sorted" ); |
| 164 | assert(!findDuplicateElement(value) && |
| 165 | "DictionaryAttr element names must be unique" ); |
| 166 | return Base::get(ctx: context, args&: value); |
| 167 | } |
| 168 | |
| 169 | /// Return the specified attribute if present, null otherwise. |
| 170 | Attribute DictionaryAttr::get(StringRef name) const { |
| 171 | auto it = impl::findAttrSorted(first: begin(), last: end(), name); |
| 172 | return it.second ? it.first->getValue() : Attribute(); |
| 173 | } |
| 174 | Attribute DictionaryAttr::get(StringAttr name) const { |
| 175 | auto it = impl::findAttrSorted(first: begin(), last: end(), name); |
| 176 | return it.second ? it.first->getValue() : Attribute(); |
| 177 | } |
| 178 | |
| 179 | /// Return the specified named attribute if present, std::nullopt otherwise. |
| 180 | std::optional<NamedAttribute> DictionaryAttr::getNamed(StringRef name) const { |
| 181 | auto it = impl::findAttrSorted(first: begin(), last: end(), name); |
| 182 | return it.second ? *it.first : std::optional<NamedAttribute>(); |
| 183 | } |
| 184 | std::optional<NamedAttribute> DictionaryAttr::getNamed(StringAttr name) const { |
| 185 | auto it = impl::findAttrSorted(first: begin(), last: end(), name); |
| 186 | return it.second ? *it.first : std::optional<NamedAttribute>(); |
| 187 | } |
| 188 | |
| 189 | /// Return whether the specified attribute is present. |
| 190 | bool DictionaryAttr::contains(StringRef name) const { |
| 191 | return impl::findAttrSorted(first: begin(), last: end(), name).second; |
| 192 | } |
| 193 | bool DictionaryAttr::contains(StringAttr name) const { |
| 194 | return impl::findAttrSorted(first: begin(), last: end(), name).second; |
| 195 | } |
| 196 | |
| 197 | DictionaryAttr::iterator DictionaryAttr::begin() const { |
| 198 | return getValue().begin(); |
| 199 | } |
| 200 | DictionaryAttr::iterator DictionaryAttr::end() const { |
| 201 | return getValue().end(); |
| 202 | } |
| 203 | size_t DictionaryAttr::size() const { return getValue().size(); } |
| 204 | |
| 205 | DictionaryAttr DictionaryAttr::getEmptyUnchecked(MLIRContext *context) { |
| 206 | return Base::get(ctx: context, args: ArrayRef<NamedAttribute>()); |
| 207 | } |
| 208 | |
| 209 | //===----------------------------------------------------------------------===// |
| 210 | // StridedLayoutAttr |
| 211 | //===----------------------------------------------------------------------===// |
| 212 | |
| 213 | /// Prints a strided layout attribute. |
| 214 | void StridedLayoutAttr::print(llvm::raw_ostream &os) const { |
| 215 | auto printIntOrQuestion = [&](int64_t value) { |
| 216 | if (ShapedType::isDynamic(dValue: value)) |
| 217 | os << "?" ; |
| 218 | else |
| 219 | os << value; |
| 220 | }; |
| 221 | |
| 222 | os << "strided<[" ; |
| 223 | llvm::interleaveComma(c: getStrides(), os, each_fn: printIntOrQuestion); |
| 224 | os << "]" ; |
| 225 | |
| 226 | if (getOffset() != 0) { |
| 227 | os << ", offset: " ; |
| 228 | printIntOrQuestion(getOffset()); |
| 229 | } |
| 230 | os << ">" ; |
| 231 | } |
| 232 | |
| 233 | /// Returns true if this layout is static, i.e. the strides and offset all have |
| 234 | /// a known value > 0. |
| 235 | bool StridedLayoutAttr::hasStaticLayout() const { |
| 236 | return ShapedType::isStatic(dValue: getOffset()) && |
| 237 | ShapedType::isStaticShape(dSizes: getStrides()); |
| 238 | } |
| 239 | |
| 240 | /// Returns the strided layout as an affine map. |
| 241 | AffineMap StridedLayoutAttr::getAffineMap() const { |
| 242 | return makeStridedLinearLayoutMap(strides: getStrides(), offset: getOffset(), context: getContext()); |
| 243 | } |
| 244 | |
| 245 | /// Checks that the type-agnostic strided layout invariants are satisfied. |
| 246 | LogicalResult |
| 247 | StridedLayoutAttr::verify(function_ref<InFlightDiagnostic()> emitError, |
| 248 | int64_t offset, ArrayRef<int64_t> strides) { |
| 249 | return success(); |
| 250 | } |
| 251 | |
| 252 | /// Checks that the type-specific strided layout invariants are satisfied. |
| 253 | LogicalResult StridedLayoutAttr::verifyLayout( |
| 254 | ArrayRef<int64_t> shape, |
| 255 | function_ref<InFlightDiagnostic()> emitError) const { |
| 256 | if (shape.size() != getStrides().size()) |
| 257 | return emitError() << "expected the number of strides to match the rank" ; |
| 258 | |
| 259 | return success(); |
| 260 | } |
| 261 | |
| 262 | LogicalResult |
| 263 | StridedLayoutAttr::getStridesAndOffset(ArrayRef<int64_t>, |
| 264 | SmallVectorImpl<int64_t> &strides, |
| 265 | int64_t &offset) const { |
| 266 | llvm::append_range(C&: strides, R: getStrides()); |
| 267 | offset = getOffset(); |
| 268 | return success(); |
| 269 | } |
| 270 | |
| 271 | //===----------------------------------------------------------------------===// |
| 272 | // StringAttr |
| 273 | //===----------------------------------------------------------------------===// |
| 274 | |
| 275 | StringAttr StringAttr::getEmptyStringAttrUnchecked(MLIRContext *context) { |
| 276 | return Base::get(ctx: context, args: "" , args: NoneType::get(context)); |
| 277 | } |
| 278 | |
| 279 | /// Twine support for StringAttr. |
| 280 | StringAttr StringAttr::get(MLIRContext *context, const Twine &twine) { |
| 281 | // Fast-path empty twine. |
| 282 | if (twine.isTriviallyEmpty()) |
| 283 | return get(context); |
| 284 | SmallVector<char, 32> tempStr; |
| 285 | return Base::get(ctx: context, args: twine.toStringRef(Out&: tempStr), args: NoneType::get(context)); |
| 286 | } |
| 287 | |
| 288 | /// Twine support for StringAttr. |
| 289 | StringAttr StringAttr::get(const Twine &twine, Type type) { |
| 290 | SmallVector<char, 32> tempStr; |
| 291 | return Base::get(ctx: type.getContext(), args: twine.toStringRef(Out&: tempStr), args&: type); |
| 292 | } |
| 293 | |
| 294 | StringRef StringAttr::getValue() const { return getImpl()->value; } |
| 295 | |
| 296 | Type StringAttr::getType() const { return getImpl()->type; } |
| 297 | |
| 298 | Dialect *StringAttr::getReferencedDialect() const { |
| 299 | return getImpl()->referencedDialect; |
| 300 | } |
| 301 | |
| 302 | //===----------------------------------------------------------------------===// |
| 303 | // FloatAttr |
| 304 | //===----------------------------------------------------------------------===// |
| 305 | |
| 306 | double FloatAttr::getValueAsDouble() const { |
| 307 | return getValueAsDouble(val: getValue()); |
| 308 | } |
| 309 | double FloatAttr::getValueAsDouble(APFloat value) { |
| 310 | if (&value.getSemantics() != &APFloat::IEEEdouble()) { |
| 311 | bool losesInfo = false; |
| 312 | value.convert(ToSemantics: APFloat::IEEEdouble(), RM: APFloat::rmNearestTiesToEven, |
| 313 | losesInfo: &losesInfo); |
| 314 | } |
| 315 | return value.convertToDouble(); |
| 316 | } |
| 317 | |
| 318 | LogicalResult FloatAttr::verify(function_ref<InFlightDiagnostic()> emitError, |
| 319 | Type type, APFloat value) { |
| 320 | // Verify that the type is correct. |
| 321 | if (!llvm::isa<FloatType>(Val: type)) |
| 322 | return emitError() << "expected floating point type" ; |
| 323 | |
| 324 | // Verify that the type semantics match that of the value. |
| 325 | if (&llvm::cast<FloatType>(Val&: type).getFloatSemantics() != |
| 326 | &value.getSemantics()) { |
| 327 | return emitError() |
| 328 | << "FloatAttr type doesn't match the type implied by its value" ; |
| 329 | } |
| 330 | return success(); |
| 331 | } |
| 332 | |
| 333 | //===----------------------------------------------------------------------===// |
| 334 | // SymbolRefAttr |
| 335 | //===----------------------------------------------------------------------===// |
| 336 | |
| 337 | SymbolRefAttr SymbolRefAttr::get(MLIRContext *ctx, StringRef value, |
| 338 | ArrayRef<FlatSymbolRefAttr> nestedRefs) { |
| 339 | return get(rootReference: StringAttr::get(context: ctx, twine: value), nestedReferences: nestedRefs); |
| 340 | } |
| 341 | |
| 342 | FlatSymbolRefAttr SymbolRefAttr::get(MLIRContext *ctx, StringRef value) { |
| 343 | return llvm::cast<FlatSymbolRefAttr>(Val: get(ctx, value, nestedRefs: {})); |
| 344 | } |
| 345 | |
| 346 | FlatSymbolRefAttr SymbolRefAttr::get(StringAttr value) { |
| 347 | return llvm::cast<FlatSymbolRefAttr>(Val: get(rootReference: value, nestedReferences: {})); |
| 348 | } |
| 349 | |
| 350 | FlatSymbolRefAttr SymbolRefAttr::get(Operation *symbol) { |
| 351 | auto symbolOp = cast<SymbolOpInterface>(Val: symbol); |
| 352 | return SymbolRefAttr::get(value: symbolOp.getNameAttr()); |
| 353 | } |
| 354 | |
| 355 | StringAttr SymbolRefAttr::getLeafReference() const { |
| 356 | ArrayRef<FlatSymbolRefAttr> nestedRefs = getNestedReferences(); |
| 357 | return nestedRefs.empty() ? getRootReference() : nestedRefs.back().getAttr(); |
| 358 | } |
| 359 | |
| 360 | //===----------------------------------------------------------------------===// |
| 361 | // IntegerAttr |
| 362 | //===----------------------------------------------------------------------===// |
| 363 | |
| 364 | int64_t IntegerAttr::getInt() const { |
| 365 | assert((getType().isIndex() || getType().isSignlessInteger()) && |
| 366 | "must be signless integer" ); |
| 367 | return getValue().getSExtValue(); |
| 368 | } |
| 369 | |
| 370 | int64_t IntegerAttr::getSInt() const { |
| 371 | assert(getType().isSignedInteger() && "must be signed integer" ); |
| 372 | return getValue().getSExtValue(); |
| 373 | } |
| 374 | |
| 375 | uint64_t IntegerAttr::getUInt() const { |
| 376 | assert(getType().isUnsignedInteger() && "must be unsigned integer" ); |
| 377 | return getValue().getZExtValue(); |
| 378 | } |
| 379 | |
| 380 | /// Return the value as an APSInt which carries the signed from the type of |
| 381 | /// the attribute. This traps on signless integers types! |
| 382 | APSInt IntegerAttr::getAPSInt() const { |
| 383 | assert(!getType().isSignlessInteger() && |
| 384 | "Signless integers don't carry a sign for APSInt" ); |
| 385 | return APSInt(getValue(), getType().isUnsignedInteger()); |
| 386 | } |
| 387 | |
| 388 | LogicalResult IntegerAttr::verify(function_ref<InFlightDiagnostic()> emitError, |
| 389 | Type type, APInt value) { |
| 390 | if (IntegerType integerType = llvm::dyn_cast<IntegerType>(Val&: type)) { |
| 391 | if (integerType.getWidth() != value.getBitWidth()) |
| 392 | return emitError() << "integer type bit width (" << integerType.getWidth() |
| 393 | << ") doesn't match value bit width (" |
| 394 | << value.getBitWidth() << ")" ; |
| 395 | return success(); |
| 396 | } |
| 397 | if (llvm::isa<IndexType>(Val: type)) { |
| 398 | if (value.getBitWidth() != IndexType::kInternalStorageBitWidth) |
| 399 | return emitError() |
| 400 | << "value bit width (" << value.getBitWidth() |
| 401 | << ") doesn't match index type internal storage bit width (" |
| 402 | << IndexType::kInternalStorageBitWidth << ")" ; |
| 403 | return success(); |
| 404 | } |
| 405 | return emitError() << "expected integer or index type" ; |
| 406 | } |
| 407 | |
| 408 | BoolAttr IntegerAttr::getBoolAttrUnchecked(IntegerType type, bool value) { |
| 409 | auto attr = Base::get(ctx: type.getContext(), args&: type, args: APInt(/*numBits=*/1, value)); |
| 410 | return llvm::cast<BoolAttr>(Val&: attr); |
| 411 | } |
| 412 | |
| 413 | //===----------------------------------------------------------------------===// |
| 414 | // BoolAttr |
| 415 | //===----------------------------------------------------------------------===// |
| 416 | |
| 417 | bool BoolAttr::getValue() const { |
| 418 | auto *storage = reinterpret_cast<IntegerAttrStorage *>(impl); |
| 419 | return storage->value.getBoolValue(); |
| 420 | } |
| 421 | |
| 422 | bool BoolAttr::classof(Attribute attr) { |
| 423 | IntegerAttr intAttr = llvm::dyn_cast<IntegerAttr>(Val&: attr); |
| 424 | return intAttr && intAttr.getType().isSignlessInteger(width: 1); |
| 425 | } |
| 426 | |
| 427 | //===----------------------------------------------------------------------===// |
| 428 | // OpaqueAttr |
| 429 | //===----------------------------------------------------------------------===// |
| 430 | |
| 431 | LogicalResult OpaqueAttr::verify(function_ref<InFlightDiagnostic()> emitError, |
| 432 | StringAttr dialect, StringRef attrData, |
| 433 | Type type) { |
| 434 | if (!Dialect::isValidNamespace(str: dialect.strref())) |
| 435 | return emitError() << "invalid dialect namespace '" << dialect << "'" ; |
| 436 | |
| 437 | // Check that the dialect is actually registered. |
| 438 | MLIRContext *context = dialect.getContext(); |
| 439 | if (!context->allowsUnregisteredDialects() && |
| 440 | !context->getLoadedDialect(name: dialect.strref())) { |
| 441 | return emitError() |
| 442 | << "#" << dialect << "<\"" << attrData << "\"> : " << type |
| 443 | << " attribute created with unregistered dialect. If this is " |
| 444 | "intended, please call allowUnregisteredDialects() on the " |
| 445 | "MLIRContext, or use -allow-unregistered-dialect with " |
| 446 | "the MLIR opt tool used" ; |
| 447 | } |
| 448 | |
| 449 | return success(); |
| 450 | } |
| 451 | |
| 452 | //===----------------------------------------------------------------------===// |
| 453 | // DenseElementsAttr Utilities |
| 454 | //===----------------------------------------------------------------------===// |
| 455 | |
| 456 | /// Get the bitwidth of a dense element type within the buffer. |
| 457 | /// DenseElementsAttr requires bitwidths to be aligned by 8. |
| 458 | static size_t getDenseElementStorageWidth(size_t origWidth) { |
| 459 | return llvm::alignTo<8>(Value: origWidth); |
| 460 | } |
| 461 | static size_t getDenseElementStorageWidth(Type elementType) { |
| 462 | return getDenseElementStorageWidth(origWidth: getDenseElementBitWidth(eltType: elementType)); |
| 463 | } |
| 464 | |
| 465 | /// Copy actual `numBytes` data from `value` (APInt) to char array(`result`) for |
| 466 | /// BE format. |
| 467 | static void copyAPIntToArrayForBEmachine(APInt value, size_t numBytes, |
| 468 | char *result) { |
| 469 | assert(llvm::endianness::native == llvm::endianness::big); |
| 470 | assert(value.getNumWords() * APInt::APINT_WORD_SIZE >= numBytes); |
| 471 | |
| 472 | // Copy the words filled with data. |
| 473 | // For example, when `value` has 2 words, the first word is filled with data. |
| 474 | // `value` (10 bytes, BE):|abcdefgh|------ij| ==> `result` (BE):|abcdefgh|--| |
| 475 | size_t numFilledWords = (value.getNumWords() - 1) * APInt::APINT_WORD_SIZE; |
| 476 | std::copy_n(first: reinterpret_cast<const char *>(value.getRawData()), |
| 477 | n: numFilledWords, result: result); |
| 478 | // Convert last word of APInt to LE format and store it in char |
| 479 | // array(`valueLE`). |
| 480 | // ex. last word of `value` (BE): |------ij| ==> `valueLE` (LE): |ji------| |
| 481 | size_t lastWordPos = numFilledWords; |
| 482 | SmallVector<char, 8> valueLE(APInt::APINT_WORD_SIZE); |
| 483 | DenseTypedElementsAttr::convertEndianOfCharForBEmachine( |
| 484 | inRawData: reinterpret_cast<const char *>(value.getRawData()) + lastWordPos, |
| 485 | outRawData: valueLE.begin(), elementBitWidth: APInt::APINT_BITS_PER_WORD, numElements: 1); |
| 486 | // Extract actual APInt data from `valueLE`, convert endianness to BE format, |
| 487 | // and store it in `result`. |
| 488 | // ex. `valueLE` (LE): |ji------| ==> `result` (BE): |abcdefgh|ij| |
| 489 | DenseTypedElementsAttr::convertEndianOfCharForBEmachine( |
| 490 | inRawData: valueLE.begin(), outRawData: result + lastWordPos, |
| 491 | elementBitWidth: (numBytes - lastWordPos) * CHAR_BIT, numElements: 1); |
| 492 | } |
| 493 | |
| 494 | /// Copy `numBytes` data from `inArray`(char array) to `result`(APINT) for BE |
| 495 | /// format. |
| 496 | static void copyArrayToAPIntForBEmachine(const char *inArray, size_t numBytes, |
| 497 | APInt &result) { |
| 498 | assert(llvm::endianness::native == llvm::endianness::big); |
| 499 | assert(result.getNumWords() * APInt::APINT_WORD_SIZE >= numBytes); |
| 500 | |
| 501 | // Copy the data that fills the word of `result` from `inArray`. |
| 502 | // For example, when `result` has 2 words, the first word will be filled with |
| 503 | // data. So, the first 8 bytes are copied from `inArray` here. |
| 504 | // `inArray` (10 bytes, BE): |abcdefgh|ij| |
| 505 | // ==> `result` (2 words, BE): |abcdefgh|--------| |
| 506 | size_t numFilledWords = (result.getNumWords() - 1) * APInt::APINT_WORD_SIZE; |
| 507 | std::copy_n( |
| 508 | first: inArray, n: numFilledWords, |
| 509 | result: const_cast<char *>(reinterpret_cast<const char *>(result.getRawData()))); |
| 510 | |
| 511 | // Convert array data which will be last word of `result` to LE format, and |
| 512 | // store it in char array(`inArrayLE`). |
| 513 | // ex. `inArray` (last two bytes, BE): |ij| ==> `inArrayLE` (LE): |ji------| |
| 514 | size_t lastWordPos = numFilledWords; |
| 515 | SmallVector<char, 8> inArrayLE(APInt::APINT_WORD_SIZE); |
| 516 | DenseTypedElementsAttr::convertEndianOfCharForBEmachine( |
| 517 | inRawData: inArray + lastWordPos, outRawData: inArrayLE.begin(), |
| 518 | elementBitWidth: (numBytes - lastWordPos) * CHAR_BIT, numElements: 1); |
| 519 | |
| 520 | // Convert `inArrayLE` to BE format, and store it in last word of `result`. |
| 521 | // ex. `inArrayLE` (LE): |ji------| ==> `result` (BE): |abcdefgh|------ij| |
| 522 | DenseTypedElementsAttr::convertEndianOfCharForBEmachine( |
| 523 | inRawData: inArrayLE.begin(), |
| 524 | outRawData: const_cast<char *>(reinterpret_cast<const char *>(result.getRawData())) + |
| 525 | lastWordPos, |
| 526 | elementBitWidth: APInt::APINT_BITS_PER_WORD, numElements: 1); |
| 527 | } |
| 528 | |
| 529 | /// Writes value to the bit position `bitPos` in array `rawData`. |
| 530 | void mlir::detail::writeBits(char *rawData, size_t bitPos, APInt value) { |
| 531 | size_t bitWidth = value.getBitWidth(); |
| 532 | |
| 533 | // The bit position is guaranteed to be byte aligned. |
| 534 | assert((bitPos % CHAR_BIT) == 0 && "expected bitPos to be 8-bit aligned" ); |
| 535 | if (llvm::endianness::native == llvm::endianness::big) { |
| 536 | // Copy from `value` to `rawData + (bitPos / CHAR_BIT)`. |
| 537 | // Copying the first `llvm::divideCeil(bitWidth, CHAR_BIT)` bytes doesn't |
| 538 | // work correctly in BE format. |
| 539 | // ex. `value` (2 words including 10 bytes) |
| 540 | // ==> BE: |abcdefgh|------ij|, LE: |hgfedcba|ji------| |
| 541 | copyAPIntToArrayForBEmachine(value, numBytes: llvm::divideCeil(Numerator: bitWidth, CHAR_BIT), |
| 542 | result: rawData + (bitPos / CHAR_BIT)); |
| 543 | } else { |
| 544 | std::copy_n(first: reinterpret_cast<const char *>(value.getRawData()), |
| 545 | n: llvm::divideCeil(Numerator: bitWidth, CHAR_BIT), |
| 546 | result: rawData + (bitPos / CHAR_BIT)); |
| 547 | } |
| 548 | } |
| 549 | |
| 550 | /// Reads the next `bitWidth` bits from the bit position `bitPos` in array |
| 551 | /// `rawData`. |
| 552 | APInt mlir::detail::readBits(const char *rawData, size_t bitPos, |
| 553 | size_t bitWidth) { |
| 554 | // The bit position is guaranteed to be byte aligned. |
| 555 | assert((bitPos % CHAR_BIT) == 0 && "expected bitPos to be 8-bit aligned" ); |
| 556 | APInt result(bitWidth, 0); |
| 557 | if (llvm::endianness::native == llvm::endianness::big) { |
| 558 | // Copy from `rawData + (bitPos / CHAR_BIT)` to `result`. |
| 559 | // Copying the first `llvm::divideCeil(bitWidth, CHAR_BIT)` bytes doesn't |
| 560 | // work correctly in BE format. |
| 561 | // ex. `result` (2 words including 10 bytes) |
| 562 | // ==> BE: |abcdefgh|------ij|, LE: |hgfedcba|ji------| This function |
| 563 | copyArrayToAPIntForBEmachine(inArray: rawData + (bitPos / CHAR_BIT), |
| 564 | numBytes: llvm::divideCeil(Numerator: bitWidth, CHAR_BIT), result); |
| 565 | } else { |
| 566 | std::copy_n(first: rawData + (bitPos / CHAR_BIT), |
| 567 | n: llvm::divideCeil(Numerator: bitWidth, CHAR_BIT), |
| 568 | result: const_cast<char *>( |
| 569 | reinterpret_cast<const char *>(result.getRawData()))); |
| 570 | } |
| 571 | return result; |
| 572 | } |
| 573 | |
| 574 | /// Returns true if 'values' corresponds to a splat, i.e. one element, or has |
| 575 | /// the same element count as 'type'. |
| 576 | template <typename Values> |
| 577 | static bool (ShapedType type, const Values &values) { |
| 578 | return (values.size() == 1) || |
| 579 | (type.getNumElements() == static_cast<int64_t>(values.size())); |
| 580 | } |
| 581 | |
| 582 | //===----------------------------------------------------------------------===// |
| 583 | // DenseElementsAttr Iterators |
| 584 | //===----------------------------------------------------------------------===// |
| 585 | |
| 586 | //===----------------------------------------------------------------------===// |
| 587 | // AttributeElementIterator |
| 588 | //===----------------------------------------------------------------------===// |
| 589 | |
| 590 | DenseElementsAttr::AttributeElementIterator::AttributeElementIterator( |
| 591 | DenseElementsAttr attr, size_t index) |
| 592 | : llvm::indexed_accessor_iterator<AttributeElementIterator, const void *, |
| 593 | Attribute, Attribute, Attribute>( |
| 594 | attr.getAsOpaquePointer(), index) {} |
| 595 | |
| 596 | Attribute DenseElementsAttr::AttributeElementIterator::operator*() const { |
| 597 | auto owner = llvm::cast<DenseElementsAttr>(Val: getFromOpaquePointer(ptr: base)); |
| 598 | Type eltTy = owner.getElementType(); |
| 599 | |
| 600 | // Handle strings specially. |
| 601 | if (llvm::isa<DenseStringElementsAttr>(Val: owner)) { |
| 602 | ArrayRef<StringRef> vals = owner.getRawStringData(); |
| 603 | return StringAttr::get(twine: owner.isSplat() ? vals.front() : vals[index], type: eltTy); |
| 604 | } |
| 605 | |
| 606 | // All other types should implement DenseElementTypeInterface. |
| 607 | auto denseEltTy = llvm::cast<DenseElementType>(Val&: eltTy); |
| 608 | ArrayRef<char> rawData = owner.getRawData(); |
| 609 | // Storage is byte-aligned: align bit size up to next byte boundary. |
| 610 | size_t bitSize = denseEltTy.getDenseElementBitSize(); |
| 611 | size_t byteSize = llvm::divideCeil(Numerator: bitSize, CHAR_BIT); |
| 612 | size_t offset = owner.isSplat() ? 0 : index * byteSize; |
| 613 | return denseEltTy.convertToAttribute(rawData: rawData.slice(N: offset, M: byteSize)); |
| 614 | } |
| 615 | |
| 616 | //===----------------------------------------------------------------------===// |
| 617 | // BoolElementIterator |
| 618 | //===----------------------------------------------------------------------===// |
| 619 | |
| 620 | DenseElementsAttr::BoolElementIterator::BoolElementIterator( |
| 621 | DenseElementsAttr attr, size_t dataIndex) |
| 622 | : DenseElementIndexedIteratorImpl<BoolElementIterator, bool, bool, bool>( |
| 623 | attr.getRawData().data(), attr.isSplat(), dataIndex) {} |
| 624 | |
| 625 | bool DenseElementsAttr::BoolElementIterator::operator*() const { |
| 626 | return static_cast<bool>(getData()[getDataIndex()]); |
| 627 | } |
| 628 | |
| 629 | //===----------------------------------------------------------------------===// |
| 630 | // IntElementIterator |
| 631 | //===----------------------------------------------------------------------===// |
| 632 | |
| 633 | DenseElementsAttr::IntElementIterator::IntElementIterator( |
| 634 | DenseElementsAttr attr, size_t dataIndex) |
| 635 | : DenseElementIndexedIteratorImpl<IntElementIterator, APInt, APInt, APInt>( |
| 636 | attr.getRawData().data(), attr.isSplat(), dataIndex), |
| 637 | bitWidth(getDenseElementBitWidth(eltType: attr.getElementType())) {} |
| 638 | |
| 639 | APInt DenseElementsAttr::IntElementIterator::operator*() const { |
| 640 | return readBits(rawData: getData(), |
| 641 | bitPos: getDataIndex() * getDenseElementStorageWidth(origWidth: bitWidth), |
| 642 | bitWidth); |
| 643 | } |
| 644 | |
| 645 | //===----------------------------------------------------------------------===// |
| 646 | // ComplexIntElementIterator |
| 647 | //===----------------------------------------------------------------------===// |
| 648 | |
| 649 | DenseElementsAttr::ComplexIntElementIterator::ComplexIntElementIterator( |
| 650 | DenseElementsAttr attr, size_t dataIndex) |
| 651 | : DenseElementIndexedIteratorImpl< |
| 652 | ComplexIntElementIterator, mlir::Complex<APInt>, mlir::Complex<APInt>, |
| 653 | mlir::Complex<APInt>>(attr.getRawData().data(), attr.isSplat(), |
| 654 | dataIndex) { |
| 655 | auto complexType = llvm::cast<ComplexType>(Val: attr.getElementType()); |
| 656 | bitWidth = getDenseElementBitWidth(eltType: complexType.getElementType()); |
| 657 | } |
| 658 | |
| 659 | mlir::Complex<APInt> |
| 660 | DenseElementsAttr::ComplexIntElementIterator::operator*() const { |
| 661 | size_t storageWidth = getDenseElementStorageWidth(origWidth: bitWidth); |
| 662 | size_t offset = getDataIndex() * storageWidth * 2; |
| 663 | return {readBits(rawData: getData(), bitPos: offset, bitWidth), |
| 664 | readBits(rawData: getData(), bitPos: offset + storageWidth, bitWidth)}; |
| 665 | } |
| 666 | |
| 667 | //===----------------------------------------------------------------------===// |
| 668 | // DenseArrayAttr |
| 669 | //===----------------------------------------------------------------------===// |
| 670 | |
| 671 | LogicalResult |
| 672 | DenseArrayAttr::verify(function_ref<InFlightDiagnostic()> emitError, |
| 673 | Type elementType, int64_t size, ArrayRef<char> rawData) { |
| 674 | if (!elementType.isIntOrIndexOrFloat()) |
| 675 | return emitError() << "expected integer or floating point element type" ; |
| 676 | int64_t dataSize = rawData.size(); |
| 677 | int64_t elementSize = |
| 678 | llvm::divideCeil(Numerator: elementType.getIntOrFloatBitWidth(), CHAR_BIT); |
| 679 | if (size * elementSize != dataSize) { |
| 680 | return emitError() << "expected data size (" << size << " elements, " |
| 681 | << elementSize |
| 682 | << " bytes each) does not match: " << dataSize |
| 683 | << " bytes" ; |
| 684 | } |
| 685 | return success(); |
| 686 | } |
| 687 | |
| 688 | namespace { |
| 689 | /// Instantiations of this class provide utilities for interacting with native |
| 690 | /// data types in the context of DenseArrayAttr. |
| 691 | template <size_t width, |
| 692 | IntegerType::SignednessSemantics signedness = IntegerType::Signless> |
| 693 | struct DenseArrayAttrIntUtil { |
| 694 | static bool checkElementType(Type eltType) { |
| 695 | auto type = llvm::dyn_cast<IntegerType>(Val&: eltType); |
| 696 | if (!type || type.getWidth() != width) |
| 697 | return false; |
| 698 | return type.getSignedness() == signedness; |
| 699 | } |
| 700 | |
| 701 | static Type getElementType(MLIRContext *ctx) { |
| 702 | return IntegerType::get(context: ctx, width, signedness); |
| 703 | } |
| 704 | |
| 705 | template <typename T> |
| 706 | static void printElement(raw_ostream &os, T value) { |
| 707 | os << value; |
| 708 | } |
| 709 | |
| 710 | template <typename T> |
| 711 | static ParseResult parseElement(AsmParser &parser, T &value) { |
| 712 | return parser.parseInteger(value); |
| 713 | } |
| 714 | }; |
| 715 | template <typename T> |
| 716 | struct DenseArrayAttrUtil; |
| 717 | |
| 718 | /// Specialization for boolean elements to print 'true' and 'false' literals for |
| 719 | /// elements. |
| 720 | template <> |
| 721 | struct DenseArrayAttrUtil<bool> : public DenseArrayAttrIntUtil<1> { |
| 722 | static void printElement(raw_ostream &os, bool value) { |
| 723 | os << (value ? "true" : "false" ); |
| 724 | } |
| 725 | }; |
| 726 | |
| 727 | /// Specialization for 8-bit integers to ensure values are printed as integers |
| 728 | /// and not characters. |
| 729 | template <> |
| 730 | struct DenseArrayAttrUtil<int8_t> : public DenseArrayAttrIntUtil<8> { |
| 731 | static void printElement(raw_ostream &os, int8_t value) { |
| 732 | os << static_cast<int>(value); |
| 733 | } |
| 734 | }; |
| 735 | template <> |
| 736 | struct DenseArrayAttrUtil<int16_t> : public DenseArrayAttrIntUtil<16> {}; |
| 737 | template <> |
| 738 | struct DenseArrayAttrUtil<int32_t> : public DenseArrayAttrIntUtil<32> {}; |
| 739 | template <> |
| 740 | struct DenseArrayAttrUtil<int64_t> : public DenseArrayAttrIntUtil<64> {}; |
| 741 | |
| 742 | /// Specialization for 32-bit floats. |
| 743 | template <> |
| 744 | struct DenseArrayAttrUtil<float> { |
| 745 | static bool checkElementType(Type eltType) { return eltType.isF32(); } |
| 746 | static Type getElementType(MLIRContext *ctx) { return Float32Type::get(context: ctx); } |
| 747 | static void printElement(raw_ostream &os, float value) { os << value; } |
| 748 | |
| 749 | /// Parse a double and cast it to a float. |
| 750 | static ParseResult parseElement(AsmParser &parser, float &value) { |
| 751 | double doubleVal; |
| 752 | if (parser.parseFloat(result&: doubleVal)) |
| 753 | return failure(); |
| 754 | value = doubleVal; |
| 755 | return success(); |
| 756 | } |
| 757 | }; |
| 758 | |
| 759 | /// Specialization for 64-bit floats. |
| 760 | template <> |
| 761 | struct DenseArrayAttrUtil<double> { |
| 762 | static bool checkElementType(Type eltType) { return eltType.isF64(); } |
| 763 | static Type getElementType(MLIRContext *ctx) { return Float64Type::get(context: ctx); } |
| 764 | static void printElement(raw_ostream &os, float value) { os << value; } |
| 765 | static ParseResult parseElement(AsmParser &parser, double &value) { |
| 766 | return parser.parseFloat(result&: value); |
| 767 | } |
| 768 | }; |
| 769 | } // namespace |
| 770 | |
| 771 | template <typename T> |
| 772 | void DenseArrayAttrImpl<T>::print(AsmPrinter &printer) const { |
| 773 | print(printer.getStream()); |
| 774 | } |
| 775 | |
| 776 | template <typename T> |
| 777 | void DenseArrayAttrImpl<T>::printWithoutBraces(raw_ostream &os) const { |
| 778 | llvm::interleaveComma(asArrayRef(), os, [&](T value) { |
| 779 | DenseArrayAttrUtil<T>::printElement(os, value); |
| 780 | }); |
| 781 | } |
| 782 | |
| 783 | template <typename T> |
| 784 | void DenseArrayAttrImpl<T>::print(raw_ostream &os) const { |
| 785 | os << "[" ; |
| 786 | printWithoutBraces(os); |
| 787 | os << "]" ; |
| 788 | } |
| 789 | |
| 790 | /// Parse a DenseArrayAttr without the braces: `1, 2, 3` |
| 791 | template <typename T> |
| 792 | Attribute DenseArrayAttrImpl<T>::parseWithoutBraces(AsmParser &parser, |
| 793 | Type odsType) { |
| 794 | SmallVector<T> data; |
| 795 | if (failed(parser.parseCommaSeparatedList([&]() { |
| 796 | T value; |
| 797 | if (DenseArrayAttrUtil<T>::parseElement(parser, value)) |
| 798 | return failure(); |
| 799 | data.push_back(value); |
| 800 | return success(); |
| 801 | }))) |
| 802 | return {}; |
| 803 | return get(context: parser.getContext(), content: data); |
| 804 | } |
| 805 | |
| 806 | /// Parse a DenseArrayAttr: `[ 1, 2, 3 ]` |
| 807 | template <typename T> |
| 808 | Attribute DenseArrayAttrImpl<T>::parse(AsmParser &parser, Type odsType) { |
| 809 | if (parser.parseLSquare()) |
| 810 | return {}; |
| 811 | // Handle empty list case. |
| 812 | if (succeeded(Result: parser.parseOptionalRSquare())) |
| 813 | return get(context: parser.getContext(), content: {}); |
| 814 | Attribute result = parseWithoutBraces(parser, odsType); |
| 815 | if (parser.parseRSquare()) |
| 816 | return {}; |
| 817 | return result; |
| 818 | } |
| 819 | |
| 820 | /// Conversion from DenseArrayAttr<T> to ArrayRef<T>. |
| 821 | template <typename T> |
| 822 | DenseArrayAttrImpl<T>::operator ArrayRef<T>() const { |
| 823 | ArrayRef<char> raw = getRawData(); |
| 824 | assert(llvm::isAddrAligned(llvm::Align(alignof(T)), raw.data())); |
| 825 | assert((raw.size() % sizeof(T)) == 0); |
| 826 | return ArrayRef<T>(reinterpret_cast<const T *>(raw.data()), |
| 827 | raw.size() / sizeof(T)); |
| 828 | } |
| 829 | |
| 830 | /// Builds a DenseArrayAttr<T> from an ArrayRef<T>. |
| 831 | template <typename T> |
| 832 | DenseArrayAttrImpl<T> DenseArrayAttrImpl<T>::get(MLIRContext *context, |
| 833 | ArrayRef<T> content) { |
| 834 | Type elementType = DenseArrayAttrUtil<T>::getElementType(context); |
| 835 | auto rawArray = ArrayRef<char>(reinterpret_cast<const char *>(content.data()), |
| 836 | content.size() * sizeof(T)); |
| 837 | return llvm::cast<DenseArrayAttrImpl<T>>( |
| 838 | Base::get(context, elementType, content.size(), rawArray)); |
| 839 | } |
| 840 | |
| 841 | template <typename T> |
| 842 | bool DenseArrayAttrImpl<T>::classof(Attribute attr) { |
| 843 | if (auto denseArray = llvm::dyn_cast<DenseArrayAttr>(Val&: attr)) |
| 844 | return DenseArrayAttrUtil<T>::checkElementType(denseArray.getElementType()); |
| 845 | return false; |
| 846 | } |
| 847 | |
| 848 | namespace mlir { |
| 849 | namespace detail { |
| 850 | // Explicit instantiation for all the supported DenseArrayAttr. |
| 851 | template class DenseArrayAttrImpl<bool>; |
| 852 | template class DenseArrayAttrImpl<int8_t>; |
| 853 | template class DenseArrayAttrImpl<int16_t>; |
| 854 | template class DenseArrayAttrImpl<int32_t>; |
| 855 | template class DenseArrayAttrImpl<int64_t>; |
| 856 | template class DenseArrayAttrImpl<float>; |
| 857 | template class DenseArrayAttrImpl<double>; |
| 858 | } // namespace detail |
| 859 | } // namespace mlir |
| 860 | |
| 861 | //===----------------------------------------------------------------------===// |
| 862 | // DenseElementsAttr |
| 863 | //===----------------------------------------------------------------------===// |
| 864 | |
| 865 | /// Method for support type inquiry through isa, cast and dyn_cast. |
| 866 | bool DenseElementsAttr::classof(Attribute attr) { |
| 867 | return llvm::isa<DenseTypedElementsAttr, DenseStringElementsAttr>(Val: attr); |
| 868 | } |
| 869 | |
| 870 | DenseElementsAttr DenseElementsAttr::get(ShapedType type, |
| 871 | ArrayRef<Attribute> values) { |
| 872 | assert(hasSameNumElementsOrSplat(type, values)); |
| 873 | Type eltType = type.getElementType(); |
| 874 | |
| 875 | // Handle strings specially. |
| 876 | if (!llvm::isa<DenseElementType>(Val: eltType)) { |
| 877 | SmallVector<StringRef, 8> stringValues; |
| 878 | stringValues.reserve(N: values.size()); |
| 879 | for (Attribute attr : values) { |
| 880 | assert(llvm::isa<StringAttr>(attr) && |
| 881 | "expected string value for non-DenseElementType element" ); |
| 882 | stringValues.push_back(Elt: llvm::cast<StringAttr>(Val&: attr).getValue()); |
| 883 | } |
| 884 | return get(type, values: stringValues); |
| 885 | } |
| 886 | |
| 887 | // All other types go through DenseElementTypeInterface. |
| 888 | auto denseEltType = llvm::dyn_cast<DenseElementType>(Val&: eltType); |
| 889 | assert(denseEltType && |
| 890 | "attempted to get DenseElementsAttr with unsupported element type" ); |
| 891 | SmallVector<char> data; |
| 892 | for (Attribute attr : values) { |
| 893 | LogicalResult result = denseEltType.convertFromAttribute(attr, result&: data); |
| 894 | if (failed(Result: result)) |
| 895 | return {}; |
| 896 | } |
| 897 | return DenseTypedElementsAttr::getRaw(type, data); |
| 898 | } |
| 899 | |
| 900 | DenseElementsAttr DenseElementsAttr::get(ShapedType type, |
| 901 | ArrayRef<bool> values) { |
| 902 | assert(hasSameNumElementsOrSplat(type, values)); |
| 903 | assert(type.getElementType().isInteger(1)); |
| 904 | return DenseTypedElementsAttr::getRaw( |
| 905 | type, data: ArrayRef<char>(reinterpret_cast<const char *>(values.data()), |
| 906 | values.size())); |
| 907 | } |
| 908 | |
| 909 | DenseElementsAttr DenseElementsAttr::get(ShapedType type, |
| 910 | ArrayRef<StringRef> values) { |
| 911 | assert(!type.getElementType().isIntOrFloat()); |
| 912 | return DenseStringElementsAttr::get(type, values); |
| 913 | } |
| 914 | |
| 915 | /// Constructs a dense integer elements attribute from an array of APInt |
| 916 | /// values. Each APInt value is expected to have the same bitwidth as the |
| 917 | /// element type of 'type'. |
| 918 | DenseElementsAttr DenseElementsAttr::get(ShapedType type, |
| 919 | ArrayRef<APInt> values) { |
| 920 | assert(type.getElementType().isIntOrIndex()); |
| 921 | assert(hasSameNumElementsOrSplat(type, values)); |
| 922 | size_t storageBitWidth = getDenseElementStorageWidth(elementType: type.getElementType()); |
| 923 | return DenseTypedElementsAttr::getRaw(type, storageWidth: storageBitWidth, values); |
| 924 | } |
| 925 | DenseElementsAttr |
| 926 | DenseElementsAttr::get(ShapedType type, ArrayRef<mlir::Complex<APInt>> values) { |
| 927 | ComplexType complex = llvm::cast<ComplexType>(Val: type.getElementType()); |
| 928 | assert(llvm::isa<IntegerType>(complex.getElementType())); |
| 929 | assert(hasSameNumElementsOrSplat(type, values)); |
| 930 | size_t storageBitWidth = getDenseElementStorageWidth(elementType: complex) / 2; |
| 931 | ArrayRef<APInt> intVals(reinterpret_cast<const APInt *>(values.data()), |
| 932 | values.size() * 2); |
| 933 | return DenseTypedElementsAttr::getRaw(type, storageWidth: storageBitWidth, values: intVals); |
| 934 | } |
| 935 | |
| 936 | // Constructs a dense float elements attribute from an array of APFloat |
| 937 | // values. Each APFloat value is expected to have the same bitwidth as the |
| 938 | // element type of 'type'. |
| 939 | DenseElementsAttr DenseElementsAttr::get(ShapedType type, |
| 940 | ArrayRef<APFloat> values) { |
| 941 | assert(llvm::isa<FloatType>(type.getElementType())); |
| 942 | assert(hasSameNumElementsOrSplat(type, values)); |
| 943 | size_t storageBitWidth = getDenseElementStorageWidth(elementType: type.getElementType()); |
| 944 | return DenseTypedElementsAttr::getRaw(type, storageWidth: storageBitWidth, values); |
| 945 | } |
| 946 | DenseElementsAttr |
| 947 | DenseElementsAttr::get(ShapedType type, |
| 948 | ArrayRef<mlir::Complex<APFloat>> values) { |
| 949 | ComplexType complex = llvm::cast<ComplexType>(Val: type.getElementType()); |
| 950 | assert(llvm::isa<FloatType>(complex.getElementType())); |
| 951 | assert(hasSameNumElementsOrSplat(type, values)); |
| 952 | ArrayRef<APFloat> apVals(reinterpret_cast<const APFloat *>(values.data()), |
| 953 | values.size() * 2); |
| 954 | size_t storageBitWidth = getDenseElementStorageWidth(elementType: complex) / 2; |
| 955 | return DenseTypedElementsAttr::getRaw(type, storageWidth: storageBitWidth, values: apVals); |
| 956 | } |
| 957 | |
| 958 | /// Construct a dense elements attribute from a raw buffer representing the |
| 959 | /// data for this attribute. Users should generally not use this methods as |
| 960 | /// the expected buffer format may not be a form the user expects. |
| 961 | DenseElementsAttr |
| 962 | DenseElementsAttr::getFromRawBuffer(ShapedType type, ArrayRef<char> rawBuffer) { |
| 963 | return DenseTypedElementsAttr::getRaw(type, data: rawBuffer); |
| 964 | } |
| 965 | |
| 966 | /// Returns true if the given buffer is a valid raw buffer for the given type. |
| 967 | bool DenseElementsAttr::isValidRawBuffer(ShapedType type, |
| 968 | ArrayRef<char> rawBuffer) { |
| 969 | size_t storageWidth = getDenseElementStorageWidth(elementType: type.getElementType()); |
| 970 | size_t rawBufferWidth = rawBuffer.size() * CHAR_BIT; |
| 971 | int64_t numElements = type.getNumElements(); |
| 972 | |
| 973 | // The raw buffer is valid if it has a single element (splat) or the right |
| 974 | // number of elements. |
| 975 | return rawBufferWidth == storageWidth || |
| 976 | rawBufferWidth == storageWidth * numElements; |
| 977 | } |
| 978 | |
| 979 | /// Check the information for a C++ data type, check if this type is valid for |
| 980 | /// the current attribute. This method is used to verify specific type |
| 981 | /// invariants that the templatized 'getValues' method cannot. |
| 982 | static bool isValidIntOrFloat(Type type, int64_t dataEltSize, bool isInt, |
| 983 | bool isSigned) { |
| 984 | // Make sure that the data element size is the same as the type element width. |
| 985 | auto denseEltBitWidth = getDenseElementBitWidth(eltType: type); |
| 986 | auto dataSize = static_cast<size_t>(dataEltSize * CHAR_BIT); |
| 987 | if (denseEltBitWidth != dataSize) { |
| 988 | LDBG() << "expected dense element bit width " << denseEltBitWidth |
| 989 | << " to match data size " << dataSize << " for type " << type; |
| 990 | return false; |
| 991 | } |
| 992 | |
| 993 | // Check that the element type is either float or integer or index. |
| 994 | if (!isInt) { |
| 995 | bool valid = llvm::isa<FloatType>(Val: type); |
| 996 | if (!valid) |
| 997 | LDBG() << "expected float type when isInt is false, but found " << type; |
| 998 | return valid; |
| 999 | } |
| 1000 | if (type.isIndex()) |
| 1001 | return true; |
| 1002 | |
| 1003 | auto intType = llvm::dyn_cast<IntegerType>(Val&: type); |
| 1004 | if (!intType) { |
| 1005 | LDBG() << "expected integer type when isInt is true, but found " << type; |
| 1006 | return false; |
| 1007 | } |
| 1008 | |
| 1009 | // Make sure signedness semantics is consistent. |
| 1010 | if (intType.isSignless()) |
| 1011 | return true; |
| 1012 | |
| 1013 | bool valid = intType.isSigned() == isSigned; |
| 1014 | if (!valid) |
| 1015 | LDBG() << "expected signedness " << isSigned << " to match type " << type; |
| 1016 | return valid; |
| 1017 | } |
| 1018 | |
| 1019 | /// Defaults down the subclass implementation. |
| 1020 | DenseElementsAttr DenseElementsAttr::getRawComplex(ShapedType type, |
| 1021 | ArrayRef<char> data, |
| 1022 | int64_t dataEltSize, |
| 1023 | bool isInt, bool isSigned) { |
| 1024 | return DenseTypedElementsAttr::getRawComplex(type, data, dataEltSize, isInt, |
| 1025 | isSigned); |
| 1026 | } |
| 1027 | DenseElementsAttr DenseElementsAttr::getRawIntOrFloat(ShapedType type, |
| 1028 | ArrayRef<char> data, |
| 1029 | int64_t dataEltSize, |
| 1030 | bool isInt, |
| 1031 | bool isSigned) { |
| 1032 | return DenseTypedElementsAttr::getRawIntOrFloat(type, data, dataEltSize, |
| 1033 | isInt, isSigned); |
| 1034 | } |
| 1035 | |
| 1036 | bool DenseElementsAttr::isValidIntOrFloat(int64_t dataEltSize, bool isInt, |
| 1037 | bool isSigned) const { |
| 1038 | return ::isValidIntOrFloat(type: getElementType(), dataEltSize, isInt, isSigned); |
| 1039 | } |
| 1040 | bool DenseElementsAttr::isValidComplex(int64_t dataEltSize, bool isInt, |
| 1041 | bool isSigned) const { |
| 1042 | return ::isValidIntOrFloat( |
| 1043 | type: llvm::cast<ComplexType>(Val: getElementType()).getElementType(), |
| 1044 | dataEltSize: dataEltSize / 2, isInt, isSigned); |
| 1045 | } |
| 1046 | |
| 1047 | /// Returns true if this attribute corresponds to a splat, i.e. if all element |
| 1048 | /// values are the same. |
| 1049 | bool DenseElementsAttr::isSplat() const { |
| 1050 | // Splat iff the data array has exactly one element. |
| 1051 | if (isa<DenseStringElementsAttr>(Val: *this)) |
| 1052 | return getRawStringData().size() == 1; |
| 1053 | // FP/Int case. |
| 1054 | size_t storageSize = llvm::divideCeil( |
| 1055 | Numerator: getDenseElementBitWidth(eltType: getType().getElementType()), CHAR_BIT); |
| 1056 | return getRawData().size() == storageSize; |
| 1057 | } |
| 1058 | |
| 1059 | /// Return if the given complex type has an integer element type. |
| 1060 | static bool isComplexOfIntType(Type type) { |
| 1061 | return llvm::isa<IntegerType>(Val: llvm::cast<ComplexType>(Val&: type).getElementType()); |
| 1062 | } |
| 1063 | |
| 1064 | auto DenseElementsAttr::tryGetComplexIntValues() const |
| 1065 | -> FailureOr<iterator_range_impl<ComplexIntElementIterator>> { |
| 1066 | if (!isComplexOfIntType(type: getElementType())) |
| 1067 | return failure(); |
| 1068 | return iterator_range_impl<ComplexIntElementIterator>( |
| 1069 | getType(), ComplexIntElementIterator(*this, 0), |
| 1070 | ComplexIntElementIterator(*this, getNumElements())); |
| 1071 | } |
| 1072 | |
| 1073 | auto DenseElementsAttr::tryGetFloatValues() const |
| 1074 | -> FailureOr<iterator_range_impl<FloatElementIterator>> { |
| 1075 | auto eltTy = llvm::dyn_cast<FloatType>(Val: getElementType()); |
| 1076 | if (!eltTy) |
| 1077 | return failure(); |
| 1078 | const auto &elementSemantics = eltTy.getFloatSemantics(); |
| 1079 | return iterator_range_impl<FloatElementIterator>( |
| 1080 | getType(), FloatElementIterator(elementSemantics, raw_int_begin()), |
| 1081 | FloatElementIterator(elementSemantics, raw_int_end())); |
| 1082 | } |
| 1083 | |
| 1084 | auto DenseElementsAttr::tryGetComplexFloatValues() const |
| 1085 | -> FailureOr<iterator_range_impl<ComplexFloatElementIterator>> { |
| 1086 | auto complexTy = llvm::dyn_cast<ComplexType>(Val: getElementType()); |
| 1087 | if (!complexTy) |
| 1088 | return failure(); |
| 1089 | auto eltTy = llvm::dyn_cast<FloatType>(Val: complexTy.getElementType()); |
| 1090 | if (!eltTy) |
| 1091 | return failure(); |
| 1092 | const auto &semantics = eltTy.getFloatSemantics(); |
| 1093 | return iterator_range_impl<ComplexFloatElementIterator>( |
| 1094 | getType(), {semantics, {*this, 0}}, |
| 1095 | {semantics, {*this, static_cast<size_t>(getNumElements())}}); |
| 1096 | } |
| 1097 | |
| 1098 | /// Return the raw storage data held by this attribute. |
| 1099 | ArrayRef<char> DenseElementsAttr::getRawData() const { |
| 1100 | return static_cast<DenseTypedElementsAttrStorage *>(impl)->data; |
| 1101 | } |
| 1102 | |
| 1103 | ArrayRef<StringRef> DenseElementsAttr::getRawStringData() const { |
| 1104 | return static_cast<DenseStringElementsAttrStorage *>(impl)->data; |
| 1105 | } |
| 1106 | |
| 1107 | /// Return a new DenseElementsAttr that has the same data as the current |
| 1108 | /// attribute, but has been reshaped to 'newType'. The new type must have the |
| 1109 | /// same total number of elements as well as element type. |
| 1110 | DenseElementsAttr DenseElementsAttr::reshape(ShapedType newType) { |
| 1111 | ShapedType curType = getType(); |
| 1112 | if (curType == newType) |
| 1113 | return *this; |
| 1114 | |
| 1115 | assert(newType.getElementType() == curType.getElementType() && |
| 1116 | "expected the same element type" ); |
| 1117 | assert(newType.getNumElements() == curType.getNumElements() && |
| 1118 | "expected the same number of elements" ); |
| 1119 | return DenseTypedElementsAttr::getRaw(type: newType, data: getRawData()); |
| 1120 | } |
| 1121 | |
| 1122 | DenseElementsAttr DenseElementsAttr::resizeSplat(ShapedType newType) { |
| 1123 | assert(isSplat() && "expected a splat type" ); |
| 1124 | |
| 1125 | ShapedType curType = getType(); |
| 1126 | if (curType == newType) |
| 1127 | return *this; |
| 1128 | |
| 1129 | assert(newType.getElementType() == curType.getElementType() && |
| 1130 | "expected the same element type" ); |
| 1131 | return DenseTypedElementsAttr::getRaw(type: newType, data: getRawData()); |
| 1132 | } |
| 1133 | |
| 1134 | /// Return a new DenseElementsAttr that has the same data as the current |
| 1135 | /// attribute, but has bitcast elements such that it is now 'newType'. The new |
| 1136 | /// type must have the same shape and element types of the same bitwidth as the |
| 1137 | /// current type. |
| 1138 | DenseElementsAttr DenseElementsAttr::bitcast(Type newElType) { |
| 1139 | ShapedType curType = getType(); |
| 1140 | Type curElType = curType.getElementType(); |
| 1141 | if (curElType == newElType) |
| 1142 | return *this; |
| 1143 | |
| 1144 | assert(getDenseElementBitWidth(newElType) == |
| 1145 | getDenseElementBitWidth(curElType) && |
| 1146 | "expected element types with the same bitwidth" ); |
| 1147 | return DenseTypedElementsAttr::getRaw(type: curType.clone(elementType: newElType), data: getRawData()); |
| 1148 | } |
| 1149 | |
| 1150 | DenseElementsAttr |
| 1151 | DenseElementsAttr::mapValues(Type newElementType, |
| 1152 | function_ref<APInt(const APInt &)> mapping) const { |
| 1153 | return llvm::cast<DenseIntElementsAttr>(Val: *this).mapValues(newElementType, |
| 1154 | mapping); |
| 1155 | } |
| 1156 | |
| 1157 | DenseElementsAttr DenseElementsAttr::mapValues( |
| 1158 | Type newElementType, function_ref<APInt(const APFloat &)> mapping) const { |
| 1159 | return llvm::cast<DenseFPElementsAttr>(Val: *this).mapValues(newElementType, |
| 1160 | mapping); |
| 1161 | } |
| 1162 | |
| 1163 | ShapedType DenseElementsAttr::getType() const { |
| 1164 | return static_cast<const DenseElementsAttributeStorage *>(impl)->type; |
| 1165 | } |
| 1166 | |
| 1167 | Type DenseElementsAttr::getElementType() const { |
| 1168 | return getType().getElementType(); |
| 1169 | } |
| 1170 | |
| 1171 | int64_t DenseElementsAttr::getNumElements() const { |
| 1172 | return getType().getNumElements(); |
| 1173 | } |
| 1174 | |
| 1175 | //===----------------------------------------------------------------------===// |
| 1176 | // DenseTypedElementsAttr |
| 1177 | //===----------------------------------------------------------------------===// |
| 1178 | |
| 1179 | /// Utility method to write a range of APInt values to a buffer. |
| 1180 | template <typename APRangeT> |
| 1181 | static void writeAPIntsToBuffer(size_t storageWidth, |
| 1182 | SmallVectorImpl<char> &data, |
| 1183 | APRangeT &&values) { |
| 1184 | size_t numValues = llvm::size(values); |
| 1185 | data.resize(N: llvm::divideCeil(Numerator: storageWidth * numValues, CHAR_BIT)); |
| 1186 | size_t offset = 0; |
| 1187 | for (auto it = values.begin(), e = values.end(); it != e; |
| 1188 | ++it, offset += storageWidth) { |
| 1189 | assert((*it).getBitWidth() <= storageWidth); |
| 1190 | writeBits(data.data(), offset, *it); |
| 1191 | } |
| 1192 | } |
| 1193 | |
| 1194 | /// Constructs a dense elements attribute from an array of raw APFloat values. |
| 1195 | /// Each APFloat value is expected to have the same bitwidth as the element |
| 1196 | /// type of 'type'. 'type' must be a vector or tensor with static shape. |
| 1197 | DenseElementsAttr DenseTypedElementsAttr::getRaw(ShapedType type, |
| 1198 | size_t storageWidth, |
| 1199 | ArrayRef<APFloat> values) { |
| 1200 | SmallVector<char> data; |
| 1201 | auto unwrapFloat = [](const APFloat &val) { return val.bitcastToAPInt(); }; |
| 1202 | writeAPIntsToBuffer(storageWidth, data, values: llvm::map_range(C&: values, F: unwrapFloat)); |
| 1203 | return DenseTypedElementsAttr::getRaw(type, data); |
| 1204 | } |
| 1205 | |
| 1206 | /// Constructs a dense elements attribute from an array of raw APInt values. |
| 1207 | /// Each APInt value is expected to have the same bitwidth as the element type |
| 1208 | /// of 'type'. |
| 1209 | DenseElementsAttr DenseTypedElementsAttr::getRaw(ShapedType type, |
| 1210 | size_t storageWidth, |
| 1211 | ArrayRef<APInt> values) { |
| 1212 | SmallVector<char> data; |
| 1213 | writeAPIntsToBuffer(storageWidth, data, values); |
| 1214 | return DenseTypedElementsAttr::getRaw(type, data); |
| 1215 | } |
| 1216 | |
| 1217 | DenseElementsAttr DenseTypedElementsAttr::getRaw(ShapedType type, |
| 1218 | ArrayRef<char> data) { |
| 1219 | assert(type.hasStaticShape() && "type must have static shape" ); |
| 1220 | assert(isValidRawBuffer(type, data)); |
| 1221 | return Base::get(ctx: type.getContext(), args&: type, args&: data); |
| 1222 | } |
| 1223 | |
| 1224 | /// Overload of the raw 'get' method that asserts that the given type is of |
| 1225 | /// complex type. This method is used to verify type invariants that the |
| 1226 | /// templatized 'get' method cannot. |
| 1227 | DenseElementsAttr DenseTypedElementsAttr::getRawComplex(ShapedType type, |
| 1228 | ArrayRef<char> data, |
| 1229 | int64_t dataEltSize, |
| 1230 | bool isInt, |
| 1231 | bool isSigned) { |
| 1232 | assert(::isValidIntOrFloat( |
| 1233 | llvm::cast<ComplexType>(type.getElementType()).getElementType(), |
| 1234 | dataEltSize / 2, isInt, isSigned) && |
| 1235 | "Try re-running with -debug-only=builtinattributes" ); |
| 1236 | |
| 1237 | int64_t numElements = data.size() / dataEltSize; |
| 1238 | (void)numElements; |
| 1239 | assert(numElements == 1 || numElements == type.getNumElements()); |
| 1240 | return getRaw(type, data); |
| 1241 | } |
| 1242 | |
| 1243 | /// Overload of the 'getRaw' method that asserts that the given type is of |
| 1244 | /// integer type. This method is used to verify type invariants that the |
| 1245 | /// templatized 'get' method cannot. |
| 1246 | DenseElementsAttr DenseTypedElementsAttr::getRawIntOrFloat(ShapedType type, |
| 1247 | ArrayRef<char> data, |
| 1248 | int64_t dataEltSize, |
| 1249 | bool isInt, |
| 1250 | bool isSigned) { |
| 1251 | assert(::isValidIntOrFloat(type.getElementType(), dataEltSize, isInt, |
| 1252 | isSigned) && |
| 1253 | "Try re-running with -debug-only=builtinattributes" ); |
| 1254 | |
| 1255 | int64_t numElements = data.size() / dataEltSize; |
| 1256 | assert(numElements == 1 || numElements == type.getNumElements()); |
| 1257 | (void)numElements; |
| 1258 | return getRaw(type, data); |
| 1259 | } |
| 1260 | |
| 1261 | void DenseTypedElementsAttr::convertEndianOfCharForBEmachine( |
| 1262 | const char *inRawData, char *outRawData, size_t elementBitWidth, |
| 1263 | size_t numElements) { |
| 1264 | using llvm::support::ulittle16_t; |
| 1265 | using llvm::support::ulittle32_t; |
| 1266 | using llvm::support::ulittle64_t; |
| 1267 | |
| 1268 | assert(llvm::endianness::native == llvm::endianness::big); |
| 1269 | // NOLINT to avoid warning message about replacing by static_assert() |
| 1270 | |
| 1271 | // Following std::copy_n always converts endianness on BE machine. |
| 1272 | switch (elementBitWidth) { |
| 1273 | case 16: { |
| 1274 | const ulittle16_t *inRawDataPos = |
| 1275 | reinterpret_cast<const ulittle16_t *>(inRawData); |
| 1276 | uint16_t *outDataPos = reinterpret_cast<uint16_t *>(outRawData); |
| 1277 | std::copy_n(first: inRawDataPos, n: numElements, result: outDataPos); |
| 1278 | break; |
| 1279 | } |
| 1280 | case 32: { |
| 1281 | const ulittle32_t *inRawDataPos = |
| 1282 | reinterpret_cast<const ulittle32_t *>(inRawData); |
| 1283 | uint32_t *outDataPos = reinterpret_cast<uint32_t *>(outRawData); |
| 1284 | std::copy_n(first: inRawDataPos, n: numElements, result: outDataPos); |
| 1285 | break; |
| 1286 | } |
| 1287 | case 64: { |
| 1288 | const ulittle64_t *inRawDataPos = |
| 1289 | reinterpret_cast<const ulittle64_t *>(inRawData); |
| 1290 | uint64_t *outDataPos = reinterpret_cast<uint64_t *>(outRawData); |
| 1291 | std::copy_n(first: inRawDataPos, n: numElements, result: outDataPos); |
| 1292 | break; |
| 1293 | } |
| 1294 | default: { |
| 1295 | size_t nBytes = elementBitWidth / CHAR_BIT; |
| 1296 | for (size_t i = 0; i < nBytes; i++) |
| 1297 | std::copy_n(first: inRawData + (nBytes - 1 - i), n: 1, result: outRawData + i); |
| 1298 | break; |
| 1299 | } |
| 1300 | } |
| 1301 | } |
| 1302 | |
| 1303 | void DenseTypedElementsAttr::convertEndianOfArrayRefForBEmachine( |
| 1304 | ArrayRef<char> inRawData, MutableArrayRef<char> outRawData, |
| 1305 | ShapedType type) { |
| 1306 | size_t numElements = type.getNumElements(); |
| 1307 | Type elementType = type.getElementType(); |
| 1308 | if (ComplexType complexTy = llvm::dyn_cast<ComplexType>(Val&: elementType)) { |
| 1309 | elementType = complexTy.getElementType(); |
| 1310 | numElements = numElements * 2; |
| 1311 | } |
| 1312 | size_t elementBitWidth = getDenseElementStorageWidth(elementType); |
| 1313 | assert(numElements * elementBitWidth == inRawData.size() * CHAR_BIT && |
| 1314 | inRawData.size() <= outRawData.size()); |
| 1315 | if (elementBitWidth <= CHAR_BIT) |
| 1316 | std::memcpy(dest: outRawData.begin(), src: inRawData.begin(), n: inRawData.size()); |
| 1317 | else |
| 1318 | convertEndianOfCharForBEmachine(inRawData: inRawData.begin(), outRawData: outRawData.begin(), |
| 1319 | elementBitWidth, numElements); |
| 1320 | } |
| 1321 | |
| 1322 | //===----------------------------------------------------------------------===// |
| 1323 | // DenseFPElementsAttr |
| 1324 | //===----------------------------------------------------------------------===// |
| 1325 | |
| 1326 | template <typename Fn, typename Attr> |
| 1327 | static ShapedType mappingHelper(Fn mapping, Attr &attr, ShapedType inType, |
| 1328 | Type newElementType, |
| 1329 | llvm::SmallVectorImpl<char> &data) { |
| 1330 | size_t bitWidth = getDenseElementBitWidth(eltType: newElementType); |
| 1331 | size_t storageBitWidth = getDenseElementStorageWidth(origWidth: bitWidth); |
| 1332 | |
| 1333 | ShapedType newArrayType = inType.cloneWith(shape: inType.getShape(), elementType: newElementType); |
| 1334 | |
| 1335 | size_t numRawElements = attr.isSplat() ? 1 : newArrayType.getNumElements(); |
| 1336 | data.resize(N: llvm::divideCeil(Numerator: storageBitWidth * numRawElements, CHAR_BIT)); |
| 1337 | |
| 1338 | // Functor used to process a single element value of the attribute. |
| 1339 | auto processElt = [&](decltype(*attr.begin()) value, size_t index) { |
| 1340 | auto newInt = mapping(value); |
| 1341 | assert(newInt.getBitWidth() == bitWidth); |
| 1342 | writeBits(data.data(), index * storageBitWidth, newInt); |
| 1343 | }; |
| 1344 | |
| 1345 | // Check for the splat case. |
| 1346 | if (attr.isSplat()) { |
| 1347 | processElt(*attr.begin(), /*index=*/0); |
| 1348 | return newArrayType; |
| 1349 | } |
| 1350 | |
| 1351 | // Otherwise, process all of the element values. |
| 1352 | uint64_t elementIdx = 0; |
| 1353 | for (auto value : attr) |
| 1354 | processElt(value, elementIdx++); |
| 1355 | return newArrayType; |
| 1356 | } |
| 1357 | |
| 1358 | DenseElementsAttr DenseFPElementsAttr::mapValues( |
| 1359 | Type newElementType, function_ref<APInt(const APFloat &)> mapping) const { |
| 1360 | llvm::SmallVector<char, 8> elementData; |
| 1361 | auto newArrayType = |
| 1362 | mappingHelper(mapping, attr: *this, inType: getType(), newElementType, data&: elementData); |
| 1363 | |
| 1364 | return getRaw(type: newArrayType, data: elementData); |
| 1365 | } |
| 1366 | |
| 1367 | /// Method for supporting type inquiry through isa, cast and dyn_cast. |
| 1368 | bool DenseFPElementsAttr::classof(Attribute attr) { |
| 1369 | if (auto denseAttr = llvm::dyn_cast<DenseElementsAttr>(Val&: attr)) |
| 1370 | return llvm::isa<FloatType>(Val: denseAttr.getType().getElementType()); |
| 1371 | return false; |
| 1372 | } |
| 1373 | |
| 1374 | //===----------------------------------------------------------------------===// |
| 1375 | // DenseIntElementsAttr |
| 1376 | //===----------------------------------------------------------------------===// |
| 1377 | |
| 1378 | DenseElementsAttr DenseIntElementsAttr::mapValues( |
| 1379 | Type newElementType, function_ref<APInt(const APInt &)> mapping) const { |
| 1380 | llvm::SmallVector<char, 8> elementData; |
| 1381 | auto newArrayType = |
| 1382 | mappingHelper(mapping, attr: *this, inType: getType(), newElementType, data&: elementData); |
| 1383 | return getRaw(type: newArrayType, data: elementData); |
| 1384 | } |
| 1385 | |
| 1386 | /// Method for supporting type inquiry through isa, cast and dyn_cast. |
| 1387 | bool DenseIntElementsAttr::classof(Attribute attr) { |
| 1388 | if (auto denseAttr = llvm::dyn_cast<DenseElementsAttr>(Val&: attr)) |
| 1389 | return denseAttr.getType().getElementType().isIntOrIndex(); |
| 1390 | return false; |
| 1391 | } |
| 1392 | |
| 1393 | //===----------------------------------------------------------------------===// |
| 1394 | // DenseResourceElementsAttr |
| 1395 | //===----------------------------------------------------------------------===// |
| 1396 | |
| 1397 | DenseResourceElementsAttr |
| 1398 | DenseResourceElementsAttr::get(ShapedType type, |
| 1399 | DenseResourceElementsHandle handle) { |
| 1400 | return Base::get(ctx: type.getContext(), args&: type, args&: handle); |
| 1401 | } |
| 1402 | |
| 1403 | DenseResourceElementsAttr DenseResourceElementsAttr::get(ShapedType type, |
| 1404 | StringRef blobName, |
| 1405 | AsmResourceBlob blob) { |
| 1406 | // Extract the builtin dialect resource manager from context and construct a |
| 1407 | // handle by inserting a new resource using the provided blob. |
| 1408 | auto &manager = |
| 1409 | DenseResourceElementsHandle::getManagerInterface(ctx: type.getContext()); |
| 1410 | return get(type, handle: manager.insert(name: blobName, blob: std::move(blob))); |
| 1411 | } |
| 1412 | |
| 1413 | ArrayRef<char> DenseResourceElementsAttr::getData() { |
| 1414 | if (AsmResourceBlob *blob = this->getRawHandle().getBlob()) |
| 1415 | return blob->getDataAs<char>(); |
| 1416 | return {}; |
| 1417 | } |
| 1418 | |
| 1419 | //===----------------------------------------------------------------------===// |
| 1420 | // DenseResourceElementsAttrBase |
| 1421 | //===----------------------------------------------------------------------===// |
| 1422 | |
| 1423 | namespace { |
| 1424 | /// Instantiations of this class provide utilities for interacting with native |
| 1425 | /// data types in the context of DenseResourceElementsAttr. |
| 1426 | template <typename T> |
| 1427 | struct DenseResourceAttrUtil; |
| 1428 | template <size_t width, bool isSigned> |
| 1429 | struct DenseResourceElementsAttrIntUtil { |
| 1430 | static bool checkElementType(Type eltType) { |
| 1431 | IntegerType type = llvm::dyn_cast<IntegerType>(Val&: eltType); |
| 1432 | if (!type || type.getWidth() != width) |
| 1433 | return false; |
| 1434 | return isSigned ? !type.isUnsigned() : !type.isSigned(); |
| 1435 | } |
| 1436 | }; |
| 1437 | template <> |
| 1438 | struct DenseResourceAttrUtil<bool> { |
| 1439 | static bool checkElementType(Type eltType) { |
| 1440 | return eltType.isSignlessInteger(width: 1); |
| 1441 | } |
| 1442 | }; |
| 1443 | template <> |
| 1444 | struct DenseResourceAttrUtil<int8_t> |
| 1445 | : public DenseResourceElementsAttrIntUtil<8, true> {}; |
| 1446 | template <> |
| 1447 | struct DenseResourceAttrUtil<uint8_t> |
| 1448 | : public DenseResourceElementsAttrIntUtil<8, false> {}; |
| 1449 | template <> |
| 1450 | struct DenseResourceAttrUtil<int16_t> |
| 1451 | : public DenseResourceElementsAttrIntUtil<16, true> {}; |
| 1452 | template <> |
| 1453 | struct DenseResourceAttrUtil<uint16_t> |
| 1454 | : public DenseResourceElementsAttrIntUtil<16, false> {}; |
| 1455 | template <> |
| 1456 | struct DenseResourceAttrUtil<int32_t> |
| 1457 | : public DenseResourceElementsAttrIntUtil<32, true> {}; |
| 1458 | template <> |
| 1459 | struct DenseResourceAttrUtil<uint32_t> |
| 1460 | : public DenseResourceElementsAttrIntUtil<32, false> {}; |
| 1461 | template <> |
| 1462 | struct DenseResourceAttrUtil<int64_t> |
| 1463 | : public DenseResourceElementsAttrIntUtil<64, true> {}; |
| 1464 | template <> |
| 1465 | struct DenseResourceAttrUtil<uint64_t> |
| 1466 | : public DenseResourceElementsAttrIntUtil<64, false> {}; |
| 1467 | template <> |
| 1468 | struct DenseResourceAttrUtil<float> { |
| 1469 | static bool checkElementType(Type eltType) { return eltType.isF32(); } |
| 1470 | }; |
| 1471 | template <> |
| 1472 | struct DenseResourceAttrUtil<double> { |
| 1473 | static bool checkElementType(Type eltType) { return eltType.isF64(); } |
| 1474 | }; |
| 1475 | } // namespace |
| 1476 | |
| 1477 | template <typename T> |
| 1478 | DenseResourceElementsAttrBase<T> |
| 1479 | DenseResourceElementsAttrBase<T>::get(ShapedType type, StringRef blobName, |
| 1480 | AsmResourceBlob blob) { |
| 1481 | // Check that the blob is in the form we were expecting. |
| 1482 | assert(blob.getDataAlignment() == alignof(T) && |
| 1483 | "alignment mismatch between expected alignment and blob alignment" ); |
| 1484 | assert(((blob.getData().size() % sizeof(T)) == 0) && |
| 1485 | "size mismatch between expected element width and blob size" ); |
| 1486 | assert(DenseResourceAttrUtil<T>::checkElementType(type.getElementType()) && |
| 1487 | "invalid shape element type for provided type `T`" ); |
| 1488 | return llvm::cast<DenseResourceElementsAttrBase<T>>( |
| 1489 | DenseResourceElementsAttr::get(type, blobName, blob: std::move(blob))); |
| 1490 | } |
| 1491 | |
| 1492 | template <typename T> |
| 1493 | std::optional<ArrayRef<T>> |
| 1494 | DenseResourceElementsAttrBase<T>::tryGetAsArrayRef() const { |
| 1495 | if (AsmResourceBlob *blob = this->getRawHandle().getBlob()) |
| 1496 | return blob->template getDataAs<T>(); |
| 1497 | return std::nullopt; |
| 1498 | } |
| 1499 | |
| 1500 | template <typename T> |
| 1501 | bool DenseResourceElementsAttrBase<T>::classof(Attribute attr) { |
| 1502 | auto resourceAttr = llvm::dyn_cast<DenseResourceElementsAttr>(Val&: attr); |
| 1503 | return resourceAttr && DenseResourceAttrUtil<T>::checkElementType( |
| 1504 | resourceAttr.getElementType()); |
| 1505 | } |
| 1506 | |
| 1507 | namespace mlir { |
| 1508 | namespace detail { |
| 1509 | // Explicit instantiation for all the supported DenseResourceElementsAttr. |
| 1510 | template class DenseResourceElementsAttrBase<bool>; |
| 1511 | template class DenseResourceElementsAttrBase<int8_t>; |
| 1512 | template class DenseResourceElementsAttrBase<int16_t>; |
| 1513 | template class DenseResourceElementsAttrBase<int32_t>; |
| 1514 | template class DenseResourceElementsAttrBase<int64_t>; |
| 1515 | template class DenseResourceElementsAttrBase<uint8_t>; |
| 1516 | template class DenseResourceElementsAttrBase<uint16_t>; |
| 1517 | template class DenseResourceElementsAttrBase<uint32_t>; |
| 1518 | template class DenseResourceElementsAttrBase<uint64_t>; |
| 1519 | template class DenseResourceElementsAttrBase<float>; |
| 1520 | template class DenseResourceElementsAttrBase<double>; |
| 1521 | } // namespace detail |
| 1522 | } // namespace mlir |
| 1523 | |
| 1524 | //===----------------------------------------------------------------------===// |
| 1525 | // SparseElementsAttr |
| 1526 | //===----------------------------------------------------------------------===// |
| 1527 | |
| 1528 | /// Get a zero APFloat for the given sparse attribute. |
| 1529 | APFloat SparseElementsAttr::getZeroAPFloat() const { |
| 1530 | auto eltType = llvm::cast<FloatType>(Val: getElementType()); |
| 1531 | return APFloat(eltType.getFloatSemantics()); |
| 1532 | } |
| 1533 | |
| 1534 | /// Get a zero APInt for the given sparse attribute. |
| 1535 | APInt SparseElementsAttr::getZeroAPInt() const { |
| 1536 | auto eltType = llvm::cast<IntegerType>(Val: getElementType()); |
| 1537 | return APInt::getZero(numBits: eltType.getWidth()); |
| 1538 | } |
| 1539 | |
| 1540 | /// Get a zero attribute for the given attribute type. |
| 1541 | Attribute SparseElementsAttr::getZeroAttr() const { |
| 1542 | auto eltType = getElementType(); |
| 1543 | |
| 1544 | // Handle floating point elements. |
| 1545 | if (llvm::isa<FloatType>(Val: eltType)) |
| 1546 | return FloatAttr::get(type: eltType, value: 0); |
| 1547 | |
| 1548 | // Handle complex elements. |
| 1549 | if (auto complexTy = llvm::dyn_cast<ComplexType>(Val&: eltType)) { |
| 1550 | auto eltType = complexTy.getElementType(); |
| 1551 | Attribute zero; |
| 1552 | if (llvm::isa<FloatType>(Val: eltType)) |
| 1553 | zero = FloatAttr::get(type: eltType, value: 0); |
| 1554 | else // must be integer |
| 1555 | zero = IntegerAttr::get(type: eltType, value: 0); |
| 1556 | return ArrayAttr::get(context: complexTy.getContext(), |
| 1557 | value: ArrayRef<Attribute>{zero, zero}); |
| 1558 | } |
| 1559 | |
| 1560 | // Handle string type. |
| 1561 | if (llvm::isa<DenseStringElementsAttr>(Val: getValues())) |
| 1562 | return StringAttr::get(twine: "" , type: eltType); |
| 1563 | |
| 1564 | // MXI8 elements use signless i8 attributes for their encoded bits. |
| 1565 | if (llvm::isa<MXI8Type>(Val: eltType)) |
| 1566 | return IntegerAttr::get(type: IntegerType::get(context: eltType.getContext(), width: 8), value: 0); |
| 1567 | |
| 1568 | // Otherwise, this is an integer. |
| 1569 | return IntegerAttr::get(type: eltType, value: 0); |
| 1570 | } |
| 1571 | |
| 1572 | /// Flatten, and return, all of the sparse indices in this attribute in |
| 1573 | /// row-major order. |
| 1574 | SmallVector<ptrdiff_t> SparseElementsAttr::getFlattenedSparseIndices() const { |
| 1575 | SmallVector<ptrdiff_t> flatSparseIndices; |
| 1576 | |
| 1577 | // The sparse indices are 64-bit integers, so we can reinterpret the raw data |
| 1578 | // as a 1-D index array. |
| 1579 | auto sparseIndices = getIndices(); |
| 1580 | auto sparseIndexValues = sparseIndices.getValues<uint64_t>(); |
| 1581 | if (sparseIndices.isSplat()) { |
| 1582 | SmallVector<uint64_t, 8> indices(getType().getRank(), |
| 1583 | *sparseIndexValues.begin()); |
| 1584 | flatSparseIndices.push_back(Elt: getFlattenedIndex(index: indices)); |
| 1585 | return flatSparseIndices; |
| 1586 | } |
| 1587 | |
| 1588 | // Otherwise, reinterpret each index as an ArrayRef when flattening. |
| 1589 | auto numSparseIndices = sparseIndices.getType().getDimSize(idx: 0); |
| 1590 | size_t rank = getType().getRank(); |
| 1591 | for (size_t i = 0, e = numSparseIndices; i != e; ++i) |
| 1592 | flatSparseIndices.push_back(Elt: getFlattenedIndex( |
| 1593 | index: {&*std::next(x: sparseIndexValues.begin(), n: i * rank), rank})); |
| 1594 | return flatSparseIndices; |
| 1595 | } |
| 1596 | |
| 1597 | LogicalResult |
| 1598 | SparseElementsAttr::verify(function_ref<InFlightDiagnostic()> emitError, |
| 1599 | ShapedType type, DenseIntElementsAttr sparseIndices, |
| 1600 | DenseElementsAttr values) { |
| 1601 | ShapedType valuesType = values.getType(); |
| 1602 | if (valuesType.getRank() != 1) |
| 1603 | return emitError() << "expected 1-d tensor for sparse element values" ; |
| 1604 | |
| 1605 | // Verify the indices and values shape. |
| 1606 | ShapedType indicesType = sparseIndices.getType(); |
| 1607 | auto emitShapeError = [&]() { |
| 1608 | return emitError() << "expected shape ([" << type.getShape() |
| 1609 | << "]); inferred shape of indices literal ([" |
| 1610 | << indicesType.getShape() |
| 1611 | << "]); inferred shape of values literal ([" |
| 1612 | << valuesType.getShape() << "])" ; |
| 1613 | }; |
| 1614 | // Verify indices shape. |
| 1615 | size_t rank = type.getRank(), indicesRank = indicesType.getRank(); |
| 1616 | if (indicesRank == 2) { |
| 1617 | if (indicesType.getDimSize(idx: 1) != static_cast<int64_t>(rank)) |
| 1618 | return emitShapeError(); |
| 1619 | } else if (indicesRank != 1 || rank != 1) { |
| 1620 | return emitShapeError(); |
| 1621 | } |
| 1622 | // Verify the values shape. |
| 1623 | int64_t numSparseIndices = indicesType.getDimSize(idx: 0); |
| 1624 | if (numSparseIndices != valuesType.getDimSize(idx: 0)) |
| 1625 | return emitShapeError(); |
| 1626 | |
| 1627 | // Verify that the sparse indices are within the value shape. |
| 1628 | auto emitIndexError = [&](unsigned indexNum, ArrayRef<uint64_t> index) { |
| 1629 | return emitError() |
| 1630 | << "sparse index #" << indexNum |
| 1631 | << " is not contained within the value shape, with index=[" << index |
| 1632 | << "], and type=" << type; |
| 1633 | }; |
| 1634 | |
| 1635 | // Handle the case where the index values are a splat. |
| 1636 | auto sparseIndexValues = sparseIndices.getValues<uint64_t>(); |
| 1637 | if (sparseIndices.isSplat()) { |
| 1638 | SmallVector<uint64_t> indices(rank, *sparseIndexValues.begin()); |
| 1639 | if (!ElementsAttr::isValidIndex(type, index: indices)) |
| 1640 | return emitIndexError(0, indices); |
| 1641 | return success(); |
| 1642 | } |
| 1643 | |
| 1644 | // Otherwise, reinterpret each index as an ArrayRef. |
| 1645 | for (size_t i = 0, e = numSparseIndices; i != e; ++i) { |
| 1646 | ArrayRef<uint64_t> index(&*std::next(x: sparseIndexValues.begin(), n: i * rank), |
| 1647 | rank); |
| 1648 | if (!ElementsAttr::isValidIndex(type, index)) |
| 1649 | return emitIndexError(i, index); |
| 1650 | } |
| 1651 | |
| 1652 | return success(); |
| 1653 | } |
| 1654 | |
| 1655 | //===----------------------------------------------------------------------===// |
| 1656 | // DistinctAttr |
| 1657 | //===----------------------------------------------------------------------===// |
| 1658 | |
| 1659 | DistinctAttr DistinctAttr::create(Attribute referencedAttr) { |
| 1660 | return Base::get(ctx: referencedAttr.getContext(), args&: referencedAttr); |
| 1661 | } |
| 1662 | |
| 1663 | Attribute DistinctAttr::getReferencedAttr() const { |
| 1664 | return getImpl()->referencedAttr; |
| 1665 | } |
| 1666 | |
| 1667 | //===----------------------------------------------------------------------===// |
| 1668 | // Attribute Utilities |
| 1669 | //===----------------------------------------------------------------------===// |
| 1670 | |
| 1671 | AffineMap mlir::makeStridedLinearLayoutMap(ArrayRef<int64_t> strides, |
| 1672 | int64_t offset, |
| 1673 | MLIRContext *context) { |
| 1674 | AffineExpr expr; |
| 1675 | unsigned nSymbols = 0; |
| 1676 | |
| 1677 | // AffineExpr for offset. |
| 1678 | // Static case. |
| 1679 | if (ShapedType::isStatic(dValue: offset)) { |
| 1680 | auto cst = getAffineConstantExpr(constant: offset, context); |
| 1681 | expr = cst; |
| 1682 | } else { |
| 1683 | // Dynamic case, new symbol for the offset. |
| 1684 | auto sym = getAffineSymbolExpr(position: nSymbols++, context); |
| 1685 | expr = sym; |
| 1686 | } |
| 1687 | |
| 1688 | // AffineExpr for strides. |
| 1689 | for (const auto &en : llvm::enumerate(First&: strides)) { |
| 1690 | auto dim = en.index(); |
| 1691 | auto stride = en.value(); |
| 1692 | auto d = getAffineDimExpr(position: dim, context); |
| 1693 | AffineExpr mult; |
| 1694 | // Static case. |
| 1695 | if (ShapedType::isStatic(dValue: stride)) |
| 1696 | mult = getAffineConstantExpr(constant: stride, context); |
| 1697 | else |
| 1698 | // Dynamic case, new symbol for each new stride. |
| 1699 | mult = getAffineSymbolExpr(position: nSymbols++, context); |
| 1700 | expr = expr + d * mult; |
| 1701 | } |
| 1702 | |
| 1703 | return AffineMap::get(dimCount: strides.size(), symbolCount: nSymbols, result: expr); |
| 1704 | } |
| 1705 | |