| 1 | //===----------------------------------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #ifndef LLVM_CLANG_LIB_CIR_CODEGEN_CIRGENBUILDER_H |
| 10 | #define LLVM_CLANG_LIB_CIR_CODEGEN_CIRGENBUILDER_H |
| 11 | |
| 12 | #include "Address.h" |
| 13 | #include "CIRGenRecordLayout.h" |
| 14 | #include "CIRGenTypeCache.h" |
| 15 | #include "mlir/Dialect/Ptr/IR/MemorySpaceInterfaces.h" |
| 16 | #include "mlir/IR/Attributes.h" |
| 17 | #include "mlir/IR/Builders.h" |
| 18 | #include "mlir/IR/BuiltinAttributes.h" |
| 19 | #include "mlir/Support/LLVM.h" |
| 20 | #include "clang/CIR/Dialect/IR/CIRDataLayout.h" |
| 21 | #include "clang/CIR/MissingFeatures.h" |
| 22 | |
| 23 | #include "clang/CIR/Dialect/Builder/CIRBaseBuilder.h" |
| 24 | #include "clang/CIR/MissingFeatures.h" |
| 25 | #include "llvm/ADT/APFloat.h" |
| 26 | #include "llvm/ADT/STLExtras.h" |
| 27 | #include "llvm/IR/FPEnv.h" |
| 28 | |
| 29 | namespace clang::CIRGen { |
| 30 | |
| 31 | class CIRGenBuilderTy : public cir::CIRBaseBuilderTy { |
| 32 | const CIRGenTypeCache &typeCache; |
| 33 | |
| 34 | llvm::StringMap<unsigned> recordNames; |
| 35 | llvm::StringMap<unsigned> globalsVersioning; |
| 36 | |
| 37 | public: |
| 38 | CIRGenBuilderTy(mlir::MLIRContext &mlirContext, const CIRGenTypeCache &tc) |
| 39 | : CIRBaseBuilderTy(mlirContext), typeCache(tc) {} |
| 40 | |
| 41 | /// Get a cir::ConstArrayAttr for a string literal. |
| 42 | /// Note: This is different from what is returned by |
| 43 | /// mlir::Builder::getStringAttr() which is an mlir::StringAttr. |
| 44 | mlir::Attribute getString(llvm::StringRef str, mlir::Type eltTy, |
| 45 | std::optional<size_t> size, |
| 46 | bool ensureNullTerm = true) { |
| 47 | size_t finalSize = size.value_or(u: str.size()); |
| 48 | |
| 49 | size_t lastNonZeroPos = str.find_last_not_of(C: '\0'); |
| 50 | // If the string is full of null bytes, emit a #cir.zero rather than |
| 51 | // a #cir.const_array. |
| 52 | if (lastNonZeroPos == llvm::StringRef::npos) { |
| 53 | auto arrayTy = cir::ArrayType::get(elementType: eltTy, size: finalSize); |
| 54 | return cir::ZeroAttr::get(type: arrayTy); |
| 55 | } |
| 56 | |
| 57 | // We emit trailing zeros for all trailing zeros, so the null-terminator in |
| 58 | // a constant is always in trailing zeros, and the null-terminator is |
| 59 | // skipped in the CIR representation. |
| 60 | size_t trailingZerosNum = finalSize - lastNonZeroPos - 1; |
| 61 | auto truncatedArrayTy = |
| 62 | cir::ArrayType::get(elementType: eltTy, size: finalSize - trailingZerosNum); |
| 63 | auto strAttr = mlir::StringAttr::get(bytes: str.drop_back(N: trailingZerosNum), |
| 64 | type: truncatedArrayTy); |
| 65 | |
| 66 | // Most C strings are null terminated, so if we are ensuring there is one, |
| 67 | // grow the array size by 1 to add a trailing zero if necessary. The 'auto' |
| 68 | // calculation of trailing zeros (the difference between the provided string |
| 69 | // and the type) will ensure we get the count correct. |
| 70 | finalSize += (ensureNullTerm && trailingZerosNum == 0); |
| 71 | |
| 72 | auto fullArrayTy = cir::ArrayType::get(elementType: eltTy, size: finalSize); |
| 73 | return cir::ConstArrayAttr::get(type: fullArrayTy, elts: strAttr); |
| 74 | } |
| 75 | |
| 76 | cir::ConstArrayAttr getConstArray(mlir::Attribute attrs, |
| 77 | cir::ArrayType arrayTy) const { |
| 78 | return cir::ConstArrayAttr::get(type: arrayTy, elts: attrs); |
| 79 | } |
| 80 | |
| 81 | mlir::Attribute getConstRecordOrZeroAttr(mlir::ArrayAttr arrayAttr, |
| 82 | cir::RecordType recordTy); |
| 83 | |
| 84 | cir::ConstRecordAttr getAnonConstRecord(mlir::ArrayAttr arrayAttr) { |
| 85 | llvm::SmallVector<mlir::Type> members; |
| 86 | for (auto &f : arrayAttr) { |
| 87 | auto ta = mlir::cast<mlir::TypedAttr>(Val: f); |
| 88 | members.push_back(Elt: ta.getType()); |
| 89 | } |
| 90 | |
| 91 | auto sTy = getAnonRecordTy(members, /*packed=*/packed: false, |
| 92 | memberKinds: cir::RecordType::getAllDataKinds(members)); |
| 93 | return cir::ConstRecordAttr::get(type: sTy, members: arrayAttr); |
| 94 | } |
| 95 | |
| 96 | cir::TypeInfoAttr getTypeInfo(mlir::ArrayAttr fieldsAttr) { |
| 97 | cir::ConstRecordAttr anonRecord = getAnonConstRecord(arrayAttr: fieldsAttr); |
| 98 | return cir::TypeInfoAttr::get(type: anonRecord.getType(), data: fieldsAttr); |
| 99 | } |
| 100 | |
| 101 | std::string getUniqueAnonRecordName() { return getUniqueRecordName(baseName: "anon" ); } |
| 102 | |
| 103 | std::string getUniqueRecordName(const std::string &baseName) { |
| 104 | auto it = recordNames.find(Key: baseName); |
| 105 | if (it == recordNames.end()) { |
| 106 | recordNames[baseName] = 0; |
| 107 | return baseName; |
| 108 | } |
| 109 | |
| 110 | return baseName + "." + std::to_string(val: recordNames[baseName]++); |
| 111 | } |
| 112 | |
| 113 | cir::LongDoubleType getLongDoubleTy(const llvm::fltSemantics &format) const { |
| 114 | if (&format == &llvm::APFloat::IEEEdouble()) |
| 115 | return cir::LongDoubleType::get(context: getContext(), underlying: typeCache.doubleTy); |
| 116 | if (&format == &llvm::APFloat::x87DoubleExtended()) |
| 117 | return cir::LongDoubleType::get(context: getContext(), underlying: typeCache.fP80Ty); |
| 118 | if (&format == &llvm::APFloat::IEEEquad()) |
| 119 | return cir::LongDoubleType::get(context: getContext(), underlying: typeCache.fP128Ty); |
| 120 | if (&format == &llvm::APFloat::PPCDoubleDouble()) |
| 121 | llvm_unreachable("NYI: PPC double-double format for long double" ); |
| 122 | llvm_unreachable("Unsupported format for long double" ); |
| 123 | } |
| 124 | |
| 125 | mlir::Type getPtrToVPtrType() { |
| 126 | return getPointerTo(ty: cir::VPtrType::get(ctx: getContext())); |
| 127 | } |
| 128 | |
| 129 | cir::FuncType getFuncType(llvm::ArrayRef<mlir::Type> params, mlir::Type retTy, |
| 130 | bool isVarArg = false) { |
| 131 | return cir::FuncType::get(inputs: params, returnType: retTy, isVarArg); |
| 132 | } |
| 133 | |
| 134 | /// Get a CIR record kind from a AST declaration tag. |
| 135 | /// Returns true if the tag kind represents a C++ class (as opposed to a |
| 136 | /// plain struct). |
| 137 | static bool tagKindIsClass(const clang::TagTypeKind kind) { |
| 138 | return kind == clang::TagTypeKind::Class; |
| 139 | } |
| 140 | |
| 141 | /// Returns true if the tag kind is a union. |
| 142 | static bool tagKindIsUnion(const clang::TagTypeKind kind) { |
| 143 | return kind == clang::TagTypeKind::Union; |
| 144 | } |
| 145 | |
| 146 | /// Get a CIR named record type. |
| 147 | /// |
| 148 | /// If a record already exists and is complete, but the client tries to fetch |
| 149 | /// it with a different set of attributes, this method will crash. |
| 150 | cir::RecordType getCompleteNamedRecordType( |
| 151 | llvm::ArrayRef<mlir::Type> members, bool packed, llvm::StringRef name, |
| 152 | llvm::ArrayRef<cir::RecordMemberKind> memberKinds) { |
| 153 | const auto nameAttr = getStringAttr(bytes: name); |
| 154 | assert(!cir::MissingFeatures::astRecordDeclAttr()); |
| 155 | |
| 156 | // Always a struct at this call site, never a class or a union. |
| 157 | auto type = cir::StructType::get(context: getContext(), members, name: nameAttr, packed, |
| 158 | /*is_class=*/is_class: false, member_kinds: memberKinds); |
| 159 | |
| 160 | // If we found an existing type, verify that either it is incomplete or |
| 161 | // it matches the requested attributes. |
| 162 | assert(!type.isIncomplete() || |
| 163 | (type.getMembers() == members && type.getPacked() == packed)); |
| 164 | |
| 165 | // Complete an incomplete record or ensure the existing complete record |
| 166 | // matches the requested attributes. |
| 167 | type.complete(members, packed, memberKinds); |
| 168 | |
| 169 | return type; |
| 170 | } |
| 171 | |
| 172 | /// Get an incomplete CIR record type. If we have a complete record |
| 173 | /// declaration, we may create an incomplete type and then add the |
| 174 | /// members, so \p rd here may be complete. |
| 175 | cir::RecordType getIncompleteRecordTy(llvm::StringRef name, |
| 176 | const clang::RecordDecl *rd) { |
| 177 | const mlir::StringAttr nameAttr = getStringAttr(bytes: name); |
| 178 | if (rd && tagKindIsUnion(kind: rd->getTagKind())) |
| 179 | return cir::UnionType::get(context: getContext(), name: nameAttr); |
| 180 | bool is_class = rd && tagKindIsClass(kind: rd->getTagKind()); |
| 181 | return cir::StructType::get(context: getContext(), name: nameAttr, is_class); |
| 182 | } |
| 183 | |
| 184 | // |
| 185 | // Operation creation helpers |
| 186 | // -------------------------- |
| 187 | // |
| 188 | using CIRBaseBuilderTy::createCopy; |
| 189 | cir::CopyOp createCopy(Address dst, Address src, bool isVolatile = false, |
| 190 | bool skipTailPadding = false) { |
| 191 | cir::CopyOp op = createCopy(dst: dst.getPointer(), src: src.getPointer(), isVolatile, |
| 192 | skipTailPadding); |
| 193 | op.setDstAlignment(dst.getAlignment().getQuantity()); |
| 194 | op.setSrcAlignment(src.getAlignment().getQuantity()); |
| 195 | return op; |
| 196 | } |
| 197 | |
| 198 | cir::MemCpyOp createMemCpy(mlir::Location loc, Address dst, Address src, |
| 199 | mlir::Value len) { |
| 200 | return cir::MemCpyOp::create( |
| 201 | builder&: *this, location: loc, dst: dst.getPointer(), src: src.getPointer(), len, |
| 202 | dst_alignment: getI64IntegerAttr(value: dst.getAlignment().getQuantity()), |
| 203 | src_alignment: getI64IntegerAttr(value: src.getAlignment().getQuantity())); |
| 204 | } |
| 205 | |
| 206 | cir::MemMoveOp createMemMove(mlir::Location loc, Address dst, Address src, |
| 207 | mlir::Value len) { |
| 208 | return cir::MemMoveOp::create( |
| 209 | builder&: *this, location: loc, dst: dst.getPointer(), src: src.getPointer(), len, |
| 210 | dst_alignment: getI64IntegerAttr(value: dst.getAlignment().getQuantity()), |
| 211 | src_alignment: getI64IntegerAttr(value: src.getAlignment().getQuantity())); |
| 212 | } |
| 213 | |
| 214 | cir::MemSetOp createMemSet(mlir::Location loc, mlir::Value dst, |
| 215 | mlir::Value val, mlir::Value len) { |
| 216 | assert(val.getType() == getUInt8Ty()); |
| 217 | return cir::MemSetOp::create(builder&: *this, location: loc, dst, alignment: {}, val, len); |
| 218 | } |
| 219 | |
| 220 | cir::MemSetOp createMemSet(mlir::Location loc, Address dst, mlir::Value val, |
| 221 | mlir::Value len) { |
| 222 | mlir::IntegerAttr align = getAlignmentAttr(alignment: dst.getAlignment()); |
| 223 | assert(val.getType() == getUInt8Ty()); |
| 224 | return cir::MemSetOp::create(builder&: *this, location: loc, dst: dst.getPointer(), alignment: align, val, len); |
| 225 | } |
| 226 | // --------------------------- |
| 227 | |
| 228 | cir::DataMemberAttr getDataMemberAttr(cir::DataMemberType ty, |
| 229 | llvm::ArrayRef<int32_t> path) { |
| 230 | return cir::DataMemberAttr::get(type: ty, path); |
| 231 | } |
| 232 | |
| 233 | cir::DataMemberAttr getNullDataMemberAttr(cir::DataMemberType ty) { |
| 234 | return cir::DataMemberAttr::get(type: ty); |
| 235 | } |
| 236 | |
| 237 | // Return true if the value is a null constant such as null pointer, (+0.0) |
| 238 | // for floating-point or zero initializer |
| 239 | bool isNullValue(mlir::Attribute attr) const { |
| 240 | if (mlir::isa<cir::ZeroAttr>(Val: attr)) |
| 241 | return true; |
| 242 | |
| 243 | if (const auto ptrVal = mlir::dyn_cast<cir::ConstPtrAttr>(Val&: attr)) |
| 244 | return ptrVal.isNullValue(); |
| 245 | |
| 246 | if (const auto intVal = mlir::dyn_cast<cir::IntAttr>(Val&: attr)) |
| 247 | return intVal.isNullValue(); |
| 248 | |
| 249 | if (const auto boolVal = mlir::dyn_cast<cir::BoolAttr>(Val&: attr)) |
| 250 | return !boolVal.getValue(); |
| 251 | |
| 252 | if (auto fpAttr = mlir::dyn_cast<cir::FPAttr>(Val&: attr)) { |
| 253 | auto fpVal = fpAttr.getValue(); |
| 254 | bool ignored; |
| 255 | llvm::APFloat fv(+0.0); |
| 256 | fv.convert(ToSemantics: fpVal.getSemantics(), RM: llvm::APFloat::rmNearestTiesToEven, |
| 257 | losesInfo: &ignored); |
| 258 | return fv.bitwiseIsEqual(RHS: fpVal); |
| 259 | } |
| 260 | if (const auto recordVal = mlir::dyn_cast<cir::ConstRecordAttr>(Val&: attr)) { |
| 261 | for (const auto elt : recordVal.getMembers()) { |
| 262 | // FIXME(cir): the record's ID should not be considered a member. |
| 263 | if (mlir::isa<mlir::StringAttr>(Val: elt)) |
| 264 | continue; |
| 265 | if (!isNullValue(attr: elt)) |
| 266 | return false; |
| 267 | } |
| 268 | return true; |
| 269 | } |
| 270 | |
| 271 | if (const auto arrayVal = mlir::dyn_cast<cir::ConstArrayAttr>(Val&: attr)) { |
| 272 | if (mlir::isa<mlir::StringAttr>(Val: arrayVal.getElts())) |
| 273 | return false; |
| 274 | |
| 275 | return llvm::all_of( |
| 276 | Range: mlir::cast<mlir::ArrayAttr>(Val: arrayVal.getElts()), |
| 277 | P: [&](const mlir::Attribute &elt) { return isNullValue(attr: elt); }); |
| 278 | } |
| 279 | return false; |
| 280 | } |
| 281 | |
| 282 | // |
| 283 | // Type helpers |
| 284 | // ------------ |
| 285 | // |
| 286 | cir::IntType getUIntNTy(int n) { |
| 287 | switch (n) { |
| 288 | case 8: |
| 289 | return getUInt8Ty(); |
| 290 | case 16: |
| 291 | return getUInt16Ty(); |
| 292 | case 32: |
| 293 | return getUInt32Ty(); |
| 294 | case 64: |
| 295 | return getUInt64Ty(); |
| 296 | default: |
| 297 | return cir::IntType::get(context: getContext(), width: n, is_signed: false); |
| 298 | } |
| 299 | } |
| 300 | |
| 301 | cir::IntType getSIntNTy(int n) { |
| 302 | switch (n) { |
| 303 | case 8: |
| 304 | return getSInt8Ty(); |
| 305 | case 16: |
| 306 | return getSInt16Ty(); |
| 307 | case 32: |
| 308 | return getSInt32Ty(); |
| 309 | case 64: |
| 310 | return getSInt64Ty(); |
| 311 | default: |
| 312 | return cir::IntType::get(context: getContext(), width: n, is_signed: true); |
| 313 | } |
| 314 | } |
| 315 | |
| 316 | cir::VoidType getVoidTy() { return typeCache.voidTy; } |
| 317 | |
| 318 | cir::IntType getBoolMemoryTy() { return getUInt8Ty(); } |
| 319 | |
| 320 | cir::IntType getSInt8Ty() { return typeCache.sInt8Ty; } |
| 321 | cir::IntType getSInt16Ty() { return typeCache.sInt16Ty; } |
| 322 | cir::IntType getSInt32Ty() { return typeCache.sInt32Ty; } |
| 323 | cir::IntType getSInt64Ty() { return typeCache.sInt64Ty; } |
| 324 | |
| 325 | cir::IntType getUInt8Ty() { return typeCache.uInt8Ty; } |
| 326 | cir::IntType getUInt16Ty() { return typeCache.uInt16Ty; } |
| 327 | cir::IntType getUInt32Ty() { return typeCache.uInt32Ty; } |
| 328 | cir::IntType getUInt64Ty() { return typeCache.uInt64Ty; } |
| 329 | |
| 330 | cir::FP16Type getFp16Ty() { return typeCache.fP16Ty; } |
| 331 | cir::BF16Type getBfloat16Ty() { return typeCache.bFloat16Ty; } |
| 332 | cir::SingleType getSingleTy() { return typeCache.floatTy; } |
| 333 | cir::DoubleType getDoubleTy() { return typeCache.doubleTy; } |
| 334 | |
| 335 | cir::ConstantOp getConstInt(mlir::Location loc, llvm::APSInt intVal); |
| 336 | |
| 337 | cir::ConstantOp getConstInt(mlir::Location loc, llvm::APInt intVal, |
| 338 | bool isUnsigned = true); |
| 339 | |
| 340 | cir::ConstantOp getConstInt(mlir::Location loc, mlir::Type t, uint64_t c); |
| 341 | |
| 342 | cir::ConstantOp getConstFP(mlir::Location loc, mlir::Type t, |
| 343 | llvm::APFloat fpVal); |
| 344 | |
| 345 | bool isInt8Ty(mlir::Type i) { |
| 346 | return i == typeCache.uInt8Ty || i == typeCache.sInt8Ty; |
| 347 | } |
| 348 | bool isInt16Ty(mlir::Type i) { |
| 349 | return i == typeCache.uInt16Ty || i == typeCache.sInt16Ty; |
| 350 | } |
| 351 | bool isInt32Ty(mlir::Type i) { |
| 352 | return i == typeCache.uInt32Ty || i == typeCache.sInt32Ty; |
| 353 | } |
| 354 | bool isInt64Ty(mlir::Type i) { |
| 355 | return i == typeCache.uInt64Ty || i == typeCache.sInt64Ty; |
| 356 | } |
| 357 | bool isInt(mlir::Type i) { return mlir::isa<cir::IntType>(Val: i); } |
| 358 | |
| 359 | cir::IntType getExtendedIntTy(cir::IntType ty, bool isSigned) { |
| 360 | switch (ty.getWidth()) { |
| 361 | case 8: |
| 362 | return isSigned ? typeCache.sInt16Ty : typeCache.uInt16Ty; |
| 363 | case 16: |
| 364 | return isSigned ? typeCache.sInt32Ty : typeCache.uInt32Ty; |
| 365 | case 32: |
| 366 | return isSigned ? typeCache.sInt64Ty : typeCache.uInt64Ty; |
| 367 | default: |
| 368 | llvm_unreachable("NYI" ); |
| 369 | } |
| 370 | } |
| 371 | |
| 372 | cir::IntType getTruncatedIntTy(cir::IntType ty, bool isSigned) { |
| 373 | switch (ty.getWidth()) { |
| 374 | case 16: |
| 375 | return isSigned ? typeCache.sInt8Ty : typeCache.uInt8Ty; |
| 376 | case 32: |
| 377 | return isSigned ? typeCache.sInt16Ty : typeCache.uInt16Ty; |
| 378 | case 64: |
| 379 | return isSigned ? typeCache.sInt32Ty : typeCache.uInt32Ty; |
| 380 | default: |
| 381 | llvm_unreachable("NYI" ); |
| 382 | } |
| 383 | } |
| 384 | |
| 385 | cir::VectorType |
| 386 | getExtendedOrTruncatedElementVectorType(cir::VectorType vt, bool isExtended, |
| 387 | bool isSigned = false) { |
| 388 | auto elementTy = mlir::dyn_cast_or_null<cir::IntType>(Val: vt.getElementType()); |
| 389 | assert(elementTy && "expected int vector" ); |
| 390 | return cir::VectorType::get(element_type: isExtended |
| 391 | ? getExtendedIntTy(ty: elementTy, isSigned) |
| 392 | : getTruncatedIntTy(ty: elementTy, isSigned), |
| 393 | size: vt.getSize()); |
| 394 | } |
| 395 | |
| 396 | // Fetch the type representing a pointer to unsigned int8 values. |
| 397 | cir::PointerType getUInt8PtrTy() { return typeCache.uInt8PtrTy; } |
| 398 | |
| 399 | /// Get a CIR anonymous struct type. |
| 400 | cir::StructType |
| 401 | getAnonRecordTy(llvm::ArrayRef<mlir::Type> members, bool packed, |
| 402 | llvm::ArrayRef<cir::RecordMemberKind> memberKinds) { |
| 403 | assert(!cir::MissingFeatures::astRecordDeclAttr()); |
| 404 | return cir::StructType::get(context: getContext(), members, packed, |
| 405 | /*is_class=*/is_class: false, member_kinds: memberKinds); |
| 406 | } |
| 407 | |
| 408 | //===--------------------------------------------------------------------===// |
| 409 | // Constant creation helpers |
| 410 | //===--------------------------------------------------------------------===// |
| 411 | cir::ConstantOp getSInt32(int32_t c, mlir::Location loc) { |
| 412 | return getConstantInt(loc, ty: getSInt32Ty(), value: c); |
| 413 | } |
| 414 | cir::ConstantOp getUInt32(uint32_t c, mlir::Location loc) { |
| 415 | return getConstantInt(loc, ty: getUInt32Ty(), value: c); |
| 416 | } |
| 417 | cir::ConstantOp getSInt64(uint64_t c, mlir::Location loc) { |
| 418 | return getConstantInt(loc, ty: getSInt64Ty(), value: c); |
| 419 | } |
| 420 | cir::ConstantOp getUInt64(uint64_t c, mlir::Location loc) { |
| 421 | return getConstantInt(loc, ty: getUInt64Ty(), value: c); |
| 422 | } |
| 423 | |
| 424 | cir::ConstantOp getZero(mlir::Location loc, mlir::Type ty) { |
| 425 | // TODO: dispatch creation for primitive types. |
| 426 | assert((mlir::isa<cir::RecordType>(ty) || mlir::isa<cir::ArrayType>(ty) || |
| 427 | mlir::isa<cir::VectorType>(ty)) && |
| 428 | "NYI for other types" ); |
| 429 | return cir::ConstantOp::create(builder&: *this, location: loc, value: cir::ZeroAttr::get(type: ty)); |
| 430 | } |
| 431 | |
| 432 | //===--------------------------------------------------------------------===// |
| 433 | // UnaryOp creation helpers |
| 434 | //===--------------------------------------------------------------------===// |
| 435 | mlir::Value createNeg(mlir::Location loc, mlir::Value value, |
| 436 | bool nsw = false) { |
| 437 | |
| 438 | if (auto intTy = mlir::dyn_cast<cir::IntType>(Val: value.getType())) { |
| 439 | // Source is a unsigned integer: first cast it to signed. |
| 440 | if (intTy.isUnsigned()) |
| 441 | value = createIntCast(src: value, newTy: getSIntNTy(n: intTy.getWidth())); |
| 442 | return createMinus(loc, input: value, nsw); |
| 443 | } |
| 444 | |
| 445 | llvm_unreachable("negation for the given type is NYI" ); |
| 446 | } |
| 447 | |
| 448 | //===--------------------------------------------------------------------===// |
| 449 | // CastOp creation helpers |
| 450 | //===--------------------------------------------------------------------===// |
| 451 | |
| 452 | // TODO: split this to createFPExt/createFPTrunc when we have dedicated cast |
| 453 | // operations. |
| 454 | mlir::Value createFloatingCast(mlir::Value v, mlir::Type destType) { |
| 455 | return cir::CastOp::create(builder&: *this, location: v.getLoc(), result: destType, |
| 456 | kind: cir::CastKind::floating, src: v, |
| 457 | fenv: getConstrainedFPAttr()); |
| 458 | } |
| 459 | |
| 460 | mlir::Value createDynCast(mlir::Location loc, mlir::Value src, |
| 461 | cir::PointerType destType, bool isRefCast, |
| 462 | cir::DynamicCastInfoAttr info) { |
| 463 | auto castKind = |
| 464 | isRefCast ? cir::DynamicCastKind::Ref : cir::DynamicCastKind::Ptr; |
| 465 | return cir::DynamicCastOp::create(builder&: *this, location: loc, result: destType, kind: castKind, src, |
| 466 | info); |
| 467 | } |
| 468 | |
| 469 | mlir::Value createDynCastToVoid(mlir::Location loc, mlir::Value src) { |
| 470 | auto srcPtrTy = mlir::cast<cir::PointerType>(Val: src.getType()); |
| 471 | cir::PointerType destTy = getVoidPtrTy(as: srcPtrTy.getAddrSpace()); |
| 472 | return cir::DynamicCastOp::create(builder&: *this, location: loc, result: destTy, |
| 473 | kind: cir::DynamicCastKind::Ptr, src, |
| 474 | info: cir::DynamicCastInfoAttr{}); |
| 475 | } |
| 476 | |
| 477 | //===--------------------------------------------------------------------===// |
| 478 | // Address creation helpers |
| 479 | //===--------------------------------------------------------------------===// |
| 480 | Address createBaseClassAddr(mlir::Location loc, Address addr, |
| 481 | mlir::Type destType, unsigned offset, |
| 482 | bool assumeNotNull) { |
| 483 | if (destType == addr.getElementType()) |
| 484 | return addr; |
| 485 | |
| 486 | auto ptrTy = getPointerTo(ty: destType, as: addr.getAddressSpace()); |
| 487 | auto baseAddr = |
| 488 | cir::BaseClassAddrOp::create(builder&: *this, location: loc, result: ptrTy, src_addr: addr.getPointer(), |
| 489 | offset: mlir::APInt(64, offset), assume_not_null: assumeNotNull); |
| 490 | return Address(baseAddr, destType, addr.getAlignment()); |
| 491 | } |
| 492 | |
| 493 | Address createDerivedClassAddr(mlir::Location loc, Address addr, |
| 494 | mlir::Type destType, unsigned offset, |
| 495 | bool assumeNotNull) { |
| 496 | if (destType == addr.getElementType()) |
| 497 | return addr; |
| 498 | |
| 499 | cir::PointerType ptrTy = getPointerTo(ty: destType, as: addr.getAddressSpace()); |
| 500 | auto derivedAddr = |
| 501 | cir::DerivedClassAddrOp::create(builder&: *this, location: loc, result: ptrTy, src_addr: addr.getPointer(), |
| 502 | offset: mlir::APInt(64, offset), assume_not_null: assumeNotNull); |
| 503 | return Address(derivedAddr, destType, addr.getAlignment()); |
| 504 | } |
| 505 | |
| 506 | //===--------------------------------------------------------------------===// |
| 507 | // Virtual Address creation helpers |
| 508 | //===--------------------------------------------------------------------===// |
| 509 | mlir::Value createVTTAddrPoint(mlir::Location loc, mlir::Type retTy, |
| 510 | mlir::Value addr, uint64_t offset) { |
| 511 | return cir::VTTAddrPointOp::create(builder&: *this, location: loc, addr: retTy, |
| 512 | name: mlir::FlatSymbolRefAttr{}, sym_addr: addr, offset); |
| 513 | } |
| 514 | |
| 515 | mlir::Value createVTTAddrPoint(mlir::Location loc, mlir::Type retTy, |
| 516 | mlir::FlatSymbolRefAttr sym, uint64_t offset) { |
| 517 | return cir::VTTAddrPointOp::create(builder&: *this, location: loc, addr: retTy, name: sym, sym_addr: mlir::Value{}, |
| 518 | offset); |
| 519 | } |
| 520 | |
| 521 | //===--------------------------------------------------------------------===// |
| 522 | // Other creation helpers |
| 523 | //===--------------------------------------------------------------------===// |
| 524 | cir::IsFPClassOp createIsFPClass(mlir::Location loc, mlir::Value src, |
| 525 | cir::FPClassTest flags) { |
| 526 | // FPClassTest occupies bits 0-9 (fcAllFlags). Sema rejects an |
| 527 | // out-of-range __builtin_isfpclass mask, so any extra bit here is an |
| 528 | // internal error; assert and mask it off so lowering stays well-formed. |
| 529 | uint32_t raw = static_cast<uint32_t>(flags); |
| 530 | uint32_t all = static_cast<uint32_t>(cir::FPClassTest::All); |
| 531 | assert((raw & ~all) == 0 && "FPClassTest mask has bits outside 0-9" ); |
| 532 | flags = static_cast<cir::FPClassTest>(raw & all); |
| 533 | return cir::IsFPClassOp::create(builder&: *this, location: loc, src, flags); |
| 534 | } |
| 535 | |
| 536 | /// Cast the element type of the given address to a different type, |
| 537 | /// preserving information like the alignment. |
| 538 | Address createElementBitCast(mlir::Location loc, Address addr, |
| 539 | mlir::Type destType) { |
| 540 | if (destType == addr.getElementType()) |
| 541 | return addr; |
| 542 | |
| 543 | auto srcPtrTy = mlir::cast<cir::PointerType>(Val: addr.getPointer().getType()); |
| 544 | auto ptrTy = getPointerTo(ty: destType, as: srcPtrTy.getAddrSpace()); |
| 545 | return Address(createBitcast(loc, src: addr.getPointer(), newTy: ptrTy), destType, |
| 546 | addr.getAlignment()); |
| 547 | } |
| 548 | |
| 549 | cir::LoadOp createLoad(mlir::Location loc, Address addr, |
| 550 | bool isVolatile = false, bool isNontemporal = false) { |
| 551 | mlir::IntegerAttr align = getAlignmentAttr(alignment: addr.getAlignment()); |
| 552 | return cir::LoadOp::create(builder&: *this, location: loc, addr: addr.getPointer(), /*isDeref=*/isDeref: false, |
| 553 | is_volatile: isVolatile, is_nontemporal: isNontemporal, |
| 554 | /*alignment=*/alignment: align, |
| 555 | /*sync_scope=*/sync_scope: cir::SyncScopeKindAttr{}, |
| 556 | /*mem_order=*/mem_order: cir::MemOrderAttr{}, |
| 557 | /*invariant=*/invariant: false); |
| 558 | } |
| 559 | |
| 560 | cir::LoadOp createAlignedLoad(mlir::Location loc, mlir::Type ty, |
| 561 | mlir::Value ptr, llvm::MaybeAlign align) { |
| 562 | if (ty != mlir::cast<cir::PointerType>(Val: ptr.getType()).getPointee()) |
| 563 | ptr = createPtrBitcast(src: ptr, newPointeeTy: ty); |
| 564 | uint64_t alignment = align ? align->value() : 0; |
| 565 | mlir::IntegerAttr alignAttr = getAlignmentAttr(alignment); |
| 566 | return cir::LoadOp::create(builder&: *this, location: loc, addr: ptr, /*isDeref=*/isDeref: false, |
| 567 | /*isVolatile=*/is_volatile: false, /*isNontemporal=*/is_nontemporal: false, |
| 568 | alignment: alignAttr, |
| 569 | /*sync_scope=*/sync_scope: cir::SyncScopeKindAttr{}, |
| 570 | /*mem_order=*/mem_order: cir::MemOrderAttr{}, |
| 571 | /*invariant=*/invariant: false); |
| 572 | } |
| 573 | |
| 574 | cir::LoadOp |
| 575 | createAlignedLoad(mlir::Location loc, mlir::Type ty, mlir::Value ptr, |
| 576 | clang::CharUnits align = clang::CharUnits::One()) { |
| 577 | return createAlignedLoad(loc, ty, ptr, align: align.getAsAlign()); |
| 578 | } |
| 579 | |
| 580 | cir::StoreOp createStore(mlir::Location loc, mlir::Value val, Address dst, |
| 581 | bool isVolatile = false, bool isNontemporal = false, |
| 582 | mlir::IntegerAttr align = {}, |
| 583 | cir::SyncScopeKindAttr scope = {}, |
| 584 | cir::MemOrderAttr order = {}) { |
| 585 | if (!align) |
| 586 | align = getAlignmentAttr(alignment: dst.getAlignment()); |
| 587 | return CIRBaseBuilderTy::createStore(loc, val, dst: dst.getPointer(), isVolatile, |
| 588 | isNontemporal, align, scope, order); |
| 589 | } |
| 590 | |
| 591 | /// Create a cir.complex.real_ptr operation that derives a pointer to the real |
| 592 | /// part of the complex value pointed to by the specified pointer value. |
| 593 | mlir::Value createComplexRealPtr(mlir::Location loc, mlir::Value value) { |
| 594 | auto srcPtrTy = mlir::cast<cir::PointerType>(Val: value.getType()); |
| 595 | auto srcComplexTy = mlir::cast<cir::ComplexType>(Val: srcPtrTy.getPointee()); |
| 596 | return cir::ComplexRealPtrOp::create( |
| 597 | builder&: *this, location: loc, result: getPointerTo(ty: srcComplexTy.getElementType()), operand: value); |
| 598 | } |
| 599 | |
| 600 | Address createComplexRealPtr(mlir::Location loc, Address addr) { |
| 601 | return Address{createComplexRealPtr(loc, value: addr.getPointer()), |
| 602 | addr.getAlignment()}; |
| 603 | } |
| 604 | |
| 605 | /// Create a cir.complex.imag_ptr operation that derives a pointer to the |
| 606 | /// imaginary part of the complex value pointed to by the specified pointer |
| 607 | /// value. |
| 608 | mlir::Value createComplexImagPtr(mlir::Location loc, mlir::Value value) { |
| 609 | auto srcPtrTy = mlir::cast<cir::PointerType>(Val: value.getType()); |
| 610 | auto srcComplexTy = mlir::cast<cir::ComplexType>(Val: srcPtrTy.getPointee()); |
| 611 | return cir::ComplexImagPtrOp::create( |
| 612 | builder&: *this, location: loc, result: getPointerTo(ty: srcComplexTy.getElementType()), operand: value); |
| 613 | } |
| 614 | |
| 615 | Address createComplexImagPtr(mlir::Location loc, Address addr) { |
| 616 | return Address{createComplexImagPtr(loc, value: addr.getPointer()), |
| 617 | addr.getAlignment()}; |
| 618 | } |
| 619 | |
| 620 | using CIRBaseBuilderTy::createGetMember; |
| 621 | Address createGetMember(mlir::Location loc, Address base, |
| 622 | llvm::StringRef name, unsigned index, |
| 623 | const cir::CIRDataLayout &layout) { |
| 624 | auto recordTy = mlir::cast<cir::RecordType>(Val: base.getElementType()); |
| 625 | |
| 626 | assert(index < recordTy.getMembers().size() && |
| 627 | "member index out of bounds" ); |
| 628 | mlir::Type memberTy = recordTy.getMembers()[index]; |
| 629 | mlir::Type memberPtrTy = getPointerTo(ty: memberTy, as: base.getAddressSpace()); |
| 630 | |
| 631 | auto memberOffset = CharUnits::fromQuantity( |
| 632 | Quantity: recordTy.getElementOffset(dataLayout: layout.layout, idx: index)); |
| 633 | |
| 634 | mlir::Value memberPtr = |
| 635 | createGetMember(loc, resultTy: memberPtrTy, base: base.getBasePointer(), name, index); |
| 636 | return Address(memberPtr, memberTy, |
| 637 | base.getAlignment().alignmentAtOffset(offset: memberOffset), |
| 638 | base.isKnownNonNull()); |
| 639 | } |
| 640 | |
| 641 | cir::GetRuntimeMemberOp createGetIndirectMember(mlir::Location loc, |
| 642 | mlir::Value objectPtr, |
| 643 | mlir::Value memberPtr) { |
| 644 | auto memberPtrTy = mlir::cast<cir::DataMemberType>(Val: memberPtr.getType()); |
| 645 | auto objectPtrTy = mlir::cast<cir::PointerType>(Val: objectPtr.getType()); |
| 646 | |
| 647 | // The member lives in the same address space as the object. |
| 648 | cir::PointerType resultTy = |
| 649 | getPointerTo(ty: memberPtrTy.getMemberTy(), as: objectPtrTy.getAddrSpace()); |
| 650 | |
| 651 | return cir::GetRuntimeMemberOp::create(builder&: *this, location: loc, result: resultTy, addr: objectPtr, |
| 652 | member: memberPtr); |
| 653 | } |
| 654 | |
| 655 | /// Create a cir.get_element or cir.ptr_stride operation to get access to an |
| 656 | /// array element. \p idx is the index of the element to access, |
| 657 | /// \p shouldDecay is true if the result should decay to a pointer to the |
| 658 | /// element type. \p flags are set on the cir.get_element emitted when |
| 659 | /// \p shouldDecay is true and \p arrayPtr points to the array type |
| 660 | /// \p eltTy, and are dropped when a cir.ptr_stride is emitted. |
| 661 | mlir::Value |
| 662 | getArrayElement(mlir::Location arrayLocBegin, mlir::Location arrayLocEnd, |
| 663 | mlir::Value arrayPtr, mlir::Type eltTy, mlir::Value idx, |
| 664 | bool shouldDecay, |
| 665 | cir::PtrOffsetFlags flags = cir::PtrOffsetFlags::None); |
| 666 | |
| 667 | /// Returns a decayed pointer to the first element of the array |
| 668 | /// pointed to by \p arrayPtr. |
| 669 | mlir::Value maybeBuildArrayDecay(mlir::Location loc, mlir::Value arrayPtr, |
| 670 | mlir::Type eltTy); |
| 671 | |
| 672 | // Convert byte offset to sequence of high-level indices suitable for |
| 673 | // GlobalViewAttr. Ideally we shouldn't deal with low-level offsets at all |
| 674 | // but currently some parts of Clang AST, which we don't want to touch just |
| 675 | // yet, return them. |
| 676 | // |
| 677 | // Returns false if the offset doesn't designate a subelement of \p ty, which |
| 678 | // happens when it lands outside of the object or in the middle of a scalar |
| 679 | // member. In that case \p indices is left in an unspecified state and the |
| 680 | // caller must describe the address with a byte offset, using a |
| 681 | // GlobalOffsetAttr, instead. |
| 682 | [[nodiscard]] bool computeGlobalViewIndicesFromFlatOffset( |
| 683 | int64_t offset, mlir::Type ty, const cir::CIRDataLayout &layout, |
| 684 | llvm::SmallVectorImpl<int64_t> &indices); |
| 685 | |
| 686 | // Convert high-level indices (e.g. from GlobalViewAttr) to byte offset. |
| 687 | uint64_t computeOffsetFromGlobalViewIndices(const cir::CIRDataLayout &layout, |
| 688 | mlir::Type ty, |
| 689 | llvm::ArrayRef<int64_t> indices); |
| 690 | |
| 691 | /// Creates a versioned global variable. If the symbol is already taken, an ID |
| 692 | /// will be appended to the symbol. The returned global must always be queried |
| 693 | /// for its name so it can be referenced correctly. |
| 694 | [[nodiscard]] cir::GlobalOp |
| 695 | createVersionedGlobal(mlir::ModuleOp module, mlir::Location loc, |
| 696 | mlir::StringRef name, mlir::Type type, bool isConstant, |
| 697 | cir::GlobalLinkageKind linkage, |
| 698 | mlir::ptr::MemorySpaceAttrInterface addrSpace = {}) { |
| 699 | // Create a unique name if the given name is already taken. |
| 700 | std::string uniqueName; |
| 701 | if (unsigned version = globalsVersioning[name.str()]++) |
| 702 | uniqueName = name.str() + "." + std::to_string(val: version); |
| 703 | else |
| 704 | uniqueName = name.str(); |
| 705 | |
| 706 | return createGlobal(mlirModule: module, loc, name: uniqueName, type, isConstant, linkage, |
| 707 | addrSpace); |
| 708 | } |
| 709 | |
| 710 | cir::StackSaveOp createStackSave(mlir::Location loc, mlir::Type ty) { |
| 711 | return cir::StackSaveOp::create(builder&: *this, location: loc, result: ty); |
| 712 | } |
| 713 | |
| 714 | cir::StackRestoreOp createStackRestore(mlir::Location loc, mlir::Value v) { |
| 715 | return cir::StackRestoreOp::create(builder&: *this, location: loc, ptr: v); |
| 716 | } |
| 717 | |
| 718 | cir::CmpThreeWayOp createThreeWayCmpTotalOrdering( |
| 719 | mlir::Location loc, mlir::Value lhs, mlir::Value rhs, |
| 720 | const llvm::APSInt <Res, const llvm::APSInt &eqRes, |
| 721 | const llvm::APSInt >Res, cir::CmpOrdering ordering) { |
| 722 | assert(ltRes.getBitWidth() == eqRes.getBitWidth() && |
| 723 | ltRes.getBitWidth() == gtRes.getBitWidth() && |
| 724 | "the three comparison results must have the same bit width" ); |
| 725 | assert((ordering == cir::CmpOrdering::Strong || |
| 726 | ordering == cir::CmpOrdering::Weak) && |
| 727 | "total ordering must be strong or weak" ); |
| 728 | cir::IntType cmpResultTy = getSIntNTy(n: ltRes.getBitWidth()); |
| 729 | auto infoAttr = cir::CmpThreeWayInfoAttr::get( |
| 730 | context: getContext(), ordering, lt: ltRes.getSExtValue(), eq: eqRes.getSExtValue(), |
| 731 | gt: gtRes.getSExtValue()); |
| 732 | return cir::CmpThreeWayOp::create(builder&: *this, location: loc, result: cmpResultTy, lhs, rhs, |
| 733 | info: infoAttr); |
| 734 | } |
| 735 | |
| 736 | cir::CmpThreeWayOp createThreeWayCmpPartialOrdering( |
| 737 | mlir::Location loc, mlir::Value lhs, mlir::Value rhs, |
| 738 | const llvm::APSInt <Res, const llvm::APSInt &eqRes, |
| 739 | const llvm::APSInt >Res, const llvm::APSInt &unorderedRes) { |
| 740 | assert(ltRes.getBitWidth() == eqRes.getBitWidth() && |
| 741 | ltRes.getBitWidth() == gtRes.getBitWidth() && |
| 742 | ltRes.getBitWidth() == unorderedRes.getBitWidth() && |
| 743 | "the four comparison results must have the same bit width" ); |
| 744 | cir::IntType cmpResultTy = getSIntNTy(n: ltRes.getBitWidth()); |
| 745 | auto infoAttr = cir::CmpThreeWayInfoAttr::get( |
| 746 | context: getContext(), lt: ltRes.getSExtValue(), eq: eqRes.getSExtValue(), |
| 747 | gt: gtRes.getSExtValue(), unordered: unorderedRes.getSExtValue()); |
| 748 | return cir::CmpThreeWayOp::create(builder&: *this, location: loc, result: cmpResultTy, lhs, rhs, |
| 749 | info: infoAttr); |
| 750 | } |
| 751 | |
| 752 | mlir::Value createSetBitfield(mlir::Location loc, mlir::Type resultType, |
| 753 | Address dstAddr, mlir::Type storageType, |
| 754 | mlir::Value src, const CIRGenBitFieldInfo &info, |
| 755 | bool isLvalueVolatile, bool useVolatile) { |
| 756 | unsigned offset = useVolatile ? info.volatileOffset : info.offset; |
| 757 | |
| 758 | // If using AAPCS and the field is volatile, load with the size of the |
| 759 | // declared field |
| 760 | storageType = |
| 761 | useVolatile ? cir::IntType::get(context: storageType.getContext(), |
| 762 | width: info.volatileStorageSize, is_signed: info.isSigned) |
| 763 | : storageType; |
| 764 | return cir::SetBitfieldOp::create( |
| 765 | builder&: *this, location: loc, type: resultType, addr: dstAddr.getPointer(), storage_type: storageType, src, |
| 766 | name: info.name, size: info.size, offset, is_signed: info.isSigned, is_volatile: isLvalueVolatile, |
| 767 | alignment: dstAddr.getAlignment().getAsAlign().value()); |
| 768 | } |
| 769 | |
| 770 | mlir::Value createGetBitfield(mlir::Location loc, mlir::Type resultType, |
| 771 | Address addr, mlir::Type storageType, |
| 772 | const CIRGenBitFieldInfo &info, |
| 773 | bool isLvalueVolatile, bool useVolatile) { |
| 774 | unsigned offset = useVolatile ? info.volatileOffset : info.offset; |
| 775 | |
| 776 | // If using AAPCS and the field is volatile, load with the size of the |
| 777 | // declared field |
| 778 | storageType = |
| 779 | useVolatile ? cir::IntType::get(context: storageType.getContext(), |
| 780 | width: info.volatileStorageSize, is_signed: info.isSigned) |
| 781 | : storageType; |
| 782 | return cir::GetBitfieldOp::create(builder&: *this, location: loc, type: resultType, addr: addr.getPointer(), |
| 783 | storage_type: storageType, name: info.name, size: info.size, offset, |
| 784 | is_signed: info.isSigned, is_volatile: isLvalueVolatile, |
| 785 | alignment: addr.getAlignment().getAsAlign().value()); |
| 786 | } |
| 787 | |
| 788 | mlir::Value createMaskedLoad(mlir::Location loc, mlir::Type ty, |
| 789 | mlir::Value ptr, llvm::Align alignment, |
| 790 | mlir::Value mask, mlir::Value passThru) { |
| 791 | assert(mlir::isa<cir::VectorType>(ty) && "Type should be vector" ); |
| 792 | assert(mask && "Mask should not be all-ones (null)" ); |
| 793 | |
| 794 | if (!passThru) |
| 795 | passThru = this->getConstant(loc, attr: cir::PoisonAttr::get(type: ty)); |
| 796 | |
| 797 | auto alignAttr = |
| 798 | this->getI64IntegerAttr(value: static_cast<int64_t>(alignment.value())); |
| 799 | |
| 800 | return cir::VecMaskedLoadOp::create(builder&: *this, location: loc, result: ty, addr: ptr, mask, pass_thru: passThru, |
| 801 | alignment: alignAttr); |
| 802 | } |
| 803 | |
| 804 | cir::VecShuffleOp |
| 805 | createVecShuffle(mlir::Location loc, mlir::Value vec1, mlir::Value vec2, |
| 806 | llvm::ArrayRef<mlir::Attribute> maskAttrs) { |
| 807 | auto vecType = mlir::cast<cir::VectorType>(Val: vec1.getType()); |
| 808 | auto resultTy = |
| 809 | cir::VectorType::getFixed(elementType: vecType.getElementType(), size: maskAttrs.size()); |
| 810 | return cir::VecShuffleOp::create(builder&: *this, location: loc, result: resultTy, vec1, vec2, |
| 811 | indices: getArrayAttr(value: maskAttrs)); |
| 812 | } |
| 813 | |
| 814 | cir::VecShuffleOp createVecShuffle(mlir::Location loc, mlir::Value vec1, |
| 815 | mlir::Type resultTy, |
| 816 | llvm::ArrayRef<int64_t> mask) { |
| 817 | cir::ConstantOp poison = |
| 818 | getConstant(loc, attr: cir::PoisonAttr::get(type: vec1.getType())); |
| 819 | auto maskAttrs = llvm::to_vector_of<mlir::Attribute>( |
| 820 | Range: llvm::map_range(C&: mask, F: [&](int32_t idx) { |
| 821 | return cir::IntAttr::get(type: getSInt32Ty(), value: idx); |
| 822 | })); |
| 823 | return cir::VecShuffleOp::create(builder&: *this, location: loc, result: resultTy, vec1, vec2: poison, |
| 824 | indices: getArrayAttr(value: maskAttrs)); |
| 825 | } |
| 826 | |
| 827 | cir::VecShuffleOp createVecShuffle(mlir::Location loc, mlir::Value vec1, |
| 828 | mlir::Value vec2, |
| 829 | llvm::ArrayRef<int64_t> mask) { |
| 830 | auto maskAttrs = llvm::to_vector_of<mlir::Attribute>( |
| 831 | Range: llvm::map_range(C&: mask, F: [&](int32_t idx) { |
| 832 | return cir::IntAttr::get(type: getSInt32Ty(), value: idx); |
| 833 | })); |
| 834 | return createVecShuffle(loc, vec1, vec2, maskAttrs); |
| 835 | } |
| 836 | |
| 837 | cir::VecShuffleOp createVecShuffle(mlir::Location loc, mlir::Value vec1, |
| 838 | llvm::ArrayRef<int64_t> mask) { |
| 839 | /// Create a unary shuffle. The second vector operand of the IR instruction |
| 840 | /// is poison. |
| 841 | cir::ConstantOp poison = |
| 842 | getConstant(loc, attr: cir::PoisonAttr::get(type: vec1.getType())); |
| 843 | return createVecShuffle(loc, vec1, vec2: poison, mask); |
| 844 | } |
| 845 | |
| 846 | cir::MatrixExtractOp (mlir::Location loc, |
| 847 | mlir::Value matrix, |
| 848 | mlir::Value rowIdx, |
| 849 | mlir::Value columnIdx) { |
| 850 | return cir::MatrixExtractOp::create(builder&: *this, location: loc, matrix, rowIdx, columnIdx); |
| 851 | } |
| 852 | |
| 853 | cir::MatrixExtractRowOp (mlir::Location loc, |
| 854 | mlir::Value matrix, |
| 855 | mlir::Value rowIdx) { |
| 856 | return cir::MatrixExtractRowOp::create(builder&: *this, location: loc, matrix, rowIdx); |
| 857 | } |
| 858 | |
| 859 | cir::MatrixInsertOp createMatrixInsert(mlir::Location loc, mlir::Value value, |
| 860 | mlir::Value matrix, mlir::Value rowIdx, |
| 861 | mlir::Value columnIdx) { |
| 862 | return cir::MatrixInsertOp::create(builder&: *this, location: loc, value, matrix, rowIdx, |
| 863 | columnIdx); |
| 864 | } |
| 865 | |
| 866 | cir::MatrixColumnMajorLoadOp createMatrixColumnMajorLoad(mlir::Location loc, |
| 867 | mlir::Type resultTy, |
| 868 | mlir::Value value, |
| 869 | mlir::Value stride, |
| 870 | bool isVolatile) { |
| 871 | return cir::MatrixColumnMajorLoadOp::create(builder&: *this, location: loc, result: resultTy, value, |
| 872 | stride, is_volatile: isVolatile); |
| 873 | } |
| 874 | |
| 875 | /// Multiply an MxK matrix by a KxN matrix, producing an MxN matrix. |
| 876 | cir::MatrixMultiplyOp createMatrixMultiply(mlir::Location loc, |
| 877 | mlir::Value lhs, mlir::Value rhs) { |
| 878 | auto lhsTy = mlir::cast<cir::MatrixType>(Val: lhs.getType()); |
| 879 | auto rhsTy = mlir::cast<cir::MatrixType>(Val: rhs.getType()); |
| 880 | auto resultTy = cir::MatrixType::get( |
| 881 | element_type: lhsTy.getElementType(), num_rows: lhsTy.getNumRows(), num_columns: rhsTy.getNumColumns()); |
| 882 | return cir::MatrixMultiplyOp::create(builder&: *this, location: loc, result: resultTy, lhs, rhs); |
| 883 | } |
| 884 | |
| 885 | cir::MatrixTransposeOp createMatrixTranspose(mlir::Location loc, |
| 886 | mlir::Value matrix) { |
| 887 | auto inputTy = mlir::cast<cir::MatrixType>(Val: matrix.getType()); |
| 888 | auto resultTy = |
| 889 | cir::MatrixType::get(element_type: inputTy.getElementType(), num_rows: inputTy.getNumColumns(), |
| 890 | num_columns: inputTy.getNumRows()); |
| 891 | return cir::MatrixTransposeOp::create(builder&: *this, location: loc, result: resultTy, value: matrix); |
| 892 | } |
| 893 | |
| 894 | cir::MatrixColumnMajorStoreOp createMatrixColumnMajorStore(mlir::Location loc, |
| 895 | mlir::Value matrix, |
| 896 | mlir::Value data, |
| 897 | mlir::Value stride, |
| 898 | bool isVolatile) { |
| 899 | return cir::MatrixColumnMajorStoreOp::create(builder&: *this, location: loc, matrix, value: data, |
| 900 | stride, is_volatile: isVolatile); |
| 901 | } |
| 902 | |
| 903 | std::pair<mlir::Value, mlir::Value> |
| 904 | splatMatrixOpOperandsIfNecessary(mlir::Location loc, mlir::Value lhs, |
| 905 | mlir::Value rhs) { |
| 906 | assert(mlir::isa<cir::MatrixType>(lhs.getType()) || |
| 907 | mlir::isa<cir::MatrixType>(rhs.getType())); |
| 908 | |
| 909 | if (!mlir::isa<cir::MatrixType>(Val: lhs.getType())) |
| 910 | lhs = cir::VecSplatOp::create(builder&: *this, location: loc, result: rhs.getType(), value: lhs); |
| 911 | else if (!mlir::isa<cir::MatrixType>(Val: rhs.getType())) |
| 912 | rhs = cir::VecSplatOp::create(builder&: *this, location: loc, result: lhs.getType(), value: rhs); |
| 913 | |
| 914 | return {lhs, rhs}; |
| 915 | } |
| 916 | |
| 917 | template <typename... Operands> |
| 918 | mlir::Value emitIntrinsicCallOp(mlir::Location loc, const llvm::StringRef str, |
| 919 | const mlir::Type &resTy, Operands &&...op) { |
| 920 | return cir::LLVMIntrinsicCallOp::create(*this, loc, |
| 921 | this->getStringAttr(bytes: str), resTy, |
| 922 | std::forward<Operands>(op)...) |
| 923 | .getResult(); |
| 924 | } |
| 925 | |
| 926 | template <typename... Operands> |
| 927 | mlir::Value emitIntrinsicCallOp(mlir::Location loc, const llvm::StringRef str, |
| 928 | const mlir::Type &resTy, |
| 929 | cir::FastMathFlagsAttr fastmath, |
| 930 | Operands &&...op) { |
| 931 | return cir::LLVMIntrinsicCallOp::create( |
| 932 | *this, loc, this->getStringAttr(bytes: str), resTy, |
| 933 | std::forward<Operands>(op)..., fastmath) |
| 934 | .getResult(); |
| 935 | } |
| 936 | |
| 937 | /// Converts \p src to \p dstTy, saturating at its bounds. The signedness of |
| 938 | /// \p dstTy selects between fptosi.sat and fptoui.sat. |
| 939 | mlir::Value createFPToIntSat(mlir::Location loc, mlir::Value src, |
| 940 | cir::IntType dstTy) { |
| 941 | return emitIntrinsicCallOp( |
| 942 | loc, str: dstTy.isSigned() ? "fptosi.sat" : "fptoui.sat" , resTy: dstTy, op&: src); |
| 943 | } |
| 944 | }; |
| 945 | |
| 946 | } // namespace clang::CIRGen |
| 947 | |
| 948 | #endif |
| 949 | |