| 1 | //==- CXXABILowering.cpp - lower C++ operations to target-specific ABI form -=// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "PassDetail.h" |
| 10 | #include "RecordTypeConverter.h" |
| 11 | #include "TargetLowering/LowerModule.h" |
| 12 | |
| 13 | #include "mlir/Dialect/OpenACC/OpenACCOpsDialect.h.inc" |
| 14 | #include "mlir/Dialect/OpenMP/OpenMPOpsDialect.h.inc" |
| 15 | #include "mlir/IR/PatternMatch.h" |
| 16 | #include "mlir/Interfaces/DataLayoutInterfaces.h" |
| 17 | #include "mlir/Pass/Pass.h" |
| 18 | #include "mlir/Transforms/DialectConversion.h" |
| 19 | #include "clang/CIR/Dialect/Builder/CIRBaseBuilder.h" |
| 20 | #include "clang/CIR/Dialect/IR/CIRAttrs.h" |
| 21 | #include "clang/CIR/Dialect/IR/CIRDataLayout.h" |
| 22 | #include "clang/CIR/Dialect/IR/CIRDialect.h" |
| 23 | #include "clang/CIR/Dialect/IR/CIROpsEnums.h" |
| 24 | #include "clang/CIR/Dialect/Passes.h" |
| 25 | #include "clang/CIR/Dialect/Transforms/CIRTransformUtils.h" |
| 26 | #include "clang/CIR/MissingFeatures.h" |
| 27 | |
| 28 | #include "llvm/ADT/TypeSwitch.h" |
| 29 | |
| 30 | using namespace mlir; |
| 31 | using namespace cir; |
| 32 | |
| 33 | namespace mlir { |
| 34 | #define GEN_PASS_DEF_CXXABILOWERING |
| 35 | #include "clang/CIR/Dialect/Passes.h.inc" |
| 36 | } // namespace mlir |
| 37 | |
| 38 | namespace { |
| 39 | // Check an attribute for legality. An attribute is only currently potentially |
| 40 | // illegal if it contains a type, member pointers are our source of illegality |
| 41 | // in regards to attributes. |
| 42 | bool isCXXABIAttributeLegal(const mlir::TypeConverter &tc, |
| 43 | mlir::Attribute attr) { |
| 44 | // If we don't have an attribute, it can't have a type! |
| 45 | if (!attr) |
| 46 | return true; |
| 47 | |
| 48 | // None of the OpenACC/OMP attributes contain a type of concern, so we can |
| 49 | // just treat them as legal. |
| 50 | if (isa<mlir::acc::OpenACCDialect, mlir::omp::OpenMPDialect>( |
| 51 | Val: attr.getDialect())) |
| 52 | return true; |
| 53 | |
| 54 | // These attributes either don't contain a type, or don't contain a type that |
| 55 | // can have a data member/method. |
| 56 | if (isa<mlir::DenseArrayAttr, mlir::FloatAttr, mlir::UnitAttr, |
| 57 | mlir::StringAttr, mlir::IntegerAttr, mlir::SymbolRefAttr, |
| 58 | cir::AnnotationAttr>(Val: attr)) |
| 59 | return true; |
| 60 | |
| 61 | // Tablegen'ed always-legal attributes: |
| 62 | if (isa< |
| 63 | #define CXX_ABI_ALWAYS_LEGAL_ATTRS |
| 64 | #include "clang/CIR/Dialect/IR/CIRLowering.inc" |
| 65 | #undef CXX_ABI_ALWAYS_LEGAL_ATTRS |
| 66 | >(Val: attr)) |
| 67 | return true; |
| 68 | |
| 69 | // A distinct attribute (such as the id of a loop) is only an identity, so |
| 70 | // there is nothing in it to lower. |
| 71 | if (isa<mlir::DistinctAttr>(Val: attr)) |
| 72 | return true; |
| 73 | |
| 74 | // Data Member and method are ALWAYS illegal. |
| 75 | if (isa<cir::DataMemberAttr, cir::DataMemberOffsetAttr, cir::MethodAttr>( |
| 76 | Val: attr)) |
| 77 | return false; |
| 78 | |
| 79 | return llvm::TypeSwitch<mlir::Attribute, bool>(attr) |
| 80 | // These attributes just have a type, so they are legal if their type is. |
| 81 | .Case<cir::ZeroAttr>( |
| 82 | caseFn: [&tc](cir::ZeroAttr za) { return tc.isLegal(type: za.getType()); }) |
| 83 | .Case<cir::PoisonAttr>( |
| 84 | caseFn: [&tc](cir::PoisonAttr pa) { return tc.isLegal(type: pa.getType()); }) |
| 85 | .Case<cir::UndefAttr>( |
| 86 | caseFn: [&tc](cir::UndefAttr uda) { return tc.isLegal(type: uda.getType()); }) |
| 87 | .Case<mlir::TypeAttr>( |
| 88 | caseFn: [&tc](mlir::TypeAttr ta) { return tc.isLegal(type: ta.getValue()); }) |
| 89 | .Case<cir::ConstPtrAttr>( |
| 90 | caseFn: [&tc](cir::ConstPtrAttr cpa) { return tc.isLegal(type: cpa.getType()); }) |
| 91 | .Case<cir::CXXCtorAttr>( |
| 92 | caseFn: [&tc](cir::CXXCtorAttr ca) { return tc.isLegal(type: ca.getType()); }) |
| 93 | .Case<cir::CXXDtorAttr>( |
| 94 | caseFn: [&tc](cir::CXXDtorAttr da) { return tc.isLegal(type: da.getType()); }) |
| 95 | .Case<cir::CXXAssignAttr>( |
| 96 | caseFn: [&tc](cir::CXXAssignAttr aa) { return tc.isLegal(type: aa.getType()); }) |
| 97 | |
| 98 | // Collection attributes are legal if ALL of the attributes in them are |
| 99 | // also legal. |
| 100 | .Case<mlir::ArrayAttr>(caseFn: [&tc](mlir::ArrayAttr array) { |
| 101 | return llvm::all_of(Range: array.getValue(), P: [&tc](mlir::Attribute attr) { |
| 102 | return isCXXABIAttributeLegal(tc, attr); |
| 103 | }); |
| 104 | }) |
| 105 | .Case<mlir::DictionaryAttr>(caseFn: [&tc](mlir::DictionaryAttr dict) { |
| 106 | return llvm::all_of(Range: dict.getValue(), P: [&tc](mlir::NamedAttribute na) { |
| 107 | return isCXXABIAttributeLegal(tc, attr: na.getValue()); |
| 108 | }); |
| 109 | }) |
| 110 | // These attributes have sub-attributes that we should check for legality. |
| 111 | .Case<cir::ConstArrayAttr>(caseFn: [&tc](cir::ConstArrayAttr array) { |
| 112 | return tc.isLegal(type: array.getType()) && |
| 113 | isCXXABIAttributeLegal(tc, attr: array.getElts()); |
| 114 | }) |
| 115 | .Case<cir::GlobalViewAttr>(caseFn: [&tc](cir::GlobalViewAttr gva) { |
| 116 | return tc.isLegal(type: gva.getType()) && |
| 117 | isCXXABIAttributeLegal(tc, attr: gva.getIndices()); |
| 118 | }) |
| 119 | .Case<cir::GlobalOffsetAttr>(caseFn: [&tc](cir::GlobalOffsetAttr goa) { |
| 120 | return tc.isLegal(type: goa.getType()); |
| 121 | }) |
| 122 | .Case<cir::VTableAttr>(caseFn: [&tc](cir::VTableAttr vta) { |
| 123 | return tc.isLegal(type: vta.getType()) && |
| 124 | isCXXABIAttributeLegal(tc, attr: vta.getData()); |
| 125 | }) |
| 126 | .Case<cir::TypeInfoAttr>(caseFn: [&tc](cir::TypeInfoAttr tia) { |
| 127 | return tc.isLegal(type: tia.getType()) && |
| 128 | isCXXABIAttributeLegal(tc, attr: tia.getData()); |
| 129 | }) |
| 130 | .Case<cir::DynamicCastInfoAttr>(caseFn: [&tc](cir::DynamicCastInfoAttr dcia) { |
| 131 | return isCXXABIAttributeLegal(tc, attr: dcia.getSrcRtti()) && |
| 132 | isCXXABIAttributeLegal(tc, attr: dcia.getDestRtti()) && |
| 133 | isCXXABIAttributeLegal(tc, attr: dcia.getRuntimeFunc()) && |
| 134 | isCXXABIAttributeLegal(tc, attr: dcia.getBadCastFunc()); |
| 135 | }) |
| 136 | .Case<cir::ConstRecordAttr>(caseFn: [&tc](cir::ConstRecordAttr cra) { |
| 137 | return tc.isLegal(type: cra.getType()) && |
| 138 | isCXXABIAttributeLegal(tc, attr: cra.getMembers()); |
| 139 | }) |
| 140 | // We did an audit of all of our attributes (both in OpenACC and CIR), so |
| 141 | // it shouldn't be dangerous to consider everything we haven't considered |
| 142 | // 'illegal'. Any 'new' attributes will end up asserting in |
| 143 | // 'rewriteAttribute' to make sure we consider them here. Otherwise, we |
| 144 | // wouldn't discover a problematic new attribute until it contains a |
| 145 | // member/method. |
| 146 | .Default(defaultResult: false); |
| 147 | } |
| 148 | |
| 149 | mlir::Attribute rewriteAttribute(const mlir::TypeConverter &tc, |
| 150 | mlir::MLIRContext *ctx, mlir::Attribute attr) { |
| 151 | // If the attribute is legal, there is no reason to rewrite it. This also |
| 152 | // filters out 'null' attributes. |
| 153 | if (isCXXABIAttributeLegal(tc, attr)) |
| 154 | return attr; |
| 155 | |
| 156 | // This switch needs to be kept in sync with the potentially-legal type switch |
| 157 | // from isCXXABIAttributeLegal. IF we miss any, this will end up causing |
| 158 | // verification/transformation issues later, often in the form of |
| 159 | // unrealized-conversion-casts. |
| 160 | |
| 161 | return llvm::TypeSwitch<mlir::Attribute, mlir::Attribute>(attr) |
| 162 | // These attributes just have a type, so convert just the type. |
| 163 | .Case<cir::ZeroAttr>(caseFn: [&tc](cir::ZeroAttr za) { |
| 164 | return cir::ZeroAttr::get(type: tc.convertType(t: za.getType())); |
| 165 | }) |
| 166 | .Case<cir::PoisonAttr>(caseFn: [&tc](cir::PoisonAttr pa) { |
| 167 | return cir::PoisonAttr::get(type: tc.convertType(t: pa.getType())); |
| 168 | }) |
| 169 | .Case<cir::UndefAttr>(caseFn: [&tc](cir::UndefAttr uda) { |
| 170 | return cir::UndefAttr::get(type: tc.convertType(t: uda.getType())); |
| 171 | }) |
| 172 | .Case<mlir::TypeAttr>(caseFn: [&tc](mlir::TypeAttr ta) { |
| 173 | return mlir::TypeAttr::get(type: tc.convertType(t: ta.getValue())); |
| 174 | }) |
| 175 | .Case<cir::ConstPtrAttr>(caseFn: [&tc](cir::ConstPtrAttr cpa) { |
| 176 | return cir::ConstPtrAttr::get(type: tc.convertType(t: cpa.getType()), |
| 177 | value: cpa.getValue()); |
| 178 | }) |
| 179 | .Case<cir::CXXCtorAttr>(caseFn: [&tc](cir::CXXCtorAttr ca) { |
| 180 | return cir::CXXCtorAttr::get(type: tc.convertType(t: ca.getType()), |
| 181 | ctorKind: ca.getCtorKind(), isTrivial: ca.getIsTrivial()); |
| 182 | }) |
| 183 | .Case<cir::CXXDtorAttr>(caseFn: [&tc](cir::CXXDtorAttr da) { |
| 184 | return cir::CXXDtorAttr::get(type: tc.convertType(t: da.getType()), |
| 185 | isTrivial: da.getIsTrivial()); |
| 186 | }) |
| 187 | .Case<cir::CXXAssignAttr>(caseFn: [&tc](cir::CXXAssignAttr aa) { |
| 188 | return cir::CXXAssignAttr::get(type: tc.convertType(t: aa.getType()), |
| 189 | assignKind: aa.getAssignKind(), isTrivial: aa.getIsTrivial()); |
| 190 | }) |
| 191 | // Collection attributes need to transform all of the attributes inside of |
| 192 | // them. |
| 193 | .Case<mlir::ArrayAttr>(caseFn: [&tc, ctx](mlir::ArrayAttr array) { |
| 194 | llvm::SmallVector<mlir::Attribute> elts; |
| 195 | for (mlir::Attribute a : array.getValue()) |
| 196 | elts.push_back(Elt: rewriteAttribute(tc, ctx, attr: a)); |
| 197 | return mlir::ArrayAttr::get(context: ctx, value: elts); |
| 198 | }) |
| 199 | .Case<mlir::DictionaryAttr>(caseFn: [&tc, ctx](mlir::DictionaryAttr dict) { |
| 200 | llvm::SmallVector<mlir::NamedAttribute> elts; |
| 201 | for (mlir::NamedAttribute na : dict.getValue()) |
| 202 | elts.emplace_back(Args: na.getName(), |
| 203 | Args: rewriteAttribute(tc, ctx, attr: na.getValue())); |
| 204 | |
| 205 | return mlir::DictionaryAttr::get(context: ctx, value: elts); |
| 206 | }) |
| 207 | // These attributes have sub-attributes that need converting too. |
| 208 | .Case<cir::ConstArrayAttr>(caseFn: [&tc, ctx](cir::ConstArrayAttr array) { |
| 209 | return cir::ConstArrayAttr::get( |
| 210 | context: ctx, type: tc.convertType(t: array.getType()), |
| 211 | elts: rewriteAttribute(tc, ctx, attr: array.getElts()), |
| 212 | trailingZerosNum: array.getTrailingZerosNum()); |
| 213 | }) |
| 214 | .Case<cir::GlobalViewAttr>(caseFn: [&tc, ctx](cir::GlobalViewAttr gva) { |
| 215 | return cir::GlobalViewAttr::get( |
| 216 | type: tc.convertType(t: gva.getType()), symbol: gva.getSymbol(), |
| 217 | indices: mlir::cast<mlir::ArrayAttr>( |
| 218 | Val: rewriteAttribute(tc, ctx, attr: gva.getIndices())), |
| 219 | addressPoint: gva.getAddressPoint()); |
| 220 | }) |
| 221 | .Case<cir::GlobalOffsetAttr>(caseFn: [&tc](cir::GlobalOffsetAttr goa) { |
| 222 | return cir::GlobalOffsetAttr::get(type: tc.convertType(t: goa.getType()), |
| 223 | symbol: goa.getSymbol(), offset: goa.getOffset()); |
| 224 | }) |
| 225 | .Case<cir::VTableAttr>(caseFn: [&tc, ctx](cir::VTableAttr vta) { |
| 226 | return cir::VTableAttr::get( |
| 227 | type: tc.convertType(t: vta.getType()), |
| 228 | data: mlir::cast<mlir::ArrayAttr>( |
| 229 | Val: rewriteAttribute(tc, ctx, attr: vta.getData())), |
| 230 | address_points: vta.getAddressPoints()); |
| 231 | }) |
| 232 | .Case<cir::TypeInfoAttr>(caseFn: [&tc, ctx](cir::TypeInfoAttr tia) { |
| 233 | return cir::TypeInfoAttr::get( |
| 234 | type: tc.convertType(t: tia.getType()), |
| 235 | data: mlir::cast<mlir::ArrayAttr>( |
| 236 | Val: rewriteAttribute(tc, ctx, attr: tia.getData()))); |
| 237 | }) |
| 238 | .Case<cir::DynamicCastInfoAttr>(caseFn: [&tc, |
| 239 | ctx](cir::DynamicCastInfoAttr dcia) { |
| 240 | return cir::DynamicCastInfoAttr::get( |
| 241 | src_rtti: mlir::cast<cir::GlobalViewAttr>( |
| 242 | Val: rewriteAttribute(tc, ctx, attr: dcia.getSrcRtti())), |
| 243 | dest_rtti: mlir::cast<cir::GlobalViewAttr>( |
| 244 | Val: rewriteAttribute(tc, ctx, attr: dcia.getDestRtti())), |
| 245 | runtime_func: dcia.getRuntimeFunc(), bad_cast_func: dcia.getBadCastFunc(), offset_hint: dcia.getOffsetHint()); |
| 246 | }) |
| 247 | .Case<cir::ConstRecordAttr>(caseFn: [&tc, ctx](cir::ConstRecordAttr cra) { |
| 248 | return cir::ConstRecordAttr::get( |
| 249 | context: ctx, type: tc.convertType(t: cra.getType()), |
| 250 | members: mlir::cast<mlir::ArrayAttr>( |
| 251 | Val: rewriteAttribute(tc, ctx, attr: cra.getMembers()))); |
| 252 | }) |
| 253 | .DefaultUnreachable(message: "unrewritten illegal attribute kind" ); |
| 254 | } |
| 255 | |
| 256 | bool areCXXABIInherentAttrsLegal(mlir::Operation *op, |
| 257 | const mlir::TypeConverter &typeConverter) { |
| 258 | bool legal = true; |
| 259 | op->getName().walkInherentAttrs( |
| 260 | op, visitor: [&](llvm::StringRef, mlir::Attribute &attr) { |
| 261 | legal &= isCXXABIAttributeLegal(tc: typeConverter, attr); |
| 262 | }); |
| 263 | return legal; |
| 264 | } |
| 265 | |
| 266 | #define GET_ABI_LOWERING_PATTERNS |
| 267 | #include "clang/CIR/Dialect/IR/CIRLowering.inc" |
| 268 | #undef GET_ABI_LOWERING_PATTERNS |
| 269 | |
| 270 | struct CXXABILoweringPass |
| 271 | : public impl::CXXABILoweringBase<CXXABILoweringPass> { |
| 272 | CXXABILoweringPass() = default; |
| 273 | void runOnOperation() override; |
| 274 | }; |
| 275 | |
| 276 | /// A generic ABI lowering rewrite pattern. This conversion pattern matches any |
| 277 | /// CIR dialect operations with at least one operand or result of an |
| 278 | /// ABI-dependent type. This conversion pattern rewrites the matched operation |
| 279 | /// by replacing all its ABI-dependent operands and results with their |
| 280 | /// lowered counterparts. |
| 281 | class CIRGenericCXXABILoweringPattern : public mlir::ConversionPattern { |
| 282 | public: |
| 283 | CIRGenericCXXABILoweringPattern(mlir::MLIRContext *context, |
| 284 | const mlir::TypeConverter &typeConverter) |
| 285 | : mlir::ConversionPattern(typeConverter, MatchAnyOpTypeTag(), |
| 286 | /*benefit=*/1, context) {} |
| 287 | |
| 288 | mlir::LogicalResult |
| 289 | matchAndRewrite(mlir::Operation *op, llvm::ArrayRef<mlir::Value> operands, |
| 290 | mlir::ConversionPatternRewriter &rewriter) const override { |
| 291 | // Do not match on operations that have dedicated ABI lowering rewrite rules |
| 292 | if (llvm::isa<cir::AllocaOp, cir::BaseDataMemberOp, cir::BaseMethodOp, |
| 293 | cir::CastOp, cir::CmpOp, cir::ConstantOp, cir::DeleteArrayOp, |
| 294 | cir::DerivedDataMemberOp, cir::DerivedMethodOp, cir::FuncOp, |
| 295 | cir::GetMethodOp, cir::GetRuntimeMemberOp, cir::GlobalOp>(Val: op)) |
| 296 | return mlir::failure(); |
| 297 | |
| 298 | const mlir::TypeConverter *typeConverter = getTypeConverter(); |
| 299 | assert(typeConverter && |
| 300 | "CIRGenericCXXABILoweringPattern requires a type converter" ); |
| 301 | bool operandsAndResultsLegal = typeConverter->isLegal(op); |
| 302 | bool regionsLegal = |
| 303 | std::all_of(first: op->getRegions().begin(), last: op->getRegions().end(), |
| 304 | pred: [typeConverter](mlir::Region ®ion) { |
| 305 | return typeConverter->isLegal(region: ®ion); |
| 306 | }); |
| 307 | bool attrsLegal = areCXXABIInherentAttrsLegal(op, typeConverter: *typeConverter); |
| 308 | |
| 309 | if (operandsAndResultsLegal && regionsLegal && attrsLegal) { |
| 310 | // The operation does not have any CXXABI-dependent operands or results, |
| 311 | // the match fails. |
| 312 | return mlir::failure(); |
| 313 | } |
| 314 | |
| 315 | mlir::OperationState loweredOpState(op->getLoc(), op->getName()); |
| 316 | loweredOpState.addOperands(newOperands: operands); |
| 317 | loweredOpState.addSuccessors(newSuccessors: op->getSuccessors()); |
| 318 | |
| 319 | // Lower inherent attributes while preserving auxiliary metadata verbatim. |
| 320 | loweredOpState.propertiesAttr = op->getPropertiesAsAttribute(); |
| 321 | loweredOpState.addAttributes(newAttributes: op->getDiscardableAttrDictionary().getValue()); |
| 322 | |
| 323 | // Lower all result types |
| 324 | llvm::SmallVector<mlir::Type> loweredResultTypes; |
| 325 | loweredResultTypes.reserve(N: op->getNumResults()); |
| 326 | for (mlir::Type result : op->getResultTypes()) |
| 327 | loweredResultTypes.push_back(Elt: typeConverter->convertType(t: result)); |
| 328 | loweredOpState.addTypes(newTypes: loweredResultTypes); |
| 329 | |
| 330 | // Convert all regions before moving any blocks. |
| 331 | for (mlir::Region ®ion : op->getRegions()) { |
| 332 | if (mlir::failed( |
| 333 | Result: rewriter.convertRegionTypes(region: ®ion, converter: *getTypeConverter()))) |
| 334 | return mlir::failure(); |
| 335 | } |
| 336 | for (mlir::Region ®ion : op->getRegions()) { |
| 337 | mlir::Region *loweredRegion = loweredOpState.addRegion(); |
| 338 | rewriter.inlineRegionBefore(region, parent&: *loweredRegion, before: loweredRegion->end()); |
| 339 | } |
| 340 | |
| 341 | // Clone the operation with lowered operand types and result types |
| 342 | mlir::Operation *loweredOp = rewriter.create(state: loweredOpState); |
| 343 | loweredOp->getName().walkInherentAttrs( |
| 344 | op: loweredOp, visitor: [&](llvm::StringRef, mlir::Attribute &attr) { |
| 345 | attr = rewriteAttribute(tc: *typeConverter, ctx: op->getContext(), attr); |
| 346 | }); |
| 347 | |
| 348 | rewriter.replaceOp(op, newOp: loweredOp); |
| 349 | return mlir::success(); |
| 350 | } |
| 351 | }; |
| 352 | |
| 353 | } // namespace |
| 354 | |
| 355 | mlir::LogicalResult CIRAllocaOpABILowering::matchAndRewrite( |
| 356 | cir::AllocaOp op, OpAdaptor adaptor, |
| 357 | mlir::ConversionPatternRewriter &rewriter) const { |
| 358 | mlir::Type allocaPtrTy = op.getType(); |
| 359 | mlir::Type loweredAllocaPtrTy = getTypeConverter()->convertType(t: allocaPtrTy); |
| 360 | |
| 361 | cir::AllocaOp loweredOp = cir::AllocaOp::create( |
| 362 | builder&: rewriter, location: op.getLoc(), addr: loweredAllocaPtrTy, name: op.getName(), |
| 363 | alignment: op.getAlignmentAttr(), /*dynAllocSize=*/adaptor.getDynAllocSize()); |
| 364 | loweredOp.setInit(op.getInit()); |
| 365 | loweredOp.setConstant(op.getConstant()); |
| 366 | |
| 367 | rewriter.replaceOp(op, newOp: loweredOp); |
| 368 | return mlir::success(); |
| 369 | } |
| 370 | |
| 371 | mlir::LogicalResult CIRCastOpABILowering::matchAndRewrite( |
| 372 | cir::CastOp op, OpAdaptor adaptor, |
| 373 | mlir::ConversionPatternRewriter &rewriter) const { |
| 374 | mlir::Type srcTy = op.getSrc().getType(); |
| 375 | |
| 376 | if (mlir::isa<cir::DataMemberType, cir::MethodType>(Val: srcTy)) { |
| 377 | switch (op.getKind()) { |
| 378 | case cir::CastKind::bitcast: { |
| 379 | mlir::Type destTy = getTypeConverter()->convertType(t: op.getType()); |
| 380 | mlir::Value loweredResult; |
| 381 | if (mlir::isa<cir::DataMemberType>(Val: srcTy)) |
| 382 | loweredResult = lowerModule->getCXXABI().lowerDataMemberBitcast( |
| 383 | op, loweredDstTy: destTy, loweredSrc: adaptor.getSrc(), builder&: rewriter); |
| 384 | else |
| 385 | loweredResult = lowerModule->getCXXABI().lowerMethodBitcast( |
| 386 | op, loweredDstTy: destTy, loweredSrc: adaptor.getSrc(), builder&: rewriter); |
| 387 | rewriter.replaceOp(op, newValues: loweredResult); |
| 388 | return mlir::success(); |
| 389 | } |
| 390 | case cir::CastKind::member_ptr_to_bool: { |
| 391 | mlir::Value loweredResult; |
| 392 | if (mlir::isa<cir::DataMemberType>(Val: srcTy)) |
| 393 | loweredResult = lowerModule->getCXXABI().lowerDataMemberToBoolCast( |
| 394 | op, loweredSrc: adaptor.getSrc(), builder&: rewriter); |
| 395 | else |
| 396 | loweredResult = lowerModule->getCXXABI().lowerMethodToBoolCast( |
| 397 | op, loweredSrc: adaptor.getSrc(), builder&: rewriter); |
| 398 | rewriter.replaceOp(op, newValues: loweredResult); |
| 399 | return mlir::success(); |
| 400 | } |
| 401 | default: |
| 402 | break; |
| 403 | } |
| 404 | } |
| 405 | |
| 406 | mlir::Value loweredResult = cir::CastOp::create( |
| 407 | builder&: rewriter, location: op.getLoc(), result: getTypeConverter()->convertType(t: op.getType()), |
| 408 | kind: adaptor.getKind(), src: adaptor.getSrc()); |
| 409 | rewriter.replaceOp(op, newValues: loweredResult); |
| 410 | return mlir::success(); |
| 411 | } |
| 412 | |
| 413 | // Helper function to lower a value for things like an initializer. |
| 414 | static mlir::TypedAttr lowerInitialValue(const LowerModule *lowerModule, |
| 415 | const mlir::DataLayout &layout, |
| 416 | const mlir::TypeConverter &tc, |
| 417 | mlir::Type ty, |
| 418 | mlir::Attribute initVal) { |
| 419 | if (mlir::isa_and_present<cir::UndefAttr>(Val: initVal)) |
| 420 | return cir::UndefAttr::get(type: tc.convertType(t: ty)); |
| 421 | |
| 422 | if (mlir::isa_and_present<cir::PoisonAttr>(Val: initVal)) |
| 423 | return cir::PoisonAttr::get(type: tc.convertType(t: ty)); |
| 424 | |
| 425 | if (mlir::isa<cir::DataMemberType>(Val: ty)) { |
| 426 | // Members without a CIR field index (e.g. no_unique_address empty fields) |
| 427 | // are represented by an explicit byte offset instead of a field path. |
| 428 | if (auto offsetVal = |
| 429 | mlir::dyn_cast_if_present<cir::DataMemberOffsetAttr>(Val&: initVal)) |
| 430 | return lowerModule->getCXXABI().lowerDataMemberOffsetConstant(attr: offsetVal, |
| 431 | layout, typeConverter: tc); |
| 432 | auto dataMemberVal = mlir::cast_if_present<cir::DataMemberAttr>(Val&: initVal); |
| 433 | return lowerModule->getCXXABI().lowerDataMemberConstant(attr: dataMemberVal, |
| 434 | layout, typeConverter: tc); |
| 435 | } |
| 436 | if (mlir::isa<cir::MethodType>(Val: ty)) { |
| 437 | auto methodVal = mlir::cast_if_present<cir::MethodAttr>(Val&: initVal); |
| 438 | return lowerModule->getCXXABI().lowerMethodConstant(attr: methodVal, layout, typeConverter: tc); |
| 439 | } |
| 440 | |
| 441 | if (auto arrTy = mlir::dyn_cast<cir::ArrayType>(Val&: ty)) { |
| 442 | auto loweredArrTy = mlir::cast<cir::ArrayType>(Val: tc.convertType(t: arrTy)); |
| 443 | |
| 444 | if (!initVal) |
| 445 | return {}; |
| 446 | |
| 447 | if (auto zeroVal = mlir::dyn_cast_if_present<cir::ZeroAttr>(Val&: initVal)) |
| 448 | return cir::ZeroAttr::get(type: loweredArrTy); |
| 449 | |
| 450 | auto arrayVal = mlir::cast<cir::ConstArrayAttr>(Val&: initVal); |
| 451 | |
| 452 | // String-literal arrays store their bytes as a StringAttr in `elts`. The |
| 453 | // backing i8 element type is never rewritten by the CXX ABI type |
| 454 | // converter, so the attribute is already legal and can be passed through |
| 455 | // unchanged. |
| 456 | if (mlir::isa<mlir::StringAttr>(Val: arrayVal.getElts())) { |
| 457 | assert(loweredArrTy == arrTy && |
| 458 | "string-literal array type should not change under CXX ABI" ); |
| 459 | return arrayVal; |
| 460 | } |
| 461 | |
| 462 | auto arrayElts = mlir::cast<ArrayAttr>(Val: arrayVal.getElts()); |
| 463 | SmallVector<mlir::Attribute> loweredElements; |
| 464 | loweredElements.reserve(N: arrTy.getSize()); |
| 465 | for (const mlir::Attribute &attr : arrayElts) { |
| 466 | auto typedAttr = cast<mlir::TypedAttr>(Val: attr); |
| 467 | loweredElements.push_back(Elt: lowerInitialValue( |
| 468 | lowerModule, layout, tc, ty: typedAttr.getType(), initVal: typedAttr)); |
| 469 | } |
| 470 | |
| 471 | return cir::ConstArrayAttr::get( |
| 472 | type: loweredArrTy, elts: mlir::ArrayAttr::get(context: ty.getContext(), value: loweredElements), |
| 473 | trailingZerosNum: arrayVal.getTrailingZerosNum()); |
| 474 | } |
| 475 | |
| 476 | if (auto recordTy = mlir::dyn_cast<cir::RecordType>(Val&: ty)) { |
| 477 | auto convertedTy = |
| 478 | mlir::dyn_cast<cir::RecordType>(Val: tc.convertType(t: recordTy)); |
| 479 | if (!convertedTy) |
| 480 | return {}; |
| 481 | |
| 482 | if (auto recVal = mlir::dyn_cast_if_present<cir::ZeroAttr>(Val&: initVal)) |
| 483 | return cir::ZeroAttr::get(type: convertedTy); |
| 484 | |
| 485 | if (auto recVal = |
| 486 | mlir::dyn_cast_if_present<cir::ConstRecordAttr>(Val&: initVal)) { |
| 487 | auto recordMembers = mlir::cast<ArrayAttr>(Val: recVal.getMembers()); |
| 488 | |
| 489 | SmallVector<mlir::Attribute> loweredMembers; |
| 490 | loweredMembers.reserve(N: recordMembers.size()); |
| 491 | |
| 492 | for (const mlir::Attribute &attr : recordMembers) { |
| 493 | auto typedAttr = cast<mlir::TypedAttr>(Val: attr); |
| 494 | loweredMembers.push_back(Elt: lowerInitialValue( |
| 495 | lowerModule, layout, tc, ty: typedAttr.getType(), initVal: typedAttr)); |
| 496 | } |
| 497 | |
| 498 | return cir::ConstRecordAttr::get( |
| 499 | type: convertedTy, members: mlir::ArrayAttr::get(context: ty.getContext(), value: loweredMembers)); |
| 500 | } |
| 501 | |
| 502 | assert(!initVal && "Record init val type not handled" ); |
| 503 | return {}; |
| 504 | } |
| 505 | |
| 506 | // Pointers can contain record types, which can change. |
| 507 | if (auto ptrTy = mlir::dyn_cast<cir::PointerType>(Val&: ty)) { |
| 508 | auto convertedTy = mlir::cast<cir::PointerType>(Val: tc.convertType(t: ptrTy)); |
| 509 | // pointers don't change other than their types. |
| 510 | |
| 511 | if (auto gva = mlir::dyn_cast_if_present<cir::GlobalViewAttr>(Val&: initVal)) |
| 512 | return cir::GlobalViewAttr::get(type: convertedTy, symbol: gva.getSymbol(), |
| 513 | indices: gva.getIndices(), addressPoint: gva.getAddressPoint()); |
| 514 | |
| 515 | if (auto goa = mlir::dyn_cast_if_present<cir::GlobalOffsetAttr>(Val&: initVal)) |
| 516 | return cir::GlobalOffsetAttr::get(type: convertedTy, symbol: goa.getSymbol(), |
| 517 | offset: goa.getOffset()); |
| 518 | |
| 519 | if (auto blockAddr = |
| 520 | mlir::dyn_cast_if_present<cir::BlockAddrInfoAttr>(Val&: initVal)) { |
| 521 | assert(convertedTy == ptrTy && "BlockAddrInfo type should not change" ); |
| 522 | return blockAddr; |
| 523 | } |
| 524 | |
| 525 | auto constPtr = mlir::cast_if_present<cir::ConstPtrAttr>(Val&: initVal); |
| 526 | if (!constPtr) |
| 527 | return {}; |
| 528 | return cir::ConstPtrAttr::get(type: convertedTy, value: constPtr.getValue()); |
| 529 | } |
| 530 | |
| 531 | assert(ty == tc.convertType(ty) && |
| 532 | "cir.global or constant operand is not an CXXABI-dependent type" ); |
| 533 | |
| 534 | // Every other type can be left alone. |
| 535 | return cast<mlir::TypedAttr>(Val&: initVal); |
| 536 | } |
| 537 | |
| 538 | mlir::LogicalResult CIRConstantOpABILowering::matchAndRewrite( |
| 539 | cir::ConstantOp op, OpAdaptor adaptor, |
| 540 | mlir::ConversionPatternRewriter &rewriter) const { |
| 541 | |
| 542 | mlir::DataLayout layout(op->getParentOfType<mlir::ModuleOp>()); |
| 543 | mlir::TypedAttr newValue = lowerInitialValue( |
| 544 | lowerModule, layout, tc: *getTypeConverter(), ty: op.getType(), initVal: op.getValue()); |
| 545 | rewriter.replaceOpWithNewOp<ConstantOp>(op, args&: newValue); |
| 546 | return mlir::success(); |
| 547 | } |
| 548 | |
| 549 | mlir::LogicalResult CIRCmpOpABILowering::matchAndRewrite( |
| 550 | cir::CmpOp op, OpAdaptor adaptor, |
| 551 | mlir::ConversionPatternRewriter &rewriter) const { |
| 552 | mlir::Type type = op.getLhs().getType(); |
| 553 | |
| 554 | mlir::Value loweredResult; |
| 555 | if (mlir::isa<cir::DataMemberType>(Val: type)) |
| 556 | loweredResult = lowerModule->getCXXABI().lowerDataMemberCmp( |
| 557 | op, loweredLhs: adaptor.getLhs(), loweredRhs: adaptor.getRhs(), builder&: rewriter); |
| 558 | else if (mlir::isa<cir::MethodType>(Val: type)) |
| 559 | loweredResult = lowerModule->getCXXABI().lowerMethodCmp( |
| 560 | op, loweredLhs: adaptor.getLhs(), loweredRhs: adaptor.getRhs(), builder&: rewriter); |
| 561 | else |
| 562 | loweredResult = cir::CmpOp::create( |
| 563 | builder&: rewriter, location: op.getLoc(), result: getTypeConverter()->convertType(t: op.getType()), |
| 564 | kind: adaptor.getKind(), lhs: adaptor.getLhs(), rhs: adaptor.getRhs()); |
| 565 | |
| 566 | rewriter.replaceOp(op, newValues: loweredResult); |
| 567 | return mlir::success(); |
| 568 | } |
| 569 | |
| 570 | mlir::LogicalResult CIRFuncOpABILowering::matchAndRewrite( |
| 571 | cir::FuncOp op, OpAdaptor adaptor, |
| 572 | mlir::ConversionPatternRewriter &rewriter) const { |
| 573 | cir::FuncType opFuncType = op.getFunctionType(); |
| 574 | mlir::TypeConverter::SignatureConversion signatureConversion( |
| 575 | opFuncType.getNumInputs()); |
| 576 | |
| 577 | for (const auto &[i, argType] : llvm::enumerate(First: opFuncType.getInputs())) { |
| 578 | mlir::Type loweredArgType = getTypeConverter()->convertType(t: argType); |
| 579 | if (!loweredArgType) |
| 580 | return mlir::failure(); |
| 581 | signatureConversion.addInputs(origInputNo: i, types: loweredArgType); |
| 582 | } |
| 583 | |
| 584 | mlir::Type loweredResultType = |
| 585 | getTypeConverter()->convertType(t: opFuncType.getReturnType()); |
| 586 | if (!loweredResultType) |
| 587 | return mlir::failure(); |
| 588 | |
| 589 | auto loweredFuncType = |
| 590 | cir::FuncType::get(inputs: signatureConversion.getConvertedTypes(), |
| 591 | returnType: loweredResultType, /*isVarArg=*/opFuncType.isVarArg()); |
| 592 | |
| 593 | // Convert the body types before moving it. |
| 594 | if (mlir::failed(Result: rewriter.convertRegionTypes( |
| 595 | region: &op.getBody(), converter: *getTypeConverter(), entryConversion: &signatureConversion))) |
| 596 | return mlir::failure(); |
| 597 | |
| 598 | // Create a new cir.func operation for the CXXABI-lowered function. |
| 599 | cir::FuncOp loweredFuncOp = rewriter.cloneWithoutRegions(op); |
| 600 | loweredFuncOp.setFunctionType(loweredFuncType); |
| 601 | |
| 602 | loweredFuncOp->getName().walkInherentAttrs( |
| 603 | op: loweredFuncOp, visitor: [&](llvm::StringRef, mlir::Attribute &attr) { |
| 604 | attr = rewriteAttribute(tc: *getTypeConverter(), ctx: op->getContext(), attr); |
| 605 | }); |
| 606 | rewriter.inlineRegionBefore(region&: op.getBody(), parent&: loweredFuncOp.getBody(), |
| 607 | before: loweredFuncOp.end()); |
| 608 | |
| 609 | rewriter.eraseOp(op); |
| 610 | return mlir::success(); |
| 611 | } |
| 612 | |
| 613 | mlir::LogicalResult CIRGlobalOpABILowering::matchAndRewrite( |
| 614 | cir::GlobalOp op, OpAdaptor adaptor, |
| 615 | mlir::ConversionPatternRewriter &rewriter) const { |
| 616 | mlir::Type ty = op.getSymType(); |
| 617 | mlir::Type loweredTy = getTypeConverter()->convertType(t: ty); |
| 618 | if (!loweredTy) |
| 619 | return mlir::failure(); |
| 620 | |
| 621 | mlir::DataLayout layout(op->getParentOfType<mlir::ModuleOp>()); |
| 622 | |
| 623 | mlir::Attribute loweredInit = lowerInitialValue( |
| 624 | lowerModule, layout, tc: *getTypeConverter(), ty, initVal: op.getInitialValueAttr()); |
| 625 | |
| 626 | cir::GlobalOp newOp = rewriter.cloneWithoutRegions(op); |
| 627 | newOp.setInitialValueAttr(loweredInit); |
| 628 | newOp.setSymType(loweredTy); |
| 629 | // Regions have to be separately moved(rather than cloned), else we cause |
| 630 | // multi-block regions/eh stuff to be double-referenced, and thus can't be |
| 631 | // removed properly during collectUnreachable. |
| 632 | rewriter.inlineRegionBefore(region&: op.getCtorRegion(), parent&: newOp.getCtorRegion(), |
| 633 | before: newOp.getCtorRegion().end()); |
| 634 | rewriter.inlineRegionBefore(region&: op.getDtorRegion(), parent&: newOp.getDtorRegion(), |
| 635 | before: newOp.getDtorRegion().end()); |
| 636 | rewriter.replaceOp(op, newOp); |
| 637 | return mlir::success(); |
| 638 | } |
| 639 | |
| 640 | mlir::LogicalResult CIRBaseDataMemberOpABILowering::matchAndRewrite( |
| 641 | cir::BaseDataMemberOp op, OpAdaptor adaptor, |
| 642 | mlir::ConversionPatternRewriter &rewriter) const { |
| 643 | mlir::Value loweredResult = lowerModule->getCXXABI().lowerBaseDataMember( |
| 644 | op, loweredSrc: adaptor.getSrc(), builder&: rewriter); |
| 645 | rewriter.replaceOp(op, newValues: loweredResult); |
| 646 | return mlir::success(); |
| 647 | } |
| 648 | |
| 649 | mlir::LogicalResult CIRBaseMethodOpABILowering::matchAndRewrite( |
| 650 | cir::BaseMethodOp op, OpAdaptor adaptor, |
| 651 | mlir::ConversionPatternRewriter &rewriter) const { |
| 652 | mlir::Value loweredResult = |
| 653 | lowerModule->getCXXABI().lowerBaseMethod(op, loweredSrc: adaptor.getSrc(), builder&: rewriter); |
| 654 | rewriter.replaceOp(op, newValues: loweredResult); |
| 655 | return mlir::success(); |
| 656 | } |
| 657 | |
| 658 | mlir::LogicalResult CIRDeleteArrayOpABILowering::matchAndRewrite( |
| 659 | cir::DeleteArrayOp op, OpAdaptor adaptor, |
| 660 | mlir::ConversionPatternRewriter &rewriter) const { |
| 661 | mlir::FlatSymbolRefAttr deleteFn = op.getDeleteFnAttr(); |
| 662 | mlir::Location loc = op->getLoc(); |
| 663 | mlir::Value loweredAddress = adaptor.getAddress(); |
| 664 | |
| 665 | cir::UsualDeleteParamsAttr deleteParams = op.getDeleteParamsAttr(); |
| 666 | if (!deleteParams) |
| 667 | deleteParams = cir::UsualDeleteParamsAttr::get(context: op.getContext(), size: false, |
| 668 | alignment: std::nullopt, type_aware_delete: false, destroying_delete: false); |
| 669 | bool cookieRequired = deleteParams.getSize() || op.getElementDtorAttr(); |
| 670 | |
| 671 | assert(!deleteParams.getDestroyingDelete() && |
| 672 | "destroying delete not legal on arrays" ); |
| 673 | assert(!deleteParams.getTypeAwareDelete() && |
| 674 | "type-aware delete not legal on arrays" ); |
| 675 | |
| 676 | const CIRCXXABI &cxxABI = lowerModule->getCXXABI(); |
| 677 | CIRBaseBuilderTy cirBuilder(rewriter); |
| 678 | |
| 679 | // Read the array cookie (or compute the void* pointer for the |
| 680 | // non-cookie case) before creating the cleanup scope. The cookie read |
| 681 | // produces values that are needed by both the destruction loop in the |
| 682 | // body region (numElements for the array.dtor) and the operator |
| 683 | // delete[] call in the cleanup region (deletePtr / numElements for the |
| 684 | // total-size computation), so it must dominate both regions. |
| 685 | mlir::Value deletePtr; |
| 686 | mlir::Value numElements; |
| 687 | auto ptrTy = mlir::cast<cir::PointerType>(Val: loweredAddress.getType()); |
| 688 | clang::CharUnits cookieSize; |
| 689 | mlir::DataLayout dl(op->getParentOfType<mlir::ModuleOp>()); |
| 690 | unsigned ptrWidth = |
| 691 | lowerModule->getTarget().getPointerWidth(AddrSpace: clang::LangAS::Default); |
| 692 | cir::IntType sizeTy = cirBuilder.getUIntNTy(n: ptrWidth); |
| 693 | |
| 694 | if (cookieRequired) { |
| 695 | clang::CharUnits elementAlign = |
| 696 | clang::CharUnits::fromQuantity(Quantity: op.getElementAlign()); |
| 697 | cxxABI.readArrayCookie(loc, elementPtr: loweredAddress, elementAlign, dataLayout: dl, builder&: cirBuilder, |
| 698 | numElements, allocPtr&: deletePtr, cookieSize); |
| 699 | } else { |
| 700 | deletePtr = cir::CastOp::create( |
| 701 | builder&: rewriter, location: loc, result: cirBuilder.getVoidPtrTy(as: ptrTy.getAddrSpace()), |
| 702 | kind: cir::CastKind::bitcast, src: loweredAddress); |
| 703 | } |
| 704 | |
| 705 | // Create a cleanup scope to wrap the ArrayDtor operation (if needed) and |
| 706 | // call the array delete operator from the cleanup region. If no exceptions |
| 707 | // are thrown during the array dtor, the normal control flow will call the |
| 708 | // delete operator. The ArrayDtor operation will get its own cleanup region |
| 709 | // when it is expanded during LoweringPrepare. If an exception is thrown, the |
| 710 | // exception handling flow will be connected to the cleanup region here to |
| 711 | // call the delete operator on the exception path. |
| 712 | mlir::FlatSymbolRefAttr dtorFn = op.getElementDtorAttr(); |
| 713 | |
| 714 | // Make sure the calls and the callees agree on calling convention. |
| 715 | auto getCalleeCallingConv = [&](mlir::FlatSymbolRefAttr callee) { |
| 716 | if (auto fn = |
| 717 | mlir::SymbolTable::lookupNearestSymbolFrom<cir::FuncOp>(from: op, symbol: callee)) |
| 718 | return fn.getCallingConv(); |
| 719 | return cir::CallingConv::C; |
| 720 | }; |
| 721 | |
| 722 | cir::CleanupKind cleanupKind = |
| 723 | op.getDtorMayThrow() ? cir::CleanupKind::All : cir::CleanupKind::Normal; |
| 724 | cir::CleanupScopeOp::create( |
| 725 | builder&: rewriter, location: loc, cleanupKind, |
| 726 | /*bodyBuilder=*/ |
| 727 | [&](mlir::OpBuilder &b, mlir::Location l) { |
| 728 | if (dtorFn) { |
| 729 | auto arrayDtor = cir::ArrayDtor::create( |
| 730 | builder&: b, location: l, addr: loweredAddress, num_elements: numElements, |
| 731 | regionBuilder: [&](mlir::OpBuilder &bb, mlir::Location ll) { |
| 732 | mlir::Value arg = |
| 733 | bb.getInsertionBlock()->addArgument(type: ptrTy, loc: ll); |
| 734 | auto dtorCall = cir::CallOp::create( |
| 735 | builder&: bb, location: ll, callee: dtorFn, resType: cir::VoidType(), operands: mlir::ValueRange{arg}); |
| 736 | dtorCall.setCallingConv(getCalleeCallingConv(dtorFn)); |
| 737 | if (!op.getDtorMayThrow()) |
| 738 | dtorCall.setNothrowAttr(bb.getUnitAttr()); |
| 739 | cir::YieldOp::create(builder&: bb, location: ll); |
| 740 | }); |
| 741 | if (op.getDtorMayThrow()) |
| 742 | arrayDtor.setDtorMayThrow(true); |
| 743 | } |
| 744 | cir::YieldOp::create(builder&: b, location: l); |
| 745 | }, |
| 746 | /*cleanupBuilder=*/ |
| 747 | [&](mlir::OpBuilder &b, mlir::Location l) { |
| 748 | llvm::SmallVector<mlir::Value> callArgs; |
| 749 | callArgs.push_back(Elt: deletePtr); |
| 750 | if (deleteParams.getSize()) { |
| 751 | uint64_t eltSizeBytes = dl.getTypeSizeInBits(t: ptrTy.getPointee()) / 8; |
| 752 | auto eltSizeVal = cir::ConstantOp::create( |
| 753 | builder&: b, location: l, value: cir::IntAttr::get(type: sizeTy, value: eltSizeBytes)); |
| 754 | mlir::Value allocSize = |
| 755 | cir::MulOp::create(builder&: b, location: l, result: sizeTy, lhs: eltSizeVal, rhs: numElements); |
| 756 | auto cookieSizeVal = cir::ConstantOp::create( |
| 757 | builder&: b, location: l, value: cir::IntAttr::get(type: sizeTy, value: cookieSize.getQuantity())); |
| 758 | allocSize = |
| 759 | cir::AddOp::create(builder&: b, location: l, result: sizeTy, lhs: allocSize, rhs: cookieSizeVal); |
| 760 | callArgs.push_back(Elt: allocSize); |
| 761 | } |
| 762 | if (deleteParams.getAlignment()) { |
| 763 | auto alignVal = cir::ConstantOp::create( |
| 764 | builder&: b, location: l, value: cir::IntAttr::get(type: sizeTy, value: *deleteParams.getAlignment())); |
| 765 | callArgs.push_back(Elt: alignVal); |
| 766 | } |
| 767 | |
| 768 | auto deleteCall = |
| 769 | cir::CallOp::create(builder&: b, location: l, callee: deleteFn, resType: cir::VoidType(), operands: callArgs); |
| 770 | deleteCall.setCallingConv(getCalleeCallingConv(deleteFn)); |
| 771 | // operator delete[] is implicitly nothrow per [basic.stc.dynamic], |
| 772 | // matching classic CodeGen's `nounwind` attribute on the call. |
| 773 | deleteCall.setNothrowAttr(b.getUnitAttr()); |
| 774 | cir::YieldOp::create(builder&: b, location: l); |
| 775 | }); |
| 776 | |
| 777 | rewriter.eraseOp(op); |
| 778 | return mlir::success(); |
| 779 | } |
| 780 | |
| 781 | mlir::LogicalResult CIRDerivedDataMemberOpABILowering::matchAndRewrite( |
| 782 | cir::DerivedDataMemberOp op, OpAdaptor adaptor, |
| 783 | mlir::ConversionPatternRewriter &rewriter) const { |
| 784 | mlir::Value loweredResult = lowerModule->getCXXABI().lowerDerivedDataMember( |
| 785 | op, loweredSrc: adaptor.getSrc(), builder&: rewriter); |
| 786 | rewriter.replaceOp(op, newValues: loweredResult); |
| 787 | return mlir::success(); |
| 788 | } |
| 789 | |
| 790 | mlir::LogicalResult CIRDerivedMethodOpABILowering::matchAndRewrite( |
| 791 | cir::DerivedMethodOp op, OpAdaptor adaptor, |
| 792 | mlir::ConversionPatternRewriter &rewriter) const { |
| 793 | mlir::Value loweredResult = lowerModule->getCXXABI().lowerDerivedMethod( |
| 794 | op, loweredSrc: adaptor.getSrc(), builder&: rewriter); |
| 795 | rewriter.replaceOp(op, newValues: loweredResult); |
| 796 | return mlir::success(); |
| 797 | } |
| 798 | |
| 799 | mlir::LogicalResult CIRDynamicCastOpABILowering::matchAndRewrite( |
| 800 | cir::DynamicCastOp op, OpAdaptor adaptor, |
| 801 | mlir::ConversionPatternRewriter &rewriter) const { |
| 802 | mlir::Value loweredResult = |
| 803 | lowerModule->getCXXABI().lowerDynamicCast(op, builder&: rewriter); |
| 804 | rewriter.replaceOp(op, newValues: loweredResult); |
| 805 | return mlir::success(); |
| 806 | } |
| 807 | |
| 808 | mlir::LogicalResult CIRGetMethodOpABILowering::matchAndRewrite( |
| 809 | cir::GetMethodOp op, OpAdaptor adaptor, |
| 810 | mlir::ConversionPatternRewriter &rewriter) const { |
| 811 | mlir::Value callee; |
| 812 | mlir::Value thisArg; |
| 813 | lowerModule->getCXXABI().lowerGetMethod( |
| 814 | op, callee, thisArg, loweredMethod: adaptor.getMethod(), loweredObjectPtr: adaptor.getObject(), rewriter); |
| 815 | rewriter.replaceOp(op, newValues: {callee, thisArg}); |
| 816 | return mlir::success(); |
| 817 | } |
| 818 | |
| 819 | mlir::LogicalResult CIRGetRuntimeMemberOpABILowering::matchAndRewrite( |
| 820 | cir::GetRuntimeMemberOp op, OpAdaptor adaptor, |
| 821 | mlir::ConversionPatternRewriter &rewriter) const { |
| 822 | mlir::Type resTy = getTypeConverter()->convertType(t: op.getType()); |
| 823 | mlir::Operation *newOp = lowerModule->getCXXABI().lowerGetRuntimeMember( |
| 824 | op, loweredResultTy: resTy, loweredAddr: adaptor.getAddr(), loweredMember: adaptor.getMember(), builder&: rewriter); |
| 825 | rewriter.replaceOp(op, newOp); |
| 826 | return mlir::success(); |
| 827 | } |
| 828 | |
| 829 | mlir::LogicalResult CIRVTableGetTypeInfoOpABILowering::matchAndRewrite( |
| 830 | cir::VTableGetTypeInfoOp op, OpAdaptor adaptor, |
| 831 | mlir::ConversionPatternRewriter &rewriter) const { |
| 832 | mlir::Value loweredResult = |
| 833 | lowerModule->getCXXABI().lowerVTableGetTypeInfo(op, builder&: rewriter); |
| 834 | rewriter.replaceOp(op, newValues: loweredResult); |
| 835 | return mlir::success(); |
| 836 | } |
| 837 | |
| 838 | namespace { |
| 839 | // A small type to handle type conversion for the the CXXABILoweringPass. |
| 840 | // Even though this is a CIR-to-CIR pass, we are eliminating some CIR types. |
| 841 | // The records that contain them are rebuilt by RecordRewritingTypeConverter. |
| 842 | class CIRABITypeConverter : public cir::RecordRewritingTypeConverter { |
| 843 | public: |
| 844 | CIRABITypeConverter(mlir::MLIRContext &ctx, mlir::DataLayout &dataLayout, |
| 845 | cir::LowerModule &lowerModule) |
| 846 | : RecordRewritingTypeConverter(ctx) { |
| 847 | addConversion(callback: [&](cir::DataMemberType type) -> mlir::Type { |
| 848 | mlir::Type abiType = |
| 849 | lowerModule.getCXXABI().lowerDataMemberType(type, typeConverter: *this); |
| 850 | return convertType(t: abiType); |
| 851 | }); |
| 852 | addConversion(callback: [&](cir::MethodType type) -> mlir::Type { |
| 853 | mlir::Type abiType = lowerModule.getCXXABI().lowerMethodType(type, typeConverter: *this); |
| 854 | return convertType(t: abiType); |
| 855 | }); |
| 856 | } |
| 857 | |
| 858 | protected: |
| 859 | // Only member pointer types are rewritten. Records that reach none of them |
| 860 | // are left alone. |
| 861 | bool typeMayChange(mlir::Type type) const override { |
| 862 | return mlir::isa<cir::DataMemberType, cir::MethodType>(Val: type); |
| 863 | } |
| 864 | }; |
| 865 | } // namespace |
| 866 | |
| 867 | static void |
| 868 | populateCXXABIConversionTarget(mlir::ConversionTarget &target, |
| 869 | const mlir::TypeConverter &typeConverter) { |
| 870 | target.addLegalOp<mlir::ModuleOp>(); |
| 871 | |
| 872 | // The ABI lowering pass is interested in CIR operations with operands or |
| 873 | // results of CXXABI-dependent types, or CIR operations with regions whose |
| 874 | // block arguments are of CXXABI-dependent types. |
| 875 | target.addDynamicallyLegalDialect<cir::CIRDialect>( |
| 876 | callback: [&typeConverter](mlir::Operation *op) { |
| 877 | if (!typeConverter.isLegal(op)) |
| 878 | return false; |
| 879 | |
| 880 | bool attrs = areCXXABIInherentAttrsLegal(op, typeConverter); |
| 881 | |
| 882 | return attrs && |
| 883 | std::all_of(first: op->getRegions().begin(), last: op->getRegions().end(), |
| 884 | pred: [&typeConverter](mlir::Region ®ion) { |
| 885 | return typeConverter.isLegal(region: ®ion); |
| 886 | }); |
| 887 | }); |
| 888 | |
| 889 | target.addDynamicallyLegalDialect<mlir::acc::OpenACCDialect>( |
| 890 | callback: [&typeConverter](mlir::Operation *op) { |
| 891 | if (!typeConverter.isLegal(op)) |
| 892 | return false; |
| 893 | |
| 894 | bool attrs = areCXXABIInherentAttrsLegal(op, typeConverter); |
| 895 | |
| 896 | return attrs && |
| 897 | std::all_of(first: op->getRegions().begin(), last: op->getRegions().end(), |
| 898 | pred: [&typeConverter](mlir::Region ®ion) { |
| 899 | return typeConverter.isLegal(region: ®ion); |
| 900 | }); |
| 901 | }); |
| 902 | |
| 903 | // TargetLowering leaves same-type casts behind. Keep them illegal so the |
| 904 | // conversion driver folds them away. |
| 905 | target.addDynamicallyLegalOp<cir::CastOp>(callback: [&typeConverter](cir::CastOp op) { |
| 906 | if (op.getSrc().getType() == op.getType()) { |
| 907 | switch (op.getKind()) { |
| 908 | case cir::CastKind::bitcast: |
| 909 | case cir::CastKind::address_space: |
| 910 | case cir::CastKind::float_complex: |
| 911 | case cir::CastKind::int_complex: |
| 912 | return false; |
| 913 | default: |
| 914 | break; |
| 915 | } |
| 916 | } |
| 917 | if (!typeConverter.isLegal(op: op.getOperation())) |
| 918 | return false; |
| 919 | // CastOp has no regions, so operand/result types and inherent |
| 920 | // attributes are the whole legality check. |
| 921 | return areCXXABIInherentAttrsLegal(op, typeConverter); |
| 922 | }); |
| 923 | |
| 924 | // Some CIR ops needs special checking for legality |
| 925 | target.addDynamicallyLegalOp<cir::FuncOp>(callback: [&typeConverter](cir::FuncOp op) { |
| 926 | bool attrs = areCXXABIInherentAttrsLegal(op, typeConverter); |
| 927 | |
| 928 | return attrs && typeConverter.isLegal(type: op.getFunctionType()); |
| 929 | }); |
| 930 | target.addDynamicallyLegalOp<cir::GlobalOp>( |
| 931 | callback: [&typeConverter](cir::GlobalOp op) { |
| 932 | return typeConverter.isLegal(type: op.getSymType()); |
| 933 | }); |
| 934 | // Operations that do not use any special types must be explicitly marked as |
| 935 | // illegal to trigger processing here. |
| 936 | target.addIllegalOp<cir::DeleteArrayOp>(); |
| 937 | target.addIllegalOp<cir::DynamicCastOp>(); |
| 938 | target.addIllegalOp<cir::VTableGetTypeInfoOp>(); |
| 939 | } |
| 940 | |
| 941 | //===----------------------------------------------------------------------===// |
| 942 | // The Pass |
| 943 | //===----------------------------------------------------------------------===// |
| 944 | |
| 945 | void CXXABILoweringPass::runOnOperation() { |
| 946 | auto mod = mlir::cast<mlir::ModuleOp>(Val: getOperation()); |
| 947 | mlir::MLIRContext *ctx = mod.getContext(); |
| 948 | |
| 949 | std::unique_ptr<cir::LowerModule> lowerModule = cir::createLowerModule(module: mod); |
| 950 | assert(lowerModule && "LowerModule shouldn't be null" ); |
| 951 | if (!lowerModule->hasTarget()) { |
| 952 | mod.emitError() << getArgument() << " requires a module with a supported '" |
| 953 | << cir::CIRDialect::getTripleAttrName() << "' attribute" ; |
| 954 | return signalPassFailure(); |
| 955 | } |
| 956 | |
| 957 | if (!lowerModule->hasCXXABI()) { |
| 958 | mod.emitError() << "C++ ABI '" |
| 959 | << clang::TargetCXXABI::getSpelling( |
| 960 | ABIKind: lowerModule->getCXXABIKind()) |
| 961 | << "' is not yet supported by CIR" ; |
| 962 | return signalPassFailure(); |
| 963 | } |
| 964 | |
| 965 | mlir::DataLayout dataLayout(mod); |
| 966 | CIRABITypeConverter typeConverter(*ctx, dataLayout, *lowerModule); |
| 967 | |
| 968 | mlir::RewritePatternSet patterns(ctx); |
| 969 | patterns.add<CIRGenericCXXABILoweringPattern>(arg: patterns.getContext(), |
| 970 | args&: typeConverter); |
| 971 | patterns.add< |
| 972 | #define GET_ABI_LOWERING_PATTERNS_LIST |
| 973 | #include "clang/CIR/Dialect/IR/CIRLowering.inc" |
| 974 | #undef GET_ABI_LOWERING_PATTERNS_LIST |
| 975 | >(arg: patterns.getContext(), args&: typeConverter, args&: dataLayout, args&: *lowerModule); |
| 976 | |
| 977 | mlir::ConversionTarget target(*ctx); |
| 978 | populateCXXABIConversionTarget(target, typeConverter); |
| 979 | |
| 980 | llvm::SmallVector<mlir::Operation *> ops; |
| 981 | ops.push_back(Elt: mod); |
| 982 | cir::collectUnreachable(parent: mod, ops); |
| 983 | |
| 984 | // One-shot conversion (no pattern rollback): rewrites materialize |
| 985 | // immediately, skipping the IRRewrite/value-mapping bookkeeping. |
| 986 | mlir::ConversionConfig config; |
| 987 | config.allowPatternRollback = false; |
| 988 | if (failed(Result: mlir::applyPartialConversion(ops, target, patterns: std::move(patterns), |
| 989 | config))) |
| 990 | signalPassFailure(); |
| 991 | |
| 992 | typeConverter.restoreRecordTypeNames(); |
| 993 | } |
| 994 | |
| 995 | std::unique_ptr<Pass> mlir::createCXXABILoweringPass() { |
| 996 | return std::make_unique<CXXABILoweringPass>(); |
| 997 | } |
| 998 | |