| 1 | //===- TargetLowering.cpp -------------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements the cir-target-lowering pass. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "RecordTypeConverter.h" |
| 14 | #include "TargetLowering/LowerModule.h" |
| 15 | #include "TargetLowering/TargetLoweringInfo.h" |
| 16 | |
| 17 | #include "mlir/IR/AttrTypeSubElements.h" |
| 18 | #include "mlir/IR/PatternMatch.h" |
| 19 | #include "mlir/Support/LLVM.h" |
| 20 | #include "mlir/Transforms/DialectConversion.h" |
| 21 | #include "clang/CIR/Dialect/IR/CIRAttrs.h" |
| 22 | #include "clang/CIR/Dialect/IR/CIRDialect.h" |
| 23 | #include "clang/CIR/Dialect/IR/CIRTypes.h" |
| 24 | #include "clang/CIR/Dialect/Passes.h" |
| 25 | #include "clang/CIR/Dialect/Transforms/CIRTransformUtils.h" |
| 26 | |
| 27 | using namespace mlir; |
| 28 | using namespace cir; |
| 29 | |
| 30 | namespace mlir { |
| 31 | #define GEN_PASS_DEF_TARGETLOWERING |
| 32 | #include "clang/CIR/Dialect/Passes.h.inc" |
| 33 | } // namespace mlir |
| 34 | |
| 35 | namespace { |
| 36 | |
| 37 | struct TargetLoweringPass |
| 38 | : public impl::TargetLoweringBase<TargetLoweringPass> { |
| 39 | TargetLoweringPass() = default; |
| 40 | void runOnOperation() override; |
| 41 | }; |
| 42 | |
| 43 | /// Converts LangAddressSpaceAttr → TargetAddressSpaceAttr inside types, |
| 44 | /// rebuilding only records that reach a language address space. |
| 45 | class TargetLoweringTypeConverter : public cir::RecordRewritingTypeConverter { |
| 46 | public: |
| 47 | TargetLoweringTypeConverter(mlir::MLIRContext &ctx, |
| 48 | const cir::TargetLoweringInfo &targetInfo) |
| 49 | : RecordRewritingTypeConverter(ctx), targetInfo(targetInfo) { |
| 50 | addConversion(callback: [this](cir::PointerType type) -> mlir::Type { |
| 51 | mlir::Type pointee = convertType(t: type.getPointee()); |
| 52 | if (!pointee) |
| 53 | return {}; |
| 54 | mlir::ptr::MemorySpaceAttrInterface addrSpace = type.getAddrSpace(); |
| 55 | if (auto langAS = |
| 56 | mlir::dyn_cast_if_present<cir::LangAddressSpaceAttr>(Val&: addrSpace)) |
| 57 | addrSpace = lowerAddrSpace(ctx: type.getContext(), langAS); |
| 58 | return cir::PointerType::get(context: type.getContext(), pointee, addrSpace); |
| 59 | }); |
| 60 | } |
| 61 | |
| 62 | mlir::ptr::MemorySpaceAttrInterface |
| 63 | lowerAddrSpace(mlir::MLIRContext *ctx, |
| 64 | cir::LangAddressSpaceAttr langAS) const { |
| 65 | unsigned targetAS = |
| 66 | targetInfo.getTargetAddrSpaceFromCIRAddrSpace(addrSpace: langAS.getValue()); |
| 67 | if (targetAS == 0) |
| 68 | return {}; |
| 69 | return cir::TargetAddressSpaceAttr::get(context: ctx, value: targetAS); |
| 70 | } |
| 71 | |
| 72 | /// Convert the types nested in \p attr, e.g. the type of a constant. |
| 73 | mlir::Attribute convertAttr(mlir::Attribute attr) const { |
| 74 | return attrReplacer.replace(attr); |
| 75 | } |
| 76 | |
| 77 | protected: |
| 78 | bool typeMayChange(mlir::Type type) const override { |
| 79 | auto ptrTy = mlir::dyn_cast<cir::PointerType>(Val&: type); |
| 80 | return ptrTy && mlir::isa_and_present<cir::LangAddressSpaceAttr>( |
| 81 | Val: ptrTy.getAddrSpace()); |
| 82 | } |
| 83 | |
| 84 | private: |
| 85 | const cir::TargetLoweringInfo &targetInfo; |
| 86 | mutable mlir::AttrTypeReplacer attrReplacer = [this] { |
| 87 | mlir::AttrTypeReplacer replacer; |
| 88 | replacer.addReplacement( |
| 89 | fn: [this](mlir::Type type) |
| 90 | -> std::optional<std::pair<mlir::Type, mlir::WalkResult>> { |
| 91 | mlir::Type converted = convertType(t: type); |
| 92 | if (!converted) |
| 93 | return std::nullopt; |
| 94 | // Already converted; don't walk into it. |
| 95 | return std::make_pair(x&: converted, y: mlir::WalkResult::skip()); |
| 96 | }); |
| 97 | return replacer; |
| 98 | }(); |
| 99 | }; |
| 100 | |
| 101 | /// A generic target lowering pattern that matches any CIR op whose operand or |
| 102 | /// result types need address space conversion. Clones the op with converted |
| 103 | /// types. |
| 104 | class CIRGenericTargetLoweringPattern : public mlir::ConversionPattern { |
| 105 | public: |
| 106 | CIRGenericTargetLoweringPattern( |
| 107 | mlir::MLIRContext *context, |
| 108 | const TargetLoweringTypeConverter &typeConverter) |
| 109 | : mlir::ConversionPattern(typeConverter, MatchAnyOpTypeTag(), |
| 110 | /*benefit=*/1, context) {} |
| 111 | |
| 112 | mlir::LogicalResult |
| 113 | matchAndRewrite(mlir::Operation *op, llvm::ArrayRef<mlir::Value> operands, |
| 114 | mlir::ConversionPatternRewriter &rewriter) const override { |
| 115 | // Do not match on operations that have dedicated lowering patterns. |
| 116 | if (llvm::isa<cir::FuncOp, cir::GlobalOp>(Val: op)) |
| 117 | return mlir::failure(); |
| 118 | |
| 119 | const mlir::TypeConverter *typeConverter = getTypeConverter(); |
| 120 | assert(typeConverter && |
| 121 | "CIRGenericTargetLoweringPattern requires a type converter" ); |
| 122 | bool operandsAndResultsLegal = typeConverter->isLegal(op); |
| 123 | bool regionsLegal = |
| 124 | std::all_of(first: op->getRegions().begin(), last: op->getRegions().end(), |
| 125 | pred: [typeConverter](mlir::Region ®ion) { |
| 126 | return typeConverter->isLegal(region: ®ion); |
| 127 | }); |
| 128 | if (operandsAndResultsLegal && regionsLegal) |
| 129 | return mlir::failure(); |
| 130 | |
| 131 | assert(op->getNumRegions() == 0 && "CIRGenericTargetLoweringPattern cannot " |
| 132 | "deal with operations with regions" ); |
| 133 | |
| 134 | mlir::OperationState loweredOpState(op->getLoc(), op->getName()); |
| 135 | loweredOpState.addOperands(newOperands: operands); |
| 136 | |
| 137 | // Preserve auxiliary metadata verbatim. |
| 138 | loweredOpState.propertiesAttr = op->getPropertiesAsAttribute(); |
| 139 | loweredOpState.addAttributes(newAttributes: op->getDiscardableAttrDictionary().getValue()); |
| 140 | |
| 141 | loweredOpState.addSuccessors(newSuccessors: op->getSuccessors()); |
| 142 | |
| 143 | llvm::SmallVector<mlir::Type> loweredResultTypes; |
| 144 | loweredResultTypes.reserve(N: op->getNumResults()); |
| 145 | for (mlir::Type result : op->getResultTypes()) |
| 146 | loweredResultTypes.push_back(Elt: typeConverter->convertType(t: result)); |
| 147 | loweredOpState.addTypes(newTypes: loweredResultTypes); |
| 148 | |
| 149 | // Convert all regions before moving any blocks. |
| 150 | for (mlir::Region ®ion : op->getRegions()) { |
| 151 | if (mlir::failed( |
| 152 | Result: rewriter.convertRegionTypes(region: ®ion, converter: *getTypeConverter()))) |
| 153 | return mlir::failure(); |
| 154 | } |
| 155 | for (mlir::Region ®ion : op->getRegions()) { |
| 156 | mlir::Region *loweredRegion = loweredOpState.addRegion(); |
| 157 | rewriter.inlineRegionBefore(region, parent&: *loweredRegion, before: loweredRegion->end()); |
| 158 | } |
| 159 | |
| 160 | mlir::Operation *loweredOp = rewriter.create(state: loweredOpState); |
| 161 | // Keep inherent attributes (e.g. allocaType, constants) in sync with the |
| 162 | // converted types. |
| 163 | const auto *tlTypeConverter = |
| 164 | static_cast<const TargetLoweringTypeConverter *>(typeConverter); |
| 165 | loweredOp->getName().walkInherentAttrs( |
| 166 | op: loweredOp, visitor: [&](llvm::StringRef, mlir::Attribute &attr) { |
| 167 | attr = tlTypeConverter->convertAttr(attr); |
| 168 | }); |
| 169 | rewriter.replaceOp(op, newOp: loweredOp); |
| 170 | return mlir::success(); |
| 171 | } |
| 172 | }; |
| 173 | |
| 174 | /// Pattern to lower GlobalOp address space attributes. GlobalOp carries |
| 175 | /// addr_space as a standalone attribute (not inside a type), so the |
| 176 | /// TypeConverter won't reach it automatically. |
| 177 | class CIRGlobalOpTargetLowering |
| 178 | : public mlir::OpConversionPattern<cir::GlobalOp> { |
| 179 | public: |
| 180 | CIRGlobalOpTargetLowering(mlir::MLIRContext *context, |
| 181 | const TargetLoweringTypeConverter &typeConverter) |
| 182 | : mlir::OpConversionPattern<cir::GlobalOp>(typeConverter, context, |
| 183 | /*benefit=*/1) {} |
| 184 | |
| 185 | mlir::LogicalResult |
| 186 | matchAndRewrite(cir::GlobalOp op, OpAdaptor adaptor, |
| 187 | mlir::ConversionPatternRewriter &rewriter) const override { |
| 188 | const auto *typeConverter = |
| 189 | static_cast<const TargetLoweringTypeConverter *>(getTypeConverter()); |
| 190 | mlir::Type loweredSymTy = typeConverter->convertType(t: op.getSymType()); |
| 191 | if (!loweredSymTy) |
| 192 | return mlir::failure(); |
| 193 | |
| 194 | // Convert the addr_space attribute. |
| 195 | mlir::ptr::MemorySpaceAttrInterface addrSpace = op.getAddrSpaceAttr(); |
| 196 | if (auto langAS = |
| 197 | mlir::dyn_cast_if_present<cir::LangAddressSpaceAttr>(Val&: addrSpace)) |
| 198 | addrSpace = typeConverter->lowerAddrSpace(ctx: op.getContext(), langAS); |
| 199 | |
| 200 | // Only rewrite if something actually changed. |
| 201 | if (loweredSymTy == op.getSymType() && addrSpace == op.getAddrSpaceAttr()) |
| 202 | return mlir::failure(); |
| 203 | |
| 204 | auto newOp = mlir::cast<cir::GlobalOp>(Val: rewriter.clone(op&: *op.getOperation())); |
| 205 | newOp.setSymType(loweredSymTy); |
| 206 | newOp.setAddrSpaceAttr(addrSpace); |
| 207 | if (mlir::Attribute init = newOp.getInitialValueAttr()) |
| 208 | newOp.setInitialValueAttr(typeConverter->convertAttr(attr: init)); |
| 209 | rewriter.replaceOp(op, newOp); |
| 210 | return mlir::success(); |
| 211 | } |
| 212 | }; |
| 213 | |
| 214 | /// Pattern to lower FuncOp types that contain address spaces. |
| 215 | class CIRFuncOpTargetLowering : public mlir::OpConversionPattern<cir::FuncOp> { |
| 216 | public: |
| 217 | using mlir::OpConversionPattern<cir::FuncOp>::OpConversionPattern; |
| 218 | |
| 219 | mlir::LogicalResult |
| 220 | matchAndRewrite(cir::FuncOp op, OpAdaptor adaptor, |
| 221 | mlir::ConversionPatternRewriter &rewriter) const override { |
| 222 | cir::FuncType opFuncType = op.getFunctionType(); |
| 223 | mlir::TypeConverter::SignatureConversion signatureConversion( |
| 224 | opFuncType.getNumInputs()); |
| 225 | |
| 226 | for (const auto &[i, argType] : llvm::enumerate(First: opFuncType.getInputs())) { |
| 227 | mlir::Type loweredArgType = getTypeConverter()->convertType(t: argType); |
| 228 | if (!loweredArgType) |
| 229 | return mlir::failure(); |
| 230 | signatureConversion.addInputs(origInputNo: i, types: loweredArgType); |
| 231 | } |
| 232 | |
| 233 | mlir::Type loweredReturnType = |
| 234 | getTypeConverter()->convertType(t: opFuncType.getReturnType()); |
| 235 | if (!loweredReturnType) |
| 236 | return mlir::failure(); |
| 237 | |
| 238 | auto loweredFuncType = cir::FuncType::get( |
| 239 | inputs: signatureConversion.getConvertedTypes(), returnType: loweredReturnType, |
| 240 | /*isVarArg=*/opFuncType.getVarArg()); |
| 241 | |
| 242 | // Nothing changed, skip. |
| 243 | if (loweredFuncType == opFuncType) |
| 244 | return mlir::failure(); |
| 245 | |
| 246 | // Convert the body types before moving it. |
| 247 | if (mlir::failed(Result: rewriter.convertRegionTypes( |
| 248 | region: &op.getBody(), converter: *getTypeConverter(), entryConversion: &signatureConversion))) |
| 249 | return mlir::failure(); |
| 250 | |
| 251 | cir::FuncOp loweredFuncOp = rewriter.cloneWithoutRegions(op); |
| 252 | loweredFuncOp.setFunctionType(loweredFuncType); |
| 253 | rewriter.inlineRegionBefore(region&: op.getBody(), parent&: loweredFuncOp.getBody(), |
| 254 | before: loweredFuncOp.end()); |
| 255 | |
| 256 | rewriter.eraseOp(op); |
| 257 | return mlir::success(); |
| 258 | } |
| 259 | }; |
| 260 | |
| 261 | } // namespace |
| 262 | |
| 263 | static void convertSyncScopeIfPresent(mlir::Operation *op, |
| 264 | cir::LowerModule &lowerModule) { |
| 265 | auto syncScopeAttr = |
| 266 | mlir::cast_if_present<cir::SyncScopeKindAttr>(Val: op->getAttr(name: "sync_scope" )); |
| 267 | if (syncScopeAttr) { |
| 268 | cir::SyncScopeKind convertedSyncScope = |
| 269 | lowerModule.getTargetLoweringInfo().convertSyncScope( |
| 270 | syncScope: syncScopeAttr.getValue()); |
| 271 | op->setAttr(name: "sync_scope" , value: cir::SyncScopeKindAttr::get(context: op->getContext(), |
| 272 | value: convertedSyncScope)); |
| 273 | } |
| 274 | } |
| 275 | |
| 276 | static void |
| 277 | populateTargetLoweringConversionTarget(mlir::ConversionTarget &target, |
| 278 | const mlir::TypeConverter &tc) { |
| 279 | target.addLegalOp<mlir::ModuleOp>(); |
| 280 | |
| 281 | target.addDynamicallyLegalDialect<cir::CIRDialect>( |
| 282 | callback: [&tc](mlir::Operation *op) { |
| 283 | if (!tc.isLegal(op)) |
| 284 | return false; |
| 285 | return std::all_of( |
| 286 | first: op->getRegions().begin(), last: op->getRegions().end(), |
| 287 | pred: [&tc](mlir::Region ®ion) { return tc.isLegal(region: ®ion); }); |
| 288 | }); |
| 289 | |
| 290 | target.addDynamicallyLegalOp<cir::FuncOp>( |
| 291 | callback: [&tc](cir::FuncOp op) { return tc.isLegal(type: op.getFunctionType()); }); |
| 292 | |
| 293 | target.addDynamicallyLegalOp<cir::GlobalOp>(callback: [&tc](cir::GlobalOp op) { |
| 294 | if (!tc.isLegal(type: op.getSymType())) |
| 295 | return false; |
| 296 | return !mlir::isa_and_present<cir::LangAddressSpaceAttr>( |
| 297 | Val: op.getAddrSpaceAttr()); |
| 298 | }); |
| 299 | } |
| 300 | |
| 301 | void TargetLoweringPass::runOnOperation() { |
| 302 | auto mod = mlir::cast<mlir::ModuleOp>(Val: getOperation()); |
| 303 | |
| 304 | std::unique_ptr<cir::LowerModule> lowerModule = cir::createLowerModule(module: mod); |
| 305 | assert(lowerModule && "LowerModule shouldn't be null" ); |
| 306 | if (!lowerModule->hasTarget()) { |
| 307 | mod.emitError() << getArgument() << " requires a module with a supported '" |
| 308 | << cir::CIRDialect::getTripleAttrName() << "' attribute" ; |
| 309 | return signalPassFailure(); |
| 310 | } |
| 311 | |
| 312 | const auto &targetInfo = lowerModule->getTargetLoweringInfo(); |
| 313 | |
| 314 | mod->walk(callback: [&](mlir::Operation *op) { |
| 315 | if (mlir::isa<cir::LoadOp, cir::StoreOp, cir::AtomicXchgOp, |
| 316 | cir::AtomicCmpXchgOp, cir::AtomicFetchOp>(Val: op)) |
| 317 | convertSyncScopeIfPresent(op, lowerModule&: *lowerModule); |
| 318 | }); |
| 319 | |
| 320 | // Address space conversion: LangAddressSpaceAttr → TargetAddressSpaceAttr. |
| 321 | TargetLoweringTypeConverter typeConverter(*mod.getContext(), targetInfo); |
| 322 | |
| 323 | mlir::RewritePatternSet patterns(mod.getContext()); |
| 324 | patterns.add<CIRGlobalOpTargetLowering>(arg: mod.getContext(), args&: typeConverter); |
| 325 | patterns.add<CIRFuncOpTargetLowering>(arg&: typeConverter, args: mod.getContext()); |
| 326 | patterns.add<CIRGenericTargetLoweringPattern>(arg: mod.getContext(), |
| 327 | args&: typeConverter); |
| 328 | |
| 329 | mlir::ConversionTarget target(*mod.getContext()); |
| 330 | populateTargetLoweringConversionTarget(target, tc: typeConverter); |
| 331 | |
| 332 | llvm::SmallVector<mlir::Operation *> ops; |
| 333 | ops.push_back(Elt: mod); |
| 334 | cir::collectUnreachable(parent: mod, ops); |
| 335 | |
| 336 | // One-shot conversion (no pattern rollback): rewrites materialize |
| 337 | // immediately, skipping the IRRewrite/value-mapping bookkeeping. |
| 338 | mlir::ConversionConfig config; |
| 339 | config.allowPatternRollback = false; |
| 340 | if (failed(Result: mlir::applyPartialConversion(ops, target, patterns: std::move(patterns), |
| 341 | config))) |
| 342 | signalPassFailure(); |
| 343 | |
| 344 | typeConverter.restoreRecordTypeNames(); |
| 345 | } |
| 346 | |
| 347 | std::unique_ptr<Pass> mlir::createTargetLoweringPass() { |
| 348 | return std::make_unique<TargetLoweringPass>(); |
| 349 | } |
| 350 | |