1//===- TargetLowering.cpp -------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the cir-target-lowering pass.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RecordTypeConverter.h"
14#include "TargetLowering/LowerModule.h"
15#include "TargetLowering/TargetLoweringInfo.h"
16
17#include "mlir/IR/AttrTypeSubElements.h"
18#include "mlir/IR/PatternMatch.h"
19#include "mlir/Support/LLVM.h"
20#include "mlir/Transforms/DialectConversion.h"
21#include "clang/CIR/Dialect/IR/CIRAttrs.h"
22#include "clang/CIR/Dialect/IR/CIRDialect.h"
23#include "clang/CIR/Dialect/IR/CIRTypes.h"
24#include "clang/CIR/Dialect/Passes.h"
25#include "clang/CIR/Dialect/Transforms/CIRTransformUtils.h"
26
27using namespace mlir;
28using namespace cir;
29
30namespace mlir {
31#define GEN_PASS_DEF_TARGETLOWERING
32#include "clang/CIR/Dialect/Passes.h.inc"
33} // namespace mlir
34
35namespace {
36
37struct TargetLoweringPass
38 : public impl::TargetLoweringBase<TargetLoweringPass> {
39 TargetLoweringPass() = default;
40 void runOnOperation() override;
41};
42
43/// Converts LangAddressSpaceAttr → TargetAddressSpaceAttr inside types,
44/// rebuilding only records that reach a language address space.
45class TargetLoweringTypeConverter : public cir::RecordRewritingTypeConverter {
46public:
47 TargetLoweringTypeConverter(mlir::MLIRContext &ctx,
48 const cir::TargetLoweringInfo &targetInfo)
49 : RecordRewritingTypeConverter(ctx), targetInfo(targetInfo) {
50 addConversion(callback: [this](cir::PointerType type) -> mlir::Type {
51 mlir::Type pointee = convertType(t: type.getPointee());
52 if (!pointee)
53 return {};
54 mlir::ptr::MemorySpaceAttrInterface addrSpace = type.getAddrSpace();
55 if (auto langAS =
56 mlir::dyn_cast_if_present<cir::LangAddressSpaceAttr>(Val&: addrSpace))
57 addrSpace = lowerAddrSpace(ctx: type.getContext(), langAS);
58 return cir::PointerType::get(context: type.getContext(), pointee, addrSpace);
59 });
60 }
61
62 mlir::ptr::MemorySpaceAttrInterface
63 lowerAddrSpace(mlir::MLIRContext *ctx,
64 cir::LangAddressSpaceAttr langAS) const {
65 unsigned targetAS =
66 targetInfo.getTargetAddrSpaceFromCIRAddrSpace(addrSpace: langAS.getValue());
67 if (targetAS == 0)
68 return {};
69 return cir::TargetAddressSpaceAttr::get(context: ctx, value: targetAS);
70 }
71
72 /// Convert the types nested in \p attr, e.g. the type of a constant.
73 mlir::Attribute convertAttr(mlir::Attribute attr) const {
74 return attrReplacer.replace(attr);
75 }
76
77protected:
78 bool typeMayChange(mlir::Type type) const override {
79 auto ptrTy = mlir::dyn_cast<cir::PointerType>(Val&: type);
80 return ptrTy && mlir::isa_and_present<cir::LangAddressSpaceAttr>(
81 Val: ptrTy.getAddrSpace());
82 }
83
84private:
85 const cir::TargetLoweringInfo &targetInfo;
86 mutable mlir::AttrTypeReplacer attrReplacer = [this] {
87 mlir::AttrTypeReplacer replacer;
88 replacer.addReplacement(
89 fn: [this](mlir::Type type)
90 -> std::optional<std::pair<mlir::Type, mlir::WalkResult>> {
91 mlir::Type converted = convertType(t: type);
92 if (!converted)
93 return std::nullopt;
94 // Already converted; don't walk into it.
95 return std::make_pair(x&: converted, y: mlir::WalkResult::skip());
96 });
97 return replacer;
98 }();
99};
100
101/// A generic target lowering pattern that matches any CIR op whose operand or
102/// result types need address space conversion. Clones the op with converted
103/// types.
104class CIRGenericTargetLoweringPattern : public mlir::ConversionPattern {
105public:
106 CIRGenericTargetLoweringPattern(
107 mlir::MLIRContext *context,
108 const TargetLoweringTypeConverter &typeConverter)
109 : mlir::ConversionPattern(typeConverter, MatchAnyOpTypeTag(),
110 /*benefit=*/1, context) {}
111
112 mlir::LogicalResult
113 matchAndRewrite(mlir::Operation *op, llvm::ArrayRef<mlir::Value> operands,
114 mlir::ConversionPatternRewriter &rewriter) const override {
115 // Do not match on operations that have dedicated lowering patterns.
116 if (llvm::isa<cir::FuncOp, cir::GlobalOp>(Val: op))
117 return mlir::failure();
118
119 const mlir::TypeConverter *typeConverter = getTypeConverter();
120 assert(typeConverter &&
121 "CIRGenericTargetLoweringPattern requires a type converter");
122 bool operandsAndResultsLegal = typeConverter->isLegal(op);
123 bool regionsLegal =
124 std::all_of(first: op->getRegions().begin(), last: op->getRegions().end(),
125 pred: [typeConverter](mlir::Region &region) {
126 return typeConverter->isLegal(region: &region);
127 });
128 if (operandsAndResultsLegal && regionsLegal)
129 return mlir::failure();
130
131 assert(op->getNumRegions() == 0 && "CIRGenericTargetLoweringPattern cannot "
132 "deal with operations with regions");
133
134 mlir::OperationState loweredOpState(op->getLoc(), op->getName());
135 loweredOpState.addOperands(newOperands: operands);
136
137 // Preserve auxiliary metadata verbatim.
138 loweredOpState.propertiesAttr = op->getPropertiesAsAttribute();
139 loweredOpState.addAttributes(newAttributes: op->getDiscardableAttrDictionary().getValue());
140
141 loweredOpState.addSuccessors(newSuccessors: op->getSuccessors());
142
143 llvm::SmallVector<mlir::Type> loweredResultTypes;
144 loweredResultTypes.reserve(N: op->getNumResults());
145 for (mlir::Type result : op->getResultTypes())
146 loweredResultTypes.push_back(Elt: typeConverter->convertType(t: result));
147 loweredOpState.addTypes(newTypes: loweredResultTypes);
148
149 // Convert all regions before moving any blocks.
150 for (mlir::Region &region : op->getRegions()) {
151 if (mlir::failed(
152 Result: rewriter.convertRegionTypes(region: &region, converter: *getTypeConverter())))
153 return mlir::failure();
154 }
155 for (mlir::Region &region : op->getRegions()) {
156 mlir::Region *loweredRegion = loweredOpState.addRegion();
157 rewriter.inlineRegionBefore(region, parent&: *loweredRegion, before: loweredRegion->end());
158 }
159
160 mlir::Operation *loweredOp = rewriter.create(state: loweredOpState);
161 // Keep inherent attributes (e.g. allocaType, constants) in sync with the
162 // converted types.
163 const auto *tlTypeConverter =
164 static_cast<const TargetLoweringTypeConverter *>(typeConverter);
165 loweredOp->getName().walkInherentAttrs(
166 op: loweredOp, visitor: [&](llvm::StringRef, mlir::Attribute &attr) {
167 attr = tlTypeConverter->convertAttr(attr);
168 });
169 rewriter.replaceOp(op, newOp: loweredOp);
170 return mlir::success();
171 }
172};
173
174/// Pattern to lower GlobalOp address space attributes. GlobalOp carries
175/// addr_space as a standalone attribute (not inside a type), so the
176/// TypeConverter won't reach it automatically.
177class CIRGlobalOpTargetLowering
178 : public mlir::OpConversionPattern<cir::GlobalOp> {
179public:
180 CIRGlobalOpTargetLowering(mlir::MLIRContext *context,
181 const TargetLoweringTypeConverter &typeConverter)
182 : mlir::OpConversionPattern<cir::GlobalOp>(typeConverter, context,
183 /*benefit=*/1) {}
184
185 mlir::LogicalResult
186 matchAndRewrite(cir::GlobalOp op, OpAdaptor adaptor,
187 mlir::ConversionPatternRewriter &rewriter) const override {
188 const auto *typeConverter =
189 static_cast<const TargetLoweringTypeConverter *>(getTypeConverter());
190 mlir::Type loweredSymTy = typeConverter->convertType(t: op.getSymType());
191 if (!loweredSymTy)
192 return mlir::failure();
193
194 // Convert the addr_space attribute.
195 mlir::ptr::MemorySpaceAttrInterface addrSpace = op.getAddrSpaceAttr();
196 if (auto langAS =
197 mlir::dyn_cast_if_present<cir::LangAddressSpaceAttr>(Val&: addrSpace))
198 addrSpace = typeConverter->lowerAddrSpace(ctx: op.getContext(), langAS);
199
200 // Only rewrite if something actually changed.
201 if (loweredSymTy == op.getSymType() && addrSpace == op.getAddrSpaceAttr())
202 return mlir::failure();
203
204 auto newOp = mlir::cast<cir::GlobalOp>(Val: rewriter.clone(op&: *op.getOperation()));
205 newOp.setSymType(loweredSymTy);
206 newOp.setAddrSpaceAttr(addrSpace);
207 if (mlir::Attribute init = newOp.getInitialValueAttr())
208 newOp.setInitialValueAttr(typeConverter->convertAttr(attr: init));
209 rewriter.replaceOp(op, newOp);
210 return mlir::success();
211 }
212};
213
214/// Pattern to lower FuncOp types that contain address spaces.
215class CIRFuncOpTargetLowering : public mlir::OpConversionPattern<cir::FuncOp> {
216public:
217 using mlir::OpConversionPattern<cir::FuncOp>::OpConversionPattern;
218
219 mlir::LogicalResult
220 matchAndRewrite(cir::FuncOp op, OpAdaptor adaptor,
221 mlir::ConversionPatternRewriter &rewriter) const override {
222 cir::FuncType opFuncType = op.getFunctionType();
223 mlir::TypeConverter::SignatureConversion signatureConversion(
224 opFuncType.getNumInputs());
225
226 for (const auto &[i, argType] : llvm::enumerate(First: opFuncType.getInputs())) {
227 mlir::Type loweredArgType = getTypeConverter()->convertType(t: argType);
228 if (!loweredArgType)
229 return mlir::failure();
230 signatureConversion.addInputs(origInputNo: i, types: loweredArgType);
231 }
232
233 mlir::Type loweredReturnType =
234 getTypeConverter()->convertType(t: opFuncType.getReturnType());
235 if (!loweredReturnType)
236 return mlir::failure();
237
238 auto loweredFuncType = cir::FuncType::get(
239 inputs: signatureConversion.getConvertedTypes(), returnType: loweredReturnType,
240 /*isVarArg=*/opFuncType.getVarArg());
241
242 // Nothing changed, skip.
243 if (loweredFuncType == opFuncType)
244 return mlir::failure();
245
246 // Convert the body types before moving it.
247 if (mlir::failed(Result: rewriter.convertRegionTypes(
248 region: &op.getBody(), converter: *getTypeConverter(), entryConversion: &signatureConversion)))
249 return mlir::failure();
250
251 cir::FuncOp loweredFuncOp = rewriter.cloneWithoutRegions(op);
252 loweredFuncOp.setFunctionType(loweredFuncType);
253 rewriter.inlineRegionBefore(region&: op.getBody(), parent&: loweredFuncOp.getBody(),
254 before: loweredFuncOp.end());
255
256 rewriter.eraseOp(op);
257 return mlir::success();
258 }
259};
260
261} // namespace
262
263static void convertSyncScopeIfPresent(mlir::Operation *op,
264 cir::LowerModule &lowerModule) {
265 auto syncScopeAttr =
266 mlir::cast_if_present<cir::SyncScopeKindAttr>(Val: op->getAttr(name: "sync_scope"));
267 if (syncScopeAttr) {
268 cir::SyncScopeKind convertedSyncScope =
269 lowerModule.getTargetLoweringInfo().convertSyncScope(
270 syncScope: syncScopeAttr.getValue());
271 op->setAttr(name: "sync_scope", value: cir::SyncScopeKindAttr::get(context: op->getContext(),
272 value: convertedSyncScope));
273 }
274}
275
276static void
277populateTargetLoweringConversionTarget(mlir::ConversionTarget &target,
278 const mlir::TypeConverter &tc) {
279 target.addLegalOp<mlir::ModuleOp>();
280
281 target.addDynamicallyLegalDialect<cir::CIRDialect>(
282 callback: [&tc](mlir::Operation *op) {
283 if (!tc.isLegal(op))
284 return false;
285 return std::all_of(
286 first: op->getRegions().begin(), last: op->getRegions().end(),
287 pred: [&tc](mlir::Region &region) { return tc.isLegal(region: &region); });
288 });
289
290 target.addDynamicallyLegalOp<cir::FuncOp>(
291 callback: [&tc](cir::FuncOp op) { return tc.isLegal(type: op.getFunctionType()); });
292
293 target.addDynamicallyLegalOp<cir::GlobalOp>(callback: [&tc](cir::GlobalOp op) {
294 if (!tc.isLegal(type: op.getSymType()))
295 return false;
296 return !mlir::isa_and_present<cir::LangAddressSpaceAttr>(
297 Val: op.getAddrSpaceAttr());
298 });
299}
300
301void TargetLoweringPass::runOnOperation() {
302 auto mod = mlir::cast<mlir::ModuleOp>(Val: getOperation());
303
304 std::unique_ptr<cir::LowerModule> lowerModule = cir::createLowerModule(module: mod);
305 assert(lowerModule && "LowerModule shouldn't be null");
306 if (!lowerModule->hasTarget()) {
307 mod.emitError() << getArgument() << " requires a module with a supported '"
308 << cir::CIRDialect::getTripleAttrName() << "' attribute";
309 return signalPassFailure();
310 }
311
312 const auto &targetInfo = lowerModule->getTargetLoweringInfo();
313
314 mod->walk(callback: [&](mlir::Operation *op) {
315 if (mlir::isa<cir::LoadOp, cir::StoreOp, cir::AtomicXchgOp,
316 cir::AtomicCmpXchgOp, cir::AtomicFetchOp>(Val: op))
317 convertSyncScopeIfPresent(op, lowerModule&: *lowerModule);
318 });
319
320 // Address space conversion: LangAddressSpaceAttr → TargetAddressSpaceAttr.
321 TargetLoweringTypeConverter typeConverter(*mod.getContext(), targetInfo);
322
323 mlir::RewritePatternSet patterns(mod.getContext());
324 patterns.add<CIRGlobalOpTargetLowering>(arg: mod.getContext(), args&: typeConverter);
325 patterns.add<CIRFuncOpTargetLowering>(arg&: typeConverter, args: mod.getContext());
326 patterns.add<CIRGenericTargetLoweringPattern>(arg: mod.getContext(),
327 args&: typeConverter);
328
329 mlir::ConversionTarget target(*mod.getContext());
330 populateTargetLoweringConversionTarget(target, tc: typeConverter);
331
332 llvm::SmallVector<mlir::Operation *> ops;
333 ops.push_back(Elt: mod);
334 cir::collectUnreachable(parent: mod, ops);
335
336 // One-shot conversion (no pattern rollback): rewrites materialize
337 // immediately, skipping the IRRewrite/value-mapping bookkeeping.
338 mlir::ConversionConfig config;
339 config.allowPatternRollback = false;
340 if (failed(Result: mlir::applyPartialConversion(ops, target, patterns: std::move(patterns),
341 config)))
342 signalPassFailure();
343
344 typeConverter.restoreRecordTypeNames();
345}
346
347std::unique_ptr<Pass> mlir::createTargetLoweringPass() {
348 return std::make_unique<TargetLoweringPass>();
349}
350