1//====- LowerToLLVM.cpp - Lowering from CIR to LLVMIR ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements lowering of CIR operations to LLVMIR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "LowerToLLVM.h"
14#include "LowerToLLVMMetadata.h"
15#include "LowerToLLVMOffloadPrintf.h"
16#include "LowerToLLVMOpenCLMetadata.h"
17
18#include <array>
19#include <optional>
20
21#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
22#include "mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h"
23#include "mlir/Dialect/DLTI/DLTI.h"
24#include "mlir/Dialect/Func/IR/FuncOps.h"
25#include "mlir/Dialect/LLVMIR/LLVMAttrs.h"
26#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
27#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
28#include "mlir/Dialect/LLVMIR/Transforms/Passes.h"
29#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
30#include "mlir/Dialect/OpenMP/Transforms/Passes.h"
31#include "mlir/Dialect/Ptr/IR/MemorySpaceInterfaces.h"
32#include "mlir/IR/BuiltinAttributes.h"
33#include "mlir/IR/BuiltinDialect.h"
34#include "mlir/IR/BuiltinOps.h"
35#include "mlir/IR/Diagnostics.h"
36#include "mlir/IR/Location.h"
37#include "mlir/IR/Matchers.h"
38#include "mlir/IR/Types.h"
39#include "mlir/Pass/Pass.h"
40#include "mlir/Pass/PassManager.h"
41#include "mlir/Support/LLVM.h"
42#include "mlir/Target/LLVMIR/Dialect/Builtin/BuiltinToLLVMIRTranslation.h"
43#include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.h"
44#include "mlir/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.h"
45#include "mlir/Target/LLVMIR/Export.h"
46#include "mlir/Transforms/DialectConversion.h"
47#include "clang/CIR/Dialect/IR/CIRAttrs.h"
48#include "clang/CIR/Dialect/IR/CIRDataLayout.h"
49#include "clang/CIR/Dialect/IR/CIRDialect.h"
50#include "clang/CIR/Dialect/IR/CIRTypes.h"
51#include "clang/CIR/Dialect/Passes.h"
52#include "clang/CIR/Dialect/Transforms/CIRTransformUtils.h"
53#include "clang/CIR/LoweringHelpers.h"
54#include "clang/CIR/MissingFeatures.h"
55#include "clang/CIR/Passes.h"
56#include "llvm/ADT/MapVector.h"
57#include "llvm/ADT/StringMap.h"
58#include "llvm/ADT/TypeSwitch.h"
59#include "llvm/IR/Module.h"
60#include "llvm/Support/AMDGPUAddrSpace.h"
61#include "llvm/Support/Casting.h"
62#include "llvm/Support/ErrorHandling.h"
63#include "llvm/Support/TimeProfiler.h"
64#include "llvm/Support/VirtualFileSystem.h"
65#include "llvm/Support/raw_ostream.h"
66#include "llvm/TargetParser/AtomicScope.h"
67
68using namespace cir;
69using namespace llvm;
70
71namespace cir {
72namespace direct {
73
74//===----------------------------------------------------------------------===//
75// Helper Methods
76//===----------------------------------------------------------------------===//
77
78namespace {
79/// If the given type is a vector or matrix type, return its element type.
80/// Otherwise return the given type unchanged.
81mlir::Type elementTypeIfVectorOrMatrix(mlir::Type type) {
82 return llvm::TypeSwitch<mlir::Type, mlir::Type>(type)
83 .Case<cir::VectorType, cir::MatrixType, mlir::VectorType>(
84 caseFn: [](auto p) { return p.getElementType(); })
85 .Default(defaultFn: [](mlir::Type p) { return p; });
86}
87} // namespace
88
89static constexpr char kTypeConversionErrorMsg[] =
90 "ClangIR lowering NYI: cannot convert ";
91
92static mlir::InFlightDiagnostic emitTypeConversionError(mlir::Operation *op,
93 mlir::Type type) {
94 return op->emitError() << kTypeConversionErrorMsg << type;
95}
96
97static mlir::InFlightDiagnostic emitTypeConversionError(mlir::Location loc,
98 mlir::Type type) {
99 return mlir::emitError(loc) << kTypeConversionErrorMsg << type;
100}
101
102/// Alignment to use for a memory access whose op carries no explicit alignment.
103/// For _BitInt the storage integer iM's ABI alignment (e.g. i128's 16)
104/// over-aligns the value, so use the _BitInt language alignment (e.g. 8).
105static uint64_t getMemoryFallbackAlignment(mlir::Operation *op,
106 mlir::Type cirType,
107 mlir::Type llvmMemType,
108 const mlir::DataLayout &dataLayout) {
109 if (auto intTy = mlir::dyn_cast<cir::IntType>(Val&: cirType);
110 intTy && intTy.isBitInt())
111 return cir::CIRDataLayout(op->getParentOfType<mlir::ModuleOp>())
112 .getTypeLanguageAlignment(ty: intTy);
113 return dataLayout.getTypeABIAlignment(t: llvmMemType);
114}
115
116static mlir::Value createIntCast(mlir::OpBuilder &bld, mlir::Value src,
117 mlir::IntegerType dstTy,
118 bool isSigned = false) {
119 mlir::Type srcTy = src.getType();
120 assert(mlir::isa<mlir::IntegerType>(srcTy));
121
122 unsigned srcWidth = mlir::cast<mlir::IntegerType>(Val&: srcTy).getWidth();
123 unsigned dstWidth = mlir::cast<mlir::IntegerType>(Val&: dstTy).getWidth();
124 mlir::Location loc = src.getLoc();
125
126 if (dstWidth > srcWidth && isSigned)
127 return mlir::LLVM::SExtOp::create(builder&: bld, location: loc, res: dstTy, arg: src);
128 if (dstWidth > srcWidth)
129 return mlir::LLVM::ZExtOp::create(builder&: bld, location: loc, res: dstTy, arg: src);
130 if (dstWidth < srcWidth)
131 return mlir::LLVM::TruncOp::create(builder&: bld, location: loc, res: dstTy, arg: src);
132 return mlir::LLVM::BitcastOp::create(builder&: bld, location: loc, res: dstTy, arg: src);
133}
134
135/// Cast a _BitInt(N) value between its literal width iN and its padded
136/// in-memory storage iM (sign/zero-extend to memory per signedness so the
137/// padding bits are well-defined, truncate back on load; matches classic
138/// CodeGen). Callers must first rule out the unsupported byte-array storage
139/// form (isSplitStorageBitInt / a null convertTypeForMemory result).
140static mlir::Value
141castBitIntMemoryStorage(mlir::ConversionPatternRewriter &rewriter,
142 const mlir::DataLayout &dataLayout, cir::IntType intTy,
143 mlir::Value value, bool toMemory) {
144 unsigned storageBits = intTy.getStorageTypeWidth(dataLayout);
145 if (storageBits == intTy.getWidth())
146 return value;
147 unsigned dstBits = toMemory ? storageBits : intTy.getWidth();
148 return createIntCast(bld&: rewriter, src: value, dstTy: rewriter.getIntegerType(width: dstBits),
149 /*isSigned=*/toMemory && intTy.isSigned());
150}
151
152static mlir::LLVM::Visibility
153lowerCIRVisibilityToLLVMVisibility(cir::VisibilityKind visibilityKind) {
154 switch (visibilityKind) {
155 case cir::VisibilityKind::Default:
156 return ::mlir::LLVM::Visibility::Default;
157 case cir::VisibilityKind::Hidden:
158 return ::mlir::LLVM::Visibility::Hidden;
159 case cir::VisibilityKind::Protected:
160 return ::mlir::LLVM::Visibility::Protected;
161 }
162 llvm_unreachable("fully covered switch");
163}
164
165static mlir::Value
166emitBoolVecConversion(mlir::ConversionPatternRewriter &rewriter,
167 mlir::Value srcVec, unsigned numElementsDst) {
168 auto srcTy = mlir::cast<mlir::VectorType>(Val: srcVec.getType());
169 unsigned numElementsSrc = srcTy.getNumElements();
170 if (numElementsSrc == numElementsDst)
171 return srcVec;
172
173 SmallVector<int32_t, 8> mask(numElementsDst, -1);
174 for (unsigned i : llvm::seq(Size: std::min(a: numElementsDst, b: numElementsSrc)))
175 mask[i] = i;
176
177 mlir::Location loc = srcVec.getLoc();
178 auto poison = mlir::LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: srcVec.getType());
179 return mlir::LLVM::ShuffleVectorOp::create(builder&: rewriter, location: loc, v1: srcVec, v2: poison,
180 mask);
181}
182
183/// Emits the value from memory as expected by its users. Should be called when
184/// the memory represetnation of a CIR type is not equal to its scalar
185/// representation.
186static mlir::Value emitFromMemory(mlir::ConversionPatternRewriter &rewriter,
187 const mlir::TypeConverter &converter,
188 mlir::DataLayout const &dataLayout,
189 cir::LoadOp op, mlir::Value value) {
190 // TODO(cir): Handle other types similarly to clang's codegen EmitFromMemory
191 if (auto boolTy = mlir::dyn_cast<cir::BoolType>(Val: op.getType())) {
192 // Create a cast value from specified size in datalayout to i1
193 assert(value.getType().isInteger(dataLayout.getTypeSizeInBits(boolTy)));
194 return createIntCast(bld&: rewriter, src: value, dstTy: rewriter.getI1Type());
195 }
196
197 // Convert the `iN` back to boolean vectors
198 if (auto vecTy = mlir::dyn_cast<cir::VectorType>(Val: op.getType())) {
199 if (mlir::isa<cir::BoolType>(Val: vecTy.getElementType())) {
200 if (vecTy.isScalable())
201 return value;
202 auto rawIntTy = mlir::cast<mlir::IntegerType>(Val: value.getType());
203 auto paddedVecTy = cir::VectorType::getFixed(elementType: vecTy.getElementType(),
204 size: rawIntTy.getWidth());
205 mlir::Type mlirVecTy = converter.convertType(t: paddedVecTy);
206 // Bitcast iP --> <P x i1>.
207 auto v = mlir::LLVM::BitcastOp::create(builder&: rewriter, location: value.getLoc(),
208 res: mlirVecTy, arg: value);
209 // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
210 return emitBoolVecConversion(rewriter, srcVec: v, numElementsDst: vecTy.getFixedSize());
211 }
212 }
213
214 // Truncate the padded storage integer back to the _BitInt's literal width.
215 if (auto intTy = mlir::dyn_cast<cir::IntType>(Val: op.getType());
216 intTy && intTy.isBitInt())
217 return castBitIntMemoryStorage(rewriter, dataLayout, intTy, value,
218 /*toMemory=*/false);
219
220 return value;
221}
222
223/// Emits a value to memory with the expected scalar type. Should be called when
224/// the memory represetnation of a CIR type is not equal to its scalar
225/// representation.
226static mlir::Value emitToMemory(mlir::ConversionPatternRewriter &rewriter,
227 mlir::DataLayout const &dataLayout,
228 mlir::Type origType, mlir::Value value) {
229
230 // TODO(cir): Handle other types similarly to clang's codegen EmitToMemory
231 if (auto boolTy = mlir::dyn_cast<cir::BoolType>(Val&: origType)) {
232 // Create zext of value from i1 to i8
233 mlir::IntegerType memType =
234 rewriter.getIntegerType(width: dataLayout.getTypeSizeInBits(t: boolTy));
235 return createIntCast(bld&: rewriter, src: value, dstTy: memType);
236 }
237
238 // Boolean vectors use:
239 // * `iN` for fixed-width vectors,
240 // * `<vscale x N x i1>` for scalable vectors,
241 // as storage type.
242 if (auto vecTy = mlir::dyn_cast<cir::VectorType>(Val&: origType)) {
243 if (mlir::isa<cir::BoolType>(Val: vecTy.getElementType())) {
244 mlir::Type resultTy;
245 if (vecTy.isScalable())
246 resultTy =
247 mlir::VectorType::get(shape: vecTy.getSize().getKnownMinValue(),
248 elementType: vecTy.getElementType(), scalableDims: vecTy.isScalable());
249 else {
250 resultTy = mlir::IntegerType::get(context: origType.getContext(),
251 width: vecTy.getBoolStorageWidth());
252 value =
253 emitBoolVecConversion(rewriter, srcVec: value, numElementsDst: vecTy.getBoolStorageWidth());
254 }
255 return mlir::LLVM::BitcastOp::create(builder&: rewriter, location: value.getLoc(), res: resultTy,
256 arg: value);
257 }
258 }
259
260 // Sign/zero-extend the _BitInt value to its padded storage integer.
261 if (auto intTy = mlir::dyn_cast<cir::IntType>(Val&: origType);
262 intTy && intTy.isBitInt())
263 return castBitIntMemoryStorage(rewriter, dataLayout, intTy, value,
264 /*toMemory=*/true);
265
266 return value;
267}
268
269mlir::LLVM::Linkage convertLinkage(cir::GlobalLinkageKind linkage) {
270 using CIR = cir::GlobalLinkageKind;
271 using LLVM = mlir::LLVM::Linkage;
272
273 switch (linkage) {
274 case CIR::AppendingLinkage:
275 return LLVM::Appending;
276 case CIR::AvailableExternallyLinkage:
277 return LLVM::AvailableExternally;
278 case CIR::CommonLinkage:
279 return LLVM::Common;
280 case CIR::ExternalLinkage:
281 return LLVM::External;
282 case CIR::ExternalWeakLinkage:
283 return LLVM::ExternWeak;
284 case CIR::InternalLinkage:
285 return LLVM::Internal;
286 case CIR::LinkOnceAnyLinkage:
287 return LLVM::Linkonce;
288 case CIR::LinkOnceODRLinkage:
289 return LLVM::LinkonceODR;
290 case CIR::PrivateLinkage:
291 return LLVM::Private;
292 case CIR::WeakAnyLinkage:
293 return LLVM::Weak;
294 case CIR::WeakODRLinkage:
295 return LLVM::WeakODR;
296 };
297 llvm_unreachable("Unknown CIR linkage type");
298}
299
300static mlir::LLVM::CConv convertCallingConv(cir::CallingConv callingConv) {
301 using CIR = cir::CallingConv;
302 using LLVM = mlir::LLVM::CConv;
303
304 switch (callingConv) {
305 case CIR::C:
306 return LLVM::C;
307 case CIR::SpirKernel:
308 return LLVM::SPIR_KERNEL;
309 case CIR::SpirFunction:
310 return LLVM::SPIR_FUNC;
311 case CIR::PTXKernel:
312 return LLVM::PTX_Kernel;
313 case CIR::AMDGPUKernel:
314 return LLVM::AMDGPU_KERNEL;
315 }
316 llvm_unreachable("Unknown calling convention");
317}
318
319static mlir::LLVM::uwtable::UWTableKind
320convertUWTableKind(cir::UnwindTableKind kind) {
321 using CIR = cir::UnwindTableKind;
322 using LLVM = mlir::LLVM::uwtable::UWTableKind;
323
324 switch (kind) {
325 case CIR::None:
326 return LLVM::None;
327 case CIR::Sync:
328 return LLVM::Sync;
329 case CIR::Async:
330 return LLVM::Async;
331 }
332 llvm_unreachable("Unknown CIR unwind table kind");
333}
334
335static mlir::LLVM::framePointerKind::FramePointerKind
336convertFramePointerKind(cir::FramePointerKind kind) {
337 using CIR = cir::FramePointerKind;
338 using LLVM = mlir::LLVM::framePointerKind::FramePointerKind;
339
340 switch (kind) {
341 case CIR::None:
342 return LLVM::None;
343 case CIR::NonLeaf:
344 return LLVM::NonLeaf;
345 case CIR::All:
346 return LLVM::All;
347 case CIR::Reserved:
348 return LLVM::Reserved;
349 case CIR::NonLeafNoReserve:
350 return LLVM::NonLeafNoReserve;
351 }
352 llvm_unreachable("Unknown CIR frame pointer kind");
353}
354
355/// The "frame-pointer" module flag holds an llvm::FramePointerKind value, as
356/// written by llvm::Module::setFramePointer.
357static llvm::FramePointerKind
358convertFramePointerKindToLLVMIR(cir::FramePointerKind kind) {
359 using CIR = cir::FramePointerKind;
360 using LLVM = llvm::FramePointerKind;
361
362 switch (kind) {
363 case CIR::None:
364 return LLVM::None;
365 case CIR::NonLeaf:
366 return LLVM::NonLeaf;
367 case CIR::All:
368 return LLVM::All;
369 case CIR::Reserved:
370 return LLVM::Reserved;
371 case CIR::NonLeafNoReserve:
372 return LLVM::NonLeafNoReserve;
373 }
374 llvm_unreachable("Unknown CIR frame pointer kind");
375}
376
377mlir::LogicalResult CIRToLLVMCopyOpLowering::matchAndRewrite(
378 cir::CopyOp op, OpAdaptor adaptor,
379 mlir::ConversionPatternRewriter &rewriter) const {
380 // Opaque pointer conversion does not validate the pointee's memory layout.
381 mlir::Type pointeeType = op.getType().getPointee();
382 if (!convertTypeForMemory(converter: *getTypeConverter(), dataLayout, type: pointeeType))
383 return emitTypeConversionError(op, type: pointeeType);
384
385 cir::CIRDataLayout cirDataLayout(op->getParentOfType<mlir::ModuleOp>());
386 const mlir::Value length = mlir::LLVM::ConstantOp::create(
387 builder&: rewriter, location: op.getLoc(), type: rewriter.getI64Type(),
388 value: op.getCopySizeInBytes(dl: cirDataLayout.layout));
389 assert(!cir::MissingFeatures::aggValueSlotVolatile());
390
391 uint64_t dstTypeAlign = cirDataLayout.getTypeLanguageAlignment(
392 ty: op.getDst().getType().getPointee());
393 uint64_t srcTypeAlign = cirDataLayout.getTypeLanguageAlignment(
394 ty: op.getSrc().getType().getPointee());
395
396 mlir::NamedAttribute dstAlignAttr = rewriter.getNamedAttr(
397 name: mlir::LLVM::LLVMDialect::getAlignAttrName(),
398 val: rewriter.getI64IntegerAttr(value: op.getDstAlignment().value_or(u&: dstTypeAlign)));
399 mlir::NamedAttribute srcAlignAttr = rewriter.getNamedAttr(
400 name: mlir::LLVM::LLVMDialect::getAlignAttrName(),
401 val: rewriter.getI64IntegerAttr(value: op.getSrcAlignment().value_or(u&: srcTypeAlign)));
402 mlir::ArrayAttr argAttrs = rewriter.getArrayAttr(value: {
403 /*dst_attrs=*/rewriter.getDictionaryAttr(value: {dstAlignAttr}),
404 /*src_attrs=*/rewriter.getDictionaryAttr(value: {srcAlignAttr}),
405 });
406
407 rewriter.replaceOpWithNewOp<mlir::LLVM::MemcpyOp>(
408 op, args: adaptor.getDst(), args: adaptor.getSrc(), args: length, args: op.getIsVolatile(),
409 /*tbaa_struct=*/args: nullptr, /*access_groups=*/args: nullptr,
410 /*alias_scopes=*/args: nullptr, /*noalias_scopes=*/args: nullptr, /*tbaa=*/args: nullptr,
411 /*arg_attrs=*/args&: argAttrs, /*res_attrs=*/args: nullptr);
412 return mlir::success();
413}
414
415template <typename CIROp, typename LLVMOp>
416static mlir::LogicalResult
417lowerMemTransferOp(CIROp op, typename CIROp::Adaptor adaptor,
418 mlir::ConversionPatternRewriter &rewriter) {
419 mlir::ArrayAttr argAttrs;
420 if (op.getDstAlignment() || op.getSrcAlignment()) {
421 mlir::NamedAttribute dstAlignAttr = rewriter.getNamedAttr(
422 name: mlir::LLVM::LLVMDialect::getAlignAttrName(),
423 val: rewriter.getI64IntegerAttr(value: op.getDstAlignment().value_or(1)));
424 mlir::NamedAttribute srcAlignAttr = rewriter.getNamedAttr(
425 name: mlir::LLVM::LLVMDialect::getAlignAttrName(),
426 val: rewriter.getI64IntegerAttr(value: op.getSrcAlignment().value_or(1)));
427 argAttrs = rewriter.getArrayAttr(value: {
428 /*dst_attrs=*/rewriter.getDictionaryAttr(value: {dstAlignAttr}),
429 /*src_attrs=*/rewriter.getDictionaryAttr(value: {srcAlignAttr}),
430 });
431 }
432 rewriter.replaceOpWithNewOp<LLVMOp>(
433 op, adaptor.getDst(), adaptor.getSrc(), adaptor.getLen(),
434 /*isVolatile=*/false, /*tbaa_struct=*/nullptr,
435 /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
436 /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr, /*arg_attrs=*/argAttrs,
437 /*res_attrs=*/nullptr);
438 return mlir::success();
439}
440
441mlir::LogicalResult CIRToLLVMMemCpyOpLowering::matchAndRewrite(
442 cir::MemCpyOp op, OpAdaptor adaptor,
443 mlir::ConversionPatternRewriter &rewriter) const {
444 return lowerMemTransferOp<cir::MemCpyOp, mlir::LLVM::MemcpyOp>(op, adaptor,
445 rewriter);
446}
447
448mlir::LogicalResult CIRToLLVMMemMoveOpLowering::matchAndRewrite(
449 cir::MemMoveOp op, OpAdaptor adaptor,
450 mlir::ConversionPatternRewriter &rewriter) const {
451 return lowerMemTransferOp<cir::MemMoveOp, mlir::LLVM::MemmoveOp>(op, adaptor,
452 rewriter);
453}
454
455mlir::LogicalResult CIRToLLVMMemSetOpLowering::matchAndRewrite(
456 cir::MemSetOp op, OpAdaptor adaptor,
457 mlir::ConversionPatternRewriter &rewriter) const {
458
459 // Capture the alignment before the replace erases `op`.
460 mlir::Attribute alignAttr = op.getAlignmentAttr();
461 auto memset = rewriter.replaceOpWithNewOp<mlir::LLVM::MemsetOp>(
462 op, args: adaptor.getDst(), args: adaptor.getVal(), args: adaptor.getLen(),
463 /*isVolatile=*/args: false);
464
465 if (alignAttr) {
466 // Construct a list full of empty attributes.
467 llvm::SmallVector<mlir::Attribute> attrs{memset.getNumOperands(),
468 rewriter.getDictionaryAttr(value: {})};
469 llvm::SmallVector<mlir::NamedAttribute> destAttrs;
470 destAttrs.push_back(
471 Elt: {mlir::LLVM::LLVMDialect::getAlignAttrName(), alignAttr});
472 attrs[memset.odsIndex_dst] = rewriter.getDictionaryAttr(value: destAttrs);
473
474 auto arrayAttr = rewriter.getArrayAttr(value: attrs);
475 memset.setArgAttrsAttr(arrayAttr);
476 }
477
478 return mlir::success();
479}
480
481static mlir::Value getLLVMIntCast(mlir::ConversionPatternRewriter &rewriter,
482 mlir::Value llvmSrc, mlir::Type llvmDstIntTy,
483 bool isUnsigned, uint64_t cirSrcWidth,
484 uint64_t cirDstIntWidth) {
485 if (cirSrcWidth == cirDstIntWidth)
486 return llvmSrc;
487
488 auto loc = llvmSrc.getLoc();
489 if (cirSrcWidth < cirDstIntWidth) {
490 if (isUnsigned)
491 return mlir::LLVM::ZExtOp::create(builder&: rewriter, location: loc, res: llvmDstIntTy, arg: llvmSrc);
492 return mlir::LLVM::SExtOp::create(builder&: rewriter, location: loc, res: llvmDstIntTy, arg: llvmSrc);
493 }
494
495 // Otherwise truncate
496 return mlir::LLVM::TruncOp::create(builder&: rewriter, location: loc, res: llvmDstIntTy, arg: llvmSrc);
497}
498
499class CIRAttrToValue {
500public:
501 CIRAttrToValue(mlir::Location parentLoc, mlir::ModuleOp parentModule,
502 mlir::ConversionPatternRewriter &rewriter,
503 mlir::SymbolTableCollection &symbolTables,
504 const mlir::TypeConverter *converter,
505 const CIRPreConversionTypeMap &preConversionTypes,
506 LLVMBlockAddressInfo *blockInfoAddr = nullptr)
507 : parentLoc(parentLoc), parentModule(parentModule), rewriter(rewriter),
508 symbolTables(symbolTables), converter(converter),
509 preConversionTypes(preConversionTypes), blockInfoAddr(blockInfoAddr) {}
510
511#define GET_CIR_ATTR_TO_VALUE_VISITOR_DECLS
512#include "clang/CIR/Dialect/IR/CIRLowering.inc"
513#undef GET_CIR_ATTR_TO_VALUE_VISITOR_DECLS
514
515private:
516 struct LoweredGlobalSymbol {
517 mlir::Value addr;
518 mlir::Type sourceType;
519 mlir::Type originalType;
520 unsigned addrSpace;
521 };
522
523 LoweredGlobalSymbol getAddressOfSymbol(mlir::FlatSymbolRefAttr symbol);
524 mlir::Value castGlobalAddrToType(mlir::Value addr, mlir::Type attrType,
525 mlir::Type sourceType, unsigned addrSpace);
526
527 mlir::Location parentLoc;
528 mlir::ModuleOp parentModule;
529 mlir::ConversionPatternRewriter &rewriter;
530 mlir::SymbolTableCollection &symbolTables;
531 const mlir::TypeConverter *converter;
532 const CIRPreConversionTypeMap &preConversionTypes;
533 // Only available when lowering global initializers that may contain block
534 // address attributes. Used to resolve a BlockAddrInfoAttr to its block tag.
535 LLVMBlockAddressInfo *blockInfoAddr;
536};
537
538/// Switches on the type of attribute and calls the appropriate conversion.
539mlir::Value
540lowerCirAttrAsValue(mlir::Location parentLoc, mlir::ModuleOp parentModule,
541 const mlir::Attribute attr,
542 mlir::ConversionPatternRewriter &rewriter,
543 mlir::SymbolTableCollection &symbolTables,
544 const mlir::TypeConverter *converter,
545 const CIRPreConversionTypeMap &preConversionTypes,
546 LLVMBlockAddressInfo *blockInfoAddr) {
547 CIRAttrToValue valueConverter(parentLoc, parentModule, rewriter, symbolTables,
548 converter, preConversionTypes, blockInfoAddr);
549 return valueConverter.visit(attr);
550}
551
552static mlir::LLVM::ModRefInfo convertModRefInfo(cir::ModRefInfo info) {
553 switch (info) {
554 case cir::ModRefInfo::NoModRef:
555 return mlir::LLVM::ModRefInfo::NoModRef;
556 case cir::ModRefInfo::Ref:
557 return mlir::LLVM::ModRefInfo::Ref;
558 case cir::ModRefInfo::Mod:
559 return mlir::LLVM::ModRefInfo::Mod;
560 case cir::ModRefInfo::ModRef:
561 return mlir::LLVM::ModRefInfo::ModRef;
562 }
563 llvm_unreachable("unhandled cir::ModRefInfo");
564}
565
566/// A null input stays null, which is how both dialects spell unknown effects.
567static mlir::LLVM::MemoryEffectsAttr
568convertMemoryEffects(mlir::MLIRContext *ctx, cir::MemoryEffectsAttr effects) {
569 if (!effects)
570 return {};
571
572 return mlir::LLVM::MemoryEffectsAttr::get(
573 context: ctx, other: convertModRefInfo(info: effects.getOther()),
574 argMem: convertModRefInfo(info: effects.getArgMem()),
575 inaccessibleMem: convertModRefInfo(info: effects.getInaccessibleMem()),
576 errnoMem: convertModRefInfo(info: effects.getErrnoMem()),
577 targetMem0: convertModRefInfo(info: effects.getTargetMem0()),
578 targetMem1: convertModRefInfo(info: effects.getTargetMem1()));
579}
580
581static void convertCallEffects(mlir::Operation *callOp, bool isNothrow,
582 cir::MemoryEffectsAttr effects,
583 mlir::LLVM::MemoryEffectsAttr &memoryEffect,
584 bool &noUnwind, bool &willReturn,
585 bool &noReturn) {
586 memoryEffect = convertMemoryEffects(ctx: callOp->getContext(), effects);
587 // CIR spells two separate cannot-unwind facts, nothrow for a callee
588 // declared not to throw and nounwind for one whose effects rule out
589 // unwinding. LLVM has only nounwind, so either one sets it.
590 noUnwind = isNothrow || callOp->hasAttr(name: CIRDialect::getNoUnwindAttrName());
591 willReturn = callOp->hasAttr(name: CIRDialect::getWillReturnAttrName());
592 noReturn = callOp->hasAttr(name: CIRDialect::getNoReturnAttrName());
593}
594
595static mlir::LLVM::CallIntrinsicOp
596createCallLLVMIntrinsicOp(mlir::ConversionPatternRewriter &rewriter,
597 mlir::Location loc, const llvm::Twine &intrinsicName,
598 mlir::Type resultTy, mlir::ValueRange operands,
599 mlir::LLVM::FastmathFlags fastmathFlags = {}) {
600 auto intrinsicNameAttr =
601 mlir::StringAttr::get(context: rewriter.getContext(), bytes: intrinsicName);
602 auto fmfAttr =
603 mlir::LLVM::FastmathFlagsAttr::get(context: rewriter.getContext(), value: fastmathFlags);
604 // CallIntrinsicOp has distinct void / result create overloads. The FMF
605 // builders take a TypeRange for results.
606 if (resultTy)
607 return mlir::LLVM::CallIntrinsicOp::create(
608 builder&: rewriter, location: loc, resultTypes: mlir::TypeRange{resultTy}, intrin: intrinsicNameAttr, args: operands,
609 fastMathFlags: fmfAttr);
610 return mlir::LLVM::CallIntrinsicOp::create(builder&: rewriter, location: loc, intrin: intrinsicNameAttr,
611 args: operands, fastMathFlags: fmfAttr);
612}
613
614static mlir::LLVM::CallIntrinsicOp replaceOpWithCallLLVMIntrinsicOp(
615 mlir::ConversionPatternRewriter &rewriter, mlir::Operation *op,
616 const llvm::Twine &intrinsicName, mlir::Type resultTy,
617 mlir::ValueRange operands, mlir::LLVM::FastmathFlags fastmathFlags = {}) {
618 mlir::LLVM::CallIntrinsicOp callIntrinOp = createCallLLVMIntrinsicOp(
619 rewriter, loc: op->getLoc(), intrinsicName, resultTy, operands, fastmathFlags);
620 rewriter.replaceOp(op, newOp: callIntrinOp.getOperation());
621 return callIntrinOp;
622}
623
624static llvm::StringRef getConstrainedRoundingMetadata(cir::FenvAttr fenv) {
625 std::optional<cir::FPDynamicRoundingMode> rounding =
626 fenv.getDynamicRoundingMode();
627 if (!rounding)
628 return "round.tonearest";
629 switch (*rounding) {
630 case cir::FPDynamicRoundingMode::ToNearest:
631 return "round.tonearest";
632 case cir::FPDynamicRoundingMode::Downward:
633 return "round.downward";
634 case cir::FPDynamicRoundingMode::Upward:
635 return "round.upward";
636 case cir::FPDynamicRoundingMode::UpwardZero:
637 return "round.towardzero";
638 case cir::FPDynamicRoundingMode::ToNearestAway:
639 return "round.tonearestaway";
640 case cir::FPDynamicRoundingMode::Unknown:
641 return "round.dynamic";
642 }
643 llvm_unreachable("unknown FP dynamic rounding mode");
644}
645
646static llvm::StringRef getConstrainedExceptMetadata(cir::FenvAttr fenv) {
647 std::optional<cir::FPExceptionMode> exceptMode = fenv.getExceptMode();
648 if (exceptMode == cir::FPExceptionMode::Masked)
649 return "fpexcept.ignore";
650 mlir::BoolAttr strictExcept = fenv.getStrictExcept();
651 if (!strictExcept)
652 return "fpexcept.ignore";
653 return strictExcept.getValue() ? "fpexcept.strict" : "fpexcept.maytrap";
654}
655
656static mlir::Value
657createFenvMetadataValue(mlir::ConversionPatternRewriter &rewriter,
658 mlir::Location loc, llvm::StringRef str) {
659 auto mdString = mlir::LLVM::MDStringAttr::get(
660 context: rewriter.getContext(), value: mlir::StringAttr::get(context: rewriter.getContext(), bytes: str));
661 return mlir::LLVM::MetadataAsValueOp::create(builder&: rewriter, location: loc, metadata: mdString);
662}
663
664mlir::LogicalResult lowerToConstrainedFPIntrinsic(
665 mlir::Operation *op, mlir::ValueRange operands, cir::FenvAttr fenv,
666 mlir::Type llvmResTy, mlir::ConversionPatternRewriter &rewriter,
667 llvm::StringRef constrainedMnemonic, bool hasRoundingMode,
668 mlir::LLVM::FastmathFlags fastmathFlags) {
669 mlir::Location loc = op->getLoc();
670 llvm::SmallVector<mlir::Value> callOperands(operands.begin(), operands.end());
671 if (hasRoundingMode)
672 callOperands.push_back(Elt: createFenvMetadataValue(
673 rewriter, loc, str: getConstrainedRoundingMetadata(fenv)));
674 callOperands.push_back(Elt: createFenvMetadataValue(
675 rewriter, loc, str: getConstrainedExceptMetadata(fenv)));
676
677 replaceOpWithCallLLVMIntrinsicOp(
678 rewriter, op, intrinsicName: "llvm.experimental.constrained." + constrainedMnemonic,
679 resultTy: llvmResTy, operands: callOperands, fastmathFlags);
680 return mlir::success();
681}
682
683static mlir::LLVM::FastmathFlags
684convertFastMathFlags(cir::FastMathFlags cirFlags);
685
686static mlir::Value createConstrainedFPIntrinsicCall(
687 mlir::ConversionPatternRewriter &rewriter, mlir::Location loc,
688 mlir::ValueRange operands, mlir::Type llvmResTy, cir::FenvAttr fenv,
689 llvm::StringRef constrainedMnemonic, bool hasRoundingMode,
690 mlir::LLVM::FastmathFlags fastmathFlags = {}) {
691 llvm::SmallVector<mlir::Value> callOperands(operands.begin(), operands.end());
692 if (hasRoundingMode)
693 callOperands.push_back(Elt: createFenvMetadataValue(
694 rewriter, loc, str: getConstrainedRoundingMetadata(fenv)));
695
696 callOperands.push_back(Elt: createFenvMetadataValue(
697 rewriter, loc, str: getConstrainedExceptMetadata(fenv)));
698
699 mlir::LLVM::CallIntrinsicOp intrinsic = createCallLLVMIntrinsicOp(
700 rewriter, loc, intrinsicName: "llvm.experimental.constrained." + constrainedMnemonic,
701 resultTy: llvmResTy, operands: callOperands, fastmathFlags);
702 return intrinsic->getResult(idx: 0);
703}
704
705template <typename LLVMOp>
706mlir::LogicalResult lowerFPOp(mlir::Operation *op, mlir::ValueRange operands,
707 cir::FastMathFlagsAttr fastmath,
708 const mlir::TypeConverter &typeConverter,
709 mlir::ConversionPatternRewriter &rewriter) {
710 mlir::Type llvmResTy = typeConverter.convertType(t: op->getResultTypes()[0]);
711 if (!llvmResTy)
712 return op->emitError(message: "expected LLVM result type for floating-point op");
713
714 auto newOp = rewriter.replaceOpWithNewOp<LLVMOp>(
715 op, mlir::TypeRange{llvmResTy}, operands,
716 cir::getDefaultProperties<LLVMOp>(op->getContext()));
717 if (fastmath)
718 mlir::cast<mlir::LLVM::FastmathFlagsInterface>(newOp.getOperation())
719 .setFastmathAttr(mlir::LLVM::FastmathFlagsAttr::get(
720 context: rewriter.getContext(), value: convertFastMathFlags(cirFlags: fastmath.getValue())));
721 return mlir::success();
722}
723
724template <typename LLVMOp>
725mlir::LogicalResult lowerConstrainableFPOp(
726 mlir::Operation *op, mlir::ValueRange operands, cir::FenvAttr fenv,
727 cir::FastMathFlagsAttr fastmath, const mlir::TypeConverter &typeConverter,
728 mlir::ConversionPatternRewriter &rewriter,
729 llvm::StringRef constrainedMnemonic, bool hasRoundingMode) {
730 if (!fenv)
731 return lowerFPOp<LLVMOp>(op, operands, fastmath, typeConverter, rewriter);
732
733 mlir::Type llvmResTy = typeConverter.convertType(t: op->getResultTypes()[0]);
734 if (!llvmResTy)
735 return op->emitError(message: "expected LLVM result type for floating-point op");
736
737 mlir::LLVM::FastmathFlags fastmathFlags = {};
738 if (fastmath)
739 fastmathFlags = convertFastMathFlags(cirFlags: fastmath.getValue());
740
741 return lowerToConstrainedFPIntrinsic(op, operands, fenv, llvmResTy, rewriter,
742 constrainedMnemonic, hasRoundingMode,
743 fastmathFlags);
744}
745
746mlir::LLVM::FastmathFlagsAttr
747convertToLLVMProperty(mlir::MLIRContext *ctx,
748 std::optional<cir::FastMathFlags> flags) {
749 return mlir::LLVM::FastmathFlagsAttr::get(
750 context: ctx, value: flags ? convertFastMathFlags(cirFlags: *flags) : mlir::LLVM::FastmathFlags{});
751}
752
753mlir::IntegerAttr convertToLLVMProperty(mlir::MLIRContext *ctx,
754 cir::FPClassTest flags) {
755 return mlir::IntegerAttr::get(type: mlir::IntegerType::get(context: ctx, width: 32),
756 value: static_cast<uint32_t>(flags));
757}
758
759mlir::IntegerAttr convertToLLVMProperty(mlir::MLIRContext *ctx, bool flag) {
760 return mlir::IntegerAttr::get(type: mlir::IntegerType::get(context: ctx, width: 1), value: flag);
761}
762
763static mlir::LLVM::FastmathFlags
764convertFastMathFlags(cir::FastMathFlags cirFlags) {
765 mlir::LLVM::FastmathFlags llvmFlags{};
766 const std::pair<cir::FastMathFlags, mlir::LLVM::FastmathFlags> flags[] = {
767 {cir::FastMathFlags::nnan, mlir::LLVM::FastmathFlags::nnan},
768 {cir::FastMathFlags::ninf, mlir::LLVM::FastmathFlags::ninf},
769 {cir::FastMathFlags::nsz, mlir::LLVM::FastmathFlags::nsz},
770 {cir::FastMathFlags::arcp, mlir::LLVM::FastmathFlags::arcp},
771 {cir::FastMathFlags::contract, mlir::LLVM::FastmathFlags::contract},
772 {cir::FastMathFlags::afn, mlir::LLVM::FastmathFlags::afn},
773 {cir::FastMathFlags::reassoc, mlir::LLVM::FastmathFlags::reassoc},
774 };
775
776 for (auto [cirFlag, llvmFlag] : flags) {
777 if (bitEnumContainsAny(bits: cirFlags, bit: cirFlag))
778 llvmFlags = llvmFlags | llvmFlag;
779 }
780
781 return llvmFlags;
782}
783
784mlir::LogicalResult CIRToLLVMLLVMIntrinsicCallOpLowering::matchAndRewrite(
785 cir::LLVMIntrinsicCallOp op, OpAdaptor adaptor,
786 mlir::ConversionPatternRewriter &rewriter) const {
787 // Result is Optional on the op, so void intrinsics have zero
788 // results; leave llvmResTy null in that case.
789 mlir::Type llvmResTy;
790 if (op->getNumResults() != 0) {
791 llvmResTy = getTypeConverter()->convertType(t: op->getResultTypes()[0]);
792 if (!llvmResTy)
793 return op.emitError(message: "expected LLVM result type");
794 }
795 StringRef name = op.getIntrinsicName();
796 mlir::LLVM::FastmathFlags fastmathFlags = {};
797 if (std::optional<cir::FastMathFlags> fastmath = op.getFastmathFlags())
798 fastmathFlags = convertFastMathFlags(cirFlags: *fastmath);
799
800 // Some LLVM intrinsics require ElementType attribute to be attached to
801 // the argument of pointer type. That prevents us from generating LLVM IR
802 // because from LLVM dialect, we have LLVM IR like the below which fails
803 // LLVM IR verification.
804 // %3 = call i64 @llvm.aarch64.ldxr.p0(ptr %2)
805 // The expected LLVM IR should be like
806 // %3 = call i64 @llvm.aarch64.ldxr.p0(ptr elementtype(i32) %2)
807 // TODO(cir): MLIR LLVM dialect should handle this part as CIR has no way
808 // to set LLVM IR attribute.
809 assert(!cir::MissingFeatures::intrinsicElementTypeSupport());
810 replaceOpWithCallLLVMIntrinsicOp(rewriter, op, intrinsicName: "llvm." + name, resultTy: llvmResTy,
811 operands: adaptor.getOperands(), fastmathFlags);
812 return mlir::success();
813}
814
815/// BoolAttr visitor.
816mlir::Value CIRAttrToValue::visitCirAttr(cir::BoolAttr boolAttr) {
817 mlir::DataLayout layout(parentModule);
818 mlir::Value boolVal = mlir::LLVM::ConstantOp::create(
819 builder&: rewriter, location: parentLoc, type: converter->convertType(t: boolAttr.getType()),
820 value: boolAttr.getValue());
821 return emitToMemory(rewriter, dataLayout: layout, origType: boolAttr.getType(), value: boolVal);
822}
823
824/// IntAttr visitor.
825mlir::Value CIRAttrToValue::visitCirAttr(cir::IntAttr intAttr) {
826 mlir::DataLayout layout(parentModule);
827
828 if (auto intTy = mlir::dyn_cast<cir::IntType>(Val: intAttr.getType());
829 intTy && intTy.isBitInt()) {
830 mlir::Type biTy = convertTypeForMemory(converter: *converter, dataLayout: layout, type: intTy);
831 return mlir::LLVM::ConstantOp::create(
832 builder&: rewriter, location: parentLoc, res: biTy,
833 value: getBitIntStorageAttr(rewriter, attr: intAttr, dataLayout: layout));
834 }
835
836 // Materialize the value at its literal width, then widen to the in-memory
837 // storage type (a no-op except for _BitInt) so aggregate members built here
838 // match the iM struct/array fields produced by convertTypeForMemory.
839 mlir::Value val = mlir::LLVM::ConstantOp::create(
840 builder&: rewriter, location: parentLoc, type: converter->convertType(t: intAttr.getType()),
841 value: intAttr.getValue());
842 return emitToMemory(rewriter, dataLayout: layout, origType: intAttr.getType(), value: val);
843}
844
845/// FPAttr visitor.
846mlir::Value CIRAttrToValue::visitCirAttr(cir::FPAttr fltAttr) {
847 return mlir::LLVM::ConstantOp::create(
848 builder&: rewriter, location: parentLoc, type: converter->convertType(t: fltAttr.getType()),
849 value: fltAttr.getValue());
850}
851
852/// ConstComplexAttr visitor.
853mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstComplexAttr complexAttr) {
854 cir::ComplexType complexType = complexAttr.getType();
855 mlir::Type complexElemTy = complexType.getElementType();
856 mlir::Type complexElemLLVMTy = converter->convertType(t: complexElemTy);
857
858 mlir::Attribute components[2];
859 if (const auto intType = mlir::dyn_cast<cir::IntType>(Val&: complexElemTy)) {
860 components[0] = rewriter.getIntegerAttr(
861 type: complexElemLLVMTy,
862 value: mlir::cast<cir::IntAttr>(Val: complexAttr.getReal()).getValue());
863 components[1] = rewriter.getIntegerAttr(
864 type: complexElemLLVMTy,
865 value: mlir::cast<cir::IntAttr>(Val: complexAttr.getImag()).getValue());
866 } else {
867 components[0] = rewriter.getFloatAttr(
868 type: complexElemLLVMTy,
869 value: mlir::cast<cir::FPAttr>(Val: complexAttr.getReal()).getValue());
870 components[1] = rewriter.getFloatAttr(
871 type: complexElemLLVMTy,
872 value: mlir::cast<cir::FPAttr>(Val: complexAttr.getImag()).getValue());
873 }
874
875 return mlir::LLVM::ConstantOp::create(
876 builder&: rewriter, location: parentLoc, res: converter->convertType(t: complexAttr.getType()),
877 value: rewriter.getArrayAttr(value: components));
878}
879
880/// ConstPtrAttr visitor.
881mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstPtrAttr ptrAttr) {
882 if (ptrAttr.isNullValue()) {
883 return mlir::LLVM::ZeroOp::create(
884 builder&: rewriter, location: parentLoc, res: converter->convertType(t: ptrAttr.getType()));
885 }
886 mlir::DataLayout layout(parentModule);
887 mlir::Value ptrVal = mlir::LLVM::ConstantOp::create(
888 builder&: rewriter, location: parentLoc,
889 type: rewriter.getIntegerType(width: layout.getTypeSizeInBits(t: ptrAttr.getType())),
890 value: ptrAttr.getValue().getInt());
891 return mlir::LLVM::IntToPtrOp::create(
892 builder&: rewriter, location: parentLoc, resultType: converter->convertType(t: ptrAttr.getType()), operands: ptrVal);
893}
894
895/// BlockAddrInfoAttr visitor.
896mlir::Value CIRAttrToValue::visitCirAttr(cir::BlockAddrInfoAttr blockAddrInfo) {
897 assert(blockInfoAddr &&
898 "block address lowering requires LLVMBlockAddressInfo");
899 // A block address is lowered to an llvm.blockaddress op that references a
900 // block tag inside the target function. The matching block tag may not have
901 // been emitted yet, in which case the address is recorded as unresolved and
902 // patched up later in resolveBlockAddressOp.
903 mlir::LLVM::BlockTagOp matchLabel =
904 blockInfoAddr->lookupBlockTag(info: blockAddrInfo);
905 mlir::LLVM::BlockTagAttr tagAttr =
906 matchLabel ? matchLabel.getTag() : mlir::LLVM::BlockTagAttr{};
907 auto blkAddr = mlir::LLVM::BlockAddressAttr::get(
908 context: rewriter.getContext(), function: blockAddrInfo.getFunc(), tag: tagAttr);
909 auto blockAddressOp = mlir::LLVM::BlockAddressOp::create(
910 builder&: rewriter, location: parentLoc,
911 res: mlir::LLVM::LLVMPointerType::get(context: rewriter.getContext()), block_addr: blkAddr);
912 if (!matchLabel)
913 blockInfoAddr->addUnresolvedBlockAddress(op: blockAddressOp, info: blockAddrInfo);
914 return blockAddressOp;
915}
916
917/// BlockAddrDiffAttr visitor.
918mlir::Value CIRAttrToValue::visitCirAttr(cir::BlockAddrDiffAttr blockAddrDiff) {
919 assert(blockInfoAddr &&
920 "block address lowering requires LLVMBlockAddressInfo");
921 // A block-address difference initializer is lowered to the difference of the
922 // two block addresses: trunc(ptrtoint(lhs) - ptrtoint(rhs)). Just like a
923 // single block address, each referenced block tag may not have been emitted
924 // yet, in which case it is recorded as unresolved and patched up later in
925 // resolveBlockAddressOp.
926 mlir::DataLayout layout(parentModule);
927 mlir::MLIRContext *ctx = rewriter.getContext();
928 auto ptrTy = mlir::LLVM::LLVMPointerType::get(context: ctx);
929
930 auto emitBlockAddr = [&](mlir::StringAttr label) -> mlir::Value {
931 auto info = cir::BlockAddrInfoAttr::get(
932 context: ctx, func_name: blockAddrDiff.getFunc().getValue(), label_name: label.getValue());
933 mlir::LLVM::BlockTagOp matchLabel = blockInfoAddr->lookupBlockTag(info);
934 mlir::LLVM::BlockTagAttr tagAttr =
935 matchLabel ? matchLabel.getTag() : mlir::LLVM::BlockTagAttr{};
936 auto blkAddr = mlir::LLVM::BlockAddressAttr::get(
937 context: ctx, function: blockAddrDiff.getFunc(), tag: tagAttr);
938 auto addrOp =
939 mlir::LLVM::BlockAddressOp::create(builder&: rewriter, location: parentLoc, res: ptrTy, block_addr: blkAddr);
940 if (!matchLabel)
941 blockInfoAddr->addUnresolvedBlockAddress(op: addrOp, info);
942 return addrOp;
943 };
944
945 mlir::Value lhsAddr = emitBlockAddr(blockAddrDiff.getLhsLabel());
946 mlir::Value rhsAddr = emitBlockAddr(blockAddrDiff.getRhsLabel());
947
948 // Compute the difference in a pointer-sized integer, then truncate to the
949 // initializer's type. LLVM is sensitive about the exact format of the
950 // address-of-label difference, so the truncation must happen after the
951 // subtraction.
952 mlir::Type intptrTy =
953 rewriter.getIntegerType(width: layout.getTypeSizeInBits(t: ptrTy));
954 mlir::Value lhsInt =
955 mlir::LLVM::PtrToIntOp::create(builder&: rewriter, location: parentLoc, res: intptrTy, arg: lhsAddr);
956 mlir::Value rhsInt =
957 mlir::LLVM::PtrToIntOp::create(builder&: rewriter, location: parentLoc, res: intptrTy, arg: rhsAddr);
958 mlir::Value diffVal =
959 mlir::LLVM::SubOp::create(builder&: rewriter, location: parentLoc, lhs: lhsInt, rhs: rhsInt);
960
961 mlir::Type resultTy = converter->convertType(t: blockAddrDiff.getType());
962 mlir::Value result = diffVal;
963 if (resultTy != intptrTy)
964 result =
965 mlir::LLVM::TruncOp::create(builder&: rewriter, location: parentLoc, res: resultTy, arg: diffVal);
966 return result;
967}
968
969// ConstArrayAttr visitor
970mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstArrayAttr attr) {
971 mlir::Type convertedTy = converter->convertType(t: attr.getType());
972 mlir::DataLayout dataLayout(parentModule);
973 mlir::Type failedType = attr.getType();
974 mlir::Type llvmTy = adjustGlobalTypeForInit(llvmType: convertedTy, init: attr, converter: *converter,
975 dataLayout, failedType: &failedType);
976 if (!llvmTy) {
977 emitTypeConversionError(loc: parentLoc, type: failedType);
978 return {};
979 }
980 mlir::Value result;
981
982 // When the array can be represented as a single constant attribute, emit one
983 // llvm.mlir.constant instead of a chain of llvm.insertvalue ops. That works
984 // for any array type, including an array of split rows, but not for a split
985 // array itself: a struct-typed constant needs scalar fields.
986 if (mlir::isa<mlir::LLVM::LLVMArrayType>(Val: llvmTy))
987 if (std::optional<mlir::Attribute> constAttr = lowerConstArrayAttr(
988 constArr: attr, symbolTables, converter, preConversionTypes, moduleOp: parentModule))
989 return mlir::LLVM::ConstantOp::create(builder&: rewriter, location: parentLoc, res: llvmTy,
990 value: *constAttr);
991
992 if (attr.hasTrailingZeros())
993 result = mlir::LLVM::ZeroOp::create(builder&: rewriter, location: parentLoc, res: llvmTy);
994 else
995 result = mlir::LLVM::UndefOp::create(builder&: rewriter, location: parentLoc, res: llvmTy);
996
997 // Iteratively lower each constant element of the array.
998 if (auto arrayAttr = mlir::dyn_cast<mlir::ArrayAttr>(Val: attr.getElts())) {
999 // Fewer struct members than elements: adjustGlobalTypeForInit grouped the
1000 // elements of an array ending in many zeros into one leading array. That
1001 // array has no trailing zeros, so it keeps its type and still lowers as
1002 // one dense constant.
1003 //
1004 // FIXME: A split array itself is a struct, which llvm.mlir.constant cannot
1005 // hold with aggregate fields, so on this path each split array costs
1006 // insertvalue ops. With many split arrays that cannot be folded into an
1007 // enclosing constant, that may be slower than lowering them unsplit as
1008 // dense arrays; consider a heuristic choosing between the two.
1009 auto structTy = mlir::dyn_cast<mlir::LLVM::LLVMStructType>(Val&: llvmTy);
1010 if (structTy && structTy.getBody().size() < arrayAttr.size()) {
1011 auto arrayTy = mlir::cast<cir::ArrayType>(Val: attr.getType());
1012 auto prefix = cir::ConstArrayAttr::get(
1013 type: cir::ArrayType::get(elementType: arrayTy.getElementType(), size: arrayAttr.size()),
1014 elts: arrayAttr);
1015 mlir::Value init = visit(attr: prefix);
1016 if (!init)
1017 return {};
1018 return mlir::LLVM::InsertValueOp::create(
1019 builder&: rewriter, location: parentLoc, container: result, value: init, position: llvm::ArrayRef<int64_t>{0});
1020 }
1021 for (auto [idx, elt] : llvm::enumerate(First&: arrayAttr)) {
1022 mlir::Value init = visit(attr: elt);
1023 if (!init)
1024 return {};
1025 result = mlir::LLVM::InsertValueOp::create(builder&: rewriter, location: parentLoc, container: result,
1026 value: init, position: idx);
1027 }
1028 } else if (auto strAttr = mlir::dyn_cast<mlir::StringAttr>(Val: attr.getElts())) {
1029 // TODO(cir): this diverges from traditional lowering. Normally the string
1030 // would be a global constant that is memcopied.
1031 auto arrayTy = mlir::dyn_cast<cir::ArrayType>(Val: strAttr.getType());
1032 assert(arrayTy && "String attribute must have an array type");
1033 mlir::Type eltTy = arrayTy.getElementType();
1034 for (auto [idx, elt] : llvm::enumerate(First&: strAttr)) {
1035 auto init = mlir::LLVM::ConstantOp::create(
1036 builder&: rewriter, location: parentLoc, type: converter->convertType(t: eltTy), value: elt);
1037 result = mlir::LLVM::InsertValueOp::create(builder&: rewriter, location: parentLoc, container: result,
1038 value: init, position: idx);
1039 }
1040 } else {
1041 llvm_unreachable("unexpected ConstArrayAttr elements");
1042 }
1043
1044 return result;
1045}
1046
1047/// ConstRecord visitor.
1048mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstRecordAttr constRecord) {
1049 mlir::Type llvmTy = converter->convertType(t: constRecord.getType());
1050 mlir::DataLayout dataLayout(parentModule);
1051 llvm::SmallVector<unsigned> paddingAddedIndexes;
1052 mlir::Type failedType = constRecord.getType();
1053 llvmTy = adjustGlobalTypeForInit(llvmType: llvmTy, init: constRecord, converter: *converter, dataLayout,
1054 paddingAddedIndexes, failedType: &failedType);
1055 if (!llvmTy) {
1056 emitTypeConversionError(loc: parentLoc, type: failedType);
1057 return {};
1058 }
1059 mlir::Value result = mlir::LLVM::UndefOp::create(builder&: rewriter, location: parentLoc, res: llvmTy);
1060
1061 uint64_t insertIdx = 0;
1062 auto paddingItr = paddingAddedIndexes.begin();
1063
1064 // Iteratively lower each constant element of the record. A record member
1065 // that owns no bytes has no LLVM field and takes no element, so the elements
1066 // run one per field.
1067 for (auto [idx, elt] : llvm::enumerate(First: constRecord.getMembers())) {
1068 if (paddingItr != paddingAddedIndexes.end() && *paddingItr == idx) {
1069 ++insertIdx;
1070 ++paddingItr;
1071 }
1072
1073 mlir::Value init = visit(attr: elt);
1074 if (!init)
1075 return {};
1076 result = mlir::LLVM::InsertValueOp::create(builder&: rewriter, location: parentLoc, container: result,
1077 value: init, position: insertIdx);
1078 ++insertIdx;
1079 }
1080
1081 return result;
1082}
1083
1084/// ConstVectorAttr visitor.
1085mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstVectorAttr attr) {
1086 const mlir::Type llvmTy = converter->convertType(t: attr.getType());
1087
1088 SmallVector<mlir::Attribute> mlirValues;
1089 for (const mlir::Attribute elementAttr : attr.getElts()) {
1090 mlir::Attribute mlirAttr;
1091 if (auto intAttr = mlir::dyn_cast<cir::IntAttr>(Val: elementAttr)) {
1092 mlirAttr = rewriter.getIntegerAttr(
1093 type: converter->convertType(t: intAttr.getType()), value: intAttr.getValue());
1094 } else if (auto floatAttr = mlir::dyn_cast<cir::FPAttr>(Val: elementAttr)) {
1095 mlirAttr = rewriter.getFloatAttr(
1096 type: converter->convertType(t: floatAttr.getType()), value: floatAttr.getValue());
1097 } else if (auto boolAttr = mlir::dyn_cast<cir::BoolAttr>(Val: elementAttr)) {
1098 mlirAttr = rewriter.getBoolAttr(value: boolAttr.getValue());
1099 } else {
1100 llvm_unreachable("vector constant with an element that is neither an "
1101 "int, a float, or a bool");
1102 }
1103 mlirValues.push_back(Elt: mlirAttr);
1104 }
1105
1106 return mlir::LLVM::ConstantOp::create(
1107 builder&: rewriter, location: parentLoc, res: llvmTy,
1108 value: mlir::DenseElementsAttr::get(type: mlir::cast<mlir::ShapedType>(Val: llvmTy),
1109 values: mlirValues));
1110}
1111
1112// GlobalViewAttr / GlobalOffsetAttr visitors.
1113CIRAttrToValue::LoweredGlobalSymbol
1114CIRAttrToValue::getAddressOfSymbol(mlir::FlatSymbolRefAttr symbol) {
1115 cir::CIRDataLayout cirDataLayout(parentModule);
1116 const mlir::DataLayout &dataLayout = cirDataLayout.layout;
1117 mlir::Type sourceType;
1118 mlir::Type originalType;
1119 unsigned addrSpace = 0;
1120 llvm::StringRef symName;
1121 mlir::Operation *sourceSymbol =
1122 symbolTables.lookupSymbolIn(symbolTableOp: parentModule, name: symbol);
1123 if (auto llvmSymbol = dyn_cast<mlir::LLVM::GlobalOp>(Val: sourceSymbol)) {
1124 if (mlir::Type preType =
1125 preConversionTypes.lookup(Key: llvmSymbol.getSymName())) {
1126 originalType = preType;
1127 sourceType = convertTypeForMemory(converter: *converter, dataLayout, type: originalType);
1128 } else {
1129 sourceType = llvmSymbol.getType();
1130 }
1131 symName = llvmSymbol.getSymName();
1132 addrSpace = llvmSymbol.getAddrSpace();
1133 } else if (auto cirSymbol = dyn_cast<cir::GlobalOp>(Val: sourceSymbol)) {
1134 originalType = cirSymbol.getSymType();
1135 sourceType = convertTypeForMemory(converter: *converter, dataLayout, type: originalType);
1136 symName = cirSymbol.getSymName();
1137 if (auto targetAS = mlir::dyn_cast_if_present<cir::TargetAddressSpaceAttr>(
1138 Val: cirSymbol.getAddrSpaceAttr()))
1139 addrSpace = targetAS.getValue();
1140 } else if (auto llvmFun = dyn_cast<mlir::LLVM::LLVMFuncOp>(Val: sourceSymbol)) {
1141 if (mlir::Type preType = preConversionTypes.lookup(Key: llvmFun.getSymName())) {
1142 originalType = preType;
1143 sourceType = converter->convertType(t: originalType);
1144 } else {
1145 sourceType = llvmFun.getFunctionType();
1146 }
1147 symName = llvmFun.getSymName();
1148 addrSpace = cirDataLayout.getProgramAddressSpace();
1149 } else if (auto fun = dyn_cast<cir::FuncOp>(Val: sourceSymbol)) {
1150 originalType = fun.getFunctionType();
1151 sourceType = converter->convertType(t: originalType);
1152 symName = fun.getSymName();
1153 addrSpace = cirDataLayout.getProgramAddressSpace();
1154 } else if (auto alias = dyn_cast<mlir::LLVM::AliasOp>(Val: sourceSymbol)) {
1155 if (mlir::Type preType = preConversionTypes.lookup(Key: alias.getSymName())) {
1156 originalType = preType;
1157 if (mlir::isa<cir::FuncType>(Val: preType))
1158 sourceType = converter->convertType(t: originalType);
1159 else
1160 sourceType = convertTypeForMemory(converter: *converter, dataLayout, type: originalType);
1161 } else {
1162 sourceType = alias.getType();
1163 }
1164 symName = alias.getSymName();
1165 addrSpace = alias.getAddrSpace();
1166 } else {
1167 llvm_unreachable("Unexpected GlobalOp type");
1168 }
1169
1170 mlir::Value addr = mlir::LLVM::AddressOfOp::create(
1171 builder&: rewriter, location: parentLoc,
1172 res: mlir::LLVM::LLVMPointerType::get(context: rewriter.getContext(), addressSpace: addrSpace),
1173 global_name: symName);
1174 return {.addr: addr, .sourceType: sourceType, .originalType: originalType, .addrSpace: addrSpace};
1175}
1176
1177mlir::Value CIRAttrToValue::castGlobalAddrToType(mlir::Value addr,
1178 mlir::Type attrType,
1179 mlir::Type sourceType,
1180 unsigned addrSpace) {
1181
1182 // We can have a global view with an integer type in the case of method
1183 // pointers. With the Itanium ABI, the #cir.method attribute is lowered to a
1184 // #cir.global_view with a pointer-sized integer representing the address of
1185 // the method.
1186 if (mlir::isa<cir::IntType>(Val: attrType)) {
1187 mlir::Type llvmDstTy = converter->convertType(t: attrType);
1188 return mlir::LLVM::PtrToIntOp::create(builder&: rewriter, location: parentLoc, res: llvmDstTy, arg: addr);
1189 }
1190
1191 if (auto ptrTy = mlir::dyn_cast<cir::PointerType>(Val&: attrType)) {
1192 auto llvmDstTy = converter->convertType<mlir::LLVM::LLVMPointerType>(t: ptrTy);
1193
1194 if (llvmDstTy.getAddressSpace() != addrSpace)
1195 addr = mlir::LLVM::AddrSpaceCastOp::create(builder&: rewriter, location: parentLoc, res: llvmDstTy,
1196 arg: addr);
1197
1198 mlir::DataLayout dataLayout(parentModule);
1199 mlir::Type llvmEltTy =
1200 convertTypeForMemory(converter: *converter, dataLayout, type: ptrTy.getPointee());
1201
1202 // No further cast needed if the pointee type already matches.
1203 if (llvmEltTy == sourceType)
1204 return addr;
1205
1206 // With opaque pointers, the pointer type is already correct (either from
1207 // the original AddressOfOp or after an addrspacecast) — skip the
1208 // redundant bitcast.
1209 if (addr.getType() == llvmDstTy)
1210 return addr;
1211
1212 return mlir::LLVM::BitcastOp::create(builder&: rewriter, location: parentLoc, res: llvmDstTy, arg: addr);
1213 }
1214
1215 if (mlir::isa<cir::VPtrType>(Val: attrType))
1216 return addr;
1217
1218 llvm_unreachable("Expecting pointer or integer type");
1219}
1220
1221mlir::Value CIRAttrToValue::visitCirAttr(cir::GlobalViewAttr globalAttr) {
1222 LoweredGlobalSymbol lowered = getAddressOfSymbol(symbol: globalAttr.getSymbol());
1223 mlir::Value addrOp = lowered.addr;
1224 mlir::Type sourceType = lowered.sourceType;
1225
1226 if (globalAttr.getIndices()) {
1227 if (!sourceType) {
1228 emitTypeConversionError(loc: parentLoc, type: lowered.originalType)
1229 << " for symbol " << globalAttr.getSymbol();
1230 return {};
1231 }
1232 llvm::SmallVector<mlir::LLVM::GEPArg> indices;
1233
1234 if (mlir::isa<mlir::LLVM::LLVMArrayType, mlir::LLVM::LLVMStructType>(
1235 Val: sourceType))
1236 indices.push_back(Elt: 0);
1237
1238 for (mlir::Attribute idx : globalAttr.getIndices()) {
1239 auto intAttr = mlir::cast<mlir::IntegerAttr>(Val&: idx);
1240 indices.push_back(Elt: intAttr.getValue().getSExtValue());
1241 }
1242 mlir::Type resTy = addrOp.getType();
1243 mlir::Type eltTy = converter->convertType(t: sourceType);
1244 auto gep =
1245 mlir::LLVM::GEPOp::create(builder&: rewriter, location: parentLoc, resultType: resTy, elementType: eltTy, basePtr: addrOp,
1246 indices, noWrapFlags: mlir::LLVM::GEPNoWrapFlags::none);
1247 if (globalAttr.getAddressPoint()) {
1248 assert(!cir::MissingFeatures::vtableRelativeLayout());
1249 if (usesRelativeVTables(mod: parentModule)) {
1250 mlir::emitError(loc: parentLoc)
1251 << "NYI: address point of a relative layout vtable";
1252 return {};
1253 }
1254
1255 // Like classic codegen, a vtable address point is inbounds, and only the
1256 // one vtable of the group it points into can be accessed through it.
1257 auto indices =
1258 globalAttr.getIndices().getAsValueRange<mlir::IntegerAttr>();
1259 assert(llvm::range_size(indices) == 2 &&
1260 "address point takes a vtable index and a slot index");
1261 auto vtableTy = mlir::cast<mlir::LLVM::LLVMArrayType>(
1262 Val: mlir::cast<mlir::LLVM::LLVMStructType>(Val&: eltTy)
1263 .getBody()[(*indices.begin()).getZExtValue()]);
1264 mlir::DataLayout layout(parentModule);
1265 int64_t slotSize = layout.getTypeSize(t: vtableTy.getElementType());
1266 int64_t offset = (*std::next(x: indices.begin())).getSExtValue() * slotSize;
1267 int64_t size = vtableTy.getNumElements() * slotSize;
1268 unsigned indexBits = *layout.getTypeIndexBitwidth(
1269 t: mlir::cast<mlir::LLVM::LLVMPointerType>(Val: addrOp.getType()));
1270 gep.setNoWrapFlags(mlir::LLVM::GEPNoWrapFlags::inbounds);
1271
1272 gep.setInrangeAttr(mlir::LLVM::ConstantRangeAttr::get(
1273 context: rewriter.getContext(),
1274 lower: llvm::APInt(indexBits, -offset, /*isSigned=*/true),
1275 upper: llvm::APInt(indexBits, size - offset, /*isSigned=*/true)));
1276 }
1277 addrOp = gep;
1278 }
1279
1280 return castGlobalAddrToType(addr: addrOp, attrType: globalAttr.getType(), sourceType,
1281 addrSpace: lowered.addrSpace);
1282}
1283
1284mlir::Value CIRAttrToValue::visitCirAttr(cir::GlobalOffsetAttr globalAttr) {
1285 LoweredGlobalSymbol lowered = getAddressOfSymbol(symbol: globalAttr.getSymbol());
1286 mlir::Value addrOp = lowered.addr;
1287
1288 // A displacement in bytes is a GEP through i8, which is what classic
1289 // codegen emits for the offset of any constant lvalue.
1290 mlir::Value offset = mlir::LLVM::ConstantOp::create(
1291 builder&: rewriter, location: parentLoc, res: rewriter.getI64Type(),
1292 value: rewriter.getI64IntegerAttr(value: globalAttr.getOffset()));
1293 addrOp = mlir::LLVM::GEPOp::create(
1294 builder&: rewriter, location: parentLoc, resultType: addrOp.getType(), elementType: rewriter.getI8Type(), basePtr: addrOp,
1295 indices: mlir::ValueRange{offset}, noWrapFlags: mlir::LLVM::GEPNoWrapFlags::none);
1296
1297 return castGlobalAddrToType(addr: addrOp, attrType: globalAttr.getType(), sourceType: lowered.sourceType,
1298 addrSpace: lowered.addrSpace);
1299}
1300
1301// TypeInfoAttr visitor.
1302mlir::Value CIRAttrToValue::visitCirAttr(cir::TypeInfoAttr typeInfoAttr) {
1303 mlir::Type llvmTy = converter->convertType(t: typeInfoAttr.getType());
1304 mlir::Value result = mlir::LLVM::UndefOp::create(builder&: rewriter, location: parentLoc, res: llvmTy);
1305
1306 for (auto [idx, elt] : llvm::enumerate(First: typeInfoAttr.getData())) {
1307 mlir::Value init = visit(attr: elt);
1308 if (!init)
1309 return {};
1310 result = mlir::LLVM::InsertValueOp::create(builder&: rewriter, location: parentLoc, container: result,
1311 value: init, position: idx);
1312 }
1313
1314 return result;
1315}
1316
1317/// UndefAttr visitor.
1318mlir::Value CIRAttrToValue::visitCirAttr(cir::UndefAttr undefAttr) {
1319 mlir::DataLayout layout(parentModule);
1320 return mlir::LLVM::UndefOp::create(
1321 builder&: rewriter, location: parentLoc,
1322 res: convertTypeForMemory(converter: *converter, dataLayout: layout, type: undefAttr.getType()));
1323}
1324
1325/// PoisonAttr visitor.
1326mlir::Value CIRAttrToValue::visitCirAttr(cir::PoisonAttr poisonAttr) {
1327 mlir::DataLayout layout(parentModule);
1328 return mlir::LLVM::PoisonOp::create(
1329 builder&: rewriter, location: parentLoc,
1330 res: convertTypeForMemory(converter: *converter, dataLayout: layout, type: poisonAttr.getType()));
1331}
1332
1333// VTableAttr visitor.
1334mlir::Value CIRAttrToValue::visitCirAttr(cir::VTableAttr vtableArr) {
1335 assert(!cir::MissingFeatures::vtableRelativeLayout());
1336 if (usesRelativeVTables(mod: parentModule)) {
1337 mlir::emitError(loc: parentLoc)
1338 << "NYI: initializer of a relative layout vtable";
1339 return {};
1340 }
1341
1342 mlir::Type llvmTy = converter->convertType(t: vtableArr.getType());
1343 mlir::Value result = mlir::LLVM::UndefOp::create(builder&: rewriter, location: parentLoc, res: llvmTy);
1344
1345 for (auto [idx, elt] : llvm::enumerate(First: vtableArr.getData())) {
1346 mlir::Value init = visit(attr: elt);
1347 if (!init)
1348 return {};
1349 result = mlir::LLVM::InsertValueOp::create(builder&: rewriter, location: parentLoc, container: result,
1350 value: init, position: idx);
1351 }
1352
1353 return result;
1354}
1355
1356/// ZeroAttr visitor.
1357mlir::Value CIRAttrToValue::visitCirAttr(cir::ZeroAttr attr) {
1358 mlir::DataLayout layout(parentModule);
1359 return mlir::LLVM::ZeroOp::create(
1360 builder&: rewriter, location: parentLoc,
1361 res: convertTypeForMemory(converter: *converter, dataLayout: layout, type: attr.getType()));
1362}
1363
1364// This class handles rewriting initializer attributes for types that do not
1365// require region initialization.
1366class GlobalInitAttrRewriter {
1367public:
1368 GlobalInitAttrRewriter(mlir::Type type, mlir::DataLayout const &dataLayout,
1369 mlir::ConversionPatternRewriter &rewriter)
1370 : llvmType(type), dataLayout(dataLayout), rewriter(rewriter) {}
1371
1372 mlir::Attribute visit(mlir::Attribute attr) {
1373 return llvm::TypeSwitch<mlir::Attribute, mlir::Attribute>(attr)
1374 .Case<cir::IntAttr, cir::FPAttr, cir::BoolAttr>(
1375 caseFn: [&](auto attrT) { return visitCirAttr(attrT); })
1376 .Default(defaultFn: [&](auto attrT) { return mlir::Attribute(); });
1377 }
1378
1379 mlir::Attribute visitCirAttr(cir::IntAttr attr) {
1380 // A split-storage _BitInt's memory representation is a byte array (see
1381 // convertTypeForMemory), not a scalar, so its constant must be built as
1382 // raw bytes rather than a single sign/zero-extended IntegerAttr.
1383 if (auto intTy = mlir::dyn_cast<cir::IntType>(Val: attr.getType());
1384 intTy && intTy.isBitInt())
1385 return getBitIntStorageAttr(rewriter, attr, dataLayout);
1386
1387 return rewriter.getIntegerAttr(type: llvmType, value: attr.getValue());
1388 }
1389
1390 mlir::Attribute visitCirAttr(cir::FPAttr attr) {
1391 return rewriter.getFloatAttr(type: llvmType, value: attr.getValue());
1392 }
1393
1394 mlir::Attribute visitCirAttr(cir::BoolAttr attr) {
1395 return rewriter.getBoolAttr(value: attr.getValue());
1396 }
1397
1398private:
1399 mlir::Type llvmType;
1400 const mlir::DataLayout &dataLayout;
1401 mlir::ConversionPatternRewriter &rewriter;
1402};
1403
1404// This pass requires the CIR to be in a "flat" state. All blocks in each
1405// function must belong to the parent region. Once scopes and control flow
1406// are implemented in CIR, a pass will be run before this one to flatten
1407// the CIR and get it into the state that this pass requires.
1408struct ConvertCIRToLLVMPass
1409 : public mlir::PassWrapper<ConvertCIRToLLVMPass,
1410 mlir::OperationPass<mlir::ModuleOp>> {
1411 void getDependentDialects(mlir::DialectRegistry &registry) const override {
1412 registry.insert<mlir::BuiltinDialect, mlir::DLTIDialect,
1413 mlir::LLVM::LLVMDialect, mlir::func::FuncDialect>();
1414 }
1415 void runOnOperation() final;
1416
1417 void processCIRAttrs(mlir::ModuleOp module);
1418
1419 void resolveBlockAddressOp(LLVMBlockAddressInfo &blockInfoAddr);
1420
1421 /// Collect (symbol_name, annotations, loc) from cir.func and cir.global ops
1422 /// before the conversion runs (the annotations attribute is dropped during
1423 /// FuncOp/GlobalOp lowering).
1424 void collectGlobalAnnotations(mlir::ModuleOp module);
1425
1426 /// Emit @llvm.global.annotations and supporting string/args constants from
1427 /// the previously-collected annotations. Mirrors what OGCG produces.
1428 void buildGlobalAnnotationsVar(mlir::ModuleOp module,
1429 LLVMAnnotationGlobals &annotationGlobals);
1430
1431 StringRef getDescription() const override {
1432 return "Convert the prepared CIR dialect module to LLVM dialect";
1433 }
1434
1435 StringRef getArgument() const override { return "cir-flat-to-llvm"; }
1436
1437private:
1438 /// One annotation entry collected pre-conversion.
1439 struct CollectedAnnotation {
1440 mlir::StringAttr symName;
1441 cir::AnnotationAttr annotation;
1442 mlir::Location loc;
1443 CollectedAnnotation(mlir::StringAttr symName,
1444 cir::AnnotationAttr annotation, mlir::Location loc)
1445 : symName(symName), annotation(annotation), loc(loc) {}
1446 };
1447 llvm::SmallVector<CollectedAnnotation> collectedAnnotations;
1448};
1449
1450mlir::LogicalResult CIRToLLVMSignBitOpLowering::matchAndRewrite(
1451 cir::SignBitOp op, OpAdaptor adaptor,
1452 mlir::ConversionPatternRewriter &rewriter) const {
1453 assert(!cir::MissingFeatures::isPPC_FP128Ty());
1454
1455 mlir::DataLayout layout(op->getParentOfType<mlir::ModuleOp>());
1456 int width = layout.getTypeSizeInBits(t: op.getInput().getType());
1457 if (auto longDoubleType =
1458 mlir::dyn_cast<cir::LongDoubleType>(Val: op.getInput().getType())) {
1459 if (mlir::isa<cir::FP80Type>(Val: longDoubleType.getUnderlying())) {
1460 // If the underlying type of LongDouble is FP80Type,
1461 // DataLayout::getTypeSizeInBits returns 128.
1462 // See https://github.com/llvm/clangir/issues/1057.
1463 // Set the width to 80 manually.
1464 width = 80;
1465 }
1466 }
1467 mlir::Type intTy = mlir::IntegerType::get(context: rewriter.getContext(), width);
1468 auto bitcast = mlir::LLVM::BitcastOp::create(builder&: rewriter, location: op->getLoc(), res: intTy,
1469 arg: adaptor.getInput());
1470
1471 auto zero = mlir::LLVM::ConstantOp::create(builder&: rewriter, location: op->getLoc(), type: intTy, value: 0);
1472 auto cmpResult = mlir::LLVM::ICmpOp::create(builder&: rewriter, location: op.getLoc(),
1473 predicate: mlir::LLVM::ICmpPredicate::slt,
1474 lhs: bitcast.getResult(), rhs: zero);
1475 rewriter.replaceOp(op, newOp: cmpResult);
1476 return mlir::success();
1477}
1478
1479mlir::LogicalResult CIRToLLVMAssumeOpLowering::matchAndRewrite(
1480 cir::AssumeOp op, OpAdaptor adaptor,
1481 mlir::ConversionPatternRewriter &rewriter) const {
1482 mlir::Value cond = adaptor.getPredicate();
1483 if (op.getBundleKind() == cir::AssumeBundleKind::None) {
1484 rewriter.replaceOpWithNewOp<mlir::LLVM::AssumeOp>(op, args&: cond);
1485 return mlir::success();
1486 }
1487
1488 llvm::StringRef tag = cir::stringifyAssumeBundleKind(op.getBundleKind());
1489 rewriter.replaceOpWithNewOp<mlir::LLVM::AssumeOp>(op, args&: cond, args&: tag,
1490 args: adaptor.getBundleArgs());
1491 return mlir::success();
1492}
1493
1494static mlir::LLVM::AtomicOrdering
1495getLLVMMemOrder(std::optional<cir::MemOrder> memorder) {
1496 if (!memorder)
1497 return mlir::LLVM::AtomicOrdering::not_atomic;
1498 switch (*memorder) {
1499 case cir::MemOrder::Relaxed:
1500 return mlir::LLVM::AtomicOrdering::monotonic;
1501 case cir::MemOrder::Consume:
1502 case cir::MemOrder::Acquire:
1503 return mlir::LLVM::AtomicOrdering::acquire;
1504 case cir::MemOrder::Release:
1505 return mlir::LLVM::AtomicOrdering::release;
1506 case cir::MemOrder::AcquireRelease:
1507 return mlir::LLVM::AtomicOrdering::acq_rel;
1508 case cir::MemOrder::SequentiallyConsistent:
1509 return mlir::LLVM::AtomicOrdering::seq_cst;
1510 }
1511 llvm_unreachable("unknown memory order");
1512}
1513
1514static llvm::AtomicScope getAtomicScope(cir::SyncScopeKind syncScope) {
1515 switch (syncScope) {
1516 case cir::SyncScopeKind::SingleThread:
1517 case cir::SyncScopeKind::HIPSingleThread:
1518 return llvm::AtomicScope::Single;
1519 case cir::SyncScopeKind::Wavefront:
1520 case cir::SyncScopeKind::HIPWavefront:
1521 case cir::SyncScopeKind::OpenCLSubGroup:
1522 return llvm::AtomicScope::Wavefront;
1523 case cir::SyncScopeKind::Workgroup:
1524 case cir::SyncScopeKind::HIPWorkgroup:
1525 case cir::SyncScopeKind::OpenCLWorkGroup:
1526 return llvm::AtomicScope::Workgroup;
1527 case cir::SyncScopeKind::Cluster:
1528 case cir::SyncScopeKind::HIPCluster:
1529 return llvm::AtomicScope::Cluster;
1530 case cir::SyncScopeKind::Device:
1531 case cir::SyncScopeKind::HIPAgent:
1532 case cir::SyncScopeKind::OpenCLDevice:
1533 return llvm::AtomicScope::Device;
1534 case cir::SyncScopeKind::System:
1535 case cir::SyncScopeKind::HIPSystem:
1536 case cir::SyncScopeKind::OpenCLAllSVMDevices:
1537 return llvm::AtomicScope::System;
1538 }
1539 llvm_unreachable("unknown sync scope");
1540}
1541
1542static llvm::StringRef getLLVMSyncScope(cir::SyncScopeKind syncScope,
1543 mlir::ModuleOp moduleOp) {
1544 llvm::AtomicScope scope = getAtomicScope(syncScope);
1545 if (auto triple = moduleOp->getAttrOfType<mlir::StringAttr>(
1546 name: cir::CIRDialect::getTripleAttrName()))
1547 if (std::optional<llvm::StringRef> name = llvm::getAtomicScopeIRString(
1548 T: llvm::Triple(triple.getValue()), S: scope))
1549 return *name;
1550 return scope == llvm::AtomicScope::Single ? "singlethread" : "";
1551}
1552
1553static std::optional<llvm::StringRef>
1554getLLVMSyncScope(std::optional<cir::SyncScopeKind> syncScope,
1555 mlir::ModuleOp moduleOp) {
1556 if (syncScope.has_value())
1557 return getLLVMSyncScope(syncScope: *syncScope, moduleOp);
1558 return std::nullopt;
1559}
1560
1561static void forwardAMDGPUNoPrivateAddrSpace(mlir::Operation *op,
1562 mlir::Operation *llvmOp,
1563 mlir::Value llvmPtr) {
1564 llvm::StringRef name = cir::CIRDialect::getAMDGPUNoPrivateAddrSpaceAttrName();
1565 auto ptrTy = mlir::cast<mlir::LLVM::LLVMPointerType>(Val: llvmPtr.getType());
1566 mlir::Attribute a = op->getAttr(name);
1567 if (a && ptrTy.getAddressSpace() == llvm::AMDGPUAS::FLAT_ADDRESS)
1568 llvmOp->setAttr(name, value: a);
1569}
1570
1571// CIRGen decides the metadata for a C++/HIP atomic from the atomic options
1572// in effect, so those markers are simply carried across.
1573static void forwardAMDGPUAtomicOptionMarkers(mlir::Operation *op,
1574 mlir::Operation *llvmOp) {
1575 for (llvm::StringRef marker :
1576 {cir::CIRDialect::getAMDGPUNoFineGrainedMemoryAttrName(),
1577 cir::CIRDialect::getAMDGPUNoRemoteMemoryAttrName(),
1578 cir::CIRDialect::getAMDGPUIgnoreDenormalModeAttrName()})
1579 if (mlir::Attribute a = op->getAttr(name: marker))
1580 llvmOp->setAttr(name: marker, value: a);
1581}
1582
1583mlir::LogicalResult CIRToLLVMAtomicCmpXchgOpLowering::matchAndRewrite(
1584 cir::AtomicCmpXchgOp op, OpAdaptor adaptor,
1585 mlir::ConversionPatternRewriter &rewriter) const {
1586 mlir::Value expected = adaptor.getExpected();
1587 mlir::Value desired = adaptor.getDesired();
1588
1589 auto cmpxchg = mlir::LLVM::AtomicCmpXchgOp::create(
1590 builder&: rewriter, location: op.getLoc(), ptr: adaptor.getPtr(), cmp: expected, val: desired,
1591 successOrdering: getLLVMMemOrder(memorder: adaptor.getSuccOrder()),
1592 failureOrdering: getLLVMMemOrder(memorder: adaptor.getFailOrder()),
1593 syncscope: getLLVMSyncScope(syncScope: op.getSyncScope(),
1594 moduleOp: op->getParentOfType<mlir::ModuleOp>()));
1595
1596 cmpxchg.setAlignment(adaptor.getAlignment());
1597 cmpxchg.setWeak(adaptor.getWeak());
1598 cmpxchg.setVolatile_(adaptor.getIsVolatile());
1599 forwardAMDGPUNoPrivateAddrSpace(op, llvmOp: cmpxchg, llvmPtr: adaptor.getPtr());
1600
1601 // Check result and apply stores accordingly.
1602 auto old = mlir::LLVM::ExtractValueOp::create(builder&: rewriter, location: op.getLoc(),
1603 container: cmpxchg.getResult(), position: 0);
1604 auto cmp = mlir::LLVM::ExtractValueOp::create(builder&: rewriter, location: op.getLoc(),
1605 container: cmpxchg.getResult(), position: 1);
1606
1607 rewriter.replaceOp(op, newValues: {old, cmp});
1608 return mlir::success();
1609}
1610
1611mlir::LogicalResult CIRToLLVMAtomicXchgOpLowering::matchAndRewrite(
1612 cir::AtomicXchgOp op, OpAdaptor adaptor,
1613 mlir::ConversionPatternRewriter &rewriter) const {
1614 assert(!cir::MissingFeatures::atomicSyncScopeID());
1615 mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(memorder: adaptor.getMemOrder());
1616 llvm::StringRef llvmSyncScope = getLLVMSyncScope(
1617 syncScope: op.getSyncScope(), moduleOp: op->getParentOfType<mlir::ModuleOp>());
1618 // Create the new op first: the forward helpers read `op`, which the
1619 // replace below erases immediately.
1620 auto xchg = mlir::LLVM::AtomicRMWOp::create(
1621 builder&: rewriter, location: op.getLoc(), binOp: mlir::LLVM::AtomicBinOp::xchg, ptr: adaptor.getPtr(),
1622 val: adaptor.getVal(), ordering: llvmOrder, syncscope: llvmSyncScope, /*alignment=*/0,
1623 isVolatile: op.getIsVolatile());
1624 forwardAMDGPUAtomicOptionMarkers(op, llvmOp: xchg);
1625 forwardAMDGPUNoPrivateAddrSpace(op, llvmOp: xchg, llvmPtr: adaptor.getPtr());
1626 rewriter.replaceOp(op, newOp: xchg);
1627 return mlir::success();
1628}
1629
1630mlir::LogicalResult CIRToLLVMAtomicTestAndSetOpLowering::matchAndRewrite(
1631 cir::AtomicTestAndSetOp op, OpAdaptor adaptor,
1632 mlir::ConversionPatternRewriter &rewriter) const {
1633 assert(!cir::MissingFeatures::atomicSyncScopeID());
1634
1635 mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(memorder: op.getMemOrder());
1636
1637 auto one = mlir::LLVM::ConstantOp::create(builder&: rewriter, location: op.getLoc(),
1638 type: rewriter.getI8Type(), value: 1);
1639 auto rmw = mlir::LLVM::AtomicRMWOp::create(
1640 builder&: rewriter, location: op.getLoc(), binOp: mlir::LLVM::AtomicBinOp::xchg, ptr: adaptor.getPtr(),
1641 val: one, ordering: llvmOrder, /*syncscope=*/llvm::StringRef(),
1642 alignment: adaptor.getAlignment().value_or(u: 0), isVolatile: op.getIsVolatile());
1643
1644 auto zero = mlir::LLVM::ConstantOp::create(builder&: rewriter, location: op.getLoc(),
1645 type: rewriter.getI8Type(), value: 0);
1646 auto cmp = mlir::LLVM::ICmpOp::create(
1647 builder&: rewriter, location: op.getLoc(), predicate: mlir::LLVM::ICmpPredicate::ne, lhs: rmw, rhs: zero);
1648
1649 rewriter.replaceOp(op, newOp: cmp);
1650 return mlir::success();
1651}
1652
1653mlir::LogicalResult CIRToLLVMAtomicClearOpLowering::matchAndRewrite(
1654 cir::AtomicClearOp op, OpAdaptor adaptor,
1655 mlir::ConversionPatternRewriter &rewriter) const {
1656 assert(!cir::MissingFeatures::atomicSyncScopeID());
1657
1658 mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(memorder: op.getMemOrder());
1659 auto zero = mlir::LLVM::ConstantOp::create(builder&: rewriter, location: op.getLoc(),
1660 type: rewriter.getI8Type(), value: 0);
1661 auto store = mlir::LLVM::StoreOp::create(
1662 builder&: rewriter, location: op.getLoc(), value: zero, addr: adaptor.getPtr(),
1663 alignment: adaptor.getAlignment().value_or(u: 0), isVolatile: op.getIsVolatile(),
1664 /*isNonTemporal=*/false, /*isInvariantGroup=*/false, ordering: llvmOrder);
1665
1666 rewriter.replaceOp(op, newOp: store);
1667 return mlir::success();
1668}
1669
1670mlir::LogicalResult CIRToLLVMAtomicFenceOpLowering::matchAndRewrite(
1671 cir::AtomicFenceOp op, OpAdaptor adaptor,
1672 mlir::ConversionPatternRewriter &rewriter) const {
1673 mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(memorder: adaptor.getOrdering());
1674
1675 auto fence = mlir::LLVM::FenceOp::create(builder&: rewriter, location: op.getLoc(), ordering: llvmOrder);
1676 fence.setSyncscope(getLLVMSyncScope(syncScope: op.getSyncscope(),
1677 moduleOp: op->getParentOfType<mlir::ModuleOp>()));
1678 if (mlir::Attribute mmra =
1679 op->getAttr(name: mlir::LLVM::LLVMDialect::getMmraAttrName()))
1680 fence->setAttr(name: mlir::LLVM::LLVMDialect::getMmraAttrName(), value: mmra);
1681
1682 rewriter.replaceOp(op, newOp: fence);
1683
1684 return mlir::success();
1685}
1686
1687static mlir::LLVM::AtomicBinOp
1688getLLVMAtomicBinOp(cir::AtomicFetchKind k, bool isInt, bool isSignedInt) {
1689 switch (k) {
1690 case cir::AtomicFetchKind::Add:
1691 return isInt ? mlir::LLVM::AtomicBinOp::add : mlir::LLVM::AtomicBinOp::fadd;
1692 case cir::AtomicFetchKind::Sub:
1693 return isInt ? mlir::LLVM::AtomicBinOp::sub : mlir::LLVM::AtomicBinOp::fsub;
1694 case cir::AtomicFetchKind::And:
1695 return mlir::LLVM::AtomicBinOp::_and;
1696 case cir::AtomicFetchKind::Xor:
1697 return mlir::LLVM::AtomicBinOp::_xor;
1698 case cir::AtomicFetchKind::Or:
1699 return mlir::LLVM::AtomicBinOp::_or;
1700 case cir::AtomicFetchKind::Nand:
1701 return mlir::LLVM::AtomicBinOp::nand;
1702 case cir::AtomicFetchKind::Max: {
1703 if (!isInt)
1704 return mlir::LLVM::AtomicBinOp::fmax;
1705 return isSignedInt ? mlir::LLVM::AtomicBinOp::max
1706 : mlir::LLVM::AtomicBinOp::umax;
1707 }
1708 case cir::AtomicFetchKind::Min: {
1709 if (!isInt)
1710 return mlir::LLVM::AtomicBinOp::fmin;
1711 return isSignedInt ? mlir::LLVM::AtomicBinOp::min
1712 : mlir::LLVM::AtomicBinOp::umin;
1713 }
1714 case cir::AtomicFetchKind::UIncWrap:
1715 return mlir::LLVM::AtomicBinOp::uinc_wrap;
1716 case cir::AtomicFetchKind::UDecWrap:
1717 return mlir::LLVM::AtomicBinOp::udec_wrap;
1718 case cir::AtomicFetchKind::Maximum:
1719 return mlir::LLVM::AtomicBinOp::fmaximum;
1720 case cir::AtomicFetchKind::Minimum:
1721 return mlir::LLVM::AtomicBinOp::fminimum;
1722 case cir::AtomicFetchKind::MaximumNum:
1723 return mlir::LLVM::AtomicBinOp::fmaximumnum;
1724 case cir::AtomicFetchKind::MinimumNum:
1725 return mlir::LLVM::AtomicBinOp::fminimumnum;
1726 }
1727 llvm_unreachable("Unknown atomic fetch opcode");
1728}
1729
1730static llvm::StringLiteral getLLVMBinopForPostAtomic(cir::AtomicFetchKind k,
1731 bool isInt) {
1732 switch (k) {
1733 case cir::AtomicFetchKind::Add:
1734 return isInt ? mlir::LLVM::AddOp::getOperationName()
1735 : mlir::LLVM::FAddOp::getOperationName();
1736 case cir::AtomicFetchKind::Sub:
1737 return isInt ? mlir::LLVM::SubOp::getOperationName()
1738 : mlir::LLVM::FSubOp::getOperationName();
1739 case cir::AtomicFetchKind::And:
1740 return mlir::LLVM::AndOp::getOperationName();
1741 case cir::AtomicFetchKind::Xor:
1742 return mlir::LLVM::XOrOp::getOperationName();
1743 case cir::AtomicFetchKind::Or:
1744 return mlir::LLVM::OrOp::getOperationName();
1745 case cir::AtomicFetchKind::Nand:
1746 // There's no nand binop in LLVM, this is later fixed with a not.
1747 return mlir::LLVM::AndOp::getOperationName();
1748 case cir::AtomicFetchKind::Max:
1749 case cir::AtomicFetchKind::Min:
1750 llvm_unreachable("handled in buildMinMaxPostOp");
1751 case cir::AtomicFetchKind::UIncWrap:
1752 case cir::AtomicFetchKind::UDecWrap:
1753 case cir::AtomicFetchKind::Maximum:
1754 case cir::AtomicFetchKind::Minimum:
1755 case cir::AtomicFetchKind::MaximumNum:
1756 case cir::AtomicFetchKind::MinimumNum:
1757 llvm_unreachable("uinc_wrap, udec_wrap, maximum, minimum, maximum_num, and "
1758 "minimum_num are always fetch_first");
1759 }
1760 llvm_unreachable("Unknown atomic fetch opcode");
1761}
1762
1763mlir::Value CIRToLLVMAtomicFetchOpLowering::buildPostOp(
1764 cir::AtomicFetchOp op, OpAdaptor adaptor,
1765 mlir::ConversionPatternRewriter &rewriter, mlir::Value rmwVal,
1766 bool isInt) const {
1767 SmallVector<mlir::Value> atomicOperands = {rmwVal, adaptor.getVal()};
1768 SmallVector<mlir::Type> atomicResTys = {rmwVal.getType()};
1769 return rewriter
1770 .create(loc: op.getLoc(),
1771 opName: rewriter.getStringAttr(
1772 bytes: getLLVMBinopForPostAtomic(k: op.getBinop(), isInt)),
1773 operands: atomicOperands, types: atomicResTys, attributes: {})
1774 ->getResult(idx: 0);
1775}
1776
1777mlir::Value CIRToLLVMAtomicFetchOpLowering::buildMinMaxPostOp(
1778 cir::AtomicFetchOp op, OpAdaptor adaptor,
1779 mlir::ConversionPatternRewriter &rewriter, mlir::Value rmwVal, bool isInt,
1780 bool isSigned) const {
1781 mlir::Location loc = op.getLoc();
1782
1783 if (!isInt) {
1784 if (op.getBinop() == cir::AtomicFetchKind::Max)
1785 return mlir::LLVM::MaxNumOp::create(builder&: rewriter, location: loc, a: rmwVal,
1786 b: adaptor.getVal());
1787 return mlir::LLVM::MinNumOp::create(builder&: rewriter, location: loc, a: rmwVal,
1788 b: adaptor.getVal());
1789 }
1790
1791 mlir::LLVM::ICmpPredicate pred;
1792 if (op.getBinop() == cir::AtomicFetchKind::Max) {
1793 pred = isSigned ? mlir::LLVM::ICmpPredicate::sgt
1794 : mlir::LLVM::ICmpPredicate::ugt;
1795 } else { // Min
1796 pred = isSigned ? mlir::LLVM::ICmpPredicate::slt
1797 : mlir::LLVM::ICmpPredicate::ult;
1798 }
1799 mlir::Value cmp = mlir::LLVM::ICmpOp::create(
1800 builder&: rewriter, location: loc,
1801 predicate: mlir::LLVM::ICmpPredicateAttr::get(context: rewriter.getContext(), val: pred), lhs: rmwVal,
1802 rhs: adaptor.getVal());
1803 return mlir::LLVM::SelectOp::create(builder&: rewriter, location: loc, condition: cmp, trueValue: rmwVal,
1804 falseValue: adaptor.getVal());
1805}
1806
1807mlir::LogicalResult CIRToLLVMAtomicFetchOpLowering::matchAndRewrite(
1808 cir::AtomicFetchOp op, OpAdaptor adaptor,
1809 mlir::ConversionPatternRewriter &rewriter) const {
1810 bool isInt = false;
1811 bool isSignedInt = false;
1812 if (auto intTy = mlir::dyn_cast<cir::IntType>(Val: op.getVal().getType())) {
1813 isInt = true;
1814 isSignedInt = intTy.isSigned();
1815 } else if (cir::isFPOrVectorOfFPType(type: op.getVal().getType())) {
1816 isInt = false;
1817 } else {
1818 return op.emitError() << "Unsupported type: " << op.getVal().getType();
1819 }
1820
1821 mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(memorder: op.getMemOrder());
1822 llvm::StringRef llvmSyncScope = getLLVMSyncScope(
1823 syncScope: op.getSyncScope(), moduleOp: op->getParentOfType<mlir::ModuleOp>());
1824 mlir::LLVM::AtomicBinOp llvmBinOp =
1825 getLLVMAtomicBinOp(k: op.getBinop(), isInt, isSignedInt);
1826 auto rmwVal = mlir::LLVM::AtomicRMWOp::create(
1827 builder&: rewriter, location: op.getLoc(), binOp: llvmBinOp, ptr: adaptor.getPtr(), val: adaptor.getVal(),
1828 ordering: llvmOrder, syncscope: llvmSyncScope, /*alignment=*/0, isVolatile: op.getIsVolatile());
1829
1830 forwardAMDGPUAtomicOptionMarkers(op, llvmOp: rmwVal);
1831 forwardAMDGPUNoPrivateAddrSpace(op, llvmOp: rmwVal, llvmPtr: adaptor.getPtr());
1832
1833 // The AMDGPU raw hardware atomic builtins need metadata for the backend to
1834 // select the native instruction. LDS atomics are always native, so the
1835 // metadata is only needed for the global and flat address spaces.
1836 if (op->hasAttr(name: cir::CIRDialect::getAMDGPURawAtomicAttrName())) {
1837 auto ptrTy =
1838 mlir::cast<mlir::LLVM::LLVMPointerType>(Val: adaptor.getPtr().getType());
1839 if (ptrTy.getAddressSpace() != llvm::AMDGPUAS::LOCAL_ADDRESS) {
1840 mlir::UnitAttr unit = rewriter.getUnitAttr();
1841 rmwVal->setAttr(name: cir::CIRDialect::getAMDGPUNoFineGrainedMemoryAttrName(),
1842 value: unit);
1843 // Denormal flushing only matters for a float add.
1844 if (llvmBinOp == mlir::LLVM::AtomicBinOp::fadd &&
1845 mlir::isa<cir::SingleType>(Val: op.getVal().getType()))
1846 rmwVal->setAttr(name: cir::CIRDialect::getAMDGPUIgnoreDenormalModeAttrName(),
1847 value: unit);
1848 }
1849 }
1850
1851 mlir::Value result = rmwVal.getResult();
1852 if (!op.getFetchFirst()) {
1853 if (op.getBinop() == cir::AtomicFetchKind::Max ||
1854 op.getBinop() == cir::AtomicFetchKind::Min)
1855 result = buildMinMaxPostOp(op, adaptor, rewriter, rmwVal: rmwVal.getRes(), isInt,
1856 isSigned: isSignedInt);
1857 else
1858 result = buildPostOp(op, adaptor, rewriter, rmwVal: rmwVal.getRes(), isInt);
1859
1860 // Compensate lack of nand binop in LLVM IR.
1861 if (op.getBinop() == cir::AtomicFetchKind::Nand) {
1862 auto negOne = mlir::LLVM::ConstantOp::create(builder&: rewriter, location: op.getLoc(),
1863 type: result.getType(), value: -1);
1864 result = mlir::LLVM::XOrOp::create(builder&: rewriter, location: op.getLoc(), lhs: result, rhs: negOne);
1865 }
1866 }
1867
1868 rewriter.replaceOp(op, newValues: result);
1869 return mlir::success();
1870}
1871
1872mlir::LogicalResult CIRToLLVMBitClrsbOpLowering::matchAndRewrite(
1873 cir::BitClrsbOp op, OpAdaptor adaptor,
1874 mlir::ConversionPatternRewriter &rewriter) const {
1875 auto zero = mlir::LLVM::ConstantOp::create(builder&: rewriter, location: op.getLoc(),
1876 type: adaptor.getInput().getType(), value: 0);
1877 auto isNeg = mlir::LLVM::ICmpOp::create(
1878 builder&: rewriter, location: op.getLoc(),
1879 predicate: mlir::LLVM::ICmpPredicateAttr::get(context: rewriter.getContext(),
1880 val: mlir::LLVM::ICmpPredicate::slt),
1881 lhs: adaptor.getInput(), rhs: zero);
1882
1883 auto negOne = mlir::LLVM::ConstantOp::create(
1884 builder&: rewriter, location: op.getLoc(), type: adaptor.getInput().getType(), value: -1);
1885 auto flipped = mlir::LLVM::XOrOp::create(builder&: rewriter, location: op.getLoc(),
1886 lhs: adaptor.getInput(), rhs: negOne);
1887
1888 auto select = mlir::LLVM::SelectOp::create(builder&: rewriter, location: op.getLoc(), condition: isNeg,
1889 trueValue: flipped, falseValue: adaptor.getInput());
1890
1891 auto resTy = getTypeConverter()->convertType(t: op.getType());
1892 auto clz = mlir::LLVM::CountLeadingZerosOp::create(
1893 builder&: rewriter, location: op.getLoc(), res: resTy, in: select, /*is_zero_poison=*/false);
1894
1895 auto one = mlir::LLVM::ConstantOp::create(builder&: rewriter, location: op.getLoc(), type: resTy, value: 1);
1896 auto res = mlir::LLVM::SubOp::create(builder&: rewriter, location: op.getLoc(), lhs: clz, rhs: one,
1897 overflowFlags: mlir::LLVM::IntegerOverflowFlags::nuw);
1898 rewriter.replaceOp(op, newOp: res);
1899
1900 return mlir::LogicalResult::success();
1901}
1902
1903mlir::LogicalResult CIRToLLVMBitFfsOpLowering::matchAndRewrite(
1904 cir::BitFfsOp op, OpAdaptor adaptor,
1905 mlir::ConversionPatternRewriter &rewriter) const {
1906 auto resTy = getTypeConverter()->convertType(t: op.getType());
1907 auto ctz = mlir::LLVM::CountTrailingZerosOp::create(builder&: rewriter, location: op.getLoc(),
1908 res: resTy, in: adaptor.getInput(),
1909 /*is_zero_poison=*/true);
1910
1911 auto one = mlir::LLVM::ConstantOp::create(builder&: rewriter, location: op.getLoc(), type: resTy, value: 1);
1912 auto ctzAddOne = mlir::LLVM::AddOp::create(builder&: rewriter, location: op.getLoc(), lhs: ctz, rhs: one);
1913
1914 auto zeroInputTy = mlir::LLVM::ConstantOp::create(
1915 builder&: rewriter, location: op.getLoc(), type: adaptor.getInput().getType(), value: 0);
1916 auto isZero = mlir::LLVM::ICmpOp::create(
1917 builder&: rewriter, location: op.getLoc(),
1918 predicate: mlir::LLVM::ICmpPredicateAttr::get(context: rewriter.getContext(),
1919 val: mlir::LLVM::ICmpPredicate::eq),
1920 lhs: adaptor.getInput(), rhs: zeroInputTy);
1921
1922 auto zero = mlir::LLVM::ConstantOp::create(builder&: rewriter, location: op.getLoc(), type: resTy, value: 0);
1923 auto res = mlir::LLVM::SelectOp::create(builder&: rewriter, location: op.getLoc(), condition: isZero, trueValue: zero,
1924 falseValue: ctzAddOne);
1925 rewriter.replaceOp(op, newOp: res);
1926
1927 return mlir::LogicalResult::success();
1928}
1929
1930mlir::LogicalResult CIRToLLVMBitParityOpLowering::matchAndRewrite(
1931 cir::BitParityOp op, OpAdaptor adaptor,
1932 mlir::ConversionPatternRewriter &rewriter) const {
1933 auto resTy = getTypeConverter()->convertType(t: op.getType());
1934 auto popcnt = mlir::LLVM::CtPopOp::create(builder&: rewriter, location: op.getLoc(), res: resTy,
1935 in: adaptor.getInput());
1936
1937 auto one = mlir::LLVM::ConstantOp::create(builder&: rewriter, location: op.getLoc(), type: resTy, value: 1);
1938 auto popcntMod2 =
1939 mlir::LLVM::AndOp::create(builder&: rewriter, location: op.getLoc(), lhs: popcnt, rhs: one);
1940 rewriter.replaceOp(op, newOp: popcntMod2);
1941
1942 return mlir::LogicalResult::success();
1943}
1944
1945mlir::LogicalResult CIRToLLVMBrCondOpLowering::matchAndRewrite(
1946 cir::BrCondOp brOp, OpAdaptor adaptor,
1947 mlir::ConversionPatternRewriter &rewriter) const {
1948 // When ZExtOp is implemented, we'll need to check if the condition is a
1949 // ZExtOp and if so, delete it if it has a single use.
1950 assert(!cir::MissingFeatures::zextOp());
1951
1952 mlir::Value i1Condition = adaptor.getCond();
1953
1954 rewriter.replaceOpWithNewOp<mlir::LLVM::CondBrOp>(
1955 op: brOp, args&: i1Condition, args: brOp.getDestTrue(), args: adaptor.getDestOperandsTrue(),
1956 args: brOp.getDestFalse(), args: adaptor.getDestOperandsFalse());
1957
1958 return mlir::success();
1959}
1960
1961mlir::Type CIRToLLVMCastOpLowering::convertTy(mlir::Type ty) const {
1962 return getTypeConverter()->convertType(t: ty);
1963}
1964
1965mlir::LogicalResult CIRToLLVMCastOpLowering::matchAndRewrite(
1966 cir::CastOp castOp, OpAdaptor adaptor,
1967 mlir::ConversionPatternRewriter &rewriter) const {
1968 // For arithmetic conversions, LLVM IR uses the same instruction to convert
1969 // both individual scalars and entire vectors. This lowering pass handles
1970 // both situations.
1971
1972 if (!getTypeConverter()->convertType(t: castOp.getType()))
1973 return emitTypeConversionError(op: castOp, type: castOp.getType());
1974
1975 switch (castOp.getKind()) {
1976 case cir::CastKind::array_to_ptrdecay: {
1977 const auto ptrTy = mlir::cast<cir::PointerType>(Val: castOp.getType());
1978 mlir::Value sourceValue = adaptor.getSrc();
1979 mlir::Type targetType = convertTy(ty: ptrTy);
1980 mlir::Type elementTy = convertTypeForMemory(converter: *getTypeConverter(), dataLayout,
1981 type: ptrTy.getPointee());
1982 if (!elementTy)
1983 return emitTypeConversionError(op: castOp, type: ptrTy.getPointee());
1984 llvm::SmallVector<mlir::LLVM::GEPArg> offset{0};
1985 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
1986 op: castOp, args&: targetType, args&: elementTy, args&: sourceValue, args&: offset);
1987 break;
1988 }
1989 case cir::CastKind::int_to_bool: {
1990 mlir::Value llvmSrcVal = adaptor.getSrc();
1991 // getZeroAttr yields a splat for vector source types so this also
1992 // handles element-wise int-to-bool conversions (e.g. an ext_vector
1993 // __builtin_convertvector to bool).
1994 mlir::Value zeroInt = mlir::LLVM::ConstantOp::create(
1995 builder&: rewriter, location: castOp.getLoc(), res: llvmSrcVal.getType(),
1996 value: rewriter.getZeroAttr(type: llvmSrcVal.getType()));
1997 rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
1998 op: castOp, args: mlir::LLVM::ICmpPredicate::ne, args&: llvmSrcVal, args&: zeroInt);
1999 break;
2000 }
2001 case cir::CastKind::integral: {
2002 mlir::Type srcType = castOp.getSrc().getType();
2003 mlir::Type dstType = castOp.getType();
2004 mlir::Value llvmSrcVal = adaptor.getSrc();
2005 mlir::Type llvmDstType = getTypeConverter()->convertType(t: dstType);
2006 cir::IntType srcIntType =
2007 mlir::cast<cir::IntType>(Val: elementTypeIfVectorOrMatrix(type: srcType));
2008 cir::IntType dstIntType =
2009 mlir::cast<cir::IntType>(Val: elementTypeIfVectorOrMatrix(type: dstType));
2010 rewriter.replaceOp(op: castOp, newValues: getLLVMIntCast(rewriter, llvmSrc: llvmSrcVal, llvmDstIntTy: llvmDstType,
2011 isUnsigned: srcIntType.isUnsigned(),
2012 cirSrcWidth: srcIntType.getWidth(),
2013 cirDstIntWidth: dstIntType.getWidth()));
2014 break;
2015 }
2016 case cir::CastKind::floating: {
2017 mlir::Value llvmSrcVal = adaptor.getSrc();
2018 mlir::Type llvmDstTy = getTypeConverter()->convertType(t: castOp.getType());
2019
2020 mlir::Type srcTy = elementTypeIfVectorOrMatrix(type: castOp.getSrc().getType());
2021 mlir::Type dstTy = elementTypeIfVectorOrMatrix(type: castOp.getType());
2022
2023 if (!mlir::isa<cir::FPTypeInterface>(Val: dstTy) ||
2024 !mlir::isa<cir::FPTypeInterface>(Val: srcTy))
2025 return castOp.emitError() << "NYI cast from " << srcTy << " to " << dstTy;
2026
2027 auto getFloatWidth = [](mlir::Type ty) -> unsigned {
2028 return mlir::cast<cir::FPTypeInterface>(Val&: ty).getWidth();
2029 };
2030
2031 bool isTrunc = getFloatWidth(srcTy) > getFloatWidth(dstTy);
2032 if (cir::FenvAttr fenv = castOp.getFenvAttr()) {
2033 // fptrunc takes rounding mode + exception behavior; fpext takes only
2034 // exception behavior.
2035 return lowerToConstrainedFPIntrinsic(
2036 op: castOp, operands: llvmSrcVal, fenv, llvmResTy: llvmDstTy, rewriter,
2037 constrainedMnemonic: isTrunc ? "fptrunc" : "fpext", /*hasRoundingMode=*/isTrunc);
2038 }
2039 if (isTrunc)
2040 rewriter.replaceOpWithNewOp<mlir::LLVM::FPTruncOp>(op: castOp, args&: llvmDstTy,
2041 args&: llvmSrcVal);
2042 else
2043 rewriter.replaceOpWithNewOp<mlir::LLVM::FPExtOp>(op: castOp, args&: llvmDstTy,
2044 args&: llvmSrcVal);
2045 return mlir::success();
2046 }
2047 case cir::CastKind::int_to_ptr: {
2048 auto dstTy = mlir::cast<cir::PointerType>(Val: castOp.getType());
2049 mlir::Value llvmSrcVal = adaptor.getSrc();
2050 mlir::Type llvmDstTy = getTypeConverter()->convertType(t: dstTy);
2051 rewriter.replaceOpWithNewOp<mlir::LLVM::IntToPtrOp>(op: castOp, args&: llvmDstTy,
2052 args&: llvmSrcVal);
2053 return mlir::success();
2054 }
2055 case cir::CastKind::ptr_to_int: {
2056 auto dstTy = mlir::cast<cir::IntType>(Val: castOp.getType());
2057 mlir::Value llvmSrcVal = adaptor.getSrc();
2058 mlir::Type llvmDstTy = getTypeConverter()->convertType(t: dstTy);
2059 rewriter.replaceOpWithNewOp<mlir::LLVM::PtrToIntOp>(op: castOp, args&: llvmDstTy,
2060 args&: llvmSrcVal);
2061 return mlir::success();
2062 }
2063 case cir::CastKind::float_to_bool: {
2064 mlir::Value llvmSrcVal = adaptor.getSrc();
2065 auto kind = mlir::LLVM::FCmpPredicate::une;
2066
2067 // Check if float is not equal to zero. getZeroAttr yields a splat
2068 // for vector source types so this also handles element-wise
2069 // float-to-bool conversions.
2070 auto zeroFloat = mlir::LLVM::ConstantOp::create(
2071 builder&: rewriter, location: castOp.getLoc(), res: llvmSrcVal.getType(),
2072 value: rewriter.getZeroAttr(type: llvmSrcVal.getType()));
2073
2074 // Extend comparison result to either bool (C++) or int (C).
2075 rewriter.replaceOpWithNewOp<mlir::LLVM::FCmpOp>(op: castOp, args&: kind, args&: llvmSrcVal,
2076 args&: zeroFloat);
2077
2078 return mlir::success();
2079 }
2080 case cir::CastKind::bool_to_int: {
2081 mlir::Type dstTy = castOp.getType();
2082 mlir::Value llvmSrcVal = adaptor.getSrc();
2083 mlir::Type llvmDstTy = getTypeConverter()->convertType(t: dstTy);
2084 // Compare element widths so this also handles vector bool -> int casts.
2085 auto srcElemTy = mlir::cast<mlir::IntegerType>(
2086 Val: elementTypeIfVectorOrMatrix(type: llvmSrcVal.getType()));
2087 auto dstElemTy =
2088 mlir::cast<cir::IntType>(Val: elementTypeIfVectorOrMatrix(type: dstTy));
2089
2090 if (srcElemTy.getWidth() == dstElemTy.getWidth())
2091 rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(op: castOp, args&: llvmDstTy,
2092 args&: llvmSrcVal);
2093 else
2094 rewriter.replaceOpWithNewOp<mlir::LLVM::ZExtOp>(op: castOp, args&: llvmDstTy,
2095 args&: llvmSrcVal);
2096 return mlir::success();
2097 }
2098 case cir::CastKind::bool_to_float: {
2099 mlir::Type dstTy = castOp.getType();
2100 mlir::Value llvmSrcVal = adaptor.getSrc();
2101 mlir::Type llvmDstTy = getTypeConverter()->convertType(t: dstTy);
2102 rewriter.replaceOpWithNewOp<mlir::LLVM::UIToFPOp>(op: castOp, args&: llvmDstTy,
2103 args&: llvmSrcVal);
2104 return mlir::success();
2105 }
2106 case cir::CastKind::int_to_float: {
2107 mlir::Type dstTy = castOp.getType();
2108 mlir::Value llvmSrcVal = adaptor.getSrc();
2109 mlir::Type llvmDstTy = getTypeConverter()->convertType(t: dstTy);
2110 bool isSigned = mlir::cast<cir::IntType>(
2111 Val: elementTypeIfVectorOrMatrix(type: castOp.getSrc().getType()))
2112 .isSigned();
2113 if (cir::FenvAttr fenv = castOp.getFenvAttr()) {
2114 return lowerToConstrainedFPIntrinsic(
2115 op: castOp, operands: llvmSrcVal, fenv, llvmResTy: llvmDstTy, rewriter,
2116 constrainedMnemonic: isSigned ? "sitofp" : "uitofp", /*hasRoundingMode=*/true);
2117 }
2118 if (isSigned)
2119 rewriter.replaceOpWithNewOp<mlir::LLVM::SIToFPOp>(op: castOp, args&: llvmDstTy,
2120 args&: llvmSrcVal);
2121 else
2122 rewriter.replaceOpWithNewOp<mlir::LLVM::UIToFPOp>(op: castOp, args&: llvmDstTy,
2123 args&: llvmSrcVal);
2124 return mlir::success();
2125 }
2126 case cir::CastKind::float_to_int: {
2127 mlir::Type dstTy = castOp.getType();
2128 mlir::Value llvmSrcVal = adaptor.getSrc();
2129 mlir::Type llvmDstTy = getTypeConverter()->convertType(t: dstTy);
2130 bool isSigned =
2131 mlir::cast<cir::IntType>(Val: elementTypeIfVectorOrMatrix(type: castOp.getType()))
2132 .isSigned();
2133 if (cir::FenvAttr fenv = castOp.getFenvAttr()) {
2134 return lowerToConstrainedFPIntrinsic(
2135 op: castOp, operands: llvmSrcVal, fenv, llvmResTy: llvmDstTy, rewriter,
2136 constrainedMnemonic: isSigned ? "fptosi" : "fptoui", /*hasRoundingMode=*/false);
2137 }
2138 if (isSigned)
2139 rewriter.replaceOpWithNewOp<mlir::LLVM::FPToSIOp>(op: castOp, args&: llvmDstTy,
2140 args&: llvmSrcVal);
2141 else
2142 rewriter.replaceOpWithNewOp<mlir::LLVM::FPToUIOp>(op: castOp, args&: llvmDstTy,
2143 args&: llvmSrcVal);
2144 return mlir::success();
2145 }
2146 case cir::CastKind::bitcast: {
2147 mlir::Type dstTy = castOp.getType();
2148 mlir::Type llvmDstTy = getTypeConverter()->convertType(t: dstTy);
2149
2150 assert(!MissingFeatures::cxxABI());
2151 assert(!MissingFeatures::dataMemberType());
2152
2153 mlir::Value llvmSrcVal = adaptor.getSrc();
2154 rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(op: castOp, args&: llvmDstTy,
2155 args&: llvmSrcVal);
2156 return mlir::success();
2157 }
2158 case cir::CastKind::ptr_to_bool: {
2159 mlir::Value llvmSrcVal = adaptor.getSrc();
2160 mlir::Value zeroPtr = mlir::LLVM::ZeroOp::create(builder&: rewriter, location: castOp.getLoc(),
2161 res: llvmSrcVal.getType());
2162 rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
2163 op: castOp, args: mlir::LLVM::ICmpPredicate::ne, args&: llvmSrcVal, args&: zeroPtr);
2164 break;
2165 }
2166 case cir::CastKind::address_space: {
2167 mlir::Type dstTy = castOp.getType();
2168 mlir::Value llvmSrcVal = adaptor.getSrc();
2169 mlir::Type llvmDstTy = getTypeConverter()->convertType(t: dstTy);
2170 rewriter.replaceOpWithNewOp<mlir::LLVM::AddrSpaceCastOp>(op: castOp, args&: llvmDstTy,
2171 args&: llvmSrcVal);
2172 break;
2173 }
2174 case cir::CastKind::member_ptr_to_bool:
2175 assert(!MissingFeatures::cxxABI());
2176 assert(!MissingFeatures::methodType());
2177 break;
2178 default: {
2179 return castOp.emitError(message: "Unhandled cast kind: ")
2180 << castOp.getKindAttrName();
2181 }
2182 }
2183
2184 return mlir::success();
2185}
2186
2187mlir::LogicalResult CIRToLLVMBuiltinIntCastOpLowering::matchAndRewrite(
2188 cir::BuiltinIntCastOp op, OpAdaptor adaptor,
2189 mlir::ConversionPatternRewriter &rewriter) const {
2190 // Both the CIR integer and the builtin integer/index lower to LLVM integer
2191 // types, so this cast becomes an integer resize. Signedness is taken from
2192 // the CIR integer side (the builtin/index side is treated as signless).
2193 bool isUnsigned = true;
2194 if (auto cirSrc = mlir::dyn_cast<cir::IntType>(Val: op.getSrc().getType()))
2195 isUnsigned = cirSrc.isUnsigned();
2196 else if (auto cirDst = mlir::dyn_cast<cir::IntType>(Val: op.getType()))
2197 isUnsigned = cirDst.isUnsigned();
2198
2199 mlir::Value llvmSrc = adaptor.getSrc();
2200 mlir::Type llvmDstTy = getTypeConverter()->convertType(t: op.getType());
2201 auto srcIntTy = mlir::cast<mlir::IntegerType>(Val: llvmSrc.getType());
2202 auto dstIntTy = mlir::cast<mlir::IntegerType>(Val&: llvmDstTy);
2203 unsigned srcWidth = srcIntTy.getWidth();
2204 unsigned dstWidth = dstIntTy.getWidth();
2205
2206 // Fixed-width builtin integers must match the CIR integer width.
2207 // If the converted LLVM widths differ, the non-CIR side must have been
2208 // 'index' type (target dependent width).
2209 assert((srcWidth == dstWidth ||
2210 mlir::isa<mlir::IndexType>(op.getSrc().getType()) ||
2211 mlir::isa<mlir::IndexType>(op.getType())) &&
2212 "only index casts may change width during lowering");
2213
2214 // For equal widths getLLVMIntCast returns the source unchanged, so casts
2215 // between CIR integers and fixed-width builtin integers lower to a no-op.
2216 rewriter.replaceOp(op, newValues: getLLVMIntCast(rewriter, llvmSrc, llvmDstIntTy: dstIntTy, isUnsigned,
2217 cirSrcWidth: srcWidth, cirDstIntWidth: dstWidth));
2218 return mlir::success();
2219}
2220
2221static mlir::Value convertToIndexTy(mlir::ConversionPatternRewriter &rewriter,
2222 mlir::ModuleOp mod, mlir::Value index,
2223 mlir::Type baseTy, cir::IntType strideTy) {
2224 mlir::Operation *indexOp = index.getDefiningOp();
2225 if (!indexOp)
2226 return index;
2227
2228 auto indexType = mlir::cast<mlir::IntegerType>(Val: index.getType());
2229 mlir::DataLayout llvmLayout(mod);
2230 std::optional<uint64_t> layoutWidth = llvmLayout.getTypeIndexBitwidth(t: baseTy);
2231
2232 // If there is no change in width, don't do anything.
2233 if (!layoutWidth || *layoutWidth == indexType.getWidth())
2234 return index;
2235
2236 // If the index comes from a subtraction, make sure the extension happens
2237 // before it. To achieve that, look at unary minus, which already got
2238 // lowered to "sub 0, x".
2239 auto sub = dyn_cast<mlir::LLVM::SubOp>(Val: indexOp);
2240 bool rewriteSub = false;
2241 if (sub) {
2242 if (auto lhsConst = dyn_cast_if_present<mlir::LLVM::ConstantOp>(
2243 Val: sub.getLhs().getDefiningOp())) {
2244 auto lhsConstInt = mlir::dyn_cast<mlir::IntegerAttr>(Val: lhsConst.getValue());
2245 if (lhsConstInt && lhsConstInt.getValue() == 0) {
2246 index = sub.getRhs();
2247 rewriteSub = true;
2248 }
2249 }
2250 }
2251
2252 auto llvmDstType = rewriter.getIntegerType(width: *layoutWidth);
2253 bool isUnsigned = strideTy && strideTy.isUnsigned();
2254 index = getLLVMIntCast(rewriter, llvmSrc: index, llvmDstIntTy: llvmDstType, isUnsigned,
2255 cirSrcWidth: indexType.getWidth(), cirDstIntWidth: *layoutWidth);
2256
2257 if (rewriteSub) {
2258 index = mlir::LLVM::SubOp::create(
2259 builder&: rewriter, location: index.getLoc(),
2260 lhs: mlir::LLVM::ConstantOp::create(builder&: rewriter, location: index.getLoc(),
2261 type: index.getType(), value: 0),
2262 rhs: index);
2263 }
2264
2265 return index;
2266}
2267
2268mlir::LogicalResult CIRToLLVMPtrStrideOpLowering::matchAndRewrite(
2269 cir::PtrStrideOp ptrStrideOp, OpAdaptor adaptor,
2270 mlir::ConversionPatternRewriter &rewriter) const {
2271
2272 const mlir::TypeConverter *tc = getTypeConverter();
2273 const mlir::Type resultTy = tc->convertType(t: ptrStrideOp.getType());
2274
2275 mlir::Type elementTy =
2276 convertTypeForMemory(converter: *tc, dataLayout, type: ptrStrideOp.getElementType());
2277 if (!elementTy)
2278 return emitTypeConversionError(op: ptrStrideOp, type: ptrStrideOp.getElementType());
2279
2280 // void and function types doesn't really have a layout to use in GEPs,
2281 // make it i8 instead.
2282 if (mlir::isa<mlir::LLVM::LLVMVoidType>(Val: elementTy) ||
2283 mlir::isa<mlir::LLVM::LLVMFunctionType>(Val: elementTy))
2284 elementTy = mlir::IntegerType::get(context: elementTy.getContext(), width: 8,
2285 signedness: mlir::IntegerType::Signless);
2286 // Zero-extend, sign-extend or trunc the pointer value.
2287 mlir::Value index = adaptor.getStride();
2288 index = convertToIndexTy(
2289 rewriter, mod: ptrStrideOp->getParentOfType<mlir::ModuleOp>(), index,
2290 baseTy: adaptor.getBase().getType(),
2291 strideTy: dyn_cast<cir::IntType>(Val: ptrStrideOp.getOperand(i: 1).getType()));
2292
2293 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
2294 op: ptrStrideOp, args: resultTy, args&: elementTy, args: adaptor.getBase(), args&: index);
2295 return mlir::success();
2296}
2297
2298mlir::LogicalResult CIRToLLVMGetElementOpLowering::matchAndRewrite(
2299 cir::GetElementOp op, OpAdaptor adaptor,
2300 mlir::ConversionPatternRewriter &rewriter) const {
2301 if (auto arrayTy =
2302 mlir::dyn_cast<cir::ArrayType>(Val: op.getBaseType().getPointee())) {
2303 const mlir::TypeConverter *converter = getTypeConverter();
2304 const mlir::Type llArrayTy = converter->convertType(t: arrayTy);
2305 if (!llArrayTy)
2306 return emitTypeConversionError(op, type: arrayTy);
2307 const mlir::Type llResultTy = converter->convertType(t: op.getType());
2308 mlir::Type elementTy =
2309 convertTypeForMemory(converter: *converter, dataLayout, type: op.getElementType());
2310
2311 // void and function types don't really have a layout to use in GEPs,
2312 // make it i8 instead.
2313 if (mlir::isa<mlir::LLVM::LLVMVoidType>(Val: elementTy) ||
2314 mlir::isa<mlir::LLVM::LLVMFunctionType>(Val: elementTy))
2315 elementTy = rewriter.getIntegerType(width: 8);
2316
2317 mlir::Value index = adaptor.getIndex();
2318 index =
2319 convertToIndexTy(rewriter, mod: op->getParentOfType<mlir::ModuleOp>(), index,
2320 baseTy: adaptor.getBase().getType(),
2321 strideTy: dyn_cast<cir::IntType>(Val: op.getOperand(i: 1).getType()));
2322
2323 // Since the base address is a pointer to an array, the first index is
2324 // always zero, and the second one selects the element. The zero takes the
2325 // element index's type, because a raw constant GEP index is always
2326 // translated as i32.
2327 mlir::Value zero = mlir::LLVM::ConstantOp::create(builder&: rewriter, location: op.getLoc(),
2328 type: index.getType(), value: 0);
2329 std::array<mlir::LLVM::GEPArg, 2> offset = {zero, index};
2330 mlir::LLVM::GEPNoWrapFlags flags = mlir::LLVM::GEPNoWrapFlags::none;
2331 if (op.getInBounds())
2332 flags |= mlir::LLVM::GEPNoWrapFlags::inbounds;
2333 if (op.getNoUnsignedWrap())
2334 flags |= mlir::LLVM::GEPNoWrapFlags::nuw;
2335 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
2336 op, args: llResultTy, args: llArrayTy, args: adaptor.getBase(), args&: offset, args&: flags);
2337 return mlir::success();
2338 }
2339
2340 op.emitError() << "NYI: GetElementOp lowering to LLVM for non-array";
2341 return mlir::failure();
2342}
2343
2344mlir::LogicalResult CIRToLLVMBaseClassAddrOpLowering::matchAndRewrite(
2345 cir::BaseClassAddrOp baseClassOp, OpAdaptor adaptor,
2346 mlir::ConversionPatternRewriter &rewriter) const {
2347 const mlir::Type resultType =
2348 getTypeConverter()->convertType(t: baseClassOp.getType());
2349 if (!resultType)
2350 return emitTypeConversionError(op: baseClassOp, type: baseClassOp.getType());
2351 mlir::Value derivedAddr = adaptor.getSrcAddr();
2352 llvm::SmallVector<mlir::LLVM::GEPArg, 1> offset = {
2353 static_cast<int32_t>(adaptor.getOffset().getZExtValue())};
2354 mlir::Type byteType = mlir::IntegerType::get(context: resultType.getContext(), width: 8,
2355 signedness: mlir::IntegerType::Signless);
2356 if (adaptor.getOffset().getZExtValue() == 0) {
2357 rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(op: baseClassOp, args: resultType,
2358 args: adaptor.getSrcAddr());
2359 return mlir::success();
2360 }
2361
2362 if (baseClassOp.getAssumeNotNull()) {
2363 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
2364 op: baseClassOp, args: resultType, args&: byteType, args&: derivedAddr, args&: offset);
2365 } else {
2366 auto loc = baseClassOp.getLoc();
2367 mlir::Value isNull = mlir::LLVM::ICmpOp::create(
2368 builder&: rewriter, location: loc, predicate: mlir::LLVM::ICmpPredicate::eq, lhs: derivedAddr,
2369 rhs: mlir::LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: derivedAddr.getType()));
2370 mlir::Value adjusted = mlir::LLVM::GEPOp::create(
2371 builder&: rewriter, location: loc, resultType, elementType: byteType, basePtr: derivedAddr, indices: offset);
2372 rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(op: baseClassOp, args&: isNull,
2373 args&: derivedAddr, args&: adjusted);
2374 }
2375 return mlir::success();
2376}
2377
2378mlir::LogicalResult CIRToLLVMDerivedClassAddrOpLowering::matchAndRewrite(
2379 cir::DerivedClassAddrOp derivedClassOp, OpAdaptor adaptor,
2380 mlir::ConversionPatternRewriter &rewriter) const {
2381 const mlir::Type resultType =
2382 getTypeConverter()->convertType(t: derivedClassOp.getType());
2383 if (!resultType)
2384 return emitTypeConversionError(op: derivedClassOp, type: derivedClassOp.getType());
2385 mlir::Value baseAddr = adaptor.getSrcAddr();
2386 // The offset is set in the operation as an unsigned value, but it must be
2387 // applied as a negative offset.
2388 int64_t offsetVal = -(adaptor.getOffset().getZExtValue());
2389 if (offsetVal == 0) {
2390 // If the offset is zero, we can just return the source address.
2391 rewriter.replaceOp(op: derivedClassOp, newValues: baseAddr);
2392 return mlir::success();
2393 }
2394 llvm::SmallVector<mlir::LLVM::GEPArg, 1> offset = {
2395 static_cast<int32_t>(offsetVal)};
2396 mlir::Type byteType = mlir::IntegerType::get(context: resultType.getContext(), width: 8,
2397 signedness: mlir::IntegerType::Signless);
2398 if (derivedClassOp.getAssumeNotNull()) {
2399 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
2400 op: derivedClassOp, args: resultType, args&: byteType, args&: baseAddr, args&: offset,
2401 args: mlir::LLVM::GEPNoWrapFlags::inbounds);
2402 } else {
2403 mlir::Location loc = derivedClassOp.getLoc();
2404 mlir::Value isNull = mlir::LLVM::ICmpOp::create(
2405 builder&: rewriter, location: loc, predicate: mlir::LLVM::ICmpPredicate::eq, lhs: baseAddr,
2406 rhs: mlir::LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: baseAddr.getType()));
2407 mlir::Value adjusted =
2408 mlir::LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType, elementType: byteType, basePtr: baseAddr,
2409 indices: offset, noWrapFlags: mlir::LLVM::GEPNoWrapFlags::inbounds);
2410 rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(op: derivedClassOp, args&: isNull,
2411 args&: baseAddr, args&: adjusted);
2412 }
2413 return mlir::success();
2414}
2415
2416mlir::LogicalResult CIRToLLVMFMaxNumOpLowering::matchAndRewrite(
2417 cir::FMaxNumOp op, OpAdaptor adaptor,
2418 mlir::ConversionPatternRewriter &rewriter) const {
2419 mlir::Type resTy = typeConverter->convertType(t: op.getType());
2420 if (cir::FenvAttr fenv = op.getFenvAttr())
2421 return lowerToConstrainedFPIntrinsic(
2422 op, operands: adaptor.getOperands(), fenv, llvmResTy: resTy, rewriter, constrainedMnemonic: "maxnum",
2423 /*hasRoundingMode=*/false, fastmathFlags: mlir::LLVM::FastmathFlags::nsz);
2424 rewriter.replaceOpWithNewOp<mlir::LLVM::MaxNumOp>(
2425 op, args&: resTy, args: adaptor.getLhs(), args: adaptor.getRhs(),
2426 args: mlir::LLVM::FastmathFlags::nsz);
2427 return mlir::success();
2428}
2429
2430mlir::LogicalResult CIRToLLVMFMinNumOpLowering::matchAndRewrite(
2431 cir::FMinNumOp op, OpAdaptor adaptor,
2432 mlir::ConversionPatternRewriter &rewriter) const {
2433 mlir::Type resTy = typeConverter->convertType(t: op.getType());
2434 if (cir::FenvAttr fenv = op.getFenvAttr())
2435 return lowerToConstrainedFPIntrinsic(
2436 op, operands: adaptor.getOperands(), fenv, llvmResTy: resTy, rewriter, constrainedMnemonic: "minnum",
2437 /*hasRoundingMode=*/false, fastmathFlags: mlir::LLVM::FastmathFlags::nsz);
2438 rewriter.replaceOpWithNewOp<mlir::LLVM::MinNumOp>(
2439 op, args&: resTy, args: adaptor.getLhs(), args: adaptor.getRhs(),
2440 args: mlir::LLVM::FastmathFlags::nsz);
2441 return mlir::success();
2442}
2443
2444mlir::LogicalResult CIRToLLVMAllocaOpLowering::matchAndRewrite(
2445 cir::AllocaOp op, OpAdaptor adaptor,
2446 mlir::ConversionPatternRewriter &rewriter) const {
2447 mlir::ModuleOp module = op->getParentOfType<mlir::ModuleOp>();
2448 mlir::Attribute langOpts =
2449 module->getAttr(name: cir::CIRDialect::getLoweringOptionsAttrName());
2450 auto loweringLangOps =
2451 dyn_cast_if_present<cir::LoweringOptionsAttr>(Val&: langOpts);
2452
2453 bool isForHLSL = loweringLangOps ? loweringLangOps.getHlsl() : false;
2454 if (isForHLSL && mlir::isa<cir::MatrixType>(Val: op.getAllocaType()) &&
2455 mlir::isa<cir::BoolType>(
2456 Val: cast<cir::MatrixType>(Val: op.getAllocaType()).getElementType())) {
2457 return op.emitError()
2458 << "NYI: Builtin matrix type with bool element type for HLSL";
2459 }
2460
2461 mlir::Value size =
2462 op.isDynamic()
2463 ? adaptor.getDynAllocSize()
2464 : mlir::LLVM::ConstantOp::create(
2465 builder&: rewriter, location: op.getLoc(),
2466 type: typeConverter->convertType(t: rewriter.getI32Type()), value: 1);
2467 mlir::Type elementTy = convertTypeForMemory(converter: *getTypeConverter(), dataLayout,
2468 type: op.getAllocaType(), isForHLSL);
2469 if (!elementTy)
2470 return op.emitError()
2471 << "NYI: lowering alloca of a type with no memory representation";
2472 mlir::Type resultTy = convertTypeForMemory(converter: *getTypeConverter(), dataLayout,
2473 type: op.getType(), isForHLSL);
2474 if (!resultTy)
2475 return emitTypeConversionError(op, type: op.getType());
2476
2477 assert(!cir::MissingFeatures::addressSpace());
2478
2479 rewriter.replaceOpWithNewOp<mlir::LLVM::AllocaOp>(op, args&: resultTy, args&: elementTy,
2480 args&: size, args: op.getAlignment());
2481
2482 return mlir::success();
2483}
2484
2485mlir::LogicalResult CIRToLLVMRotateOpLowering::matchAndRewrite(
2486 cir::RotateOp op, OpAdaptor adaptor,
2487 mlir::ConversionPatternRewriter &rewriter) const {
2488 // Note that LLVM intrinsic calls to @llvm.fsh{r,l}.i* have the same type as
2489 // the operand.
2490 mlir::Value input = adaptor.getInput();
2491 if (op.isRotateLeft())
2492 rewriter.replaceOpWithNewOp<mlir::LLVM::FshlOp>(op, args&: input, args&: input,
2493 args: adaptor.getAmount());
2494 else
2495 rewriter.replaceOpWithNewOp<mlir::LLVM::FshrOp>(op, args&: input, args&: input,
2496 args: adaptor.getAmount());
2497 return mlir::LogicalResult::success();
2498}
2499
2500/// The `llvm.byval`, `llvm.sret`, and `llvm.byref` argument attributes carry
2501/// the pointee type as a TypeAttr. After the CallConvLowering pass that type
2502/// is still a CIR type; remap it to the lowered LLVM type so translation to
2503/// LLVM IR does not encounter a CIR type in an attribute. The pointee must
2504/// name the in-memory storage type, not the value type, since it sizes the
2505/// byval copy and the sret slot: a _BitInt narrower than its padded storage
2506/// integer would otherwise understate both. Returns std::nullopt when a
2507/// pointee has no memory representation, leaving the caller to emit the NYI
2508/// error, and the input unchanged when there is nothing to convert.
2509static std::optional<mlir::ArrayAttr> convertTypedArgAttrs(
2510 mlir::ArrayAttr argAttrs, const mlir::TypeConverter &converter,
2511 const mlir::DataLayout &dataLayout, mlir::MLIRContext *ctx) {
2512 if (!argAttrs)
2513 return argAttrs;
2514 bool changed = false;
2515 SmallVector<mlir::Attribute> loweredArgAttrs;
2516 loweredArgAttrs.reserve(N: argAttrs.size());
2517 for (mlir::Attribute a : argAttrs) {
2518 auto dict = cast<mlir::DictionaryAttr>(Val&: a);
2519 SmallVector<mlir::NamedAttribute> entries(dict.begin(), dict.end());
2520 for (mlir::NamedAttribute &entry : entries) {
2521 StringRef name = entry.getName().strref();
2522 if (name != mlir::LLVM::LLVMDialect::getByValAttrName() &&
2523 name != mlir::LLVM::LLVMDialect::getStructRetAttrName() &&
2524 name != mlir::LLVM::LLVMDialect::getByRefAttrName())
2525 continue;
2526 auto typeAttr = dyn_cast<mlir::TypeAttr>(Val: entry.getValue());
2527 if (!typeAttr)
2528 continue;
2529 mlir::Type lowered =
2530 convertTypeForMemory(converter, dataLayout, type: typeAttr.getValue());
2531 if (!lowered)
2532 return std::nullopt;
2533 if (lowered != typeAttr.getValue()) {
2534 entry.setValue(mlir::TypeAttr::get(type: lowered));
2535 changed = true;
2536 }
2537 }
2538 loweredArgAttrs.push_back(Elt: mlir::DictionaryAttr::get(context: ctx, value: entries));
2539 }
2540 return changed ? mlir::ArrayAttr::get(context: ctx, value: loweredArgAttrs) : argAttrs;
2541}
2542
2543struct LoweredCallAttributes {
2544 mlir::ArrayAttr argAttrs;
2545 mlir::ArrayAttr resAttrs;
2546 SmallVector<mlir::NamedAttribute> discardableAttrs;
2547};
2548
2549static mlir::LogicalResult lowerCallAttributes(
2550 cir::CIRCallOpInterface op, const mlir::TypeConverter &converter,
2551 const mlir::DataLayout &dataLayout, LoweredCallAttributes &result) {
2552 if (auto argAttrs = mlir::dyn_cast_if_present<mlir::ArrayAttr>(
2553 Val: op->getInherentAttr(name: CIRDialect::getArgAttrsAttrName())
2554 .value_or(u: mlir::Attribute{}))) {
2555 std::optional<mlir::ArrayAttr> lowered =
2556 convertTypedArgAttrs(argAttrs, converter, dataLayout, ctx: op->getContext());
2557 if (!lowered)
2558 return op->emitError() << "NYI: lowering a byval/sret/byref call "
2559 "argument whose pointee type has no "
2560 "memory representation";
2561 result.argAttrs = *lowered;
2562 }
2563 result.resAttrs = mlir::dyn_cast_if_present<mlir::ArrayAttr>(
2564 Val: op->getInherentAttr(name: CIRDialect::getResAttrsAttrName())
2565 .value_or(u: mlir::Attribute{}));
2566
2567 for (mlir::NamedAttribute attr : op->getDiscardableAttrs()) {
2568 if (attr.getName() == CIRDialect::getNoUnwindAttrName() ||
2569 attr.getName() == CIRDialect::getNoReturnAttrName())
2570 continue;
2571 assert(!cir::MissingFeatures::opFuncExtraAttrs());
2572 result.discardableAttrs.push_back(Elt: attr);
2573 }
2574 return mlir::success();
2575}
2576
2577template <typename CallLikeOp>
2578static void setLoweredCallAttributes(CallLikeOp op,
2579 const LoweredCallAttributes &attributes) {
2580 if (attributes.argAttrs)
2581 op.setArgAttrsAttr(attributes.argAttrs);
2582 if (attributes.resAttrs)
2583 op.setResAttrsAttr(attributes.resAttrs);
2584 for (mlir::NamedAttribute attr : attributes.discardableAttrs) {
2585 if (op->getInherentAttr(attr.getName()).has_value())
2586 op->setInherentAttr(attr.getName(), attr.getValue());
2587 else
2588 op->setDiscardableAttr(attr.getName(), attr.getValue());
2589 }
2590}
2591
2592static mlir::LogicalResult rewriteCallOrInvoke(
2593 mlir::Operation *op, mlir::ValueRange callOperands,
2594 mlir::ConversionPatternRewriter &rewriter,
2595 const mlir::TypeConverter *converter, const mlir::DataLayout &dataLayout,
2596 mlir::SymbolTableCollection &symbolTables,
2597 mlir::FlatSymbolRefAttr calleeAttr, mlir::Block *continueBlock = nullptr,
2598 mlir::Block *landingPadBlock = nullptr) {
2599 llvm::SmallVector<mlir::Type, 8> llvmResults;
2600 mlir::ValueTypeRange<mlir::ResultRange> cirResults = op->getResultTypes();
2601 auto call = cast<cir::CIRCallOpInterface>(Val: op);
2602
2603 if (converter->convertTypes(types: cirResults, results&: llvmResults).failed())
2604 return mlir::failure();
2605
2606 mlir::LLVM::MemoryEffectsAttr memoryEffects;
2607 bool noUnwind = false;
2608 bool willReturn = false;
2609 bool noReturn = false;
2610 convertCallEffects(callOp: op, isNothrow: call.getNothrow(), effects: call.getMemoryEffectsAttr(),
2611 memoryEffect&: memoryEffects, noUnwind, willReturn, noReturn);
2612
2613 LoweredCallAttributes attributes;
2614 if (mlir::failed(
2615 Result: lowerCallAttributes(op: call, converter: *converter, dataLayout, result&: attributes)))
2616 return mlir::failure();
2617
2618 mlir::LLVM::LLVMFunctionType llvmFnTy;
2619
2620 // Temporary to handle the case where we need to prepend an operand if the
2621 // callee is an alias.
2622 SmallVector<mlir::Value> adjustedCallOperands;
2623
2624 if (calleeAttr) { // direct call
2625 mlir::Operation *callee =
2626 symbolTables.lookupNearestSymbolFrom(from: op, symbol: calleeAttr);
2627 if (auto fn = mlir::dyn_cast<mlir::FunctionOpInterface>(Val: callee)) {
2628 llvmFnTy = converter->convertType<mlir::LLVM::LLVMFunctionType>(
2629 t: fn.getFunctionType());
2630 assert(llvmFnTy && "Failed to convert function type");
2631 } else if (auto alias = mlir::cast<mlir::LLVM::AliasOp>(Val: callee)) {
2632 // If the callee was an alias. In that case,
2633 // we need to prepend the address of the alias to the operands. The
2634 // way aliases work in the LLVM dialect is a little counter-intuitive.
2635 // The AliasOp itself is a pseudo-function that returns the address of
2636 // the global value being aliased, but when we generate the call we
2637 // need to insert an operation that gets the address of the AliasOp.
2638 // This all gets sorted out when the LLVM dialect is lowered to LLVM IR.
2639 auto symAttr = mlir::cast<mlir::FlatSymbolRefAttr>(Val&: calleeAttr);
2640 auto addrOfAlias =
2641 mlir::LLVM::AddressOfOp::create(
2642 builder&: rewriter, location: op->getLoc(),
2643 res: mlir::LLVM::LLVMPointerType::get(context: rewriter.getContext()), global_name: symAttr)
2644 .getResult();
2645 adjustedCallOperands.push_back(Elt: addrOfAlias);
2646
2647 // Now add the regular operands and assign this to the range value.
2648 llvm::append_range(C&: adjustedCallOperands, R&: callOperands);
2649 callOperands = adjustedCallOperands;
2650
2651 // Clear the callee attribute because we're calling an alias.
2652 calleeAttr = {};
2653 llvmFnTy = mlir::cast<mlir::LLVM::LLVMFunctionType>(Val: alias.getType());
2654 } else {
2655 // Was this an ifunc?
2656 return op->emitError(message: "Unexpected callee type!");
2657 }
2658 } else { // indirect call
2659 assert(!op->getOperands().empty() &&
2660 "operands list must no be empty for the indirect call");
2661 auto calleeTy = op->getOperands().front().getType();
2662 auto calleePtrTy = cast<cir::PointerType>(Val&: calleeTy);
2663 auto calleeFuncTy = cast<cir::FuncType>(Val: calleePtrTy.getPointee());
2664 llvm::append_range(C&: adjustedCallOperands, R&: callOperands);
2665 llvmFnTy = cast<mlir::LLVM::LLVMFunctionType>(
2666 Val: converter->convertType(t: calleeFuncTy));
2667 }
2668
2669 mlir::LLVM::CConv cconv = convertCallingConv(callingConv: call.getCallingConv());
2670
2671 // The replace below erases the root op, so read what the CallOp branch
2672 // needs first.
2673 const bool mustTail = op->hasAttr(name: CIRDialect::getMustTailAttrName());
2674 const std::optional<cir::InlineKind> inlineKind = call.getInlineKind();
2675 if (landingPadBlock) {
2676 assert(!cir::MissingFeatures::opCallInvokeAttrs());
2677 auto newOp = rewriter.replaceOpWithNewOp<mlir::LLVM::InvokeOp>(
2678 op, args&: llvmFnTy, args&: calleeAttr, args&: callOperands, args&: continueBlock,
2679 args: mlir::ValueRange{}, args&: landingPadBlock, args: mlir::ValueRange{});
2680 newOp.setCConv(cconv);
2681 setLoweredCallAttributes(op: newOp, attributes);
2682 } else {
2683 auto newOp = rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
2684 op, args&: llvmFnTy, args&: calleeAttr, args&: callOperands);
2685 newOp.setCConv(cconv);
2686 setLoweredCallAttributes(op: newOp, attributes);
2687 if (memoryEffects)
2688 newOp.setMemoryEffectsAttr(memoryEffects);
2689 newOp.setNoUnwind(noUnwind);
2690 newOp.setWillReturn(willReturn);
2691 newOp.setNoreturn(noReturn);
2692 if (mustTail)
2693 newOp.setTailCallKind(mlir::LLVM::TailCallKind::MustTail);
2694
2695 if (inlineKind) {
2696 newOp.setNoInline(*inlineKind == cir::InlineKind::NoInline);
2697 newOp.setInlineHint(*inlineKind == cir::InlineKind::InlineHint);
2698 newOp.setAlwaysInline(*inlineKind == cir::InlineKind::AlwaysInline);
2699 }
2700 }
2701
2702 return mlir::success();
2703}
2704
2705mlir::LogicalResult CIRToLLVMCallOpLowering::matchAndRewrite(
2706 cir::CallOp op, OpAdaptor adaptor,
2707 mlir::ConversionPatternRewriter &rewriter) const {
2708 return rewriteCallOrInvoke(op: op.getOperation(), callOperands: adaptor.getOperands(), rewriter,
2709 converter: getTypeConverter(), dataLayout, symbolTables,
2710 calleeAttr: op.getCalleeAttr());
2711}
2712
2713mlir::LogicalResult CIRToLLVMTryCallOpLowering::matchAndRewrite(
2714 cir::TryCallOp op, OpAdaptor adaptor,
2715 mlir::ConversionPatternRewriter &rewriter) const {
2716 return rewriteCallOrInvoke(op: op.getOperation(), callOperands: adaptor.getOperands(), rewriter,
2717 converter: getTypeConverter(), dataLayout, symbolTables,
2718 calleeAttr: op.getCalleeAttr(), continueBlock: op.getNormalDest(),
2719 landingPadBlock: op.getUnwindDest());
2720}
2721
2722mlir::LogicalResult CIRToLLVMReturnAddrOpLowering::matchAndRewrite(
2723 cir::ReturnAddrOp op, OpAdaptor adaptor,
2724 mlir::ConversionPatternRewriter &rewriter) const {
2725 const mlir::Type llvmPtrTy = getTypeConverter()->convertType(t: op.getType());
2726 replaceOpWithCallLLVMIntrinsicOp(rewriter, op, intrinsicName: "llvm.returnaddress",
2727 resultTy: llvmPtrTy, operands: adaptor.getOperands());
2728 return mlir::success();
2729}
2730
2731mlir::LogicalResult CIRToLLVMFrameAddrOpLowering::matchAndRewrite(
2732 cir::FrameAddrOp op, OpAdaptor adaptor,
2733 mlir::ConversionPatternRewriter &rewriter) const {
2734 const mlir::Type llvmPtrTy = getTypeConverter()->convertType(t: op.getType());
2735 if (!llvmPtrTy)
2736 return emitTypeConversionError(op, type: op.getType());
2737 replaceOpWithCallLLVMIntrinsicOp(rewriter, op, intrinsicName: "llvm.frameaddress", resultTy: llvmPtrTy,
2738 operands: adaptor.getOperands());
2739 return mlir::success();
2740}
2741
2742mlir::LogicalResult CIRToLLVMClearCacheOpLowering::matchAndRewrite(
2743 cir::ClearCacheOp op, OpAdaptor adaptor,
2744 mlir::ConversionPatternRewriter &rewriter) const {
2745 mlir::Value begin = adaptor.getBegin();
2746 mlir::Value end = adaptor.getEnd();
2747 auto intrinNameAttr =
2748 mlir::StringAttr::get(context: op.getContext(), bytes: "llvm.clear_cache");
2749 rewriter.replaceOpWithNewOp<mlir::LLVM::CallIntrinsicOp>(
2750 op, args: mlir::Type{}, args&: intrinNameAttr, args: mlir::ValueRange{begin, end});
2751
2752 return mlir::success();
2753}
2754
2755mlir::LogicalResult CIRToLLVMAddrOfReturnAddrOpLowering::matchAndRewrite(
2756 cir::AddrOfReturnAddrOp op, OpAdaptor adaptor,
2757 mlir::ConversionPatternRewriter &rewriter) const {
2758 const mlir::Type llvmPtrTy = getTypeConverter()->convertType(t: op.getType());
2759 if (!llvmPtrTy)
2760 return emitTypeConversionError(op, type: op.getType());
2761 replaceOpWithCallLLVMIntrinsicOp(rewriter, op, intrinsicName: "llvm.addressofreturnaddress",
2762 resultTy: llvmPtrTy, operands: adaptor.getOperands());
2763 return mlir::success();
2764}
2765
2766mlir::LogicalResult CIRToLLVMAMDGPUFeaturePredicateOpLowering::matchAndRewrite(
2767 cir::AMDGPUFeaturePredicateOp op, OpAdaptor adaptor,
2768 mlir::ConversionPatternRewriter &rewriter) const {
2769 mlir::Location loc = op.getLoc();
2770 mlir::MLIRContext *ctx = rewriter.getContext();
2771 mlir::Value specId =
2772 mlir::LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: rewriter.getI32Type(), value: -1);
2773 mlir::Value defaultValue =
2774 mlir::LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: rewriter.getI1Type(), value: 0);
2775 auto predicate = mlir::LLVM::MDNodeAttr::get(
2776 context: ctx, operands: mlir::LLVM::MDStringAttr::get(context: ctx, value: op.getPredicateAttr()));
2777 mlir::Value md =
2778 mlir::LLVM::MetadataAsValueOp::create(builder&: rewriter, location: loc, metadata: predicate);
2779 replaceOpWithCallLLVMIntrinsicOp(
2780 rewriter, op, intrinsicName: "llvm.spv.named.boolean.spec.constant",
2781 resultTy: getTypeConverter()->convertType(t: op.getType()),
2782 operands: mlir::ValueRange{specId, defaultValue, md});
2783 return mlir::success();
2784}
2785
2786mlir::LogicalResult CIRToLLVMLoadOpLowering::matchAndRewrite(
2787 cir::LoadOp op, OpAdaptor adaptor,
2788 mlir::ConversionPatternRewriter &rewriter) const {
2789 const mlir::Type llvmTy =
2790 convertTypeForLoadStore(converter: *getTypeConverter(), dataLayout, type: op.getType());
2791 if (!llvmTy)
2792 return emitTypeConversionError(op, type: op.getType());
2793 mlir::LLVM::AtomicOrdering ordering = getLLVMMemOrder(memorder: op.getMemOrder());
2794 std::optional<size_t> opAlign = op.getAlignment();
2795 unsigned alignment = (unsigned)opAlign.value_or(
2796 u: getMemoryFallbackAlignment(op, cirType: op.getType(), llvmMemType: llvmTy, dataLayout));
2797
2798 assert(!cir::MissingFeatures::lowerModeOptLevel());
2799
2800 std::optional<llvm::StringRef> llvmSyncScope = getLLVMSyncScope(
2801 syncScope: op.getSyncScope(), moduleOp: op->getParentOfType<mlir::ModuleOp>());
2802
2803 mlir::LLVM::LoadOp newLoad = mlir::LLVM::LoadOp::create(
2804 builder&: rewriter, location: op->getLoc(), type: llvmTy, addr: adaptor.getAddr(), alignment,
2805 isVolatile: op.getIsVolatile(), /*isNonTemporal=*/op.getIsNontemporal(),
2806 /*isInvariant=*/op.getInvariant(), /*isInvariantGroup=*/false,
2807 /*isNoundef=*/false, ordering, syncscope: llvmSyncScope.value_or(u: std::string()));
2808 if (mlir::Attribute domain = op->getAttr(name: "cir.riscv_nontemporal_domain"))
2809 newLoad->setAttr(name: "cir.riscv_nontemporal_domain", value: domain);
2810
2811 // Convert adapted result to its original type if needed.
2812 mlir::Value result = emitFromMemory(rewriter, converter: *getTypeConverter(), dataLayout,
2813 op, value: newLoad.getResult());
2814 rewriter.replaceOp(op, newValues: result);
2815 assert(!cir::MissingFeatures::opLoadStoreTbaa());
2816 return mlir::LogicalResult::success();
2817}
2818
2819mlir::LogicalResult
2820cir::direct::CIRToLLVMVecMaskedLoadOpLowering::matchAndRewrite(
2821 cir::VecMaskedLoadOp op, OpAdaptor adaptor,
2822 mlir::ConversionPatternRewriter &rewriter) const {
2823 const mlir::Type llvmResTy =
2824 convertTypeForLoadStore(converter: *getTypeConverter(), dataLayout, type: op.getType());
2825 if (!llvmResTy)
2826 return op.emitError()
2827 << "NYI: lowering masked load of a type with no memory "
2828 "representation";
2829
2830 std::optional<size_t> opAlign = op.getAlignment();
2831 unsigned alignment =
2832 (unsigned)opAlign.value_or(u: dataLayout.getTypeABIAlignment(t: llvmResTy));
2833
2834 auto newLoad = mlir::LLVM::MaskedLoadOp::create(
2835 builder&: rewriter, location: op.getLoc(), res: llvmResTy, data: adaptor.getAddr(), mask: adaptor.getMask(),
2836 pass_thru: adaptor.getPassThru(), alignment);
2837
2838 rewriter.replaceOp(op, newValues: newLoad.getResult());
2839 return mlir::success();
2840}
2841
2842mlir::LogicalResult CIRToLLVMStoreOpLowering::matchAndRewrite(
2843 cir::StoreOp op, OpAdaptor adaptor,
2844 mlir::ConversionPatternRewriter &rewriter) const {
2845 mlir::LLVM::AtomicOrdering memorder = getLLVMMemOrder(memorder: op.getMemOrder());
2846 mlir::Type valueType = op.getValue().getType();
2847 const mlir::Type llvmTy =
2848 convertTypeForLoadStore(converter: *getTypeConverter(), dataLayout, type: valueType);
2849 if (!llvmTy)
2850 return op.emitError()
2851 << "NYI: lowering store of a type with no memory representation";
2852 std::optional<size_t> opAlign = op.getAlignment();
2853 unsigned alignment = (unsigned)opAlign.value_or(
2854 u: getMemoryFallbackAlignment(op, cirType: valueType, llvmMemType: llvmTy, dataLayout));
2855
2856 assert(!cir::MissingFeatures::lowerModeOptLevel());
2857
2858 // Convert adapted value to its memory type if needed.
2859 mlir::Value value = emitToMemory(rewriter, dataLayout,
2860 origType: op.getValue().getType(), value: adaptor.getValue());
2861 assert(!cir::MissingFeatures::opLoadStoreTbaa());
2862
2863 std::optional<llvm::StringRef> llvmSyncScope = getLLVMSyncScope(
2864 syncScope: op.getSyncScope(), moduleOp: op->getParentOfType<mlir::ModuleOp>());
2865
2866 mlir::LLVM::StoreOp storeOp = mlir::LLVM::StoreOp::create(
2867 builder&: rewriter, location: op->getLoc(), value, addr: adaptor.getAddr(), alignment,
2868 isVolatile: op.getIsVolatile(),
2869 /*isNonTemporal=*/op.getIsNontemporal(), /*isInvariantGroup=*/false,
2870 ordering: memorder, syncscope: llvmSyncScope.value_or(u: std::string()));
2871 if (mlir::Attribute domain = op->getAttr(name: "cir.riscv_nontemporal_domain"))
2872 storeOp->setAttr(name: "cir.riscv_nontemporal_domain", value: domain);
2873 rewriter.replaceOp(op, newOp: storeOp);
2874 assert(!cir::MissingFeatures::opLoadStoreTbaa());
2875 return mlir::LogicalResult::success();
2876}
2877
2878static mlir::Type getConstArrayBaseElementType(mlir::Type ty) {
2879 while (auto arrTy = mlir::dyn_cast<cir::ArrayType>(Val&: ty))
2880 ty = arrTy.getElementType();
2881 return ty;
2882}
2883
2884static bool isBulkLowerableConstArrayBaseElement(mlir::Type baseElemTy) {
2885 return mlir::isa<cir::PointerType, cir::IntType, cir::BoolType,
2886 cir::FPTypeInterface, cir::RecordType>(Val: baseElemTy);
2887}
2888
2889mlir::LogicalResult CIRToLLVMConstantOpLowering::matchAndRewrite(
2890 cir::ConstantOp op, OpAdaptor adaptor,
2891 mlir::ConversionPatternRewriter &rewriter) const {
2892 mlir::Attribute attr = op.getValue();
2893 if (!getTypeConverter()->convertType(t: op.getType()))
2894 return emitTypeConversionError(op, type: op.getType());
2895 mlir::ModuleOp moduleOp = op->getParentOfType<mlir::ModuleOp>();
2896
2897 if (mlir::isa<cir::PoisonAttr>(Val: attr)) {
2898 rewriter.replaceOpWithNewOp<mlir::LLVM::PoisonOp>(
2899 op, args: getTypeConverter()->convertType(t: op.getType()));
2900 return mlir::success();
2901 }
2902
2903 if (mlir::isa<cir::UndefAttr>(Val: attr)) {
2904 rewriter.replaceOpWithNewOp<mlir::LLVM::UndefOp>(
2905 op, args: getTypeConverter()->convertType(t: op.getType()));
2906 return mlir::success();
2907 }
2908
2909 if (mlir::isa<mlir::IntegerType>(Val: op.getType())) {
2910 // Verified cir.const operations cannot actually be of these types, but the
2911 // lowering pass may generate temporary cir.const operations with these
2912 // types. This is OK since MLIR allows unverified operations to be alive
2913 // during a pass as long as they don't live past the end of the pass.
2914 attr = op.getValue();
2915 } else if (mlir::isa<cir::BoolType>(Val: op.getType())) {
2916 int value = mlir::cast<cir::BoolAttr>(Val: op.getValue()).getValue();
2917 attr = rewriter.getIntegerAttr(type: typeConverter->convertType(t: op.getType()),
2918 value);
2919 } else if (mlir::isa<cir::IntType>(Val: op.getType())) {
2920 // Lower GlobalViewAttr to llvm.mlir.addressof + llvm.mlir.ptrtoint and
2921 // BlockAddrDiffAttr to the difference of two llvm.blockaddress.
2922 if (mlir::isa<cir::GlobalViewAttr, cir::BlockAddrDiffAttr>(Val: op.getValue())) {
2923 auto newOp = lowerCirAttrAsValue(
2924 parentLoc: op->getLoc(), parentModule: moduleOp, attr: op.getValue(), rewriter, symbolTables,
2925 converter: getTypeConverter(), preConversionTypes, blockInfoAddr: &blockInfoAddr);
2926 rewriter.replaceOp(op, newValues: newOp);
2927 return mlir::success();
2928 }
2929
2930 attr = rewriter.getIntegerAttr(
2931 type: typeConverter->convertType(t: op.getType()),
2932 value: mlir::cast<cir::IntAttr>(Val: op.getValue()).getValue());
2933 } else if (mlir::isa<cir::FPTypeInterface>(Val: op.getType())) {
2934 attr = rewriter.getFloatAttr(
2935 type: typeConverter->convertType(t: op.getType()),
2936 value: mlir::cast<cir::FPAttr>(Val: op.getValue()).getValue());
2937 } else if (mlir::isa<cir::PointerType>(Val: op.getType())) {
2938 // Optimize with dedicated LLVM op for null pointers.
2939 if (mlir::isa<cir::ConstPtrAttr>(Val: op.getValue())) {
2940 if (mlir::cast<cir::ConstPtrAttr>(Val: op.getValue()).isNullValue()) {
2941 rewriter.replaceOpWithNewOp<mlir::LLVM::ZeroOp>(
2942 op, args: typeConverter->convertType(t: op.getType()));
2943 return mlir::success();
2944 }
2945 }
2946 // Lower GlobalViewAttr and GlobalOffsetAttr to llvm.mlir.addressof,
2947 // non-null ConstPtrAttr to llvm.inttoptr and BlockAddrInfoAttr to
2948 // llvm.blockaddress.
2949 if (mlir::isa<cir::GlobalViewAttr, cir::GlobalOffsetAttr, cir::ConstPtrAttr,
2950 cir::BlockAddrInfoAttr>(Val: op.getValue())) {
2951 auto newOp = lowerCirAttrAsValue(
2952 parentLoc: op->getLoc(), parentModule: moduleOp, attr: op.getValue(), rewriter, symbolTables,
2953 converter: getTypeConverter(), preConversionTypes, blockInfoAddr: &blockInfoAddr);
2954 if (!newOp)
2955 return mlir::failure();
2956 rewriter.replaceOp(op, newValues: newOp);
2957 return mlir::success();
2958 }
2959 attr = op.getValue();
2960 } else if (const auto arrTy = mlir::dyn_cast<cir::ArrayType>(Val: op.getType())) {
2961 // Unlike a global initializer, the constant must keep its type.
2962 mlir::Attribute value = spellOutSplitTrailingZeros(attr: op.getValue());
2963 const auto constArr = mlir::dyn_cast<cir::ConstArrayAttr>(Val&: value);
2964 if (!constArr && !isa<cir::ZeroAttr, cir::UndefAttr>(Val: value))
2965 return op.emitError() << "array does not have a constant initializer";
2966
2967 std::optional<mlir::Attribute> denseAttr;
2968 if (constArr &&
2969 (denseAttr = lowerConstArrayAttr(constArr, symbolTables, converter: typeConverter,
2970 preConversionTypes, moduleOp))) {
2971 attr = denseAttr.value();
2972 } else {
2973 const mlir::Value initVal =
2974 lowerCirAttrAsValue(parentLoc: op->getLoc(), parentModule: moduleOp, attr: value, rewriter,
2975 symbolTables, converter: typeConverter, preConversionTypes);
2976 if (!initVal)
2977 return mlir::failure();
2978 rewriter.replaceOp(op, newValues: initVal);
2979 return mlir::success();
2980 }
2981 } else if (const auto recordAttr =
2982 mlir::dyn_cast<cir::ConstRecordAttr>(Val: op.getValue())) {
2983 auto initVal = lowerCirAttrAsValue(
2984 parentLoc: op->getLoc(), parentModule: moduleOp, attr: spellOutSplitTrailingZeros(attr: recordAttr),
2985 rewriter, symbolTables, converter: typeConverter, preConversionTypes);
2986 if (!initVal)
2987 return mlir::failure();
2988 rewriter.replaceOp(op, newValues: initVal);
2989 return mlir::success();
2990 } else if (const auto vecTy = mlir::dyn_cast<cir::VectorType>(Val: op.getType())) {
2991 mlir::Value initVal = lowerCirAttrAsValue(
2992 parentLoc: op->getLoc(), parentModule: moduleOp, attr: op.getValue(), rewriter, symbolTables,
2993 converter: getTypeConverter(), preConversionTypes);
2994 if (!initVal)
2995 return mlir::failure();
2996 rewriter.replaceOp(op, newValues: initVal);
2997 return mlir::success();
2998 } else if (mlir::isa<cir::RecordType>(Val: op.getType())) {
2999 if (mlir::isa<cir::ZeroAttr, cir::UndefAttr>(Val: attr)) {
3000 mlir::Value initVal =
3001 lowerCirAttrAsValue(parentLoc: op->getLoc(), parentModule: moduleOp, attr, rewriter,
3002 symbolTables, converter: typeConverter, preConversionTypes);
3003 if (!initVal)
3004 return mlir::failure();
3005 rewriter.replaceOp(op, newValues: initVal);
3006 return mlir::success();
3007 }
3008 return op.emitError() << "unsupported lowering for record constant type "
3009 << op.getType();
3010 } else if (auto complexTy = mlir::dyn_cast<cir::ComplexType>(Val: op.getType())) {
3011 mlir::Type complexElemTy = complexTy.getElementType();
3012 mlir::Type complexElemLLVMTy = typeConverter->convertType(t: complexElemTy);
3013
3014 if (auto zeroInitAttr = mlir::dyn_cast<cir::ZeroAttr>(Val: op.getValue())) {
3015 mlir::TypedAttr zeroAttr = rewriter.getZeroAttr(type: complexElemLLVMTy);
3016 mlir::ArrayAttr array = rewriter.getArrayAttr(value: {zeroAttr, zeroAttr});
3017 rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(
3018 op, args: getTypeConverter()->convertType(t: op.getType()), args&: array);
3019 return mlir::success();
3020 }
3021
3022 if (mlir::isa<cir::UndefAttr>(Val: op.getValue())) {
3023 rewriter.replaceOpWithNewOp<mlir::LLVM::UndefOp>(
3024 op, args: getTypeConverter()->convertType(t: op.getType()));
3025 return mlir::success();
3026 }
3027
3028 auto complexAttr = mlir::cast<cir::ConstComplexAttr>(Val: op.getValue());
3029
3030 mlir::Attribute components[2];
3031 if (mlir::isa<cir::IntType>(Val: complexElemTy)) {
3032 components[0] = rewriter.getIntegerAttr(
3033 type: complexElemLLVMTy,
3034 value: mlir::cast<cir::IntAttr>(Val: complexAttr.getReal()).getValue());
3035 components[1] = rewriter.getIntegerAttr(
3036 type: complexElemLLVMTy,
3037 value: mlir::cast<cir::IntAttr>(Val: complexAttr.getImag()).getValue());
3038 } else {
3039 components[0] = rewriter.getFloatAttr(
3040 type: complexElemLLVMTy,
3041 value: mlir::cast<cir::FPAttr>(Val: complexAttr.getReal()).getValue());
3042 components[1] = rewriter.getFloatAttr(
3043 type: complexElemLLVMTy,
3044 value: mlir::cast<cir::FPAttr>(Val: complexAttr.getImag()).getValue());
3045 }
3046
3047 attr = rewriter.getArrayAttr(value: components);
3048 } else {
3049 return op.emitError() << "unsupported constant type " << op.getType();
3050 }
3051
3052 rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(
3053 op, args: getTypeConverter()->convertType(t: op.getType()), args&: attr);
3054
3055 return mlir::success();
3056}
3057
3058static uint64_t getTypeAllocSize(mlir::Type type, mlir::Operation &op) {
3059 mlir::DataLayout layout(op.getParentOfType<mlir::ModuleOp>());
3060 // For LLVM purposes we treat void as u8.
3061 if (isa<cir::VoidType>(Val: type))
3062 type = cir::IntType::get(context: type.getContext(), width: 8, /*isSigned=*/is_signed: false);
3063 return layout.getTypeAllocSize(t: type);
3064}
3065
3066mlir::LogicalResult CIRToLLVMPrefetchOpLowering::matchAndRewrite(
3067 cir::PrefetchOp op, OpAdaptor adaptor,
3068 mlir::ConversionPatternRewriter &rewriter) const {
3069 rewriter.replaceOpWithNewOp<mlir::LLVM::Prefetch>(
3070 op, args: adaptor.getAddr(), args: adaptor.getIsWrite(), args: adaptor.getLocality(),
3071 /*DataCache=*/args: 1);
3072 return mlir::success();
3073}
3074
3075mlir::LogicalResult CIRToLLVMPtrDiffOpLowering::matchAndRewrite(
3076 cir::PtrDiffOp op, OpAdaptor adaptor,
3077 mlir::ConversionPatternRewriter &rewriter) const {
3078 mlir::Type pointeeType = op.getLhs().getType().getPointee();
3079 if (!convertTypeForMemory(converter: *getTypeConverter(), dataLayout, type: pointeeType))
3080 return emitTypeConversionError(op, type: pointeeType);
3081
3082 auto dstTy = mlir::cast<cir::IntType>(Val: op.getType());
3083 mlir::Type llvmDstTy = getTypeConverter()->convertType(t: dstTy);
3084
3085 auto lhs = mlir::LLVM::PtrToIntOp::create(builder&: rewriter, location: op.getLoc(), res: llvmDstTy,
3086 arg: adaptor.getLhs());
3087 auto rhs = mlir::LLVM::PtrToIntOp::create(builder&: rewriter, location: op.getLoc(), res: llvmDstTy,
3088 arg: adaptor.getRhs());
3089
3090 auto diff =
3091 mlir::LLVM::SubOp::create(builder&: rewriter, location: op.getLoc(), res: llvmDstTy, lhs, rhs);
3092
3093 assert(!cir::MissingFeatures::llvmLoweringPtrDiffConsidersPointee());
3094 uint64_t typeSize = getTypeAllocSize(type: pointeeType, op&: *op);
3095
3096 // Avoid silly division by 1.
3097 mlir::Value resultVal = diff.getResult();
3098 if (typeSize != 1) {
3099 auto typeSizeVal = mlir::LLVM::ConstantOp::create(builder&: rewriter, location: op.getLoc(),
3100 type: llvmDstTy, value: typeSize);
3101
3102 if (dstTy.isUnsigned()) {
3103 auto uDiv =
3104 mlir::LLVM::UDivOp::create(builder&: rewriter, location: op.getLoc(), lhs: diff, rhs: typeSizeVal);
3105 uDiv.setIsExact(true);
3106 resultVal = uDiv.getResult();
3107 } else {
3108 auto sDiv =
3109 mlir::LLVM::SDivOp::create(builder&: rewriter, location: op.getLoc(), lhs: diff, rhs: typeSizeVal);
3110 sDiv.setIsExact(true);
3111 resultVal = sDiv.getResult();
3112 }
3113 }
3114 rewriter.replaceOp(op, newValues: resultVal);
3115 return mlir::success();
3116}
3117
3118mlir::LogicalResult CIRToLLVMExpectOpLowering::matchAndRewrite(
3119 cir::ExpectOp op, OpAdaptor adaptor,
3120 mlir::ConversionPatternRewriter &rewriter) const {
3121 // TODO(cir): do not generate LLVM intrinsics under -O0
3122 assert(!cir::MissingFeatures::optInfoAttr());
3123
3124 std::optional<llvm::APFloat> prob = op.getProb();
3125 if (prob)
3126 rewriter.replaceOpWithNewOp<mlir::LLVM::ExpectWithProbabilityOp>(
3127 op, args: adaptor.getVal(), args: adaptor.getExpected(), args&: prob.value());
3128 else
3129 rewriter.replaceOpWithNewOp<mlir::LLVM::ExpectOp>(op, args: adaptor.getVal(),
3130 args: adaptor.getExpected());
3131 return mlir::success();
3132}
3133
3134/// Return true for discardable CIR markers handled explicitly below.
3135static bool isHandledDiscardableFuncAttr(mlir::NamedAttribute attr) {
3136 return attr.getName() == CIRDialect::getNoReturnAttrName() ||
3137 attr.getName() == CIRDialect::getNoUnwindAttrName() ||
3138 attr.getName() == CIRDialect::getWillReturnAttrName() ||
3139 attr.getName() == CIRDialect::getStrictFPAttrName() ||
3140 attr.getName() == CIRDialect::getNoRecurseAttrName() ||
3141 attr.getName() == CIRDialect::getMustProgressAttrName() ||
3142 attr.getName() == CIRDialect::getNoBuiltinAttrName() ||
3143 attr.getName() == CIRDialect::getOptNoneAttrName() ||
3144 attr.getName() == CIRDialect::getNakedAttrName() ||
3145 attr.getName() == CIRDialect::getSYCLModuleIdAttrName() ||
3146 attr.getName() == CIRDialect::getDontCallErrorAttrName() ||
3147 attr.getName() == CIRDialect::getDontCallWarnAttrName() ||
3148 attr.getName() == CIRDialect::getRuntimeFuncAttrName();
3149}
3150
3151/// Lower `cir.func` attributes for an `LLVMFuncOp` or `LLVM::AliasOp`.
3152/// Drop attributes populated by the destination op builder. If
3153/// `includeFunctionOnlyAttrs` is false, also omit attributes that are only
3154/// valid on functions. Fails when an argument attribute's pointee type has
3155/// no memory representation.
3156mlir::LogicalResult CIRToLLVMFuncOpLowering::lowerFuncAttributes(
3157 cir::FuncOp func, bool includeFunctionOnlyAttrs,
3158 SmallVectorImpl<mlir::NamedAttribute> &result) const {
3159 FunctionMetadataLowering metadataLowering(func.getContext());
3160 for (mlir::NamedAttribute attr : func->getDiscardableAttrs()) {
3161 if (isHandledDiscardableFuncAttr(attr))
3162 continue;
3163 if (metadataLowering.lower(attr, includeFunctionOnlyAttrs))
3164 continue;
3165 assert(!cir::MissingFeatures::opFuncExtraAttrs());
3166 result.push_back(Elt: attr);
3167 }
3168
3169 if (mlir::StringAttr visibility = func.getSymVisibilityAttr())
3170 result.emplace_back(Args: func.getSymVisibilityAttrName(), Args&: visibility);
3171
3172 if (includeFunctionOnlyAttrs) {
3173 if (mlir::ArrayAttr argAttrs = func.getArgAttrsAttr()) {
3174 std::optional<mlir::ArrayAttr> lowered = convertTypedArgAttrs(
3175 argAttrs, converter: *getTypeConverter(), dataLayout, ctx: getContext());
3176 if (!lowered)
3177 return func.emitError() << "NYI: lowering a byval/sret/byref "
3178 "argument whose pointee type has no "
3179 "memory representation";
3180 result.emplace_back(Args: func.getArgAttrsAttrName(), Args&: *lowered);
3181 }
3182 if (mlir::ArrayAttr resAttrs = func.getResAttrsAttr())
3183 result.emplace_back(Args: func.getResAttrsAttrName(), Args&: resAttrs);
3184 }
3185
3186 if (includeFunctionOnlyAttrs)
3187 metadataLowering.appendAttrs(result);
3188
3189 return mlir::success();
3190}
3191
3192mlir::LogicalResult CIRToLLVMFuncOpLowering::matchAndRewriteAlias(
3193 cir::FuncOp op, llvm::StringRef aliasee, mlir::Type ty, OpAdaptor adaptor,
3194 mlir::ConversionPatternRewriter &rewriter) const {
3195 SmallVector<mlir::NamedAttribute, 4> attributes;
3196 if (mlir::failed(Result: lowerFuncAttributes(func: op, /*includeFunctionOnlyAttrs=*/false,
3197 result&: attributes)))
3198 return mlir::failure();
3199
3200 mlir::Location loc = op.getLoc();
3201 // Create the alias before replacing: without pattern rollback the
3202 // replace erases the root op immediately. The driver does not
3203 // invalidate the cached symbol table either, so build it before the
3204 // replacement exists and maintain it explicitly below.
3205 mlir::ModuleOp modOp = op->getParentOfType<mlir::ModuleOp>();
3206 symbolTables.getSymbolTable(op: modOp);
3207 auto aliasOp = mlir::LLVM::AliasOp::create(
3208 builder&: rewriter, location: loc, type: ty, linkage: convertLinkage(linkage: op.getLinkage()), name: op.getName(),
3209 dsoLocal: op.getDsoLocal(), tls_mode: mlir::LLVM::ThreadLocalMode::NotThreadLocal,
3210 attrs: attributes);
3211 aliasOp.setVisibility_(
3212 lowerCIRVisibilityToLLVMVisibility(visibilityKind: op.getGlobalVisibility()));
3213 if (op.getUnnamedAddr())
3214 aliasOp.setUnnamedAddr(mlir::LLVM::UnnamedAddr::Global);
3215 {
3216 mlir::OpBuilder::InsertionGuard guard(rewriter);
3217 rewriter.createBlock(parent: &aliasOp.getInitializerRegion());
3218 cir::CIRDataLayout cirDataLayout(
3219 aliasOp->getParentOfType<mlir::ModuleOp>());
3220 mlir::Type ptrTy = mlir::LLVM::LLVMPointerType::get(
3221 context: ty.getContext(), addressSpace: cirDataLayout.getProgramAddressSpace());
3222 auto addrOp =
3223 mlir::LLVM::AddressOfOp::create(builder&: rewriter, location: loc, res: ptrTy, global_name: aliasee);
3224 mlir::LLVM::ReturnOp::create(builder&: rewriter, location: loc, arg: addrOp);
3225 }
3226 symbolTables.getSymbolTable(op: modOp).remove(op);
3227 rewriter.replaceOp(op, newOp: aliasOp);
3228 symbolTables.getSymbolTable(op: modOp).insert(symbol: aliasOp);
3229
3230 return mlir::success();
3231}
3232
3233mlir::LogicalResult CIRToLLVMFuncOpLowering::matchAndRewrite(
3234 cir::FuncOp op, OpAdaptor adaptor,
3235 mlir::ConversionPatternRewriter &rewriter) const {
3236
3237 cir::FuncType fnType = op.getFunctionType();
3238 bool isDsoLocal = op.getDsoLocal();
3239 mlir::TypeConverter::SignatureConversion signatureConversion(
3240 fnType.getNumInputs());
3241
3242 for (const auto &argType : llvm::enumerate(First: fnType.getInputs())) {
3243 mlir::Type convertedType = typeConverter->convertType(t: argType.value());
3244 if (!convertedType)
3245 return emitTypeConversionError(op, type: argType.value());
3246 signatureConversion.addInputs(origInputNo: argType.index(), types: convertedType);
3247 }
3248
3249 mlir::Type resultType =
3250 getTypeConverter()->convertType(t: fnType.getReturnType());
3251 if (!resultType)
3252 return emitTypeConversionError(op, type: fnType.getReturnType());
3253
3254 // Create the LLVM function operation.
3255 mlir::Type llvmFnTy = mlir::LLVM::LLVMFunctionType::get(
3256 result: resultType, arguments: signatureConversion.getConvertedTypes(),
3257 /*isVarArg=*/fnType.isVarArg());
3258
3259 // If this is an alias, it needs to be lowered to llvm::AliasOp.
3260 if (std::optional<llvm::StringRef> aliasee = op.getAliasee())
3261 return matchAndRewriteAlias(op, aliasee: *aliasee, ty: llvmFnTy, adaptor, rewriter);
3262
3263 // LLVMFuncOp expects a single FileLine Location instead of a fused
3264 // location.
3265 mlir::Location loc = op.getLoc();
3266 if (mlir::FusedLoc fusedLoc = mlir::dyn_cast<mlir::FusedLoc>(Val&: loc))
3267 loc = fusedLoc.getLocations()[0];
3268 assert((mlir::isa<mlir::FileLineColLoc>(loc) ||
3269 mlir::isa<mlir::UnknownLoc>(loc)) &&
3270 "expected single location or unknown location here");
3271
3272 mlir::LLVM::Linkage linkage = convertLinkage(linkage: op.getLinkage());
3273 mlir::LLVM::CConv cconv = convertCallingConv(callingConv: op.getCallingConv());
3274 SmallVector<mlir::NamedAttribute, 4> attributes;
3275 if (mlir::failed(Result: lowerFuncAttributes(func: op, /*includeFunctionOnlyAttrs=*/true,
3276 result&: attributes)))
3277 return mlir::failure();
3278
3279 // Convert the body types here: this mutates the op, so it runs after the
3280 // last fallible read above and before any op creation below. A failure
3281 // leaves the body unmoved for the driver to report.
3282 if (failed(Result: rewriter.convertRegionTypes(region: &op.getBody(), converter: *typeConverter,
3283 entryConversion: &signatureConversion)))
3284 return mlir::failure();
3285
3286 mlir::SymbolRefAttr comdatAttr = getComdatAttr(op, builder&: rewriter);
3287
3288 // The driver does not invalidate the cached symbol table, so build it
3289 // before the replacement exists and maintain it explicitly below.
3290 mlir::ModuleOp modOp = op->getParentOfType<mlir::ModuleOp>();
3291 symbolTables.getSymbolTable(op: modOp);
3292 mlir::LLVM::LLVMFuncOp fn = mlir::LLVM::LLVMFuncOp::create(
3293 builder&: rewriter, location: loc, name: op.getName(), type: llvmFnTy, linkage, dsoLocal: isDsoLocal, cconv,
3294 comdat: comdatAttr, attrs: attributes);
3295
3296 assert(!cir::MissingFeatures::opFuncMultipleReturnVals());
3297
3298 if (op.getUnnamedAddr())
3299 fn.setUnnamedAddr(mlir::LLVM::UnnamedAddr::Global);
3300
3301 if (cir::MemoryEffectsAttr effects = op.getMemoryEffectsAttr())
3302 fn.setMemoryEffectsAttr(convertMemoryEffects(ctx: fn.getContext(), effects));
3303
3304 // As for calls, either cannot-unwind fact maps to LLVM's nounwind.
3305 if (op->hasAttr(name: CIRDialect::getNoUnwindAttrName()) ||
3306 op->hasAttr(name: CIRDialect::getNoThrowAttrName()))
3307 fn.setNoUnwind(true);
3308 if (op->hasAttr(name: CIRDialect::getWillReturnAttrName()))
3309 fn.setWillReturn(true);
3310
3311 if (op->hasAttr(name: CIRDialect::getNoReturnAttrName()))
3312 fn.setNoreturn(true);
3313
3314 if (op->hasAttr(name: CIRDialect::getOptNoneAttrName()))
3315 fn.setOptimizeNone(true);
3316
3317 if (std::optional<cir::UnwindTableKind> uwtableKind = op.getUwtable())
3318 fn.setUwtableKindAttr(mlir::LLVM::UWTableKindAttr::get(
3319 context: fn.getContext(), uwtableKind: convertUWTableKind(kind: *uwtableKind)));
3320
3321 if (std::optional<cir::FramePointerKind> framePointerKind =
3322 op.getFramePointer())
3323 fn.setFramePointerAttr(mlir::LLVM::FramePointerKindAttr::get(
3324 context: fn.getContext(), framePointerKind: convertFramePointerKind(kind: *framePointerKind)));
3325
3326 // Function attributes with no dedicated field on the LLVM dialect's
3327 // LLVMFuncOp are routed through the `passthrough` array. The MLIR LLVM IR
3328 // translator forwards `passthrough` entries to LLVM IR as function
3329 // attributes.
3330 SmallVector<mlir::Attribute> passthrough;
3331 for (llvm::StringRef flagAttr :
3332 {CIRDialect::getStrictFPAttrName(), CIRDialect::getNoRecurseAttrName(),
3333 CIRDialect::getMustProgressAttrName(),
3334 CIRDialect::getNoBuiltinAttrName(), CIRDialect::getNakedAttrName()})
3335 if (op->hasAttr(name: flagAttr))
3336 passthrough.push_back(Elt: rewriter.getStringAttr(bytes: flagAttr));
3337
3338 if (auto moduleId = op->getAttrOfType<mlir::StringAttr>(
3339 name: CIRDialect::getSYCLModuleIdAttrName()))
3340 passthrough.push_back(Elt: rewriter.getArrayAttr(
3341 value: {rewriter.getStringAttr(bytes: CIRDialect::getSYCLModuleIdAttrName()),
3342 moduleId}));
3343
3344 for (llvm::StringRef dontCallAttr : {CIRDialect::getDontCallErrorAttrName(),
3345 CIRDialect::getDontCallWarnAttrName()})
3346 if (auto diagnostic = op->getAttrOfType<mlir::StringAttr>(name: dontCallAttr))
3347 passthrough.push_back(Elt: rewriter.getArrayAttr(
3348 value: {rewriter.getStringAttr(bytes: dontCallAttr), diagnostic}));
3349
3350 // Every function but a runtime function gets the TU-wide default function
3351 // attributes.
3352 if (!op->hasAttr(name: CIRDialect::getRuntimeFuncAttrName())) {
3353 if (auto langOpts = op->getParentOfType<mlir::ModuleOp>()
3354 ->getAttrOfType<cir::LoweringOptionsAttr>(
3355 name: CIRDialect::getLoweringOptionsAttrName())) {
3356 // A strictfp function may use a pragma to observe FP exceptions locally,
3357 // so the TU-wide default does not apply to it.
3358 if (langOpts.getNoTrappingMath() &&
3359 !op->hasAttr(name: CIRDialect::getStrictFPAttrName()))
3360 passthrough.push_back(
3361 Elt: rewriter.getStrArrayAttr(values: {"no-trapping-math", "true"}));
3362 passthrough.push_back(Elt: rewriter.getStrArrayAttr(
3363 values: {"stack-protector-buffer-size",
3364 llvm::utostr(X: langOpts.getSspBufferSize())}));
3365 }
3366 }
3367
3368 if (op.getCoroutineKind())
3369 passthrough.push_back(Elt: rewriter.getStringAttr(bytes: "presplitcoroutine"));
3370 if (!passthrough.empty())
3371 fn.setPassthroughAttr(rewriter.getArrayAttr(value: passthrough));
3372
3373 if (std::optional<cir::InlineKind> inlineKind = op.getInlineKind()) {
3374 fn.setNoInline(*inlineKind == cir::InlineKind::NoInline);
3375 fn.setInlineHint(*inlineKind == cir::InlineKind::InlineHint);
3376 fn.setAlwaysInline(*inlineKind == cir::InlineKind::AlwaysInline);
3377 }
3378
3379 if (std::optional<llvm::StringRef> personality = op.getPersonality())
3380 fn.setPersonality(*personality);
3381
3382 fn.setVisibility_(
3383 lowerCIRVisibilityToLLVMVisibility(visibilityKind: op.getGlobalVisibility()));
3384
3385 fn.setAlignment(op.getAlignment());
3386 assert(!MissingFeatures::opFuncPreferredAlignment());
3387
3388 symbolTables.getSymbolTable(op: modOp).remove(op);
3389 rewriter.inlineRegionBefore(region&: op.getBody(), parent&: fn.getBody(), before: fn.end());
3390
3391 rewriter.eraseOp(op);
3392 symbolTables.getSymbolTable(op: modOp).insert(symbol: fn);
3393
3394 return mlir::LogicalResult::success();
3395}
3396
3397mlir::LogicalResult CIRToLLVMGetGlobalOpLowering::matchAndRewrite(
3398 cir::GetGlobalOp op, OpAdaptor adaptor,
3399 mlir::ConversionPatternRewriter &rewriter) const {
3400 // FIXME(cir): Premature DCE to avoid lowering stuff we're not using.
3401 // CIRGen should mitigate this and not emit the get_global.
3402 if (op->getUses().empty()) {
3403 rewriter.eraseOp(op);
3404 return mlir::success();
3405 }
3406
3407 mlir::Type type = getTypeConverter()->convertType(t: op.getType());
3408 if (!type)
3409 return emitTypeConversionError(op, type: op.getType());
3410 mlir::Operation *newop = mlir::LLVM::AddressOfOp::create(
3411 builder&: rewriter, location: op.getLoc(), res: type, global_name: op.getName());
3412
3413 if (op.getTls()) {
3414 // Handle access to TLS via intrinsic.
3415 newop = mlir::LLVM::ThreadlocalAddressOp::create(builder&: rewriter, location: op.getLoc(),
3416 res: type, global: newop->getResult(idx: 0));
3417 }
3418
3419 rewriter.replaceOp(op, newOp: newop);
3420 return mlir::success();
3421}
3422
3423llvm::SmallVector<mlir::NamedAttribute>
3424CIRToLLVMGlobalOpLowering::lowerGlobalAttributes(
3425 cir::GlobalOp op, mlir::ConversionPatternRewriter &rewriter) const {
3426 SmallVector<mlir::NamedAttribute> attributes;
3427
3428 if (mlir::StringAttr sectionAttr = op.getSectionAttr())
3429 attributes.push_back(Elt: rewriter.getNamedAttr(name: "section", val: sectionAttr));
3430
3431 mlir::LLVM::VisibilityAttr visibility = mlir::LLVM::VisibilityAttr::get(
3432 context: getContext(),
3433 val: lowerCIRVisibilityToLLVMVisibility(visibilityKind: op.getGlobalVisibility()));
3434 attributes.push_back(Elt: rewriter.getNamedAttr(name: "visibility_", val: visibility));
3435
3436 if (op.getUnnamedAddr())
3437 attributes.push_back(Elt: rewriter.getNamedAttr(
3438 name: "unnamed_addr", val: mlir::LLVM::UnnamedAddrAttr::get(
3439 context: getContext(), val: mlir::LLVM::UnnamedAddr::Global)));
3440
3441 if (op->getAttr(name: CUDAExternallyInitializedAttr::getMnemonic()))
3442 attributes.push_back(Elt: rewriter.getNamedAttr(name: "externally_initialized",
3443 val: rewriter.getUnitAttr()));
3444
3445 return attributes;
3446}
3447
3448static mlir::LLVM::ThreadLocalMode
3449convertTlsModelAttrToLLVM(TLSModelAttr attr) {
3450 // assert that we can just static-cast these.
3451#define CHECK_ENUM(CIR, LLVM_VAL) \
3452 static_assert(static_cast<unsigned>(TLSModel::CIR) == \
3453 static_cast<unsigned>(mlir::LLVM::ThreadLocalMode::LLVM_VAL))
3454 CHECK_ENUM(GeneralDynamic, GeneralDynamic);
3455 CHECK_ENUM(LocalDynamic, LocalDynamic);
3456 CHECK_ENUM(InitialExec, InitialExec);
3457 CHECK_ENUM(LocalExec, LocalExec);
3458#undef CHECK_ENUM
3459
3460 if (!attr)
3461 return mlir::LLVM::ThreadLocalMode::NotThreadLocal;
3462
3463 return static_cast<mlir::LLVM::ThreadLocalMode>(attr.getValue());
3464}
3465
3466/// Create the region-initialized LLVM global beside the CIR op and update
3467/// insertion point to the end of the initializer block. Returns the new op,
3468/// or null on failure. The caller owns the replacement: it runs the
3469/// initializer visit with the old op still in the table, then swaps the
3470/// table and erases the old op on success.
3471mlir::LLVM::GlobalOp
3472CIRToLLVMGlobalOpLowering::setupRegionInitializedLLVMGlobalOp(
3473 cir::GlobalOp op, mlir::ConversionPatternRewriter &rewriter) const {
3474 mlir::Type llvmType =
3475 convertTypeForMemory(converter: *getTypeConverter(), dataLayout, type: op.getSymType());
3476
3477 // Keep the global's type in sync with the value built by CIRAttrToValue: a
3478 // flexible array member initializer requires an oversized anonymous struct.
3479 mlir::Type failedType = op.getSymType();
3480 if (std::optional<mlir::Attribute> init = op.getInitialValue())
3481 llvmType = adjustGlobalTypeForInit(llvmType, init: *init, converter: *getTypeConverter(),
3482 dataLayout, failedType: &failedType);
3483 if (!llvmType) {
3484 emitTypeConversionError(op, type: failedType);
3485 return {};
3486 }
3487
3488 // FIXME: These default values are placeholders until the the equivalent
3489 // attributes are available on cir.global ops. This duplicates code
3490 // in CIRToLLVMGlobalOpLowering::matchAndRewrite() but that will go
3491 // away when the placeholders are no longer needed.
3492 const bool isConst = op.getConstant();
3493 unsigned addrSpace = 0;
3494 if (auto targetAS = mlir::dyn_cast_if_present<cir::TargetAddressSpaceAttr>(
3495 Val: op.getAddrSpaceAttr()))
3496 addrSpace = targetAS.getValue();
3497 const bool isDsoLocal = op.getDsoLocal();
3498 mlir::LLVM::ThreadLocalMode threadLocalMode =
3499 convertTlsModelAttrToLLVM(attr: op.getTlsModelAttr());
3500 const uint64_t alignment = op.getAlignment().value_or(u: 0);
3501 const mlir::LLVM::Linkage linkage = convertLinkage(linkage: op.getLinkage());
3502 const StringRef symbol = op.getSymName();
3503
3504 SmallVector<mlir::NamedAttribute> attributes =
3505 lowerGlobalAttributes(op, rewriter);
3506
3507 mlir::LLVM::GlobalOp newGlobalOp = mlir::LLVM::GlobalOp::create(
3508 builder&: rewriter, location: op.getLoc(), type: llvmType, isConstant: isConst, linkage, name: symbol, value: nullptr,
3509 alignment, addrSpace, dsoLocal: isDsoLocal, tls_mode: threadLocalMode, comdat: mlir::SymbolRefAttr{},
3510 attrs: attributes);
3511 rewriter.createBlock(parent: &newGlobalOp.getRegion());
3512 rewriter.setInsertionPointToEnd(newGlobalOp.getInitializerBlock());
3513 return newGlobalOp;
3514}
3515
3516mlir::LogicalResult
3517CIRToLLVMGlobalOpLowering::matchAndRewriteRegionInitializedGlobal(
3518 cir::GlobalOp op, mlir::Attribute init,
3519 mlir::ConversionPatternRewriter &rewriter) const {
3520 // TODO: Generalize this handling when more types are needed here.
3521 assert((isa<cir::BlockAddrDiffAttr, cir::BlockAddrInfoAttr,
3522 cir::ConstArrayAttr, cir::ConstRecordAttr, cir::ConstVectorAttr,
3523 cir::ConstPtrAttr, cir::ConstComplexAttr, cir::GlobalOffsetAttr,
3524 cir::GlobalViewAttr, cir::TypeInfoAttr, cir::UndefAttr,
3525 cir::PoisonAttr, cir::VTableAttr, cir::ZeroAttr>(init)));
3526
3527 // TODO(cir): once LLVM's dialect has proper equivalent attributes this
3528 // should be updated. For now, we use a custom op to initialize globals
3529 // to the appropriate value.
3530 const mlir::Location loc = op.getLoc();
3531 mlir::ModuleOp moduleOp = op->getParentOfType<mlir::ModuleOp>();
3532 // Build the cached table before the replacement exists, so the swap below
3533 // drops a real entry instead of depending on duplicate tolerance.
3534 auto &symbolTable = symbolTables.getSymbolTable(op: moduleOp);
3535 // Create the replacement beside the root op, not via replaceOp: the table
3536 // swap below and the visit both need the old op alive.
3537 rewriter.setInsertionPoint(op);
3538 mlir::LLVM::GlobalOp newGlobalOp =
3539 setupRegionInitializedLLVMGlobalOp(op, rewriter);
3540 if (!newGlobalOp)
3541 return mlir::failure();
3542 // Keep the old op in the table during the visit so self-references
3543 // resolve to the CIR type, not the possibly adjusted LLVM type.
3544 // Pass blockInfoAddr so that block address initializers (either as the whole
3545 // initializer or nested inside an aggregate) can be resolved by the
3546 // BlockAddrInfoAttr visitor.
3547 CIRAttrToValue valueConverter(loc, moduleOp, rewriter, symbolTables,
3548 typeConverter, preConversionTypes,
3549 &blockInfoAddr);
3550 mlir::Value value = valueConverter.visit(attr: init);
3551 if (!value) {
3552 // Drop the speculative replacement; the table still holds the live
3553 // root op for the driver to report on. Everything created so far
3554 // lives inside the new global's region, so erasing it restores the
3555 // module exactly: module-level ops must wait for a successful visit
3556 // (see the comdat below).
3557 rewriter.eraseOp(op: newGlobalOp);
3558 return mlir::failure();
3559 }
3560 mlir::LLVM::ReturnOp::create(builder&: rewriter, location: loc, arg: value);
3561 // Attach comdat only after the visit succeeds, so the failure path
3562 // above only has the new global itself to drop.
3563 if (mlir::SymbolRefAttr comdatAttr = getComdatAttr(op, builder&: rewriter))
3564 newGlobalOp.setComdatAttr(comdatAttr);
3565 symbolTable.remove(op);
3566 symbolTable.insert(symbol: newGlobalOp);
3567 rewriter.eraseOp(op);
3568 return mlir::success();
3569}
3570
3571// Replace a CIR global with an LLVM global, maintaining the cached symbol
3572// table. The table entry is dropped before creation so the replacement
3573// reuses the name.
3574static mlir::LLVM::GlobalOp
3575replaceCirGlobal(mlir::SymbolTableCollection &symbolTables, cir::GlobalOp op,
3576 mlir::Type llvmType, bool isConst, mlir::LLVM::Linkage linkage,
3577 llvm::StringRef symbol, mlir::Attribute init,
3578 uint64_t alignment, unsigned addrSpace, bool isDsoLocal,
3579 mlir::LLVM::ThreadLocalMode threadLocalMode,
3580 mlir::SymbolRefAttr comdatAttr,
3581 llvm::ArrayRef<mlir::NamedAttribute> attributes,
3582 mlir::ConversionPatternRewriter &rewriter) {
3583 mlir::ModuleOp modOp = op->getParentOfType<mlir::ModuleOp>();
3584 symbolTables.getSymbolTable(op: modOp).remove(op);
3585 auto newGlobalOp = rewriter.replaceOpWithNewOp<mlir::LLVM::GlobalOp>(
3586 op, args&: llvmType, args&: isConst, args&: linkage, args&: symbol, args&: init, args&: alignment, args&: addrSpace,
3587 args&: isDsoLocal, args&: threadLocalMode, args&: comdatAttr, args&: attributes);
3588 symbolTables.getSymbolTable(op: modOp).insert(symbol: newGlobalOp);
3589 return newGlobalOp;
3590}
3591
3592mlir::LogicalResult CIRToLLVMGlobalOpLowering::matchAndRewrite(
3593 cir::GlobalOp op, OpAdaptor adaptor,
3594 mlir::ConversionPatternRewriter &rewriter) const {
3595 if (mlir::isa_and_present<cir::LangAddressSpaceAttr>(Val: op.getAddrSpaceAttr()))
3596 return op.emitError()
3597 << "cannot lower a global with a language address space";
3598
3599 // If this global requires non-trivial initialization or destruction,
3600 // that needs to be moved to runtime handlers during LoweringPrepare.
3601 if (!op.getCtorRegion().empty() || !op.getDtorRegion().empty())
3602 return op.emitError() << "GlobalOp ctor and dtor regions should be removed "
3603 "in LoweringPrepare";
3604
3605 std::optional<mlir::Attribute> init = op.getInitialValue();
3606
3607 // Fetch required values to create LLVM op.
3608 const mlir::Type cirSymType = op.getSymType();
3609
3610 // This is the LLVM dialect type.
3611 mlir::Type llvmType =
3612 convertTypeForMemory(converter: *getTypeConverter(), dataLayout, type: cirSymType);
3613 if (!llvmType)
3614 return op.emitError()
3615 << "NYI: lowering global of a type with no memory representation";
3616
3617 // A flexible array member initializer makes the constant larger than the
3618 // record's declared type, so the global must use an oversized anonymous
3619 // struct instead.
3620 mlir::Type failedType = cirSymType;
3621 if (init.has_value())
3622 llvmType = adjustGlobalTypeForInit(llvmType, init: *init, converter: *getTypeConverter(),
3623 dataLayout, failedType: &failedType);
3624 if (!llvmType)
3625 return emitTypeConversionError(op, type: failedType);
3626
3627 // FIXME: These default values are placeholders until the the equivalent
3628 // attributes are available on cir.global ops.
3629 const bool isConst = op.getConstant();
3630 unsigned addrSpace = 0;
3631 if (auto targetAS = mlir::dyn_cast_if_present<cir::TargetAddressSpaceAttr>(
3632 Val: op.getAddrSpaceAttr()))
3633 addrSpace = targetAS.getValue();
3634 const bool isDsoLocal = op.getDsoLocal();
3635 mlir::LLVM::ThreadLocalMode threadLocalMode =
3636 convertTlsModelAttrToLLVM(attr: op.getTlsModelAttr());
3637 const uint64_t alignment = op.getAlignment().value_or(u: 0);
3638 const mlir::LLVM::Linkage linkage = convertLinkage(linkage: op.getLinkage());
3639 const StringRef symbol = op.getSymName();
3640 SmallVector<mlir::NamedAttribute> attributes =
3641 lowerGlobalAttributes(op, rewriter);
3642
3643 // If this is a variable alias, lower it to llvm.mlir.alias.
3644 if (std::optional<llvm::StringRef> aliasee = op.getAliasee()) {
3645 mlir::Location loc = op.getLoc();
3646 // Create the alias before replacing: without pattern rollback the
3647 // replace erases the root op immediately. The driver does not
3648 // invalidate the cached symbol table either, so build it before the
3649 // replacement exists and maintain it explicitly below.
3650 mlir::ModuleOp modOp = op->getParentOfType<mlir::ModuleOp>();
3651 symbolTables.getSymbolTable(op: modOp);
3652 auto aliasOp =
3653 mlir::LLVM::AliasOp::create(builder&: rewriter, location: loc, type: llvmType, linkage, name: symbol,
3654 dsoLocal: isDsoLocal, tls_mode: threadLocalMode, attrs: attributes);
3655 {
3656 mlir::OpBuilder::InsertionGuard guard(rewriter);
3657 rewriter.createBlock(parent: &aliasOp.getInitializerRegion());
3658 mlir::Type ptrTy =
3659 mlir::LLVM::LLVMPointerType::get(context: getContext(), addressSpace: addrSpace);
3660 auto addrOp =
3661 mlir::LLVM::AddressOfOp::create(builder&: rewriter, location: loc, res: ptrTy, global_name: *aliasee);
3662 mlir::LLVM::ReturnOp::create(builder&: rewriter, location: loc, arg: addrOp);
3663 }
3664 symbolTables.getSymbolTable(op: modOp).remove(op);
3665 rewriter.replaceOp(op, newOp: aliasOp);
3666 symbolTables.getSymbolTable(op: modOp).insert(symbol: aliasOp);
3667 return mlir::success();
3668 }
3669
3670 if (init.has_value()) {
3671 if (mlir::isa<cir::FPAttr, cir::IntAttr, cir::BoolAttr>(Val: init.value())) {
3672 GlobalInitAttrRewriter initRewriter(llvmType, dataLayout, rewriter);
3673 init = initRewriter.visit(attr: init.value());
3674 // If initRewriter returned a null attribute, init will have a value but
3675 // the value will be null. If that happens, initRewriter didn't handle the
3676 // attribute type. It probably needs to be added to
3677 // GlobalInitAttrRewriter.
3678 if (!init.value()) {
3679 op.emitError() << "unsupported initializer '" << init.value() << "'";
3680 return mlir::failure();
3681 }
3682 } else if (auto constArr =
3683 mlir::dyn_cast<cir::ConstArrayAttr>(Val&: init.value())) {
3684 // Bulk-emit llvm.mlir.global when lowerConstArrayAttr can build the
3685 // whole initializer as one aggregate attribute (no insertvalue
3686 // region). Leaf type must match what lowerConstArrayAttr handles
3687 // (pointers, integers, bools, floats, and string literals with
3688 // trailing_zeros).
3689 if (isBulkLowerableConstArrayBaseElement(
3690 baseElemTy: getConstArrayBaseElementType(ty: constArr.getType()))) {
3691 mlir::ModuleOp modOp = op->getParentOfType<mlir::ModuleOp>();
3692 if (std::optional<mlir::Attribute> bulkInit =
3693 lowerConstArrayAttr(constArr, symbolTables, converter: typeConverter,
3694 preConversionTypes, moduleOp: modOp)) {
3695 mlir::SymbolRefAttr comdatAttr = getComdatAttr(op, builder&: rewriter);
3696 replaceCirGlobal(symbolTables, op, llvmType, isConst, linkage, symbol,
3697 init: bulkInit.value(), alignment, addrSpace, isDsoLocal,
3698 threadLocalMode, comdatAttr, attributes, rewriter);
3699 return mlir::success();
3700 }
3701 }
3702 return matchAndRewriteRegionInitializedGlobal(op, init: init.value(), rewriter);
3703 } else if (auto constRecord =
3704 mlir::dyn_cast<cir::ConstRecordAttr>(Val&: init.value())) {
3705 // Bulk-emit llvm.mlir.global when every member of the record can be
3706 // lowered to a constant attribute. The LLVM dialect global translation
3707 // turns an ArrayAttr (one element per struct field) into an
3708 // llvm::ConstantStruct, so the whole initializer becomes a single
3709 // attribute on the global instead of an insertvalue region.
3710 mlir::ModuleOp modOp = op->getParentOfType<mlir::ModuleOp>();
3711 if (std::optional<mlir::Attribute> bulkInit =
3712 lowerConstRecordAttr(constRecord, symbolTables, converter: typeConverter,
3713 preConversionTypes, moduleOp: modOp)) {
3714 mlir::SymbolRefAttr comdatAttr = getComdatAttr(op, builder&: rewriter);
3715 replaceCirGlobal(symbolTables, op, llvmType, isConst, linkage, symbol,
3716 init: bulkInit.value(), alignment, addrSpace, isDsoLocal,
3717 threadLocalMode, comdatAttr, attributes, rewriter);
3718 return mlir::success();
3719 }
3720 return matchAndRewriteRegionInitializedGlobal(op, init: init.value(), rewriter);
3721 } else if (mlir::isa<cir::BlockAddrDiffAttr, cir::BlockAddrInfoAttr,
3722 cir::ConstVectorAttr, cir::ConstRecordAttr,
3723 cir::ConstPtrAttr, cir::ConstComplexAttr,
3724 cir::GlobalOffsetAttr, cir::GlobalViewAttr,
3725 cir::TypeInfoAttr, cir::UndefAttr, cir::PoisonAttr,
3726 cir::VTableAttr, cir::ZeroAttr>(Val: init.value())) {
3727 // TODO(cir): once LLVM's dialect has proper equivalent attributes this
3728 // should be updated. For now, we use a custom op to initialize globals
3729 // to the appropriate value.
3730 return matchAndRewriteRegionInitializedGlobal(op, init: init.value(), rewriter);
3731 } else {
3732 // We will only get here if new initializer types are added and this
3733 // code is not updated to handle them.
3734 op.emitError() << "unsupported initializer '" << init.value() << "'";
3735 return mlir::failure();
3736 }
3737 }
3738
3739 mlir::SymbolRefAttr comdatAttr = getComdatAttr(op, builder&: rewriter);
3740 replaceCirGlobal(symbolTables, op, llvmType, isConst, linkage, symbol,
3741 init: init.value_or(u: mlir::Attribute()), alignment, addrSpace,
3742 isDsoLocal, threadLocalMode, comdatAttr, attributes,
3743 rewriter);
3744
3745 return mlir::success();
3746}
3747
3748static mlir::SymbolRefAttr
3749getComdatAttrHelper(mlir::ModuleOp modOp, mlir::OpBuilder &builder,
3750 StringRef symName, mlir::LLVM::ComdatOp &comdatOp,
3751 mlir::SymbolTableCollection &symbolTables) {
3752 mlir::OpBuilder::InsertionGuard guard(builder);
3753 StringRef comdatName = "__llvm_comdat";
3754 if (!comdatOp) {
3755 // The GlobalOp and FuncOp lowering patterns each cache their own
3756 // 'comdatOp', but both now share a single module-level comdat region, so
3757 // whichever pattern gets here second has to find the existing one rather
3758 // than create a duplicate symbol.
3759 comdatOp = modOp.lookupSymbol<mlir::LLVM::ComdatOp>(name: comdatName);
3760 }
3761
3762 if (!comdatOp) {
3763 builder.setInsertionPointToStart(modOp.getBody());
3764 comdatOp =
3765 mlir::LLVM::ComdatOp::create(builder, location: modOp.getLoc(), symName: comdatName);
3766 }
3767
3768 // Cached, and shared by both patterns: a linear scan of the comdat region
3769 // per symbol is quadratic in the number of comdats.
3770 mlir::SymbolTable &selectors = symbolTables.getSymbolTable(op: comdatOp);
3771 if (auto comdatSelector =
3772 selectors.lookup<mlir::LLVM::ComdatSelectorOp>(name: symName)) {
3773 return mlir::SymbolRefAttr::get(
3774 ctx: builder.getContext(), value: comdatName,
3775 nestedRefs: mlir::FlatSymbolRefAttr::get(value: comdatSelector.getSymNameAttr()));
3776 }
3777
3778 builder.setInsertionPointToStart(&comdatOp.getBody().back());
3779 auto selectorOp = mlir::LLVM::ComdatSelectorOp::create(
3780 builder, location: comdatOp.getLoc(), sym_name: symName, comdat: mlir::LLVM::comdat::Comdat::Any,
3781 /*sym_visibility=*/nullptr);
3782 selectors.insert(symbol: selectorOp);
3783 return mlir::SymbolRefAttr::get(
3784 ctx: builder.getContext(), value: comdatName,
3785 nestedRefs: mlir::FlatSymbolRefAttr::get(value: selectorOp.getSymNameAttr()));
3786}
3787
3788mlir::SymbolRefAttr
3789CIRToLLVMGlobalOpLowering::getComdatAttr(cir::GlobalOp &op,
3790 mlir::OpBuilder &builder) const {
3791 std::optional<llvm::StringRef> comdat = op.getComdat();
3792 if (!comdat)
3793 return mlir::SymbolRefAttr{};
3794 llvm::StringRef comdatKey = comdat->empty() ? op.getSymName() : *comdat;
3795 return getComdatAttrHelper(modOp: op->getParentOfType<mlir::ModuleOp>(), builder,
3796 symName: comdatKey, comdatOp, symbolTables);
3797}
3798
3799mlir::SymbolRefAttr
3800CIRToLLVMFuncOpLowering::getComdatAttr(cir::FuncOp &op,
3801 mlir::OpBuilder &builder) const {
3802 std::optional<llvm::StringRef> comdat = op.getComdat();
3803 if (!comdat)
3804 return mlir::SymbolRefAttr{};
3805 llvm::StringRef comdatKey = comdat->empty() ? op.getSymName() : *comdat;
3806 return getComdatAttrHelper(modOp: op->getParentOfType<mlir::ModuleOp>(), builder,
3807 symName: comdatKey, comdatOp, symbolTables);
3808}
3809
3810mlir::LogicalResult CIRToLLVMSwitchFlatOpLowering::matchAndRewrite(
3811 cir::SwitchFlatOp op, OpAdaptor adaptor,
3812 mlir::ConversionPatternRewriter &rewriter) const {
3813
3814 llvm::SmallVector<mlir::APInt, 8> caseValues;
3815 for (mlir::Attribute val : op.getCaseValues()) {
3816 auto intAttr = cast<cir::IntAttr>(Val&: val);
3817 caseValues.push_back(Elt: intAttr.getValue());
3818 }
3819
3820 llvm::SmallVector<mlir::Block *, 8> caseDestinations;
3821
3822 for (mlir::Block *x : op.getCaseDestinations())
3823 caseDestinations.push_back(Elt: x);
3824
3825 // Use the converted operands so they match the type-converted successor
3826 // block arguments.
3827 llvm::SmallVector<mlir::ValueRange> caseOperands = adaptor.getCaseOperands();
3828
3829 // Set switch op to branch to the newly created blocks.
3830 rewriter.setInsertionPoint(op);
3831 rewriter.replaceOpWithNewOp<mlir::LLVM::SwitchOp>(
3832 op, args: adaptor.getCondition(), args: op.getDefaultDestination(),
3833 args: adaptor.getDefaultOperands(), args&: caseValues, args&: caseDestinations, args&: caseOperands);
3834 return mlir::success();
3835}
3836
3837static mlir::LLVM::IntegerOverflowFlags nswFlag(bool nsw) {
3838 return nsw ? mlir::LLVM::IntegerOverflowFlags::nsw
3839 : mlir::LLVM::IntegerOverflowFlags::none;
3840}
3841
3842template <typename CIROp, typename LLVMIntOp>
3843static mlir::LogicalResult
3844lowerIncDecOp(CIROp op, typename CIROp::Adaptor adaptor,
3845 mlir::ConversionPatternRewriter &rewriter) {
3846 mlir::Type llvmType = adaptor.getInput().getType();
3847 mlir::Location loc = op.getLoc();
3848
3849 auto maybeNSW = nswFlag(op.getNoSignedWrap());
3850 mlir::LLVM::ConstantOp one;
3851 if (mlir::isa<cir::VectorType>(op.getType())) {
3852 mlir::ShapedType shapedTy = mlir::cast<mlir::ShapedType>(Val&: llvmType);
3853 mlir::APInt oneAP(shapedTy.getElementTypeBitWidth(), 1);
3854 auto oneVec = mlir::DenseElementsAttr::get(type: shapedTy, list: {oneAP});
3855 one = mlir::LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: llvmType, value: oneVec);
3856 } else {
3857 one = mlir::LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: llvmType, value: 1);
3858 }
3859 rewriter.replaceOpWithNewOp<LLVMIntOp>(op, adaptor.getInput(), one, maybeNSW);
3860 return mlir::success();
3861}
3862
3863mlir::LogicalResult CIRToLLVMIncOpLowering::matchAndRewrite(
3864 cir::IncOp op, OpAdaptor adaptor,
3865 mlir::ConversionPatternRewriter &rewriter) const {
3866 return lowerIncDecOp<cir::IncOp, mlir::LLVM::AddOp>(op, adaptor, rewriter);
3867}
3868
3869mlir::LogicalResult CIRToLLVMDecOpLowering::matchAndRewrite(
3870 cir::DecOp op, OpAdaptor adaptor,
3871 mlir::ConversionPatternRewriter &rewriter) const {
3872 return lowerIncDecOp<cir::DecOp, mlir::LLVM::SubOp>(op, adaptor, rewriter);
3873}
3874
3875mlir::LogicalResult CIRToLLVMMinusOpLowering::matchAndRewrite(
3876 cir::MinusOp op, OpAdaptor adaptor,
3877 mlir::ConversionPatternRewriter &rewriter) const {
3878 bool isVector = mlir::isa<cir::VectorType>(Val: op.getType());
3879 mlir::Type llvmType = adaptor.getInput().getType();
3880 mlir::Location loc = op.getLoc();
3881
3882 auto maybeNSW = nswFlag(nsw: op.getNoSignedWrap());
3883 mlir::Value zero;
3884 if (isVector)
3885 zero = mlir::LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: llvmType);
3886 else
3887 zero = mlir::LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: llvmType, value: 0);
3888 rewriter.replaceOpWithNewOp<mlir::LLVM::SubOp>(op, args&: zero, args: adaptor.getInput(),
3889 args&: maybeNSW);
3890 return mlir::success();
3891}
3892
3893mlir::LogicalResult CIRToLLVMNotOpLowering::matchAndRewrite(
3894 cir::NotOp op, OpAdaptor adaptor,
3895 mlir::ConversionPatternRewriter &rewriter) const {
3896 mlir::Type elementType = elementTypeIfVectorOrMatrix(type: op.getType());
3897 bool isVector = mlir::isa<cir::VectorType>(Val: op.getType());
3898 mlir::Type llvmType = adaptor.getInput().getType();
3899 mlir::Location loc = op.getLoc();
3900
3901 if (mlir::isa<cir::IntType>(Val: elementType)) {
3902 mlir::Value minusOne;
3903 if (isVector) {
3904 const unsigned eltWidth =
3905 mlir::cast<cir::IntType>(Val&: elementType).getWidth();
3906 mlir::DenseElementsAttr denseVec =
3907 mlir::DenseElementsAttr::get(type: mlir::cast<mlir::ShapedType>(Val&: llvmType),
3908 list: {APInt::getAllOnes(numBits: eltWidth)});
3909 minusOne =
3910 mlir::LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: llvmType, value: denseVec);
3911 } else {
3912 minusOne = mlir::LLVM::ConstantOp::create(
3913 builder&: rewriter, location: loc, type: llvmType,
3914 value: APInt::getAllOnes(numBits: cast<mlir::IntegerType>(Val&: llvmType).getWidth()));
3915 }
3916 rewriter.replaceOpWithNewOp<mlir::LLVM::XOrOp>(op, args: adaptor.getInput(),
3917 args&: minusOne);
3918 return mlir::success();
3919 }
3920
3921 if (mlir::isa<cir::BoolType>(Val: elementType)) {
3922 mlir::Value one;
3923 if (isVector) {
3924 mlir::DenseElementsAttr denseVec = mlir::DenseElementsAttr::get(
3925 type: mlir::cast<mlir::ShapedType>(Val&: llvmType), list: {APInt(1, 1)});
3926 one = mlir::LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: llvmType, value: denseVec);
3927 } else {
3928 one = mlir::LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: llvmType, value: 1);
3929 }
3930 rewriter.replaceOpWithNewOp<mlir::LLVM::XOrOp>(op, args: adaptor.getInput(), args&: one);
3931 return mlir::success();
3932 }
3933
3934 return op.emitError() << "Unsupported type for bitwise NOT";
3935}
3936
3937static bool isIntTypeUnsigned(mlir::Type type) {
3938 // TODO: Ideally, we should only need to check cir::IntType here.
3939 return mlir::isa<cir::IntType>(Val: type)
3940 ? mlir::cast<cir::IntType>(Val&: type).isUnsigned()
3941 : mlir::cast<mlir::IntegerType>(Val&: type).isUnsigned();
3942}
3943
3944//===----------------------------------------------------------------------===//
3945// Binary Op Lowering
3946//===----------------------------------------------------------------------===//
3947
3948mlir::LLVM::IntegerOverflowFlags
3949convertToLLVMProperty(mlir::MLIRContext *, cir::OverflowFlags flags) {
3950 mlir::LLVM::IntegerOverflowFlags result =
3951 mlir::LLVM::IntegerOverflowFlags::none;
3952 if (bitEnumContainsAny(bits: flags, bit: cir::OverflowFlags::NoSignedWrap))
3953 result = result | mlir::LLVM::IntegerOverflowFlags::nsw;
3954 if (bitEnumContainsAny(bits: flags, bit: cir::OverflowFlags::NoUnsignedWrap))
3955 result = result | mlir::LLVM::IntegerOverflowFlags::nuw;
3956 return result;
3957}
3958
3959/// Lower an integer op to its signed or unsigned LLVM counterpart.
3960template <typename UIntOp, typename SIntOp, typename CIROp>
3961static mlir::LogicalResult
3962lowerIntBinaryOp(CIROp op, mlir::Value lhs, mlir::Value rhs,
3963 mlir::ConversionPatternRewriter &rewriter) {
3964 const mlir::Type eltType = elementTypeIfVectorOrMatrix(op.getRhs().getType());
3965 assert(cir::isIntOrBoolType(eltType) &&
3966 "integer binary op expects integer operand types");
3967 if (isIntTypeUnsigned(type: eltType))
3968 rewriter.replaceOpWithNewOp<UIntOp>(op, lhs, rhs);
3969 else
3970 rewriter.replaceOpWithNewOp<SIntOp>(op, lhs, rhs);
3971 return mlir::success();
3972}
3973
3974mlir::LogicalResult CIRToLLVMAddSatOpLowering::matchAndRewrite(
3975 cir::AddSatOp op, OpAdaptor adaptor,
3976 mlir::ConversionPatternRewriter &rewriter) const {
3977 return lowerIntBinaryOp<mlir::LLVM::UAddSat, mlir::LLVM::SAddSat>(
3978 op, lhs: adaptor.getLhs(), rhs: adaptor.getRhs(), rewriter);
3979}
3980
3981mlir::LogicalResult CIRToLLVMSubSatOpLowering::matchAndRewrite(
3982 cir::SubSatOp op, OpAdaptor adaptor,
3983 mlir::ConversionPatternRewriter &rewriter) const {
3984 return lowerIntBinaryOp<mlir::LLVM::USubSat, mlir::LLVM::SSubSat>(
3985 op, lhs: adaptor.getLhs(), rhs: adaptor.getRhs(), rewriter);
3986}
3987
3988mlir::LogicalResult CIRToLLVMDivOpLowering::matchAndRewrite(
3989 cir::DivOp op, OpAdaptor adaptor,
3990 mlir::ConversionPatternRewriter &rewriter) const {
3991 return lowerIntBinaryOp<mlir::LLVM::UDivOp, mlir::LLVM::SDivOp>(
3992 op, lhs: adaptor.getLhs(), rhs: adaptor.getRhs(), rewriter);
3993}
3994
3995mlir::LogicalResult CIRToLLVMRemOpLowering::matchAndRewrite(
3996 cir::RemOp op, OpAdaptor adaptor,
3997 mlir::ConversionPatternRewriter &rewriter) const {
3998 return lowerIntBinaryOp<mlir::LLVM::URemOp, mlir::LLVM::SRemOp>(
3999 op, lhs: adaptor.getLhs(), rhs: adaptor.getRhs(), rewriter);
4000}
4001
4002template <typename CIROp, typename UIntOp, typename SIntOp>
4003static mlir::LogicalResult
4004lowerMinMaxOp(CIROp op, typename CIROp::Adaptor adaptor,
4005 mlir::ConversionPatternRewriter &rewriter) {
4006 const mlir::Value lhs = adaptor.getLhs();
4007 const mlir::Value rhs = adaptor.getRhs();
4008 if (isIntTypeUnsigned(elementTypeIfVectorOrMatrix(op.getRhs().getType())))
4009 rewriter.replaceOpWithNewOp<UIntOp>(op, lhs, rhs);
4010 else
4011 rewriter.replaceOpWithNewOp<SIntOp>(op, lhs, rhs);
4012 return mlir::success();
4013}
4014
4015mlir::LogicalResult CIRToLLVMMaxOpLowering::matchAndRewrite(
4016 cir::MaxOp op, OpAdaptor adaptor,
4017 mlir::ConversionPatternRewriter &rewriter) const {
4018 return lowerMinMaxOp<cir::MaxOp, mlir::LLVM::UMaxOp, mlir::LLVM::SMaxOp>(
4019 op, adaptor, rewriter);
4020}
4021
4022mlir::LogicalResult CIRToLLVMMinOpLowering::matchAndRewrite(
4023 cir::MinOp op, OpAdaptor adaptor,
4024 mlir::ConversionPatternRewriter &rewriter) const {
4025 return lowerMinMaxOp<cir::MinOp, mlir::LLVM::UMinOp, mlir::LLVM::SMinOp>(
4026 op, adaptor, rewriter);
4027}
4028
4029/// Convert from a CIR comparison kind to an LLVM IR integral comparison kind.
4030static mlir::LLVM::ICmpPredicate
4031convertCmpKindToICmpPredicate(cir::CmpOpKind kind, bool isSigned) {
4032 using CIR = cir::CmpOpKind;
4033 using LLVMICmp = mlir::LLVM::ICmpPredicate;
4034 switch (kind) {
4035 case CIR::eq:
4036 return LLVMICmp::eq;
4037 case CIR::ne:
4038 return LLVMICmp::ne;
4039 case CIR::lt:
4040 return (isSigned ? LLVMICmp::slt : LLVMICmp::ult);
4041 case CIR::le:
4042 return (isSigned ? LLVMICmp::sle : LLVMICmp::ule);
4043 case CIR::gt:
4044 return (isSigned ? LLVMICmp::sgt : LLVMICmp::ugt);
4045 case CIR::ge:
4046 return (isSigned ? LLVMICmp::sge : LLVMICmp::uge);
4047 case CIR::one:
4048 case CIR::uno:
4049 llvm_unreachable("FP-only comparison used with integer type");
4050 }
4051 llvm_unreachable("Unknown CmpOpKind");
4052}
4053
4054/// Convert from a CIR comparison kind to an LLVM IR floating-point comparison
4055/// kind.
4056static mlir::LLVM::FCmpPredicate
4057convertCmpKindToFCmpPredicate(cir::CmpOpKind kind) {
4058 using CIR = cir::CmpOpKind;
4059 using LLVMFCmp = mlir::LLVM::FCmpPredicate;
4060 switch (kind) {
4061 case CIR::eq:
4062 return LLVMFCmp::oeq;
4063 case CIR::ne:
4064 return LLVMFCmp::une;
4065 case CIR::lt:
4066 return LLVMFCmp::olt;
4067 case CIR::le:
4068 return LLVMFCmp::ole;
4069 case CIR::gt:
4070 return LLVMFCmp::ogt;
4071 case CIR::ge:
4072 return LLVMFCmp::oge;
4073 case CIR::one:
4074 return LLVMFCmp::one;
4075 case CIR::uno:
4076 return LLVMFCmp::uno;
4077 }
4078 llvm_unreachable("Unknown CmpOpKind");
4079}
4080
4081/// Convert from a CIR fp comparison kind to an LLVM IR floating-point
4082/// comparison kind.
4083static mlir::LLVM::FCmpPredicate
4084convertCmpKindToFCmpPredicate(cir::FCmpOpKind kind) {
4085 using CIR = cir::FCmpOpKind;
4086 using LLVMFCmp = mlir::LLVM::FCmpPredicate;
4087 switch (kind) {
4088 case CIR::_false:
4089 return LLVMFCmp::_false;
4090 case CIR::oeq:
4091 return LLVMFCmp::oeq;
4092 case CIR::ogt:
4093 return LLVMFCmp::ogt;
4094 case CIR::oge:
4095 return LLVMFCmp::oge;
4096 case CIR::olt:
4097 return LLVMFCmp::olt;
4098 case CIR::ole:
4099 return LLVMFCmp::ole;
4100 case CIR::one:
4101 return LLVMFCmp::one;
4102 case CIR::ord:
4103 return LLVMFCmp::ord;
4104 case CIR::ueq:
4105 return LLVMFCmp::ueq;
4106 case CIR::ugt:
4107 return LLVMFCmp::ugt;
4108 case CIR::uge:
4109 return LLVMFCmp::uge;
4110 case CIR::ult:
4111 return LLVMFCmp::ult;
4112 case CIR::ule:
4113 return LLVMFCmp::ule;
4114 case CIR::une:
4115 return LLVMFCmp::une;
4116 case CIR::uno:
4117 return LLVMFCmp::uno;
4118 case CIR::_true:
4119 return LLVMFCmp::_true;
4120 }
4121 llvm_unreachable("Unknown FCmpOpKind");
4122}
4123
4124static llvm::StringRef
4125convertCmpKindToConstrainedFCmpPredicate(cir::CmpOpKind kind) {
4126 using CIR = cir::CmpOpKind;
4127 switch (kind) {
4128 case CIR::eq:
4129 return "oeq";
4130 case CIR::ne:
4131 return "une";
4132 case CIR::lt:
4133 return "olt";
4134 case CIR::le:
4135 return "ole";
4136 case CIR::gt:
4137 return "ogt";
4138 case CIR::ge:
4139 return "oge";
4140 case CIR::one:
4141 return "one";
4142 case CIR::uno:
4143 return "uno";
4144 }
4145 llvm_unreachable("Unknown CmpOpKind");
4146}
4147
4148static bool isSignalingConstrainedFCmp(cir::CmpOpKind kind) {
4149 using CIR = cir::CmpOpKind;
4150 switch (kind) {
4151 case CIR::lt:
4152 case CIR::le:
4153 case CIR::gt:
4154 case CIR::ge:
4155 return true;
4156 case CIR::eq:
4157 case CIR::ne:
4158 case CIR::one:
4159 case CIR::uno:
4160 return false;
4161 }
4162 llvm_unreachable("Unknown CmpOpKind");
4163}
4164
4165static llvm::StringRef
4166convertCmpKindToConstrainedFCmpPredicate(cir::FCmpOpKind kind) {
4167 using CIR = cir::FCmpOpKind;
4168 switch (kind) {
4169 case CIR::_false:
4170 return "false";
4171 case CIR::oeq:
4172 return "oeq";
4173 case CIR::ogt:
4174 return "ogt";
4175 case CIR::oge:
4176 return "oge";
4177 case CIR::olt:
4178 return "olt";
4179 case CIR::ole:
4180 return "ole";
4181 case CIR::one:
4182 return "one";
4183 case CIR::ord:
4184 return "ord";
4185 case CIR::ueq:
4186 return "ueq";
4187 case CIR::ugt:
4188 return "ugt";
4189 case CIR::uge:
4190 return "uge";
4191 case CIR::ult:
4192 return "ult";
4193 case CIR::ule:
4194 return "ule";
4195 case CIR::une:
4196 return "une";
4197 case CIR::uno:
4198 return "uno";
4199 case CIR::_true:
4200 return "true";
4201 }
4202 llvm_unreachable("Unknown FCmpOpKind");
4203}
4204
4205template <typename OpKindTy>
4206static mlir::LLVM::CallIntrinsicOp
4207createConstrainedFCmpCall(mlir::ConversionPatternRewriter &rewriter,
4208 mlir::Location loc, mlir::Value lhs, mlir::Value rhs,
4209 OpKindTy kind, bool isSignaling, cir::FenvAttr fenv,
4210 mlir::Type llvmResTy) {
4211 llvm::SmallVector<mlir::Value, 4> callOperands = {
4212 lhs, rhs,
4213 createFenvMetadataValue(rewriter, loc,
4214 convertCmpKindToConstrainedFCmpPredicate(kind)),
4215 createFenvMetadataValue(rewriter, loc,
4216 str: getConstrainedExceptMetadata(fenv))};
4217 llvm::StringRef intrinsicName = isSignaling
4218 ? "llvm.experimental.constrained.fcmps"
4219 : "llvm.experimental.constrained.fcmp";
4220 return createCallLLVMIntrinsicOp(rewriter, loc, intrinsicName, resultTy: llvmResTy,
4221 operands: callOperands);
4222}
4223
4224mlir::LogicalResult CIRToLLVMCmpOpLowering::matchAndRewrite(
4225 cir::CmpOp cmpOp, OpAdaptor adaptor,
4226 mlir::ConversionPatternRewriter &rewriter) const {
4227 mlir::Type type = cmpOp.getLhs().getType();
4228
4229 if (mlir::isa<cir::IntType, mlir::IntegerType>(Val: type)) {
4230 bool isSigned = mlir::isa<cir::IntType>(Val: type)
4231 ? mlir::cast<cir::IntType>(Val&: type).isSigned()
4232 : mlir::cast<mlir::IntegerType>(Val&: type).isSigned();
4233 mlir::LLVM::ICmpPredicate kind =
4234 convertCmpKindToICmpPredicate(kind: cmpOp.getKind(), isSigned);
4235 rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
4236 op: cmpOp, args&: kind, args: adaptor.getLhs(), args: adaptor.getRhs());
4237 return mlir::success();
4238 }
4239
4240 if (mlir::isa<cir::BoolType, cir::PointerType, cir::VPtrType>(Val: type)) {
4241 // Booleans, including enums with a boolean underlying type, compare as
4242 // unsigned integers, as do pointers and !cir.vptr, which lowers to a
4243 // pointer.
4244 mlir::LLVM::ICmpPredicate kind =
4245 convertCmpKindToICmpPredicate(kind: cmpOp.getKind(), /*isSigned=*/false);
4246 rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
4247 op: cmpOp, args&: kind, args: adaptor.getLhs(), args: adaptor.getRhs());
4248 return mlir::success();
4249 }
4250
4251 if (mlir::isa<cir::FPTypeInterface>(Val: type)) {
4252 mlir::Type llvmResTy = getTypeConverter()->convertType(t: cmpOp.getType());
4253 if (cir::FenvAttr fenv = cmpOp.getFenvAttr()) {
4254 mlir::LLVM::CallIntrinsicOp call = createConstrainedFCmpCall(
4255 rewriter, loc: cmpOp.getLoc(), lhs: adaptor.getLhs(), rhs: adaptor.getRhs(),
4256 kind: cmpOp.getKind(), isSignaling: isSignalingConstrainedFCmp(kind: cmpOp.getKind()), fenv,
4257 llvmResTy);
4258 rewriter.replaceOp(op: cmpOp, newValues: call.getResult(i: 0));
4259 return mlir::success();
4260 }
4261 mlir::LLVM::FCmpPredicate kind =
4262 convertCmpKindToFCmpPredicate(kind: cmpOp.getKind());
4263 rewriter.replaceOpWithNewOp<mlir::LLVM::FCmpOp>(
4264 op: cmpOp, args&: kind, args: adaptor.getLhs(), args: adaptor.getRhs());
4265 return mlir::success();
4266 }
4267
4268 if (mlir::isa<cir::ComplexType>(Val: type)) {
4269 mlir::Value lhs = adaptor.getLhs();
4270 mlir::Value rhs = adaptor.getRhs();
4271 mlir::Location loc = cmpOp.getLoc();
4272
4273 auto complexType = mlir::cast<cir::ComplexType>(Val: cmpOp.getLhs().getType());
4274 mlir::Type complexElemTy =
4275 getTypeConverter()->convertType(t: complexType.getElementType());
4276
4277 auto lhsReal = mlir::LLVM::ExtractValueOp::create(
4278 builder&: rewriter, location: loc, res: complexElemTy, container: lhs, position: ArrayRef(int64_t{0}));
4279 auto lhsImag = mlir::LLVM::ExtractValueOp::create(
4280 builder&: rewriter, location: loc, res: complexElemTy, container: lhs, position: ArrayRef(int64_t{1}));
4281 auto rhsReal = mlir::LLVM::ExtractValueOp::create(
4282 builder&: rewriter, location: loc, res: complexElemTy, container: rhs, position: ArrayRef(int64_t{0}));
4283 auto rhsImag = mlir::LLVM::ExtractValueOp::create(
4284 builder&: rewriter, location: loc, res: complexElemTy, container: rhs, position: ArrayRef(int64_t{1}));
4285
4286 if (cmpOp.getKind() == cir::CmpOpKind::eq) {
4287 if (complexElemTy.isInteger()) {
4288 auto realCmp = mlir::LLVM::ICmpOp::create(
4289 builder&: rewriter, location: loc, predicate: mlir::LLVM::ICmpPredicate::eq, lhs: lhsReal, rhs: rhsReal);
4290 auto imagCmp = mlir::LLVM::ICmpOp::create(
4291 builder&: rewriter, location: loc, predicate: mlir::LLVM::ICmpPredicate::eq, lhs: lhsImag, rhs: rhsImag);
4292 rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(op: cmpOp, args&: realCmp, args&: imagCmp);
4293 return mlir::success();
4294 }
4295
4296 auto realCmp = mlir::LLVM::FCmpOp::create(
4297 builder&: rewriter, location: loc, predicate: mlir::LLVM::FCmpPredicate::oeq, lhs: lhsReal, rhs: rhsReal);
4298 auto imagCmp = mlir::LLVM::FCmpOp::create(
4299 builder&: rewriter, location: loc, predicate: mlir::LLVM::FCmpPredicate::oeq, lhs: lhsImag, rhs: rhsImag);
4300 rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(op: cmpOp, args&: realCmp, args&: imagCmp);
4301 return mlir::success();
4302 }
4303
4304 if (cmpOp.getKind() == cir::CmpOpKind::ne) {
4305 if (complexElemTy.isInteger()) {
4306 auto realCmp = mlir::LLVM::ICmpOp::create(
4307 builder&: rewriter, location: loc, predicate: mlir::LLVM::ICmpPredicate::ne, lhs: lhsReal, rhs: rhsReal);
4308 auto imagCmp = mlir::LLVM::ICmpOp::create(
4309 builder&: rewriter, location: loc, predicate: mlir::LLVM::ICmpPredicate::ne, lhs: lhsImag, rhs: rhsImag);
4310 rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(op: cmpOp, args&: realCmp, args&: imagCmp);
4311 return mlir::success();
4312 }
4313
4314 auto realCmp = mlir::LLVM::FCmpOp::create(
4315 builder&: rewriter, location: loc, predicate: mlir::LLVM::FCmpPredicate::une, lhs: lhsReal, rhs: rhsReal);
4316 auto imagCmp = mlir::LLVM::FCmpOp::create(
4317 builder&: rewriter, location: loc, predicate: mlir::LLVM::FCmpPredicate::une, lhs: lhsImag, rhs: rhsImag);
4318 rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(op: cmpOp, args&: realCmp, args&: imagCmp);
4319 return mlir::success();
4320 }
4321 }
4322
4323 return cmpOp.emitError() << "unsupported type for CmpOp: " << type;
4324}
4325
4326/// Shared lowering logic for checked binary arithmetic overflow operations.
4327/// The \p opStr parameter specifies the arithmetic operation name used in the
4328/// LLVM intrinsic (e.g., "add", "sub", "mul").
4329template <typename OpTy>
4330static mlir::LogicalResult
4331lowerBinOpOverflow(OpTy op, typename OpTy::Adaptor adaptor,
4332 mlir::ConversionPatternRewriter &rewriter,
4333 const mlir::TypeConverter *typeConverter,
4334 llvm::StringRef opStr) {
4335 mlir::Location loc = op.getLoc();
4336 cir::IntType operandTy = op.getLhs().getType();
4337 // The result type may be a `cir.bool`, which behaves as a 1-bit unsigned
4338 // integer for the purposes of the checked arithmetic.
4339 mlir::Type resultTy = op.getResult().getType();
4340 auto resultIntTy = mlir::dyn_cast<cir::IntType>(Val&: resultTy);
4341 unsigned resultWidth = resultIntTy ? resultIntTy.getWidth() : 1;
4342 bool resultSigned = resultIntTy && resultIntTy.getIsSigned();
4343
4344 bool sign = operandTy.getIsSigned() || resultSigned;
4345 unsigned width =
4346 std::max(a: operandTy.getWidth() + (sign && operandTy.isUnsigned()),
4347 b: resultWidth + (sign && !resultSigned));
4348
4349 mlir::IntegerType encompassedLLVMTy = rewriter.getIntegerType(width);
4350
4351 mlir::Value lhs = adaptor.getLhs();
4352 mlir::Value rhs = adaptor.getRhs();
4353 if (operandTy.getWidth() < width) {
4354 if (operandTy.isSigned()) {
4355 lhs = mlir::LLVM::SExtOp::create(builder&: rewriter, location: loc, res: encompassedLLVMTy, arg: lhs);
4356 rhs = mlir::LLVM::SExtOp::create(builder&: rewriter, location: loc, res: encompassedLLVMTy, arg: rhs);
4357 } else {
4358 lhs = mlir::LLVM::ZExtOp::create(builder&: rewriter, location: loc, res: encompassedLLVMTy, arg: lhs);
4359 rhs = mlir::LLVM::ZExtOp::create(builder&: rewriter, location: loc, res: encompassedLLVMTy, arg: rhs);
4360 }
4361 }
4362
4363 // The intrinsic name is `@llvm.{s|u}{op}.with.overflow.i{width}`
4364 std::string intrinName = ("llvm." + llvm::Twine(sign ? 's' : 'u') + opStr +
4365 ".with.overflow.i" + llvm::Twine(width))
4366 .str();
4367 auto intrinNameAttr = mlir::StringAttr::get(op.getContext(), intrinName);
4368
4369 mlir::IntegerType overflowLLVMTy = rewriter.getI1Type();
4370 auto intrinRetTy = mlir::LLVM::LLVMStructType::getLiteral(
4371 context: rewriter.getContext(), types: {encompassedLLVMTy, overflowLLVMTy});
4372
4373 auto callLLVMIntrinOp = mlir::LLVM::CallIntrinsicOp::create(
4374 rewriter, loc, intrinRetTy, intrinNameAttr, mlir::ValueRange{lhs, rhs});
4375 mlir::Value intrinRet = callLLVMIntrinOp.getResult(0);
4376
4377 mlir::Value result = mlir::LLVM::ExtractValueOp::create(
4378 builder&: rewriter, location: loc, container: intrinRet, position: ArrayRef<int64_t>{0})
4379 .getResult();
4380 mlir::Value overflow = mlir::LLVM::ExtractValueOp::create(
4381 builder&: rewriter, location: loc, container: intrinRet, position: ArrayRef<int64_t>{1})
4382 .getResult();
4383
4384 if (resultWidth < width) {
4385 mlir::Type resultLLVMTy = typeConverter->convertType(t: resultTy);
4386 auto truncResult =
4387 mlir::LLVM::TruncOp::create(builder&: rewriter, location: loc, res: resultLLVMTy, arg: result);
4388
4389 // Extend the truncated result back to the encompassing type to check for
4390 // any overflows during the truncation.
4391 mlir::Value truncResultExt;
4392 if (resultSigned)
4393 truncResultExt = mlir::LLVM::SExtOp::create(
4394 builder&: rewriter, location: loc, res: encompassedLLVMTy, arg: truncResult);
4395 else
4396 truncResultExt = mlir::LLVM::ZExtOp::create(
4397 builder&: rewriter, location: loc, res: encompassedLLVMTy, arg: truncResult);
4398 auto truncOverflow = mlir::LLVM::ICmpOp::create(
4399 builder&: rewriter, location: loc, predicate: mlir::LLVM::ICmpPredicate::ne, lhs: truncResultExt, rhs: result);
4400
4401 result = truncResult;
4402 overflow = mlir::LLVM::OrOp::create(builder&: rewriter, location: loc, lhs: overflow, rhs: truncOverflow);
4403 }
4404
4405 mlir::Type boolLLVMTy =
4406 typeConverter->convertType(op.getOverflow().getType());
4407 if (boolLLVMTy != rewriter.getI1Type())
4408 overflow = mlir::LLVM::ZExtOp::create(builder&: rewriter, location: loc, res: boolLLVMTy, arg: overflow);
4409
4410 rewriter.replaceOp(op, mlir::ValueRange{result, overflow});
4411
4412 return mlir::success();
4413}
4414
4415mlir::LogicalResult CIRToLLVMAddOverflowOpLowering::matchAndRewrite(
4416 cir::AddOverflowOp op, OpAdaptor adaptor,
4417 mlir::ConversionPatternRewriter &rewriter) const {
4418 return lowerBinOpOverflow(op, adaptor, rewriter, typeConverter: getTypeConverter(), opStr: "add");
4419}
4420
4421mlir::LogicalResult CIRToLLVMSubOverflowOpLowering::matchAndRewrite(
4422 cir::SubOverflowOp op, OpAdaptor adaptor,
4423 mlir::ConversionPatternRewriter &rewriter) const {
4424 return lowerBinOpOverflow(op, adaptor, rewriter, typeConverter: getTypeConverter(), opStr: "sub");
4425}
4426
4427mlir::LogicalResult CIRToLLVMMulOverflowOpLowering::matchAndRewrite(
4428 cir::MulOverflowOp op, OpAdaptor adaptor,
4429 mlir::ConversionPatternRewriter &rewriter) const {
4430 return lowerBinOpOverflow(op, adaptor, rewriter, typeConverter: getTypeConverter(), opStr: "mul");
4431}
4432
4433mlir::LogicalResult CIRToLLVMFrexpOpLowering::matchAndRewrite(
4434 cir::FrexpOp op, OpAdaptor adaptor,
4435 mlir::ConversionPatternRewriter &rewriter) const {
4436 mlir::Location loc = op.getLoc();
4437 mlir::Type fpLLVMTy =
4438 getTypeConverter()->convertType(t: op.getResult().getType());
4439 mlir::Type intLLVMTy = getTypeConverter()->convertType(t: op.getExp().getType());
4440
4441 auto structTy = mlir::LLVM::LLVMStructType::getLiteral(context: rewriter.getContext(),
4442 types: {fpLLVMTy, intLLVMTy});
4443
4444 auto callOp = createCallLLVMIntrinsicOp(rewriter, loc, intrinsicName: "llvm.frexp", resultTy: structTy,
4445 operands: adaptor.getSrc());
4446 mlir::Value result = callOp.getResult(i: 0);
4447
4448 mlir::Value mantissa =
4449 mlir::LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: result, position: 0);
4450 mlir::Value exponent =
4451 mlir::LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: result, position: 1);
4452 rewriter.replaceOp(op, newValues: mlir::ValueRange{mantissa, exponent});
4453 return mlir::success();
4454}
4455
4456mlir::LogicalResult CIRToLLVMModfOpLowering::matchAndRewrite(
4457 cir::ModfOp op, OpAdaptor adaptor,
4458 mlir::ConversionPatternRewriter &rewriter) const {
4459 mlir::Location loc = op.getLoc();
4460 mlir::Type fpLLVMTy =
4461 getTypeConverter()->convertType(t: op.getFractional().getType());
4462
4463 auto structTy = mlir::LLVM::LLVMStructType::getLiteral(context: rewriter.getContext(),
4464 types: {fpLLVMTy, fpLLVMTy});
4465
4466 auto callOp = createCallLLVMIntrinsicOp(rewriter, loc, intrinsicName: "llvm.modf", resultTy: structTy,
4467 operands: adaptor.getSrc());
4468 mlir::Value result = callOp.getResult(i: 0);
4469
4470 mlir::Value fractional =
4471 mlir::LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: result, position: 0);
4472 mlir::Value integral =
4473 mlir::LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: result, position: 1);
4474 rewriter.replaceOp(op, newValues: mlir::ValueRange{fractional, integral});
4475 return mlir::success();
4476}
4477
4478mlir::LogicalResult CIRToLLVMShiftOpLowering::matchAndRewrite(
4479 cir::ShiftOp op, OpAdaptor adaptor,
4480 mlir::ConversionPatternRewriter &rewriter) const {
4481 assert((op.getValue().getType() == op.getType()) &&
4482 "inconsistent operands' types NYI");
4483
4484 const mlir::Type llvmTy = getTypeConverter()->convertType(t: op.getType());
4485 mlir::Value amt = adaptor.getAmount();
4486 mlir::Value val = adaptor.getValue();
4487
4488 auto cirAmtTy = mlir::dyn_cast<cir::IntType>(Val: op.getAmount().getType());
4489 bool isUnsigned;
4490 if (cirAmtTy) {
4491 auto cirValTy = mlir::cast<cir::IntType>(Val: op.getValue().getType());
4492 isUnsigned = cirValTy.isUnsigned();
4493
4494 // Ensure shift amount is the same type as the value. Some undefined
4495 // behavior might occur in the casts below as per [C99 6.5.7.3].
4496 // Vector type shift amount needs no cast as type consistency is expected to
4497 // be already be enforced at CIRGen.
4498 if (cirAmtTy)
4499 amt = getLLVMIntCast(rewriter, llvmSrc: amt, llvmDstIntTy: llvmTy, isUnsigned: true, cirSrcWidth: cirAmtTy.getWidth(),
4500 cirDstIntWidth: cirValTy.getWidth());
4501 } else {
4502 auto cirValVTy = mlir::cast<cir::VectorType>(Val: op.getValue().getType());
4503 isUnsigned =
4504 mlir::cast<cir::IntType>(Val: cirValVTy.getElementType()).isUnsigned();
4505 }
4506
4507 // Lower to the proper LLVM shift operation.
4508 if (op.getIsShiftleft()) {
4509 rewriter.replaceOpWithNewOp<mlir::LLVM::ShlOp>(op, args: llvmTy, args&: val, args&: amt);
4510 return mlir::success();
4511 }
4512
4513 if (isUnsigned)
4514 rewriter.replaceOpWithNewOp<mlir::LLVM::LShrOp>(op, args: llvmTy, args&: val, args&: amt);
4515 else
4516 rewriter.replaceOpWithNewOp<mlir::LLVM::AShrOp>(op, args: llvmTy, args&: val, args&: amt);
4517 return mlir::success();
4518}
4519
4520mlir::LogicalResult CIRToLLVMSelectOpLowering::matchAndRewrite(
4521 cir::SelectOp op, OpAdaptor adaptor,
4522 mlir::ConversionPatternRewriter &rewriter) const {
4523 // Two special cases in the LLVMIR codegen of select op:
4524 // - select %0, %1, false => and %0, %1
4525 // - select %0, true, %1 => or %0, %1
4526 // Match the (possibly already converted) adaptor operands: without pattern
4527 // rollback the producer may be an LLVM constant by the time this runs.
4528 if (mlir::isa<cir::BoolType>(Val: op.getType())) {
4529 if (mlir::matchPattern(value: adaptor.getFalseValue(), pattern: mlir::m_Zero())) {
4530 // select %0, %1, false => and %0, %1
4531 rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(op, args: adaptor.getCondition(),
4532 args: adaptor.getTrueValue());
4533 return mlir::success();
4534 }
4535 if (mlir::matchPattern(value: adaptor.getTrueValue(), pattern: mlir::m_One())) {
4536 // select %0, true, %1 => or %0, %1
4537 rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(op, args: adaptor.getCondition(),
4538 args: adaptor.getFalseValue());
4539 return mlir::success();
4540 }
4541 }
4542
4543 mlir::Value llvmCondition = adaptor.getCondition();
4544 rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(
4545 op, args&: llvmCondition, args: adaptor.getTrueValue(), args: adaptor.getFalseValue());
4546
4547 return mlir::success();
4548}
4549
4550static void prepareTypeConverter(mlir::LLVMTypeConverter &converter,
4551 mlir::DataLayout &dataLayout) {
4552 converter.addConversion(callback: [&](cir::PointerType type) -> mlir::Type {
4553 mlir::ptr::MemorySpaceAttrInterface addrSpaceAttr = type.getAddrSpace();
4554 unsigned numericAS = 0;
4555
4556 if (mlir::isa_and_present<cir::LangAddressSpaceAttr>(Val: addrSpaceAttr))
4557 return {};
4558
4559 if (auto targetAsAttr =
4560 mlir::dyn_cast_if_present<cir::TargetAddressSpaceAttr>(
4561 Val&: addrSpaceAttr))
4562 numericAS = targetAsAttr.getValue();
4563 return mlir::LLVM::LLVMPointerType::get(context: type.getContext(), addressSpace: numericAS);
4564 });
4565 converter.addConversion(callback: [&](cir::VPtrType type) -> mlir::Type {
4566 assert(!cir::MissingFeatures::addressSpace());
4567 return mlir::LLVM::LLVMPointerType::get(context: type.getContext());
4568 });
4569 // On the device side, surface reference is represented as an object handle
4570 // in 64-bit integer.
4571 converter.addConversion(callback: [&](cir::CUDADeviceSurfaceType type) -> mlir::Type {
4572 return mlir::IntegerType::get(context: type.getContext(), width: 64);
4573 });
4574 converter.addConversion(callback: [&](cir::CUDADeviceTextureType type) -> mlir::Type {
4575 return mlir::IntegerType::get(context: type.getContext(), width: 64);
4576 });
4577 converter.addConversion(callback: [&](cir::ArrayType type) -> mlir::Type {
4578 mlir::Type ty =
4579 convertTypeForMemory(converter, dataLayout, type: type.getElementType());
4580 // A null element type means an unsupported member (e.g. a _BitInt with
4581 // byte-array storage); propagate the conversion failure.
4582 if (!ty)
4583 return {};
4584 return mlir::LLVM::LLVMArrayType::get(elementType: ty, numElements: type.getSize());
4585 });
4586 converter.addConversion(callback: [&](cir::VectorType type) -> mlir::Type {
4587 // Vector-of-_BitInt memory layout is not modeled here (the element would
4588 // stay at its literal width); report a conversion failure rather than emit
4589 // an unvalidated vector.
4590 if (auto intTy = mlir::dyn_cast<cir::IntType>(Val: type.getElementType());
4591 intTy && intTy.isBitInt())
4592 return {};
4593 const mlir::Type ty = converter.convertType(t: type.getElementType());
4594 if (!ty)
4595 return {};
4596 return mlir::VectorType::get(shape: type.getSize().getKnownMinValue(), elementType: ty,
4597 scalableDims: {type.isScalable()});
4598 });
4599 converter.addConversion(callback: [&](cir::MatrixType type) -> mlir::Type {
4600 const uint64_t size = type.getNumRows() * type.getNumColumns();
4601 const mlir::Type elemTy = converter.convertType(t: type.getElementType());
4602 return mlir::VectorType::get(shape: size, elementType: elemTy);
4603 });
4604 converter.addConversion(callback: [&](cir::BoolType type) -> mlir::Type {
4605 return mlir::IntegerType::get(context: type.getContext(), width: 1,
4606 signedness: mlir::IntegerType::Signless);
4607 });
4608 converter.addConversion(callback: [&](cir::IntType type) -> mlir::Type {
4609 // LLVM doesn't work with signed types, so we drop the CIR signs here.
4610 return mlir::IntegerType::get(context: type.getContext(), width: type.getWidth());
4611 });
4612 converter.addConversion(callback: [&](cir::SingleType type) -> mlir::Type {
4613 return mlir::Float32Type::get(context: type.getContext());
4614 });
4615 converter.addConversion(callback: [&](cir::DoubleType type) -> mlir::Type {
4616 return mlir::Float64Type::get(context: type.getContext());
4617 });
4618 converter.addConversion(callback: [&](cir::FP80Type type) -> mlir::Type {
4619 return mlir::Float80Type::get(context: type.getContext());
4620 });
4621 converter.addConversion(callback: [&](cir::FP128Type type) -> mlir::Type {
4622 return mlir::Float128Type::get(context: type.getContext());
4623 });
4624 converter.addConversion(callback: [&](cir::LongDoubleType type) -> mlir::Type {
4625 return converter.convertType(t: type.getUnderlying());
4626 });
4627 converter.addConversion(callback: [&](cir::FP16Type type) -> mlir::Type {
4628 return mlir::Float16Type::get(context: type.getContext());
4629 });
4630 converter.addConversion(callback: [&](cir::BF16Type type) -> mlir::Type {
4631 return mlir::BFloat16Type::get(context: type.getContext());
4632 });
4633 converter.addConversion(callback: [&](cir::ComplexType type) -> mlir::Type {
4634 // A complex type is lowered to an LLVM struct that contains the real and
4635 // imaginary part as data fields.
4636 mlir::Type elementTy = converter.convertType(t: type.getElementType());
4637 mlir::Type structFields[2] = {elementTy, elementTy};
4638 return mlir::LLVM::LLVMStructType::getLiteral(context: type.getContext(),
4639 types: structFields);
4640 });
4641 converter.addConversion(callback: [&](cir::FuncType type) -> std::optional<mlir::Type> {
4642 auto result = converter.convertType(t: type.getReturnType());
4643 if (!result)
4644 return std::nullopt;
4645 llvm::SmallVector<mlir::Type> arguments;
4646 arguments.reserve(N: type.getNumInputs());
4647 if (converter.convertTypes(types: type.getInputs(), results&: arguments).failed())
4648 return std::nullopt;
4649 auto varArg = type.isVarArg();
4650 return mlir::LLVM::LLVMFunctionType::get(result, arguments, isVarArg: varArg);
4651 });
4652 converter.addConversion(callback: [&](cir::StructType type) -> mlir::Type {
4653 llvm::SmallVector<mlir::Type> llvmMembers;
4654 for (mlir::Type ty : type.getMembers()) {
4655 // A bit-field that shares an earlier field's access unit, and a
4656 // zero-width bit-field, own no bytes, so they get no field of their own.
4657 // StructType::getLLVMFieldIndex maps the two numberings.
4658 if (!cir::memberOwnsBytes(memberTy: ty))
4659 continue;
4660 mlir::Type memberTy = convertTypeForMemory(converter, dataLayout,
4661 type: cir::memberStorageType(memberTy: ty));
4662 // A null member means an unsupported type (e.g. a _BitInt with byte-array
4663 // storage); propagate the conversion failure instead of building an
4664 // invalid struct body.
4665 if (!memberTy)
4666 return {};
4667 llvmMembers.push_back(Elt: memberTy);
4668 }
4669
4670 mlir::LLVM::LLVMStructType llvmStruct;
4671 if (type.getName()) {
4672 llvmStruct = mlir::LLVM::LLVMStructType::getIdentified(
4673 context: type.getContext(), name: type.getPrefixedName());
4674 if (llvmStruct.setBody(types: llvmMembers, isPacked: type.getPacked()).failed())
4675 llvm_unreachable("Failed to set body of record");
4676 } else {
4677 llvmStruct = mlir::LLVM::LLVMStructType::getLiteral(
4678 context: type.getContext(), types: llvmMembers, isPacked: type.getPacked());
4679 }
4680 return llvmStruct;
4681 });
4682 // Unions are lowered as only the largest member.
4683 converter.addConversion(callback: [&](cir::UnionType type) -> mlir::Type {
4684 llvm::SmallVector<mlir::Type> llvmMembers;
4685 if (!type.getMembers().empty())
4686 if (auto storage = type.getUnionStorageType(dataLayout)) {
4687 mlir::Type storageTy =
4688 convertTypeForMemory(converter, dataLayout, type: storage);
4689 if (!storageTy)
4690 return {};
4691 llvmMembers.push_back(Elt: storageTy);
4692 }
4693 if (mlir::Type pad = type.getPadding()) {
4694 mlir::Type padTy = convertTypeForMemory(converter, dataLayout, type: pad);
4695 if (!padTy)
4696 return {};
4697 llvmMembers.push_back(Elt: padTy);
4698 }
4699
4700 mlir::LLVM::LLVMStructType llvmStruct;
4701 if (type.getName()) {
4702 llvmStruct = mlir::LLVM::LLVMStructType::getIdentified(
4703 context: type.getContext(), name: type.getPrefixedName());
4704 if (llvmStruct.setBody(types: llvmMembers, isPacked: type.getPacked()).failed())
4705 llvm_unreachable("Failed to set body of record");
4706 } else {
4707 llvmStruct = mlir::LLVM::LLVMStructType::getLiteral(
4708 context: type.getContext(), types: llvmMembers, isPacked: type.getPacked());
4709 }
4710 return llvmStruct;
4711 });
4712 converter.addConversion(callback: [&](cir::VoidType type) -> mlir::Type {
4713 return mlir::LLVM::LLVMVoidType::get(ctx: type.getContext());
4714 });
4715}
4716static void buildCtorDtorList(
4717 mlir::ModuleOp module, StringRef globalXtorName, StringRef llvmXtorName,
4718 llvm::function_ref<cir::GlobalCtorDtorEntry(mlir::Attribute)> createXtor) {
4719 llvm::SmallVector<cir::GlobalCtorDtorEntry> globalXtors;
4720 if (auto attr =
4721 module->getDiscardableAttrOfType<mlir::ArrayAttr>(name: globalXtorName))
4722 for (mlir::Attribute element : attr)
4723 globalXtors.emplace_back(Args: createXtor(element));
4724
4725 if (globalXtors.empty())
4726 return;
4727
4728 mlir::OpBuilder builder(module.getContext());
4729 builder.setInsertionPointToEnd(&module.getBodyRegion().back());
4730
4731 // Create a global array llvm.global_ctors with element type of
4732 // struct { i32, ptr, ptr }
4733 cir::CIRDataLayout cirDataLayout(module);
4734 auto ctorPFTy = mlir::LLVM::LLVMPointerType::get(
4735 context: builder.getContext(), addressSpace: cirDataLayout.getProgramAddressSpace());
4736 llvm::SmallVector<mlir::Type> ctorStructFields;
4737 ctorStructFields.push_back(Elt: builder.getI32Type());
4738 ctorStructFields.push_back(Elt: ctorPFTy);
4739 ctorStructFields.push_back(Elt: ctorPFTy);
4740
4741 auto ctorStructTy = mlir::LLVM::LLVMStructType::getLiteral(
4742 context: builder.getContext(), types: ctorStructFields);
4743 auto ctorStructArrayTy =
4744 mlir::LLVM::LLVMArrayType::get(elementType: ctorStructTy, numElements: globalXtors.size());
4745
4746 mlir::Location loc = module.getLoc();
4747 auto newGlobalOp = mlir::LLVM::GlobalOp::create(
4748 builder, location: loc, type: ctorStructArrayTy, /*constant=*/isConstant: false,
4749 linkage: mlir::LLVM::Linkage::Appending, name: llvmXtorName, value: mlir::Attribute());
4750
4751 builder.createBlock(parent: &newGlobalOp.getRegion());
4752 builder.setInsertionPointToEnd(newGlobalOp.getInitializerBlock());
4753
4754 mlir::Value result =
4755 mlir::LLVM::UndefOp::create(builder, location: loc, res: ctorStructArrayTy);
4756
4757 for (auto [index, xtorInfo] : llvm::enumerate(First&: globalXtors)) {
4758 mlir::Value structInit =
4759 mlir::LLVM::UndefOp::create(builder, location: loc, res: ctorStructTy);
4760 mlir::Value initPriority = mlir::LLVM::ConstantOp::create(
4761 builder, location: loc, type: ctorStructFields[0], value: xtorInfo.priority);
4762 mlir::Value initFuncAddr = mlir::LLVM::AddressOfOp::create(
4763 builder, location: loc, res: ctorStructFields[1], global_name: xtorInfo.name);
4764 mlir::Value initAssociate =
4765 xtorInfo.associated.empty()
4766 ? mlir::LLVM::ZeroOp::create(builder, location: loc, res: ctorStructFields[2])
4767 .getResult()
4768 : mlir::LLVM::AddressOfOp::create(builder, location: loc, res: ctorStructFields[2],
4769 global_name: xtorInfo.associated)
4770 .getResult();
4771 // Literal zero makes the InsertValueOp::create ambiguous.
4772 llvm::SmallVector<int64_t> zero{0};
4773 structInit = mlir::LLVM::InsertValueOp::create(builder, location: loc, container: structInit,
4774 value: initPriority, position: zero);
4775 structInit = mlir::LLVM::InsertValueOp::create(builder, location: loc, container: structInit,
4776 value: initFuncAddr, position: 1);
4777 structInit = mlir::LLVM::InsertValueOp::create(builder, location: loc, container: structInit,
4778 value: initAssociate, position: 2);
4779 result = mlir::LLVM::InsertValueOp::create(builder, location: loc, container: result, value: structInit,
4780 position: index);
4781 }
4782
4783 mlir::LLVM::ReturnOp::create(builder, location: loc, arg: result);
4784}
4785
4786mlir::LogicalResult CIRToLLVMObjSizeOpLowering::matchAndRewrite(
4787 cir::ObjSizeOp op, OpAdaptor adaptor,
4788 mlir::ConversionPatternRewriter &rewriter) const {
4789 mlir::Type llvmResTy = getTypeConverter()->convertType(t: op.getType());
4790 mlir::Location loc = op->getLoc();
4791
4792 mlir::IntegerType i1Ty = rewriter.getI1Type();
4793
4794 auto i1Val = [&rewriter, &loc, &i1Ty](bool val) {
4795 return mlir::LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i1Ty, value: val);
4796 };
4797
4798 replaceOpWithCallLLVMIntrinsicOp(rewriter, op, intrinsicName: "llvm.objectsize", resultTy: llvmResTy,
4799 operands: {
4800 adaptor.getPtr(),
4801 i1Val(op.getMin()),
4802 i1Val(op.getNullunknown()),
4803 i1Val(op.getDynamic()),
4804 });
4805
4806 return mlir::LogicalResult::success();
4807}
4808
4809//===----------------------------------------------------------------------===//
4810// @llvm.global.annotations emission
4811//===----------------------------------------------------------------------===//
4812
4813/// Annotation strings and arguments, shared by all annotations in a module.
4814struct LLVMAnnotationGlobals {
4815 explicit LLVMAnnotationGlobals(mlir::ModuleOp module) {
4816 assert(!cir::MissingFeatures::annotationConstantAddrSpace());
4817 auto asAttr = mlir::dyn_cast_if_present<mlir::IntegerAttr>(
4818 Val: mlir::DataLayout(module).getGlobalMemorySpace());
4819 addrSpace = asAttr ? asAttr.getUInt() : 0;
4820 }
4821
4822 unsigned addrSpace;
4823 llvm::StringMap<mlir::LLVM::GlobalOp> stringCache;
4824 llvm::StringMap<mlir::LLVM::GlobalOp> argStringCache;
4825 llvm::MapVector<mlir::ArrayAttr, mlir::LLVM::GlobalOp> argsCache;
4826};
4827
4828namespace {
4829constexpr StringRef llvmMetadataSectionName = "llvm.metadata";
4830
4831/// Get-or-create a private constant string global in the llvm.metadata
4832/// section, deduplicated by string content.
4833mlir::LLVM::GlobalOp
4834getOrCreateAnnotationStringGlobal(mlir::OpBuilder &builder, mlir::Location loc,
4835 mlir::ModuleOp module, llvm::StringRef str,
4836 llvm::StringMap<mlir::LLVM::GlobalOp> &cache,
4837 bool isArg, unsigned addrSpace) {
4838 auto it = cache.find(Key: str);
4839 if (it != cache.end())
4840 return it->second;
4841
4842 auto i8Ty = mlir::IntegerType::get(context: module.getContext(), width: 8);
4843 auto arrayTy = mlir::LLVM::LLVMArrayType::get(elementType: i8Ty, numElements: str.size() + 1);
4844 std::string name = ".str";
4845 if (!cache.empty())
4846 name += "." + std::to_string(val: cache.size());
4847 name += ".annotation";
4848 if (isArg)
4849 name += ".arg";
4850
4851 mlir::LLVM::GlobalOp strGlobal = mlir::LLVM::GlobalOp::create(
4852 builder, location: loc, type: arrayTy, /*isConstant=*/true, linkage: mlir::LLVM::Linkage::Private,
4853 name, value: mlir::StringAttr::get(context: module.getContext(), bytes: std::string(str) + '\0'),
4854 /*alignment=*/isArg ? 1 : 0, addrSpace);
4855 if (!isArg)
4856 strGlobal.setSection(llvmMetadataSectionName);
4857 strGlobal.setUnnamedAddr(mlir::LLVM::UnnamedAddr::Global);
4858 strGlobal.setDsoLocal(true);
4859 cache[str] = strGlobal;
4860 return strGlobal;
4861}
4862
4863/// Get-or-create a private constant struct holding the annotation arguments,
4864/// deduplicated by ArrayAttr identity.
4865mlir::LLVM::GlobalOp getOrCreateAnnotationArgsVar(
4866 mlir::OpBuilder &builder, mlir::Location loc, mlir::ModuleOp module,
4867 mlir::ArrayAttr argsAttr,
4868 llvm::StringMap<mlir::LLVM::GlobalOp> &argStringCache,
4869 llvm::MapVector<mlir::ArrayAttr, mlir::LLVM::GlobalOp> &argsCache,
4870 unsigned addrSpace) {
4871 auto it = argsCache.find(Key: argsAttr);
4872 if (it != argsCache.end())
4873 return it->second;
4874
4875 auto ptrTy =
4876 mlir::LLVM::LLVMPointerType::get(context: builder.getContext(), addressSpace: addrSpace);
4877
4878 llvm::SmallVector<mlir::Type> fieldTypes;
4879 for (mlir::Attribute arg : argsAttr) {
4880 if (mlir::isa<mlir::StringAttr>(Val: arg))
4881 fieldTypes.push_back(Elt: ptrTy);
4882 else if (auto intAttr = mlir::dyn_cast<mlir::IntegerAttr>(Val&: arg))
4883 fieldTypes.push_back(Elt: intAttr.getType());
4884 else
4885 llvm_unreachable("Unsupported annotation arg type");
4886 }
4887
4888 auto structTy =
4889 mlir::LLVM::LLVMStructType::getLiteral(context: builder.getContext(), types: fieldTypes);
4890 std::string name = ".args";
4891 if (!argsCache.empty())
4892 name += "." + std::to_string(val: argsCache.size());
4893 name += ".annotation";
4894
4895 mlir::LLVM::GlobalOp argsGlobal = mlir::LLVM::GlobalOp::create(
4896 builder, location: loc, type: structTy, /*isConstant=*/true, linkage: mlir::LLVM::Linkage::Private,
4897 name, value: mlir::Attribute(), /*alignment=*/0, addrSpace);
4898 argsGlobal.setSection(llvmMetadataSectionName);
4899 argsGlobal.setUnnamedAddr(mlir::LLVM::UnnamedAddr::Global);
4900 argsGlobal.setDsoLocal(true);
4901
4902 // Build the initializer block.
4903 argsGlobal.getRegion().push_back(block: new mlir::Block());
4904 mlir::OpBuilder initBuilder(module.getContext());
4905 initBuilder.setInsertionPointToEnd(argsGlobal.getInitializerBlock());
4906
4907 mlir::Value structInit =
4908 mlir::LLVM::UndefOp::create(builder&: initBuilder, location: loc, res: structTy);
4909 for (auto [idx, arg] : llvm::enumerate(First&: argsAttr)) {
4910 if (auto strArg = mlir::dyn_cast<mlir::StringAttr>(Val: arg)) {
4911 mlir::LLVM::GlobalOp strGlobal = getOrCreateAnnotationStringGlobal(
4912 builder, loc, module, str: strArg.getValue(), cache&: argStringCache,
4913 /*isArg=*/true, addrSpace);
4914 mlir::LLVM::AddressOfOp strAddr = mlir::LLVM::AddressOfOp::create(
4915 builder&: initBuilder, location: loc, res: ptrTy, global_name: strGlobal.getSymName());
4916 structInit = mlir::LLVM::InsertValueOp::create(builder&: initBuilder, location: loc,
4917 container: structInit, value: strAddr, position: idx);
4918 } else if (auto intArg = mlir::dyn_cast<mlir::IntegerAttr>(Val: arg)) {
4919 mlir::LLVM::ConstantOp intConst = mlir::LLVM::ConstantOp::create(
4920 builder&: initBuilder, location: loc, type: intArg.getType(), value: intArg.getValue());
4921 structInit = mlir::LLVM::InsertValueOp::create(builder&: initBuilder, location: loc,
4922 container: structInit, value: intConst, position: idx);
4923 } else {
4924 llvm_unreachable("Unsupported annotation arg type");
4925 }
4926 }
4927 mlir::LLVM::ReturnOp::create(builder&: initBuilder, location: loc, arg: structInit);
4928
4929 argsCache[argsAttr] = argsGlobal;
4930 return argsGlobal;
4931}
4932
4933/// Resolve a possibly-fused MLIR Location to a FileLineColLoc, returning
4934/// {filename, line}. Returns {empty, 0} if no usable file location is found.
4935std::pair<llvm::StringRef, unsigned> extractFileLine(mlir::Location loc) {
4936 mlir::Location resolved = loc;
4937 if (auto fused = mlir::dyn_cast<mlir::FusedLoc>(Val&: resolved)) {
4938 if (!fused.getLocations().empty())
4939 resolved = fused.getLocations()[0];
4940 }
4941 if (auto fl = mlir::dyn_cast<mlir::FileLineColLoc>(Val&: resolved))
4942 return {fl.getFilename().getValue(), fl.getLine()};
4943 return {"", 0};
4944}
4945
4946/// The name, file, line, and args operands of the llvm.*annotation intrinsics.
4947struct AnnotationOperands {
4948 mlir::Value name, file, line, args;
4949};
4950AnnotationOperands getAnnotationOperands(mlir::OpBuilder &builder,
4951 mlir::Location loc,
4952 cir::AnnotationAttr annotation,
4953 LLVMAnnotationGlobals &globals) {
4954 auto module = builder.getInsertionBlock()
4955 ->getParentOp()
4956 ->getParentOfType<mlir::ModuleOp>();
4957 auto [filename, line] = extractFileLine(loc);
4958 mlir::ArrayAttr args = annotation.getArgs();
4959 mlir::LLVM::GlobalOp nameGlobal, fileGlobal, argsGlobal;
4960 {
4961 mlir::OpBuilder::InsertionGuard guard(builder);
4962 builder.setInsertionPointToEnd(module.getBody());
4963 nameGlobal = getOrCreateAnnotationStringGlobal(
4964 builder, loc, module, str: annotation.getName().getValue(),
4965 cache&: globals.stringCache, /*isArg=*/false, addrSpace: globals.addrSpace);
4966 fileGlobal = getOrCreateAnnotationStringGlobal(
4967 builder, loc, module, str: filename, cache&: globals.stringCache, /*isArg=*/false,
4968 addrSpace: globals.addrSpace);
4969 if (args && !args.empty())
4970 argsGlobal = getOrCreateAnnotationArgsVar(
4971 builder, loc, module, argsAttr: args, argStringCache&: globals.argStringCache, argsCache&: globals.argsCache,
4972 addrSpace: globals.addrSpace);
4973 }
4974
4975 auto ptrTy =
4976 mlir::LLVM::LLVMPointerType::get(context: builder.getContext(), addressSpace: globals.addrSpace);
4977 AnnotationOperands ops;
4978 ops.name = mlir::LLVM::AddressOfOp::create(builder, location: loc, res: ptrTy,
4979 global_name: nameGlobal.getSymName());
4980 ops.file = mlir::LLVM::AddressOfOp::create(builder, location: loc, res: ptrTy,
4981 global_name: fileGlobal.getSymName());
4982 ops.line =
4983 mlir::LLVM::ConstantOp::create(builder, location: loc, type: builder.getI32Type(), value: line);
4984 if (argsGlobal)
4985 ops.args = mlir::LLVM::AddressOfOp::create(builder, location: loc, res: ptrTy,
4986 global_name: argsGlobal.getSymName());
4987 else
4988 ops.args = mlir::LLVM::ZeroOp::create(builder, location: loc, res: ptrTy);
4989 return ops;
4990}
4991} // namespace
4992
4993mlir::LogicalResult CIRToLLVMVarAnnotationOpLowering::matchAndRewrite(
4994 cir::VarAnnotationOp op, OpAdaptor adaptor,
4995 mlir::ConversionPatternRewriter &rewriter) const {
4996 AnnotationOperands ops = getAnnotationOperands(
4997 builder&: rewriter, loc: op.getLoc(), annotation: op.getAnnotation(), globals&: annotationGlobals);
4998 rewriter.replaceOpWithNewOp<mlir::LLVM::VarAnnotation>(
4999 op, args: adaptor.getAddr(), args&: ops.name, args&: ops.file, args&: ops.line, args&: ops.args);
5000 return mlir::success();
5001}
5002
5003mlir::LogicalResult CIRToLLVMPtrAnnotationOpLowering::matchAndRewrite(
5004 cir::PtrAnnotationOp op, OpAdaptor adaptor,
5005 mlir::ConversionPatternRewriter &rewriter) const {
5006 AnnotationOperands ops = getAnnotationOperands(
5007 builder&: rewriter, loc: op.getLoc(), annotation: op.getAnnotation(), globals&: annotationGlobals);
5008 rewriter.replaceOpWithNewOp<mlir::LLVM::PtrAnnotation>(
5009 op, args: adaptor.getAddr().getType(), args: adaptor.getAddr(), args&: ops.name, args&: ops.file,
5010 args&: ops.line, args&: ops.args);
5011 return mlir::success();
5012}
5013
5014mlir::LogicalResult CIRToLLVMAnnotationOpLowering::matchAndRewrite(
5015 cir::AnnotationOp op, OpAdaptor adaptor,
5016 mlir::ConversionPatternRewriter &rewriter) const {
5017 AnnotationOperands ops = getAnnotationOperands(
5018 builder&: rewriter, loc: op.getLoc(), annotation: op.getAnnotation(), globals&: annotationGlobals);
5019 rewriter.replaceOpWithNewOp<mlir::LLVM::Annotation>(
5020 op, args: adaptor.getValue().getType(), args: adaptor.getValue(), args&: ops.name, args&: ops.file,
5021 args&: ops.line);
5022 return mlir::success();
5023}
5024
5025void ConvertCIRToLLVMPass::collectGlobalAnnotations(mlir::ModuleOp module) {
5026 auto handleArray = [&](mlir::StringAttr symName, mlir::ArrayAttr arr,
5027 mlir::Location loc) {
5028 if (!arr)
5029 return;
5030 for (mlir::Attribute a : arr)
5031 if (auto annot = mlir::dyn_cast<cir::AnnotationAttr>(Val&: a))
5032 collectedAnnotations.emplace_back(Args&: symName, Args&: annot, Args&: loc);
5033 };
5034
5035 // Walk in IR order: GlobalOps first (they appear before functions in the
5036 // module body), then FuncOps. This matches OGCG's emission order.
5037 module.walk(callback: [&](cir::GlobalOp op) {
5038 handleArray(op.getSymNameAttr(), op.getAnnotationsAttr(), op.getLoc());
5039 });
5040 module.walk(callback: [&](cir::FuncOp op) {
5041 handleArray(op.getSymNameAttr(), op.getAnnotationsAttr(), op.getLoc());
5042 });
5043}
5044
5045void ConvertCIRToLLVMPass::buildGlobalAnnotationsVar(
5046 mlir::ModuleOp module, LLVMAnnotationGlobals &annotationGlobals) {
5047 if (collectedAnnotations.empty())
5048 return;
5049
5050 mlir::MLIRContext *ctx = module.getContext();
5051 mlir::OpBuilder builder(ctx);
5052 builder.setInsertionPointToEnd(&module.getBodyRegion().back());
5053
5054 unsigned addrSpace = annotationGlobals.addrSpace;
5055 auto ptrTy = mlir::LLVM::LLVMPointerType::get(context: ctx, addressSpace: addrSpace);
5056 auto i32Ty = builder.getI32Type();
5057
5058 // Each entry: { ptr, ptr, ptr, i32, ptr }.
5059 auto entryTy = mlir::LLVM::LLVMStructType::getLiteral(
5060 context: ctx, types: {ptrTy, ptrTy, ptrTy, i32Ty, ptrTy});
5061 auto arrayTy =
5062 mlir::LLVM::LLVMArrayType::get(elementType: entryTy, numElements: collectedAnnotations.size());
5063
5064 mlir::Location moduleLoc = module.getLoc();
5065 auto annotationsGlobal = mlir::LLVM::GlobalOp::create(
5066 builder, location: moduleLoc, type: arrayTy, /*isConstant=*/false,
5067 linkage: mlir::LLVM::Linkage::Appending, name: "llvm.global.annotations",
5068 value: mlir::Attribute(), /*alignment=*/0, addrSpace);
5069 annotationsGlobal.setSection(llvmMetadataSectionName);
5070
5071 // Strings/args constants must come *before* @llvm.global.annotations to
5072 // match OGCG output order. Insert them just before the annotations global.
5073 mlir::OpBuilder constsBuilder(ctx);
5074 constsBuilder.setInsertionPoint(annotationsGlobal);
5075
5076 llvm::StringMap<mlir::LLVM::GlobalOp> &stringCache =
5077 annotationGlobals.stringCache;
5078 llvm::StringMap<mlir::LLVM::GlobalOp> &argStringCache =
5079 annotationGlobals.argStringCache;
5080 llvm::MapVector<mlir::ArrayAttr, mlir::LLVM::GlobalOp> &argsCache =
5081 annotationGlobals.argsCache;
5082
5083 // Build the initializer block of @llvm.global.annotations.
5084 annotationsGlobal.getRegion().push_back(block: new mlir::Block());
5085 mlir::OpBuilder initBuilder(ctx);
5086 initBuilder.setInsertionPointToEnd(annotationsGlobal.getInitializerBlock());
5087
5088 mlir::Value arrayVal =
5089 mlir::LLVM::UndefOp::create(builder&: initBuilder, location: moduleLoc, res: arrayTy);
5090
5091 for (auto [idx, entry] : llvm::enumerate(First&: collectedAnnotations)) {
5092 mlir::Value entryVal =
5093 mlir::LLVM::UndefOp::create(builder&: initBuilder, location: moduleLoc, res: entryTy);
5094
5095 // Field 0: ptr to the annotated symbol. (Literal zero is ambiguous on
5096 // InsertValueOp::create, wrap in a SmallVector.)
5097 llvm::SmallVector<int64_t> zero{0};
5098 unsigned symAddrSpace = 0;
5099 if (auto global = mlir::dyn_cast_if_present<mlir::LLVM::GlobalOp>(
5100 Val: mlir::SymbolTable::lookupSymbolIn(op: module, symbol: entry.symName)))
5101 symAddrSpace = global.getAddrSpace();
5102 mlir::Value symAddr = mlir::LLVM::AddressOfOp::create(
5103 builder&: initBuilder, location: moduleLoc,
5104 res: mlir::LLVM::LLVMPointerType::get(context: ctx, addressSpace: symAddrSpace),
5105 global_name: entry.symName.getValue());
5106 if (symAddrSpace != addrSpace)
5107 symAddr = mlir::LLVM::AddrSpaceCastOp::create(builder&: initBuilder, location: moduleLoc,
5108 res: ptrTy, arg: symAddr);
5109 entryVal = mlir::LLVM::InsertValueOp::create(builder&: initBuilder, location: moduleLoc,
5110 container: entryVal, value: symAddr, position: zero);
5111
5112 // Field 1: ptr to the annotation name string.
5113 mlir::LLVM::GlobalOp nameGlobal = getOrCreateAnnotationStringGlobal(
5114 builder&: constsBuilder, loc: moduleLoc, module, str: entry.annotation.getName().getValue(),
5115 cache&: stringCache, /*isArg=*/false, addrSpace);
5116 mlir::LLVM::AddressOfOp nameAddr = mlir::LLVM::AddressOfOp::create(
5117 builder&: initBuilder, location: moduleLoc, res: ptrTy, global_name: nameGlobal.getSymName());
5118 entryVal = mlir::LLVM::InsertValueOp::create(builder&: initBuilder, location: moduleLoc,
5119 container: entryVal, value: nameAddr, position: 1);
5120
5121 // Fields 2 and 3: ptr to filename string and line number.
5122 auto [filename, line] = extractFileLine(loc: entry.loc);
5123 mlir::LLVM::GlobalOp fileGlobal = getOrCreateAnnotationStringGlobal(
5124 builder&: constsBuilder, loc: moduleLoc, module, str: filename, cache&: stringCache,
5125 /*isArg=*/false, addrSpace);
5126 mlir::LLVM::AddressOfOp fileAddr = mlir::LLVM::AddressOfOp::create(
5127 builder&: initBuilder, location: moduleLoc, res: ptrTy, global_name: fileGlobal.getSymName());
5128 entryVal = mlir::LLVM::InsertValueOp::create(builder&: initBuilder, location: moduleLoc,
5129 container: entryVal, value: fileAddr, position: 2);
5130 mlir::LLVM::ConstantOp lineConst =
5131 mlir::LLVM::ConstantOp::create(builder&: initBuilder, location: moduleLoc, type: i32Ty, value: line);
5132 entryVal = mlir::LLVM::InsertValueOp::create(builder&: initBuilder, location: moduleLoc,
5133 container: entryVal, value: lineConst, position: 3);
5134
5135 // Field 4: ptr to args, or null if none.
5136 mlir::ArrayAttr args = entry.annotation.getArgs();
5137 mlir::Value argsField;
5138 if (!args || args.empty()) {
5139 argsField = mlir::LLVM::ZeroOp::create(builder&: initBuilder, location: moduleLoc, res: ptrTy);
5140 } else {
5141 mlir::LLVM::GlobalOp argsGlobal =
5142 getOrCreateAnnotationArgsVar(builder&: constsBuilder, loc: moduleLoc, module, argsAttr: args,
5143 argStringCache, argsCache, addrSpace);
5144 argsField = mlir::LLVM::AddressOfOp::create(builder&: initBuilder, location: moduleLoc, res: ptrTy,
5145 global_name: argsGlobal.getSymName());
5146 }
5147 entryVal = mlir::LLVM::InsertValueOp::create(builder&: initBuilder, location: moduleLoc,
5148 container: entryVal, value: argsField, position: 4);
5149
5150 arrayVal = mlir::LLVM::InsertValueOp::create(builder&: initBuilder, location: moduleLoc,
5151 container: arrayVal, value: entryVal, position: idx);
5152 }
5153
5154 mlir::LLVM::ReturnOp::create(builder&: initBuilder, location: moduleLoc, arg: arrayVal);
5155}
5156
5157void ConvertCIRToLLVMPass::resolveBlockAddressOp(
5158 LLVMBlockAddressInfo &blockInfoAddr) {
5159
5160 mlir::ModuleOp module = getOperation();
5161 mlir::OpBuilder opBuilder(module.getContext());
5162 for (auto &[blockAddOp, blockInfo] :
5163 blockInfoAddr.getUnresolvedBlockAddress()) {
5164 mlir::LLVM::BlockTagOp resolvedLabel =
5165 blockInfoAddr.lookupBlockTag(info: blockInfo);
5166 assert(resolvedLabel && "expected BlockTagOp to already be emitted");
5167 mlir::FlatSymbolRefAttr fnSym = blockInfo.getFunc();
5168 auto blkAddTag = mlir::LLVM::BlockAddressAttr::get(
5169 context: opBuilder.getContext(), function: fnSym, tag: resolvedLabel.getTagAttr());
5170 blockAddOp.setBlockAddrAttr(blkAddTag);
5171 }
5172 blockInfoAddr.clearUnresolvedMap();
5173}
5174
5175void ConvertCIRToLLVMPass::processCIRAttrs(mlir::ModuleOp module) {
5176 // Lower the module attributes to LLVM equivalents.
5177 if (mlir::Attribute tripleAttr =
5178 module->getAttr(name: cir::CIRDialect::getTripleAttrName()))
5179 module->setAttr(name: mlir::LLVM::LLVMDialect::getTargetTripleAttrName(),
5180 value: tripleAttr);
5181
5182 if (mlir::Attribute asmAttr =
5183 module->getAttr(name: cir::CIRDialect::getModuleLevelAsmAttrName()))
5184 module->setAttr(name: mlir::LLVM::LLVMDialect::getModuleLevelAsmAttrName(),
5185 value: asmAttr);
5186
5187 if (auto framePointerAttr = module->getAttrOfType<cir::FramePointerKindAttr>(
5188 name: cir::CIRDialect::getModuleFramePointerAttrName())) {
5189 mlir::OpBuilder builder(module.getContext());
5190 builder.setInsertionPointToEnd(module.getBody());
5191 auto flag = mlir::LLVM::ModuleFlagAttr::get(
5192 context: module.getContext(), behavior: mlir::LLVM::ModFlagBehavior::Max,
5193 key: builder.getStringAttr(bytes: "frame-pointer"),
5194 value: builder.getI32IntegerAttr(value: static_cast<int32_t>(
5195 convertFramePointerKindToLLVMIR(kind: framePointerAttr.getValue()))));
5196 mlir::LLVM::ModuleFlagsOp::create(builder, location: module.getLoc(),
5197 flags: builder.getArrayAttr(value: {flag}));
5198 }
5199
5200 lowerOpenCLModuleMetadataAttrs(module);
5201}
5202
5203void ConvertCIRToLLVMPass::runOnOperation() {
5204 llvm::TimeTraceScope scope("Convert CIR to LLVM Pass");
5205
5206 mlir::ModuleOp module = getOperation();
5207 // CIR type of each symbol before lowering, for lookups that find the
5208 // already-lowered LLVM symbol after replacement. This must cover every
5209 // symbol kind that can be replaced: a kind missing from this loop
5210 // quietly falls back to the adjusted LLVM type in readers (here and in
5211 // LoweringHelpers).
5212 // Aliases are covered: they are GlobalOp and FuncOp ops with an
5213 // aliasee, not separate symbol kinds.
5214 CIRPreConversionTypeMap preConversionTypes;
5215 for (mlir::Operation &op : module.getBody()->getOperations()) {
5216 if (auto globalOp = mlir::dyn_cast<cir::GlobalOp>(Val&: op))
5217 preConversionTypes[globalOp.getSymName()] = globalOp.getSymType();
5218 else if (auto funcOp = mlir::dyn_cast<cir::FuncOp>(Val&: op))
5219 preConversionTypes[funcOp.getSymName()] = funcOp.getFunctionType();
5220 }
5221 mlir::DataLayout dl(module);
5222 mlir::LLVMTypeConverter converter(&getContext());
5223 prepareTypeConverter(converter, dataLayout&: dl);
5224
5225 /// Tracks the state required to lower CIR `LabelOp` and `BlockAddressOp`.
5226 /// Maps labels to their corresponding `BlockTagOp` and keeps bookkeeping
5227 /// of unresolved `BlockAddressOp`s until they are matched with the
5228 /// corresponding `BlockTagOp` in `resolveBlockAddressOp`.
5229 LLVMBlockAddressInfo blockInfoAddr;
5230 /// Cached symbol table collection used by call lowering patterns to avoid
5231 /// repeated O(M) module-wide symbol scans for every call site.
5232 mlir::SymbolTableCollection symbolTables;
5233 LLVMAnnotationGlobals annotationGlobals(module);
5234 mlir::RewritePatternSet patterns(&getContext());
5235 patterns.add<CIRToLLVMVarAnnotationOpLowering,
5236 CIRToLLVMPtrAnnotationOpLowering, CIRToLLVMAnnotationOpLowering>(
5237 arg&: converter, args: patterns.getContext(), args&: dl, args&: symbolTables, args&: annotationGlobals);
5238 patterns.add<CIRToLLVMBlockAddressOpLowering, CIRToLLVMLabelOpLowering>(
5239 arg&: converter, args: patterns.getContext(), args&: dl, args&: symbolTables, args&: blockInfoAddr);
5240 patterns.add<CIRToLLVMGlobalOpLowering, CIRToLLVMConstantOpLowering>(
5241 arg&: converter, args: patterns.getContext(), args&: dl, args&: symbolTables, args&: blockInfoAddr,
5242 args&: preConversionTypes);
5243
5244 patterns.add<
5245#define GET_LLVM_LOWERING_PATTERNS_LIST
5246#include "clang/CIR/Dialect/IR/CIRLowering.inc"
5247#undef GET_LLVM_LOWERING_PATTERNS_LIST
5248 >(arg&: converter, args: patterns.getContext(), args&: dl, args&: symbolTables);
5249 // Patterns reading replaced symbols take the pre-conversion type map.
5250 patterns
5251 .add<CIRToLLVMVTableAddrPointOpLowering, CIRToLLVMVTTAddrPointOpLowering>(
5252 arg&: converter, args: patterns.getContext(), args&: dl, args&: symbolTables,
5253 args&: preConversionTypes);
5254
5255 processCIRAttrs(module);
5256
5257 // Collect annotation info from cir.func / cir.global before conversion;
5258 // the annotations attribute is filtered out during FuncOp/GlobalOp lowering.
5259 collectGlobalAnnotations(module);
5260
5261 // Resolve constant printf strings while their producers are still CIR.
5262 annotateAMDGPUPrintfStrings(module, symbolTables);
5263
5264 mlir::ConversionTarget target(getContext());
5265 target.addLegalOp<mlir::ModuleOp>();
5266 target.addLegalDialect<mlir::LLVM::LLVMDialect>();
5267 mlir::configureOpenMPToLLVMConversionLegality(target, typeConverter: converter);
5268 target.addLegalDialect<mlir::omp::OpenMPDialect>();
5269 mlir::populateOpenMPToLLVMConversionPatterns(converter, patterns);
5270 target.addIllegalDialect<mlir::BuiltinDialect, cir::CIRDialect,
5271 mlir::func::FuncDialect>();
5272
5273 llvm::SmallVector<mlir::Operation *> ops;
5274 ops.push_back(Elt: module);
5275 cir::collectUnreachable(parent: module, ops);
5276
5277 // One-shot conversion (no pattern rollback): rewrites materialize
5278 // immediately, skipping the IRRewrite/value-mapping bookkeeping. The
5279 // CIR patterns here are compatible: nothing dereferences the root op
5280 // after replacement, and the cached symbol table is maintained
5281 // explicitly at every module-symbol replacement.
5282 mlir::ConversionConfig config;
5283 config.allowPatternRollback = false;
5284 if (failed(
5285 Result: applyPartialConversion(ops, target, patterns: std::move(patterns), config))) {
5286 signalPassFailure();
5287 return;
5288 }
5289
5290 // Drop CIR-native data-layout entries.
5291 if (auto dlSpec = mlir::dyn_cast_or_null<mlir::DataLayoutSpecAttr>(
5292 Val: module->getAttr(name: mlir::DLTIDialect::kDataLayoutAttrName))) {
5293 llvm::SmallVector<mlir::DataLayoutEntryInterface> kept;
5294 for (mlir::DataLayoutEntryInterface entry : dlSpec.getEntries()) {
5295 if (entry.isTypeEntry()) {
5296 mlir::Type key = mlir::cast<mlir::Type>(Val: entry.getKey());
5297 if (mlir::isa<cir::PointerType, cir::IntType, cir::VectorType>(Val: key))
5298 continue;
5299 }
5300 kept.push_back(Elt: entry);
5301 }
5302 module->setAttr(name: mlir::DLTIDialect::kDataLayoutAttrName,
5303 value: mlir::DataLayoutSpecAttr::get(context: module.getContext(), entries: kept));
5304 }
5305
5306 // Emit the llvm.global_ctors array.
5307 buildCtorDtorList(
5308 module, globalXtorName: cir::CIRDialect::getGlobalCtorsAttrName(), llvmXtorName: "llvm.global_ctors",
5309 createXtor: [](mlir::Attribute attr) {
5310 auto ctorAttr = mlir::cast<cir::GlobalCtorAttr>(Val&: attr);
5311 StringRef associated;
5312 if (mlir::FlatSymbolRefAttr assoc = ctorAttr.getAssociated())
5313 associated = assoc.getValue();
5314 return cir::GlobalCtorDtorEntry(ctorAttr.getName(),
5315 ctorAttr.getPriority(), associated);
5316 });
5317 // Emit the llvm.global_dtors array.
5318 buildCtorDtorList(
5319 module, globalXtorName: cir::CIRDialect::getGlobalDtorsAttrName(), llvmXtorName: "llvm.global_dtors",
5320 createXtor: [](mlir::Attribute attr) {
5321 auto dtorAttr = mlir::cast<cir::GlobalDtorAttr>(Val&: attr);
5322 StringRef associated;
5323 if (mlir::FlatSymbolRefAttr assoc = dtorAttr.getAssociated())
5324 associated = assoc.getValue();
5325 return cir::GlobalCtorDtorEntry(dtorAttr.getName(),
5326 dtorAttr.getPriority(), associated);
5327 });
5328 // Emit @llvm.global.annotations from the previously-collected entries.
5329 buildGlobalAnnotationsVar(module, annotationGlobals);
5330
5331 resolveBlockAddressOp(blockInfoAddr);
5332}
5333
5334mlir::LogicalResult CIRToLLVMBrOpLowering::matchAndRewrite(
5335 cir::BrOp op, OpAdaptor adaptor,
5336 mlir::ConversionPatternRewriter &rewriter) const {
5337 rewriter.replaceOpWithNewOp<mlir::LLVM::BrOp>(op, args: adaptor.getOperands(),
5338 args: op.getDest());
5339 return mlir::LogicalResult::success();
5340}
5341
5342mlir::LogicalResult CIRToLLVMGetMemberOpLowering::matchAndRewrite(
5343 cir::GetMemberOp op, OpAdaptor adaptor,
5344 mlir::ConversionPatternRewriter &rewriter) const {
5345 mlir::Type llResTy = getTypeConverter()->convertType(t: op.getType());
5346 if (!llResTy)
5347 return emitTypeConversionError(op, type: op.getType());
5348 mlir::Type pointee = op.getAddrTy().getPointee();
5349
5350 if (mlir::isa<cir::UnionType>(Val: pointee)) {
5351 // Union members share the address space, so we just need a bitcast to
5352 // conform to type-checking.
5353 rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(op, args&: llResTy,
5354 args: adaptor.getAddr());
5355 return mlir::success();
5356 }
5357
5358 auto structTy = mlir::cast<cir::StructType>(Val&: pointee);
5359 // Since the base address is a pointer to an aggregate, the first offset
5360 // is always zero. The second offset tells us which member it will access.
5361 // Members that own no bytes are left out of the LLVM body, so the CIR member
5362 // index has to be mapped rather than used directly.
5363 llvm::SmallVector<mlir::LLVM::GEPArg, 2> offset{
5364 0, static_cast<int32_t>(structTy.getLLVMFieldIndex(idx: op.getIndex()))};
5365 const mlir::Type elementTy = getTypeConverter()->convertType(t: structTy);
5366 if (!elementTy)
5367 return emitTypeConversionError(op, type: structTy);
5368 // Struct member accesses are always inbounds and nuw: the base pointer
5369 // is valid and the member offset is a positive, constant offset within
5370 // the struct layout, so it cannot wrap. This matches LLVM's
5371 // IRBuilder::CreateStructGEP.
5372 mlir::LLVM::GEPNoWrapFlags flags =
5373 mlir::LLVM::GEPNoWrapFlags::inbounds | mlir::LLVM::GEPNoWrapFlags::nuw;
5374 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
5375 op, args&: llResTy, args: elementTy, args: adaptor.getAddr(), args&: offset, args&: flags);
5376 return mlir::success();
5377}
5378
5379mlir::LogicalResult CIRToLLVMExtractMemberOpLowering::matchAndRewrite(
5380 cir::ExtractMemberOp op, OpAdaptor adaptor,
5381 mlir::ConversionPatternRewriter &rewriter) const {
5382 if (mlir::isa<cir::UnionType>(Val: op.getRecord().getType())) {
5383 op.emitError(message: "cir.extract_member cannot extract member from a union");
5384 return mlir::failure();
5385 }
5386
5387 auto structTy = mlir::cast<cir::StructType>(Val: op.getRecord().getType());
5388 std::int64_t indices[1] = {
5389 static_cast<std::int64_t>(structTy.getLLVMFieldIndex(idx: op.getIndex()))};
5390 rewriter.replaceOpWithNewOp<mlir::LLVM::ExtractValueOp>(
5391 op, args: adaptor.getRecord(), args&: indices);
5392 return mlir::success();
5393}
5394
5395mlir::LogicalResult CIRToLLVMInsertMemberOpLowering::matchAndRewrite(
5396 cir::InsertMemberOp op, OpAdaptor adaptor,
5397 mlir::ConversionPatternRewriter &rewriter) const {
5398 if (mlir::isa<cir::UnionType>(Val: op.getRecord().getType())) {
5399 op.emitError(message: "cir.update_member cannot update member of a union");
5400 return mlir::failure();
5401 }
5402
5403 auto structTy = mlir::cast<cir::StructType>(Val: op.getRecord().getType());
5404 std::int64_t indices[1] = {
5405 static_cast<std::int64_t>(structTy.getLLVMFieldIndex(idx: op.getIndex()))};
5406 rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>(
5407 op, args: adaptor.getRecord(), args: adaptor.getValue(), args&: indices);
5408 return mlir::success();
5409}
5410
5411void createLLVMFuncOpIfNotExist(mlir::ConversionPatternRewriter &rewriter,
5412 mlir::SymbolTableCollection &symbolTables,
5413 mlir::Operation *srcOp, llvm::StringRef fnName,
5414 mlir::Type fnTy, mlir::ArrayAttr argAttrs,
5415 mlir::ArrayAttr resAttrs) {
5416 mlir::ModuleOp modOp = srcOp->getParentOfType<mlir::ModuleOp>();
5417 mlir::Operation *sourceSymbol = symbolTables.lookupSymbolIn(
5418 symbolTableOp: modOp, symbol: mlir::StringAttr::get(context: fnTy.getContext(), bytes: fnName));
5419 if (!sourceSymbol) {
5420 mlir::OpBuilder::InsertionGuard guard(rewriter);
5421 auto enclosingFnOp = srcOp->getParentOfType<mlir::LLVM::LLVMFuncOp>();
5422 rewriter.setInsertionPoint(enclosingFnOp);
5423 auto fn =
5424 mlir::LLVM::LLVMFuncOp::create(builder&: rewriter, location: srcOp->getLoc(), name: fnName, type: fnTy);
5425 if (argAttrs)
5426 fn.setArgAttrsAttr(argAttrs);
5427 if (resAttrs)
5428 fn.setResAttrsAttr(resAttrs);
5429 // Update the symbol table.
5430 symbolTables.getSymbolTable(op: fn->getParentOp()).insert(symbol: fn);
5431 }
5432}
5433
5434mlir::LogicalResult CIRToLLVMThrowOpLowering::matchAndRewrite(
5435 cir::ThrowOp op, OpAdaptor adaptor,
5436 mlir::ConversionPatternRewriter &rewriter) const {
5437 mlir::Location loc = op.getLoc();
5438 auto voidTy = mlir::LLVM::LLVMVoidType::get(ctx: getContext());
5439
5440 // TODO(cir): set the runtime calling convention on the runtime calls below.
5441 assert(!cir::MissingFeatures::opFuncCallingConv());
5442
5443 if (op.rethrows()) {
5444 auto funcTy = mlir::LLVM::LLVMFunctionType::get(result: voidTy, arguments: {});
5445
5446 // Get or create `declare void @__cxa_rethrow()`
5447 const llvm::StringRef functionName = "__cxa_rethrow";
5448 createLLVMFuncOpIfNotExist(rewriter, symbolTables, srcOp: op, fnName: functionName,
5449 fnTy: funcTy);
5450
5451 auto cxaRethrow = mlir::LLVM::CallOp::create(
5452 builder&: rewriter, location: loc, results: mlir::TypeRange{}, callee: functionName);
5453
5454 rewriter.replaceOp(op, newOp: cxaRethrow);
5455 return mlir::success();
5456 }
5457
5458 auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(context: rewriter.getContext());
5459 auto fnTy = mlir::LLVM::LLVMFunctionType::get(
5460 result: voidTy, arguments: {llvmPtrTy, llvmPtrTy, llvmPtrTy});
5461
5462 // Get or create `declare void @__cxa_throw(ptr, ptr, ptr)`
5463 const llvm::StringRef fnName = "__cxa_throw";
5464 createLLVMFuncOpIfNotExist(rewriter, symbolTables, srcOp: op, fnName, fnTy);
5465
5466 mlir::Value typeInfo = mlir::LLVM::AddressOfOp::create(
5467 builder&: rewriter, location: loc, res: mlir::LLVM::LLVMPointerType::get(context: rewriter.getContext()),
5468 global_name: adaptor.getTypeInfoAttr());
5469
5470 mlir::Value dtor;
5471 if (op.getDtor()) {
5472 dtor = mlir::LLVM::AddressOfOp::create(builder&: rewriter, location: loc, res: llvmPtrTy,
5473 global_name: adaptor.getDtorAttr());
5474 } else {
5475 dtor = mlir::LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: llvmPtrTy);
5476 }
5477
5478 auto cxaThrowCall = mlir::LLVM::CallOp::create(
5479 builder&: rewriter, location: loc, results: mlir::TypeRange{}, callee: fnName,
5480 args: mlir::ValueRange{adaptor.getExceptionPtr(), typeInfo, dtor});
5481
5482 rewriter.replaceOp(op, newOp: cxaThrowCall);
5483 return mlir::success();
5484}
5485
5486mlir::LogicalResult CIRToLLVMAllocExceptionOpLowering::matchAndRewrite(
5487 cir::AllocExceptionOp op, OpAdaptor adaptor,
5488 mlir::ConversionPatternRewriter &rewriter) const {
5489 // Get or create `declare ptr @__cxa_allocate_exception(i64)`
5490 StringRef fnName = "__cxa_allocate_exception";
5491 auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(context: rewriter.getContext());
5492 auto int64Ty = mlir::IntegerType::get(context: rewriter.getContext(), width: 64);
5493 auto fnTy = mlir::LLVM::LLVMFunctionType::get(result: llvmPtrTy, arguments: {int64Ty});
5494
5495 createLLVMFuncOpIfNotExist(rewriter, symbolTables, srcOp: op, fnName, fnTy);
5496 auto exceptionSize = mlir::LLVM::ConstantOp::create(builder&: rewriter, location: op.getLoc(),
5497 value: adaptor.getSizeAttr());
5498
5499 auto allocaExceptionCall = mlir::LLVM::CallOp::create(
5500 builder&: rewriter, location: op.getLoc(), results: mlir::TypeRange{llvmPtrTy}, callee: fnName,
5501 args: mlir::ValueRange{exceptionSize});
5502
5503 rewriter.replaceOp(op, newOp: allocaExceptionCall);
5504 return mlir::success();
5505}
5506
5507static mlir::LLVM::LLVMStructType
5508getLLVMLandingPadStructTy(mlir::ConversionPatternRewriter &rewriter) {
5509 // Create the landing pad type: struct { ptr, i32 }
5510 mlir::MLIRContext *ctx = rewriter.getContext();
5511 auto llvmPtr = mlir::LLVM::LLVMPointerType::get(context: ctx);
5512 llvm::SmallVector<mlir::Type> structFields = {llvmPtr, rewriter.getI32Type()};
5513 return mlir::LLVM::LLVMStructType::getLiteral(context: ctx, types: structFields);
5514}
5515
5516mlir::LogicalResult CIRToLLVMEhInflightOpLowering::matchAndRewrite(
5517 cir::EhInflightOp op, OpAdaptor adaptor,
5518 mlir::ConversionPatternRewriter &rewriter) const {
5519 auto llvmFn = op->getParentOfType<mlir::LLVM::LLVMFuncOp>();
5520 assert(llvmFn && "expected LLVM function parent");
5521 mlir::Block *entryBlock = &llvmFn.getRegion().front();
5522 assert(entryBlock->isEntryBlock());
5523
5524 mlir::ArrayAttr catchListAttr = op.getCatchTypeListAttr();
5525 mlir::ArrayAttr filterListAttr = op.getFilterTypeListAttr();
5526 mlir::SmallVector<mlir::Value> landingPadClauses;
5527
5528 auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(context: rewriter.getContext());
5529 mlir::Location loc = op.getLoc();
5530
5531 // %landingpad = landingpad { ptr, i32 }
5532 // Note that since llvm.landingpad has to be the first operation on the
5533 // block, any needed value for its operands has to be added somewhere else.
5534 if (catchListAttr) {
5535 // catch ptr @_ZTIi
5536 // catch ptr @_ZTIPKc
5537 for (mlir::Attribute catchAttr : catchListAttr) {
5538 auto symAttr = cast<mlir::FlatSymbolRefAttr>(Val&: catchAttr);
5539 // Generate `llvm.mlir.addressof` for each symbol, and place those
5540 // operations in the LLVM function entry basic block.
5541 mlir::OpBuilder::InsertionGuard guard(rewriter);
5542 rewriter.setInsertionPointToStart(entryBlock);
5543 mlir::Value addrOp = mlir::LLVM::AddressOfOp::create(
5544 builder&: rewriter, location: loc, res: llvmPtrTy, global_name: symAttr.getValue());
5545 landingPadClauses.push_back(Elt: addrOp);
5546 }
5547 }
5548
5549 // Emit a catch-all clause (catch ptr null) when:
5550 // - The catch_all attribute is set (typed catches + catch-all), or
5551 // - No typed catches, no cleanup, and no filter (legacy pure catch-all)
5552 if (op.getCatchAll() ||
5553 (!catchListAttr && !op.getCleanup() && !filterListAttr)) {
5554 mlir::OpBuilder::InsertionGuard guard(rewriter);
5555 rewriter.setInsertionPointToStart(entryBlock);
5556 mlir::Value nullOp = mlir::LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: llvmPtrTy);
5557 landingPadClauses.push_back(Elt: nullOp);
5558 }
5559
5560 if (filterListAttr) {
5561 // Filter clauses are array-typed landingpad operands:
5562 // filter [N x ptr] [ptr @_ZTIi, ...]
5563 // An empty list is throw() and lowers to [0 x ptr] zeroinitializer.
5564 mlir::OpBuilder::InsertionGuard guard(rewriter);
5565 rewriter.setInsertionPointToStart(entryBlock);
5566 auto filterArrayTy =
5567 mlir::LLVM::LLVMArrayType::get(elementType: llvmPtrTy, numElements: filterListAttr.size());
5568 mlir::Value filterVal;
5569 if (filterListAttr.empty()) {
5570 filterVal = mlir::LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: filterArrayTy);
5571 } else {
5572 filterVal = mlir::LLVM::UndefOp::create(builder&: rewriter, location: loc, res: filterArrayTy);
5573 for (auto [i, filterAttr] : llvm::enumerate(First&: filterListAttr)) {
5574 auto symAttr = mlir::cast<mlir::FlatSymbolRefAttr>(Val: filterAttr);
5575 mlir::Value addrOp = mlir::LLVM::AddressOfOp::create(
5576 builder&: rewriter, location: loc, res: llvmPtrTy, global_name: symAttr.getValue());
5577 filterVal = mlir::LLVM::InsertValueOp::create(
5578 builder&: rewriter, location: loc, container: filterVal, value: addrOp,
5579 position: llvm::ArrayRef<int64_t>{static_cast<int64_t>(i)});
5580 }
5581 }
5582 landingPadClauses.push_back(Elt: filterVal);
5583 }
5584
5585 // %slot = extractvalue { ptr, i32 } %x, 0
5586 // %selector = extractvalue { ptr, i32 } %x, 1
5587 mlir::LLVM::LLVMStructType llvmLandingPadStructTy =
5588 getLLVMLandingPadStructTy(rewriter);
5589 auto landingPadOp = mlir::LLVM::LandingpadOp::create(
5590 builder&: rewriter, location: loc, resultType: llvmLandingPadStructTy, operands: landingPadClauses);
5591
5592 // The LLVM cleanup flag is only needed when there is no catch-all handler,
5593 // since catch-all (catch ptr null) already ensures the personality function
5594 // enters the landing pad for all exception types.
5595 if (op.getCleanup() && !op.getCatchAll())
5596 landingPadOp.setCleanup(true);
5597
5598 mlir::Value slot =
5599 mlir::LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: landingPadOp, position: 0);
5600 mlir::Value selector =
5601 mlir::LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: landingPadOp, position: 1);
5602 rewriter.replaceOp(op, newValues: mlir::ValueRange{slot, selector});
5603
5604 return mlir::success();
5605}
5606
5607mlir::LogicalResult CIRToLLVMResumeFlatOpLowering::matchAndRewrite(
5608 cir::ResumeFlatOp op, OpAdaptor adaptor,
5609 mlir::ConversionPatternRewriter &rewriter) const {
5610 // %lpad.val = insertvalue { ptr, i32 } poison, ptr %exception_ptr, 0
5611 // %lpad.val2 = insertvalue { ptr, i32 } %lpad.val, i32 %selector, 1
5612 // resume { ptr, i32 } %lpad.val2
5613 mlir::Type llvmLandingPadStructTy = getLLVMLandingPadStructTy(rewriter);
5614 mlir::Value poison = mlir::LLVM::PoisonOp::create(builder&: rewriter, location: op.getLoc(),
5615 res: llvmLandingPadStructTy);
5616
5617 SmallVector<int64_t> slotIdx = {0};
5618 mlir::Value slot = mlir::LLVM::InsertValueOp::create(
5619 builder&: rewriter, location: op.getLoc(), container: poison, value: adaptor.getExceptionPtr(), position: slotIdx);
5620
5621 SmallVector<int64_t> selectorIdx = {1};
5622 mlir::Value selector = mlir::LLVM::InsertValueOp::create(
5623 builder&: rewriter, location: op.getLoc(), container: slot, value: adaptor.getTypeId(), position: selectorIdx);
5624
5625 rewriter.replaceOpWithNewOp<mlir::LLVM::ResumeOp>(op, args&: selector);
5626 return mlir::success();
5627}
5628
5629mlir::LogicalResult CIRToLLVMEhTypeIdOpLowering::matchAndRewrite(
5630 cir::EhTypeIdOp op, OpAdaptor adaptor,
5631 mlir::ConversionPatternRewriter &rewriter) const {
5632 mlir::Value addrOp = mlir::LLVM::AddressOfOp::create(
5633 builder&: rewriter, location: op.getLoc(),
5634 res: mlir::LLVM::LLVMPointerType::get(context: rewriter.getContext()),
5635 global_name: op.getTypeSymAttr());
5636 rewriter.replaceOpWithNewOp<mlir::LLVM::EhTypeidForOp>(
5637 op, args: rewriter.getI32Type(), args&: addrOp);
5638 return mlir::success();
5639}
5640
5641mlir::LogicalResult CIRToLLVMEhSetjmpOpLowering::matchAndRewrite(
5642 cir::EhSetjmpOp op, OpAdaptor adaptor,
5643 mlir::ConversionPatternRewriter &rewriter) const {
5644 mlir::Type returnType = typeConverter->convertType(t: op.getType());
5645 mlir::LLVM::CallIntrinsicOp newOp =
5646 createCallLLVMIntrinsicOp(rewriter, loc: op.getLoc(), intrinsicName: "llvm.eh.sjlj.setjmp",
5647 resultTy: returnType, operands: adaptor.getEnv());
5648 rewriter.replaceOp(op, newOp);
5649 return mlir::success();
5650}
5651
5652mlir::LogicalResult CIRToLLVMEhLongjmpOpLowering::matchAndRewrite(
5653 cir::EhLongjmpOp op, OpAdaptor adaptor,
5654 mlir::ConversionPatternRewriter &rewriter) const {
5655 replaceOpWithCallLLVMIntrinsicOp(rewriter, op, intrinsicName: "llvm.eh.sjlj.longjmp",
5656 /*resultTy=*/{}, operands: adaptor.getOperands());
5657 return mlir::success();
5658}
5659
5660mlir::LogicalResult CIRToLLVMTrapOpLowering::matchAndRewrite(
5661 cir::TrapOp op, OpAdaptor adaptor,
5662 mlir::ConversionPatternRewriter &rewriter) const {
5663 mlir::Location loc = op->getLoc();
5664 rewriter.eraseOp(op);
5665
5666 mlir::LLVM::Trap::create(builder&: rewriter, location: loc);
5667
5668 // Note that the call to llvm.trap is not a terminator in LLVM dialect.
5669 // So we must emit an additional llvm.unreachable to terminate the current
5670 // block.
5671 mlir::LLVM::UnreachableOp::create(builder&: rewriter, location: loc);
5672
5673 return mlir::success();
5674}
5675
5676static mlir::Value
5677getValueForVTableSymbol(mlir::Operation *op,
5678 mlir::ConversionPatternRewriter &rewriter,
5679 mlir::SymbolTableCollection &symbolTables,
5680 const mlir::TypeConverter *converter,
5681 const CIRPreConversionTypeMap &preConversionTypes,
5682 mlir::FlatSymbolRefAttr nameAttr, mlir::Type &eltType) {
5683 auto module = op->getParentOfType<mlir::ModuleOp>();
5684 mlir::Operation *symbol = symbolTables.lookupSymbolIn(symbolTableOp: module, name: nameAttr);
5685 if (auto llvmSymbol = mlir::dyn_cast<mlir::LLVM::GlobalOp>(Val: symbol)) {
5686 if (mlir::Type preType = preConversionTypes.lookup(Key: nameAttr.getValue())) {
5687 eltType = converter->convertType(t: preType);
5688 if (!eltType) {
5689 emitTypeConversionError(op, type: preType)
5690 << " for vtable symbol " << nameAttr;
5691 return {};
5692 }
5693 } else {
5694 eltType = llvmSymbol.getType();
5695 }
5696 } else if (auto cirSymbol = mlir::dyn_cast<cir::GlobalOp>(Val: symbol)) {
5697 eltType = converter->convertType(t: cirSymbol.getSymType());
5698 if (!eltType) {
5699 emitTypeConversionError(op, type: cirSymbol.getSymType())
5700 << " for vtable symbol " << nameAttr;
5701 return {};
5702 }
5703 } else {
5704 op->emitError() << "unexpected symbol type for " << symbol;
5705 return {};
5706 }
5707
5708 return mlir::LLVM::AddressOfOp::create(
5709 builder&: rewriter, location: op->getLoc(),
5710 res: mlir::LLVM::LLVMPointerType::get(context: op->getContext()), global_name: nameAttr.getValue());
5711}
5712
5713mlir::LogicalResult CIRToLLVMVTableAddrPointOpLowering::matchAndRewrite(
5714 cir::VTableAddrPointOp op, OpAdaptor adaptor,
5715 mlir::ConversionPatternRewriter &rewriter) const {
5716 assert(!cir::MissingFeatures::vtableRelativeLayout());
5717 if (usesRelativeVTables(mod: op->getParentOfType<mlir::ModuleOp>()))
5718 return op.emitError(message: "NYI: address point of a relative layout vtable");
5719
5720 const mlir::TypeConverter *converter = getTypeConverter();
5721 mlir::Type targetType = converter->convertType(t: op.getType());
5722 llvm::SmallVector<mlir::LLVM::GEPArg> offsets;
5723 mlir::Type eltType;
5724 mlir::Value symAddr =
5725 getValueForVTableSymbol(op, rewriter, symbolTables, converter,
5726 preConversionTypes, nameAttr: op.getNameAttr(), eltType);
5727 if (!symAddr)
5728 return mlir::failure();
5729
5730 offsets = llvm::SmallVector<mlir::LLVM::GEPArg>{
5731 0, op.getAddressPointAttr().getIndex(),
5732 op.getAddressPointAttr().getOffset()};
5733
5734 assert(eltType && "Shouldn't ever be missing an eltType here");
5735 mlir::LLVM::GEPNoWrapFlags inboundsNuw =
5736 mlir::LLVM::GEPNoWrapFlags::inbounds | mlir::LLVM::GEPNoWrapFlags::nuw;
5737 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(op, args&: targetType, args&: eltType,
5738 args&: symAddr, args&: offsets, args&: inboundsNuw);
5739 return mlir::success();
5740}
5741
5742mlir::LogicalResult CIRToLLVMVTableGetVPtrOpLowering::matchAndRewrite(
5743 cir::VTableGetVPtrOp op, OpAdaptor adaptor,
5744 mlir::ConversionPatternRewriter &rewriter) const {
5745 // cir.vtable.get_vptr is equivalent to a bitcast from the source object
5746 // pointer to the vptr type. Since the LLVM dialect uses opaque pointers
5747 // we can just replace uses of this operation with the original pointer.
5748 mlir::Value srcVal = adaptor.getSrc();
5749 rewriter.replaceOp(op, newValues: srcVal);
5750 return mlir::success();
5751}
5752
5753mlir::LogicalResult CIRToLLVMVTableGetVirtualFnOpLowering::matchAndRewrite(
5754 cir::VTableGetVirtualFnOp op, OpAdaptor adaptor,
5755 mlir::ConversionPatternRewriter &rewriter) const {
5756 assert(!cir::MissingFeatures::vtableRelativeLayout());
5757 if (usesRelativeVTables(mod: op->getParentOfType<mlir::ModuleOp>()))
5758 return op.emitError(
5759 message: "NYI: virtual function lookup in a relative layout vtable");
5760
5761 mlir::Type resultType = getTypeConverter()->convertType(t: op.getType());
5762 if (!resultType)
5763 return emitTypeConversionError(op, type: op.getType());
5764
5765 // The entry is the function pointer itself: load it from its slot.
5766 auto ptrTy = mlir::LLVM::LLVMPointerType::get(context: rewriter.getContext());
5767 mlir::Value slot = mlir::LLVM::GEPOp::create(
5768 builder&: rewriter, location: op.getLoc(), resultType: ptrTy, elementType: ptrTy, basePtr: adaptor.getVptr(),
5769 indices: llvm::ArrayRef<mlir::LLVM::GEPArg>{static_cast<int32_t>(op.getIndex())},
5770 noWrapFlags: mlir::LLVM::GEPNoWrapFlags::inbounds);
5771 rewriter.replaceOpWithNewOp<mlir::LLVM::LoadOp>(
5772 op, args&: resultType, args&: slot, args: dataLayout.getTypeABIAlignment(t: ptrTy),
5773 /*isVolatile=*/args: false, /*isNonTemporal=*/args: false,
5774 /*isInvariant=*/args: op.getInvariant(), /*isInvariantGroup=*/args: false);
5775 return mlir::success();
5776}
5777
5778mlir::LogicalResult
5779CIRToLLVMVTableGetVirtualFnByOffsetOpLowering::matchAndRewrite(
5780 cir::VTableGetVirtualFnByOffsetOp op, OpAdaptor adaptor,
5781 mlir::ConversionPatternRewriter &rewriter) const {
5782 assert(!cir::MissingFeatures::vtableRelativeLayout());
5783 if (usesRelativeVTables(mod: op->getParentOfType<mlir::ModuleOp>()))
5784 return op.emitError(
5785 message: "NYI: virtual function lookup by offset in a relative layout vtable");
5786
5787 mlir::Type resultType = getTypeConverter()->convertType(t: op.getType());
5788 if (!resultType)
5789 return emitTypeConversionError(op, type: op.getType());
5790
5791 // The entry is the function pointer itself: load it from its slot, addressed
5792 // by its byte offset.
5793 auto ptrTy = mlir::LLVM::LLVMPointerType::get(context: rewriter.getContext());
5794 mlir::Value slot = mlir::LLVM::GEPOp::create(
5795 builder&: rewriter, location: op.getLoc(), resultType: ptrTy, elementType: rewriter.getI8Type(), basePtr: adaptor.getVptr(),
5796 indices: llvm::ArrayRef<mlir::LLVM::GEPArg>{adaptor.getOffset()});
5797 rewriter.replaceOpWithNewOp<mlir::LLVM::LoadOp>(
5798 op, args&: resultType, args&: slot, args: dataLayout.getTypeABIAlignment(t: ptrTy));
5799 return mlir::success();
5800}
5801
5802mlir::LogicalResult CIRToLLVMVTableGetOffsetOpLowering::matchAndRewrite(
5803 cir::VTableGetOffsetOp op, OpAdaptor adaptor,
5804 mlir::ConversionPatternRewriter &rewriter) const {
5805 assert(!cir::MissingFeatures::vtableRelativeLayout());
5806 if (usesRelativeVTables(mod: op->getParentOfType<mlir::ModuleOp>()))
5807 return op.emitError(message: "NYI: offset lookup in a relative layout vtable");
5808
5809 mlir::Type resultType = getTypeConverter()->convertType(t: op.getType());
5810 if (!resultType)
5811 return emitTypeConversionError(op, type: op.getType());
5812
5813 // The entry is the offset itself: load it from its slot, addressed by its
5814 // byte offset.
5815 auto ptrTy = mlir::LLVM::LLVMPointerType::get(context: rewriter.getContext());
5816 int64_t byteOffset =
5817 op.getIndex() * cir::getVTableEntrySize(
5818 mod: op->getParentOfType<mlir::ModuleOp>(), dataLayout);
5819 mlir::Value byteOffsetVal = mlir::LLVM::ConstantOp::create(
5820 builder&: rewriter, location: op.getLoc(), res: rewriter.getI64Type(),
5821 value: rewriter.getI64IntegerAttr(value: byteOffset));
5822 mlir::Value slot = mlir::LLVM::GEPOp::create(
5823 builder&: rewriter, location: op.getLoc(), resultType: ptrTy, elementType: rewriter.getI8Type(), basePtr: adaptor.getVptr(),
5824 indices: llvm::ArrayRef<mlir::LLVM::GEPArg>{byteOffsetVal});
5825 rewriter.replaceOpWithNewOp<mlir::LLVM::LoadOp>(
5826 op, args&: resultType, args&: slot, args: dataLayout.getTypeABIAlignment(t: ptrTy));
5827 return mlir::success();
5828}
5829
5830mlir::LogicalResult CIRToLLVMVTableGetEntryOpLowering::matchAndRewrite(
5831 cir::VTableGetEntryOp op, OpAdaptor adaptor,
5832 mlir::ConversionPatternRewriter &rewriter) const {
5833 assert(!cir::MissingFeatures::vtableRelativeLayout());
5834 if (usesRelativeVTables(mod: op->getParentOfType<mlir::ModuleOp>()))
5835 return op.emitError(
5836 message: "NYI: pointer entry lookup in a relative layout vtable");
5837
5838 mlir::Type resultType = getTypeConverter()->convertType(t: op.getType());
5839 if (!resultType)
5840 return emitTypeConversionError(op, type: op.getType());
5841
5842 // The entry is the pointer itself: load it from its slot.
5843 auto ptrTy = mlir::LLVM::LLVMPointerType::get(context: rewriter.getContext());
5844 mlir::Value index = mlir::LLVM::ConstantOp::create(
5845 builder&: rewriter, location: op.getLoc(), res: rewriter.getI64Type(),
5846 value: rewriter.getI64IntegerAttr(value: op.getIndex()));
5847 mlir::Value slot = mlir::LLVM::GEPOp::create(
5848 builder&: rewriter, location: op.getLoc(), resultType: ptrTy, elementType: ptrTy, basePtr: adaptor.getVptr(),
5849 indices: llvm::ArrayRef<mlir::LLVM::GEPArg>{index},
5850 noWrapFlags: mlir::LLVM::GEPNoWrapFlags::inbounds);
5851 rewriter.replaceOpWithNewOp<mlir::LLVM::LoadOp>(
5852 op, args&: resultType, args&: slot, args: dataLayout.getTypeABIAlignment(t: ptrTy));
5853 return mlir::success();
5854}
5855
5856mlir::LogicalResult CIRToLLVMVTTAddrPointOpLowering::matchAndRewrite(
5857 cir::VTTAddrPointOp op, OpAdaptor adaptor,
5858 mlir::ConversionPatternRewriter &rewriter) const {
5859 const mlir::Type resultType = getTypeConverter()->convertType(t: op.getType());
5860 if (!resultType)
5861 return emitTypeConversionError(op, type: op.getType());
5862 llvm::SmallVector<mlir::LLVM::GEPArg> offsets;
5863 mlir::Type eltType;
5864 mlir::Value llvmAddr = adaptor.getSymAddr();
5865
5866 if (op.getSymAddr()) {
5867 if (op.getOffset() == 0) {
5868 rewriter.replaceOp(op, newValues: {llvmAddr});
5869 return mlir::success();
5870 }
5871
5872 offsets.push_back(Elt: adaptor.getOffset());
5873 eltType = mlir::LLVM::LLVMPointerType::get(context: rewriter.getContext());
5874 } else {
5875 llvmAddr =
5876 getValueForVTableSymbol(op, rewriter, symbolTables, converter: getTypeConverter(),
5877 preConversionTypes, nameAttr: op.getNameAttr(), eltType);
5878 if (!llvmAddr)
5879 return mlir::failure();
5880 assert(eltType && "Shouldn't ever be missing an eltType here");
5881 offsets.push_back(Elt: 0);
5882 offsets.push_back(Elt: adaptor.getOffset());
5883 }
5884 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
5885 op, args: resultType, args&: eltType, args&: llvmAddr, args&: offsets,
5886 args: mlir::LLVM::GEPNoWrapFlags::inbounds);
5887 return mlir::success();
5888}
5889
5890mlir::LogicalResult CIRToLLVMVecCreateOpLowering::matchAndRewrite(
5891 cir::VecCreateOp op, OpAdaptor adaptor,
5892 mlir::ConversionPatternRewriter &rewriter) const {
5893 // Start with an 'undef' value for the vector. Then 'insertelement' for
5894 // each of the vector elements.
5895 const cir::VectorType vecTy = op.getType();
5896 const mlir::Type llvmTy = typeConverter->convertType(t: vecTy);
5897 const mlir::Location loc = op.getLoc();
5898 mlir::Value result = mlir::LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: llvmTy);
5899 const uint64_t vectorSize = vecTy.getFixedSize();
5900 assert(vectorSize == op.getElements().size() &&
5901 "cir.vec.create op count doesn't match vector type elements count");
5902
5903 for (uint64_t i = 0; i < vectorSize; ++i) {
5904 const mlir::Value indexValue =
5905 mlir::LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: rewriter.getI64Type(), value: i);
5906 result = mlir::LLVM::InsertElementOp::create(
5907 builder&: rewriter, location: loc, vector: result, value: adaptor.getElements()[i], position: indexValue);
5908 }
5909
5910 rewriter.replaceOp(op, newValues: result);
5911 return mlir::success();
5912}
5913
5914mlir::LogicalResult CIRToLLVMVecCmpOpLowering::matchAndRewrite(
5915 cir::VecCmpOp op, OpAdaptor adaptor,
5916 mlir::ConversionPatternRewriter &rewriter) const {
5917 mlir::Type elementType = op.getLhs().getType().getElementType();
5918 mlir::Value bitResult;
5919 if (auto intType = mlir::dyn_cast<cir::IntType>(Val&: elementType)) {
5920 bitResult = mlir::LLVM::ICmpOp::create(
5921 builder&: rewriter, location: op.getLoc(),
5922 predicate: convertCmpKindToICmpPredicate(kind: op.getKind(), isSigned: intType.isSigned()),
5923 lhs: adaptor.getLhs(), rhs: adaptor.getRhs());
5924 } else if (auto boolType = mlir::dyn_cast<cir::BoolType>(Val&: elementType)) {
5925 bitResult = mlir::LLVM::ICmpOp::create(
5926 builder&: rewriter, location: op.getLoc(),
5927 predicate: convertCmpKindToICmpPredicate(kind: op.getKind(), /*isSigned=*/false),
5928 lhs: adaptor.getLhs(), rhs: adaptor.getRhs());
5929 } else {
5930 return op.emitError() << "unsupported type for VecCmpOp: " << elementType;
5931 }
5932
5933 // LLVM IR vector comparison returns a vector of i1. This one-bit vector
5934 // must be sign-extended to the correct result type, unless a vector of i1 is
5935 // the type we need.
5936 mlir::Type vecElemTy = op.getType().getElementType();
5937 if (isa<cir::IntType>(Val: vecElemTy) &&
5938 cast<cir::IntType>(Val&: vecElemTy).getWidth() > 1)
5939 rewriter.replaceOpWithNewOp<mlir::LLVM::SExtOp>(
5940 op, args: typeConverter->convertType(t: op.getType()), args&: bitResult);
5941 else
5942 rewriter.replaceOp(op, newValues: bitResult);
5943 return mlir::success();
5944}
5945
5946mlir::LogicalResult CIRToLLVMVecFCmpOpLowering::matchAndRewrite(
5947 cir::VecFCmpOp op, OpAdaptor adaptor,
5948 mlir::ConversionPatternRewriter &rewriter) const {
5949 mlir::Type elementType = op.getLhs().getType().getElementType();
5950 if (!mlir::isa<cir::FPTypeInterface>(Val: elementType))
5951 return op.emitError() << "unsupported type for VecFCmpOp: " << elementType;
5952
5953 mlir::Value bitResult;
5954 if (cir::FenvAttr fenv = op.getFenvAttr()) {
5955 llvm::ElementCount size = op.getLhs().getType().getSize();
5956 auto i1VecTy = mlir::VectorType::get(
5957 shape: size.getKnownMinValue(), elementType: rewriter.getI1Type(), scalableDims: {size.isScalable()});
5958 mlir::LLVM::CallIntrinsicOp CFCmpCall = createConstrainedFCmpCall(
5959 rewriter, loc: op.getLoc(), lhs: adaptor.getLhs(), rhs: adaptor.getRhs(), kind: op.getKind(),
5960 isSignaling: op.getSignaling(), fenv, llvmResTy: i1VecTy);
5961 bitResult = CFCmpCall.getResult(i: 0);
5962 } else {
5963 bitResult = mlir::LLVM::FCmpOp::create(
5964 builder&: rewriter, location: op.getLoc(), predicate: convertCmpKindToFCmpPredicate(kind: op.getKind()),
5965 lhs: adaptor.getLhs(), rhs: adaptor.getRhs());
5966 }
5967
5968 // LLVM IR vector comparison returns a vector of i1. This one-bit vector
5969 // must be sign-extended to the correct result type, unless a vector of i1 is
5970 // the type we need.
5971 mlir::Type vecElemTy = op.getType().getElementType();
5972 if (isa<cir::IntType>(Val: vecElemTy) &&
5973 cast<cir::IntType>(Val&: vecElemTy).getWidth() > 1)
5974 rewriter.replaceOpWithNewOp<mlir::LLVM::SExtOp>(
5975 op, args: typeConverter->convertType(t: op.getType()), args&: bitResult);
5976 else
5977 rewriter.replaceOp(op, newValues: bitResult);
5978 return mlir::success();
5979}
5980
5981mlir::LogicalResult CIRToLLVMVecSplatOpLowering::matchAndRewrite(
5982 cir::VecSplatOp op, OpAdaptor adaptor,
5983 mlir::ConversionPatternRewriter &rewriter) const {
5984 // Vector splat can be implemented with an `insertelement` and a
5985 // `shufflevector`, which is better than an `insertelement` for each
5986 // element in the vector. Start with an undef vector. Insert the value into
5987 // the first element. Then use a `shufflevector` with a mask of all 0 to
5988 // fill out the entire vector with that value.
5989 mlir::Type llvmTy = typeConverter->convertType(t: op.getType());
5990 mlir::Location loc = op.getLoc();
5991 mlir::Value poison = mlir::LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: llvmTy);
5992
5993 mlir::Value elementValue = adaptor.getValue();
5994 if (elementValue.getDefiningOp<mlir::LLVM::PoisonOp>()) {
5995 // If the splat value is poison, then we can just use poison value
5996 // for the entire vector.
5997 rewriter.replaceOp(op, newValues: poison);
5998 return mlir::success();
5999 }
6000
6001 if (auto constValue = elementValue.getDefiningOp<mlir::LLVM::ConstantOp>()) {
6002 if (auto intAttr = dyn_cast<mlir::IntegerAttr>(Val: constValue.getValue())) {
6003 mlir::DenseIntElementsAttr denseVec = mlir::DenseIntElementsAttr::get(
6004 type: mlir::cast<mlir::ShapedType>(Val&: llvmTy), arg: intAttr.getValue());
6005 rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(
6006 op, args: denseVec.getType(), args&: denseVec);
6007 return mlir::success();
6008 }
6009
6010 if (auto fpAttr = dyn_cast<mlir::FloatAttr>(Val: constValue.getValue())) {
6011 mlir::DenseFPElementsAttr denseVec = mlir::DenseFPElementsAttr::get(
6012 type: mlir::cast<mlir::ShapedType>(Val&: llvmTy), arg: fpAttr.getValue());
6013 rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(
6014 op, args: denseVec.getType(), args&: denseVec);
6015 return mlir::success();
6016 }
6017 }
6018
6019 mlir::Value indexValue =
6020 mlir::LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: rewriter.getI64Type(), value: 0);
6021 mlir::Value oneElement = mlir::LLVM::InsertElementOp::create(
6022 builder&: rewriter, location: loc, vector: poison, value: elementValue, position: indexValue);
6023 SmallVector<int32_t> zeroValues(
6024 mlir::cast<mlir::VectorType>(Val&: llvmTy).getNumElements(), 0);
6025 rewriter.replaceOpWithNewOp<mlir::LLVM::ShuffleVectorOp>(op, args&: oneElement,
6026 args&: poison, args&: zeroValues);
6027 return mlir::success();
6028}
6029
6030mlir::LogicalResult CIRToLLVMVecShuffleOpLowering::matchAndRewrite(
6031 cir::VecShuffleOp op, OpAdaptor adaptor,
6032 mlir::ConversionPatternRewriter &rewriter) const {
6033 // LLVM::ShuffleVectorOp takes an ArrayRef of int for the list of indices.
6034 // Convert the ClangIR ArrayAttr of IntAttr constants into a
6035 // SmallVector<int>.
6036 SmallVector<int, 8> indices;
6037 std::transform(
6038 first: op.getIndices().begin(), last: op.getIndices().end(),
6039 result: std::back_inserter(x&: indices), unary_op: [](mlir::Attribute intAttr) {
6040 return mlir::cast<cir::IntAttr>(Val&: intAttr).getValue().getSExtValue();
6041 });
6042 rewriter.replaceOpWithNewOp<mlir::LLVM::ShuffleVectorOp>(
6043 op, args: adaptor.getVec1(), args: adaptor.getVec2(), args&: indices);
6044 return mlir::success();
6045}
6046
6047mlir::LogicalResult CIRToLLVMVecShuffleDynamicOpLowering::matchAndRewrite(
6048 cir::VecShuffleDynamicOp op, OpAdaptor adaptor,
6049 mlir::ConversionPatternRewriter &rewriter) const {
6050 // LLVM IR does not have an operation that corresponds to this form of
6051 // the built-in.
6052 // __builtin_shufflevector(V, I)
6053 // is implemented as this pseudocode, where the for loop is unrolled
6054 // and N is the number of elements:
6055 //
6056 // result = undef
6057 // maskbits = NextPowerOf2(N - 1)
6058 // masked = I & maskbits
6059 // for (i in 0 <= i < N)
6060 // result[i] = V[masked[i]]
6061 mlir::Location loc = op.getLoc();
6062 mlir::Value input = adaptor.getVec();
6063 mlir::Type llvmIndexVecType =
6064 getTypeConverter()->convertType(t: op.getIndices().getType());
6065 mlir::Type llvmIndexType = getTypeConverter()->convertType(
6066 t: op.getIndices().getType().getElementType());
6067 uint64_t numElements = op.getVec().getType().getFixedSize();
6068
6069 uint64_t maskBits = llvm::NextPowerOf2(A: numElements - 1) - 1;
6070 mlir::Value maskValue = mlir::LLVM::ConstantOp::create(
6071 builder&: rewriter, location: loc, res: llvmIndexType,
6072 value: rewriter.getIntegerAttr(type: llvmIndexType, value: maskBits));
6073 mlir::Value maskVector =
6074 mlir::LLVM::UndefOp::create(builder&: rewriter, location: loc, res: llvmIndexVecType);
6075
6076 for (uint64_t i = 0; i < numElements; ++i) {
6077 mlir::Value idxValue =
6078 mlir::LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: rewriter.getI64Type(), value: i);
6079 maskVector = mlir::LLVM::InsertElementOp::create(builder&: rewriter, location: loc, vector: maskVector,
6080 value: maskValue, position: idxValue);
6081 }
6082
6083 mlir::Value maskedIndices = mlir::LLVM::AndOp::create(
6084 builder&: rewriter, location: loc, res: llvmIndexVecType, lhs: adaptor.getIndices(), rhs: maskVector);
6085 mlir::Value result = mlir::LLVM::UndefOp::create(
6086 builder&: rewriter, location: loc, res: getTypeConverter()->convertType(t: op.getVec().getType()));
6087 for (uint64_t i = 0; i < numElements; ++i) {
6088 mlir::Value iValue =
6089 mlir::LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: rewriter.getI64Type(), value: i);
6090 mlir::Value indexValue = mlir::LLVM::ExtractElementOp::create(
6091 builder&: rewriter, location: loc, vector: maskedIndices, position: iValue);
6092 mlir::Value valueAtIndex =
6093 mlir::LLVM::ExtractElementOp::create(builder&: rewriter, location: loc, vector: input, position: indexValue);
6094 result = mlir::LLVM::InsertElementOp::create(builder&: rewriter, location: loc, vector: result,
6095 value: valueAtIndex, position: iValue);
6096 }
6097 rewriter.replaceOp(op, newValues: result);
6098 return mlir::success();
6099}
6100
6101mlir::LogicalResult CIRToLLVMVecTernaryOpLowering::matchAndRewrite(
6102 cir::VecTernaryOp op, OpAdaptor adaptor,
6103 mlir::ConversionPatternRewriter &rewriter) const {
6104 mlir::Value bitVec = adaptor.getCond();
6105
6106 if (!isa<cir::BoolType>(Val: op.getCond().getType().getElementType())) {
6107 // Convert `cond` into a vector of i1, then use that in a `select` op.
6108 bitVec = mlir::LLVM::ICmpOp::create(
6109 builder&: rewriter, location: op.getLoc(), predicate: mlir::LLVM::ICmpPredicate::ne, lhs: adaptor.getCond(),
6110 rhs: mlir::LLVM::ZeroOp::create(
6111 builder&: rewriter, location: op.getCond().getLoc(),
6112 res: typeConverter->convertType(t: op.getCond().getType())));
6113 }
6114
6115 rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(
6116 op, args&: bitVec, args: adaptor.getLhs(), args: adaptor.getRhs());
6117 return mlir::success();
6118}
6119
6120/// Computes the column-major flat index (columnIdx * numRows) + rowIdx.
6121static mlir::Value
6122computeMatrixFlatIndex(mlir::ConversionPatternRewriter &rewriter,
6123 mlir::Location loc, cir::MatrixType matrixTy,
6124 mlir::Value rowIdx, mlir::Value columnIdx) {
6125 rowIdx = createIntCast(bld&: rewriter, src: rowIdx, dstTy: rewriter.getI64Type(), isSigned: false);
6126 columnIdx = createIntCast(bld&: rewriter, src: columnIdx, dstTy: rewriter.getI64Type(), isSigned: false);
6127 mlir::Type indexTy = rowIdx.getType();
6128 mlir::Value numRows = mlir::LLVM::ConstantOp::create(
6129 builder&: rewriter, location: loc, res: indexTy,
6130 value: rewriter.getIntegerAttr(type: indexTy, value: matrixTy.getNumRows()));
6131 mlir::Value flatIndex = mlir::LLVM::MulOp::create(
6132 builder&: rewriter, location: loc, operands: {columnIdx, numRows}, /*properties=*/{},
6133 /*discardableAttributes=*/{});
6134 return mlir::LLVM::AddOp::create(builder&: rewriter, location: loc, lhs: flatIndex, rhs: rowIdx,
6135 /*overflowFlags=*/{});
6136}
6137
6138mlir::LogicalResult CIRToLLVMMatrixExtractOpLowering::matchAndRewrite(
6139 cir::MatrixExtractOp op, OpAdaptor adaptor,
6140 mlir::ConversionPatternRewriter &rewriter) const {
6141 mlir::Value flatIndex =
6142 computeMatrixFlatIndex(rewriter, loc: op.getLoc(), matrixTy: op.getMatrix().getType(),
6143 rowIdx: adaptor.getRowIdx(), columnIdx: adaptor.getColumnIdx());
6144 rewriter.replaceOpWithNewOp<mlir::LLVM::ExtractElementOp>(
6145 op, args: adaptor.getMatrix(), args&: flatIndex);
6146 return mlir::success();
6147}
6148
6149mlir::LogicalResult CIRToLLVMMatrixExtractRowOpLowering::matchAndRewrite(
6150 cir::MatrixExtractRowOp op, OpAdaptor adaptor,
6151 mlir::ConversionPatternRewriter &rewriter) const {
6152 cir::MatrixType matrixTy = op.getMatrix().getType();
6153 mlir::Type elementType =
6154 typeConverter->convertType(t: matrixTy.getElementType());
6155 mlir::Type resultTy =
6156 mlir::VectorType::get(shape: matrixTy.getNumColumns(), elementType);
6157 mlir::Value rowVec =
6158 mlir::LLVM::PoisonOp::create(builder&: rewriter, location: op->getLoc(), res: resultTy);
6159
6160 for (unsigned col = 0; col != matrixTy.getNumColumns(); ++col) {
6161 mlir::Value columnIdx = mlir::LLVM::ConstantOp::create(
6162 builder&: rewriter, location: op.getLoc(), res: rewriter.getI32Type(),
6163 value: rewriter.getIntegerAttr(type: rewriter.getI32Type(), value: col));
6164 mlir::Value flatIndex = computeMatrixFlatIndex(
6165 rewriter, loc: op.getLoc(), matrixTy, rowIdx: adaptor.getRowIdx(), columnIdx);
6166
6167 auto element = mlir::LLVM::ExtractElementOp::create(
6168 builder&: rewriter, location: op->getLoc(), res: elementType, vector: adaptor.getMatrix(), position: flatIndex);
6169 rowVec = mlir::LLVM::InsertElementOp::create(builder&: rewriter, location: op->getLoc(), vector: rowVec,
6170 value: element, position: columnIdx);
6171 }
6172 rewriter.replaceOp(op, newValues: rowVec);
6173 return mlir::success();
6174}
6175
6176mlir::LogicalResult CIRToLLVMMatrixInsertOpLowering::matchAndRewrite(
6177 cir::MatrixInsertOp op, OpAdaptor adaptor,
6178 mlir::ConversionPatternRewriter &rewriter) const {
6179 mlir::Value flatIndex =
6180 computeMatrixFlatIndex(rewriter, loc: op.getLoc(), matrixTy: op.getMatrix().getType(),
6181 rowIdx: adaptor.getRowIdx(), columnIdx: adaptor.getColumnIdx());
6182 rewriter.replaceOpWithNewOp<mlir::LLVM::InsertElementOp>(
6183 op, args: adaptor.getMatrix(), args: adaptor.getValue(), args&: flatIndex);
6184 return mlir::success();
6185}
6186
6187mlir::LogicalResult CIRToLLVMMatrixColumnMajorLoadOpLowering::matchAndRewrite(
6188 cir::MatrixColumnMajorLoadOp op, OpAdaptor adaptor,
6189 mlir::ConversionPatternRewriter &rewriter) const {
6190 cir::MatrixType resultMatrixTy = op.getResult().getType();
6191 mlir::Type resultTy = typeConverter->convertType(t: resultMatrixTy);
6192 rewriter.replaceOpWithNewOp<mlir::LLVM::MatrixColumnMajorLoadOp>(
6193 op, args&: resultTy, args: adaptor.getValue(), args: adaptor.getStride(),
6194 args: rewriter.getBoolAttr(value: op.getIsVolatile()),
6195 args: rewriter.getI32IntegerAttr(value: resultMatrixTy.getNumRows()),
6196 args: rewriter.getI32IntegerAttr(value: resultMatrixTy.getNumColumns()));
6197 return mlir::success();
6198}
6199
6200mlir::LogicalResult CIRToLLVMMatrixColumnMajorStoreOpLowering::matchAndRewrite(
6201 cir::MatrixColumnMajorStoreOp op, OpAdaptor adaptor,
6202 mlir::ConversionPatternRewriter &rewriter) const {
6203 cir::MatrixType matrixTy = op.getMatrix().getType();
6204 rewriter.replaceOpWithNewOp<mlir::LLVM::MatrixColumnMajorStoreOp>(
6205 op, args: adaptor.getMatrix(), args: adaptor.getValue(), args: adaptor.getStride(),
6206 args: rewriter.getBoolAttr(value: op.getIsVolatile()),
6207 args: rewriter.getI32IntegerAttr(value: matrixTy.getNumRows()),
6208 args: rewriter.getI32IntegerAttr(value: matrixTy.getNumColumns()));
6209 return mlir::success();
6210}
6211
6212mlir::LogicalResult CIRToLLVMMatrixMultiplyOpLowering::matchAndRewrite(
6213 cir::MatrixMultiplyOp op, OpAdaptor adaptor,
6214 mlir::ConversionPatternRewriter &rewriter) const {
6215 cir::MatrixType lhsTy = op.getLhs().getType();
6216 cir::MatrixType rhsTy = op.getRhs().getType();
6217 mlir::Type resultTy = typeConverter->convertType(t: op.getResult().getType());
6218 rewriter.replaceOpWithNewOp<mlir::LLVM::MatrixMultiplyOp>(
6219 op, args&: resultTy, args: adaptor.getLhs(), args: adaptor.getRhs(), args: lhsTy.getNumRows(),
6220 args: lhsTy.getNumColumns(), args: rhsTy.getNumColumns());
6221 return mlir::success();
6222}
6223
6224mlir::LogicalResult CIRToLLVMMatrixTransposeOpLowering::matchAndRewrite(
6225 cir::MatrixTransposeOp op, OpAdaptor adaptor,
6226 mlir::ConversionPatternRewriter &rewriter) const {
6227 cir::MatrixType resultMatrixTy = op.getValue().getType();
6228 mlir::Type resultTy = typeConverter->convertType(t: resultMatrixTy);
6229 rewriter.replaceOpWithNewOp<mlir::LLVM::MatrixTransposeOp>(
6230 op, args&: resultTy, args: adaptor.getValue(), args: resultMatrixTy.getNumRows(),
6231 args: resultMatrixTy.getNumColumns());
6232 return mlir::success();
6233}
6234
6235mlir::LogicalResult CIRToLLVMComplexCreateOpLowering::matchAndRewrite(
6236 cir::ComplexCreateOp op, OpAdaptor adaptor,
6237 mlir::ConversionPatternRewriter &rewriter) const {
6238 mlir::Type complexLLVMTy =
6239 getTypeConverter()->convertType(t: op.getResult().getType());
6240 auto initialComplex =
6241 mlir::LLVM::UndefOp::create(builder&: rewriter, location: op->getLoc(), res: complexLLVMTy);
6242
6243 auto realComplex = mlir::LLVM::InsertValueOp::create(
6244 builder&: rewriter, location: op->getLoc(), container: initialComplex, value: adaptor.getReal(),
6245 position: ArrayRef(int64_t{0}));
6246
6247 auto complex = mlir::LLVM::InsertValueOp::create(
6248 builder&: rewriter, location: op->getLoc(), container: realComplex, value: adaptor.getImag(),
6249 position: ArrayRef(int64_t{1}));
6250
6251 rewriter.replaceOp(op, newOp: complex);
6252 return mlir::success();
6253}
6254
6255mlir::LogicalResult CIRToLLVMComplexRealOpLowering::matchAndRewrite(
6256 cir::ComplexRealOp op, OpAdaptor adaptor,
6257 mlir::ConversionPatternRewriter &rewriter) const {
6258 mlir::Value operand = adaptor.getOperand();
6259 if (mlir::isa<cir::ComplexType>(Val: op.getOperand().getType())) {
6260 operand = rewriter.createOrFold<mlir::LLVM::ExtractValueOp>(
6261 location: op.getLoc(), args&: operand, args: llvm::ArrayRef<std::int64_t>{0});
6262 }
6263 rewriter.replaceOp(op, newValues: operand);
6264 return mlir::success();
6265}
6266
6267mlir::LogicalResult CIRToLLVMComplexImagOpLowering::matchAndRewrite(
6268 cir::ComplexImagOp op, OpAdaptor adaptor,
6269 mlir::ConversionPatternRewriter &rewriter) const {
6270 mlir::Type resultLLVMTy = getTypeConverter()->convertType(t: op.getType());
6271 mlir::Value operand = adaptor.getOperand();
6272 mlir::Location loc = op.getLoc();
6273
6274 if (mlir::isa<cir::ComplexType>(Val: op.getOperand().getType())) {
6275 operand = rewriter.createOrFold<mlir::LLVM::ExtractValueOp>(
6276 location: loc, args&: operand, args: llvm::ArrayRef<std::int64_t>{1});
6277 } else {
6278 mlir::TypedAttr zeroAttr = rewriter.getZeroAttr(type: resultLLVMTy);
6279 operand =
6280 mlir::LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: resultLLVMTy, value: zeroAttr);
6281 }
6282
6283 rewriter.replaceOp(op, newValues: operand);
6284 return mlir::success();
6285}
6286
6287mlir::LogicalResult CIRToLLVMComplexAddOpLowering::matchAndRewrite(
6288 cir::ComplexAddOp op, OpAdaptor adaptor,
6289 mlir::ConversionPatternRewriter &rewriter) const {
6290 mlir::Value lhs = adaptor.getLhs();
6291 mlir::Value rhs = adaptor.getRhs();
6292 mlir::Location loc = op.getLoc();
6293
6294 auto complexType = mlir::cast<cir::ComplexType>(Val: op.getLhs().getType());
6295 mlir::Type complexElemTy =
6296 getTypeConverter()->convertType(t: complexType.getElementType());
6297 auto lhsReal = mlir::LLVM::ExtractValueOp::create(
6298 builder&: rewriter, location: loc, res: complexElemTy, container: lhs, position: ArrayRef(int64_t{0}));
6299 auto lhsImag = mlir::LLVM::ExtractValueOp::create(
6300 builder&: rewriter, location: loc, res: complexElemTy, container: lhs, position: ArrayRef(int64_t{1}));
6301 auto rhsReal = mlir::LLVM::ExtractValueOp::create(
6302 builder&: rewriter, location: loc, res: complexElemTy, container: rhs, position: ArrayRef(int64_t{0}));
6303 auto rhsImag = mlir::LLVM::ExtractValueOp::create(
6304 builder&: rewriter, location: loc, res: complexElemTy, container: rhs, position: ArrayRef(int64_t{1}));
6305
6306 mlir::Value newReal;
6307 mlir::Value newImag;
6308 if (complexElemTy.isInteger()) {
6309 newReal = mlir::LLVM::AddOp::create(builder&: rewriter, location: loc, res: complexElemTy, lhs: lhsReal,
6310 rhs: rhsReal);
6311 newImag = mlir::LLVM::AddOp::create(builder&: rewriter, location: loc, res: complexElemTy, lhs: lhsImag,
6312 rhs: rhsImag);
6313 } else {
6314 assert(!cir::MissingFeatures::fastMathFlags());
6315 assert(!cir::MissingFeatures::fpConstraints());
6316 newReal = mlir::LLVM::FAddOp::create(builder&: rewriter, location: loc, res: complexElemTy, lhs: lhsReal,
6317 rhs: rhsReal);
6318 newImag = mlir::LLVM::FAddOp::create(builder&: rewriter, location: loc, res: complexElemTy, lhs: lhsImag,
6319 rhs: rhsImag);
6320 }
6321
6322 mlir::Type complexLLVMTy =
6323 getTypeConverter()->convertType(t: op.getResult().getType());
6324 auto initialComplex =
6325 mlir::LLVM::PoisonOp::create(builder&: rewriter, location: op->getLoc(), res: complexLLVMTy);
6326
6327 auto realComplex = mlir::LLVM::InsertValueOp::create(
6328 builder&: rewriter, location: op->getLoc(), container: initialComplex, value: newReal, position: ArrayRef(int64_t{0}));
6329
6330 rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>(
6331 op, args&: realComplex, args&: newImag, args: ArrayRef(int64_t{1}));
6332
6333 return mlir::success();
6334}
6335
6336mlir::LogicalResult CIRToLLVMComplexSubOpLowering::matchAndRewrite(
6337 cir::ComplexSubOp op, OpAdaptor adaptor,
6338 mlir::ConversionPatternRewriter &rewriter) const {
6339 mlir::Value lhs = adaptor.getLhs();
6340 mlir::Value rhs = adaptor.getRhs();
6341 mlir::Location loc = op.getLoc();
6342
6343 auto complexType = mlir::cast<cir::ComplexType>(Val: op.getLhs().getType());
6344 mlir::Type complexElemTy =
6345 getTypeConverter()->convertType(t: complexType.getElementType());
6346 auto lhsReal = mlir::LLVM::ExtractValueOp::create(
6347 builder&: rewriter, location: loc, res: complexElemTy, container: lhs, position: ArrayRef(int64_t{0}));
6348 auto lhsImag = mlir::LLVM::ExtractValueOp::create(
6349 builder&: rewriter, location: loc, res: complexElemTy, container: lhs, position: ArrayRef(int64_t{1}));
6350 auto rhsReal = mlir::LLVM::ExtractValueOp::create(
6351 builder&: rewriter, location: loc, res: complexElemTy, container: rhs, position: ArrayRef(int64_t{0}));
6352 auto rhsImag = mlir::LLVM::ExtractValueOp::create(
6353 builder&: rewriter, location: loc, res: complexElemTy, container: rhs, position: ArrayRef(int64_t{1}));
6354
6355 mlir::Value newReal;
6356 mlir::Value newImag;
6357 if (complexElemTy.isInteger()) {
6358 newReal = mlir::LLVM::SubOp::create(builder&: rewriter, location: loc, res: complexElemTy, lhs: lhsReal,
6359 rhs: rhsReal);
6360 newImag = mlir::LLVM::SubOp::create(builder&: rewriter, location: loc, res: complexElemTy, lhs: lhsImag,
6361 rhs: rhsImag);
6362 } else {
6363 assert(!cir::MissingFeatures::fastMathFlags());
6364 assert(!cir::MissingFeatures::fpConstraints());
6365 newReal = mlir::LLVM::FSubOp::create(builder&: rewriter, location: loc, res: complexElemTy, lhs: lhsReal,
6366 rhs: rhsReal);
6367 newImag = mlir::LLVM::FSubOp::create(builder&: rewriter, location: loc, res: complexElemTy, lhs: lhsImag,
6368 rhs: rhsImag);
6369 }
6370
6371 mlir::Type complexLLVMTy =
6372 getTypeConverter()->convertType(t: op.getResult().getType());
6373 auto initialComplex =
6374 mlir::LLVM::PoisonOp::create(builder&: rewriter, location: op->getLoc(), res: complexLLVMTy);
6375
6376 auto realComplex = mlir::LLVM::InsertValueOp::create(
6377 builder&: rewriter, location: op->getLoc(), container: initialComplex, value: newReal, position: ArrayRef(int64_t{0}));
6378
6379 rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>(
6380 op, args&: realComplex, args&: newImag, args: ArrayRef(int64_t{1}));
6381
6382 return mlir::success();
6383}
6384
6385mlir::LogicalResult CIRToLLVMComplexFAddOpLowering::matchAndRewrite(
6386 cir::ComplexFAddOp op, OpAdaptor adaptor,
6387 mlir::ConversionPatternRewriter &rewriter) const {
6388 mlir::Value lhs = adaptor.getLhs();
6389 mlir::Value rhs = adaptor.getRhs();
6390 mlir::Location loc = op.getLoc();
6391
6392 auto complexType = mlir::cast<cir::ComplexType>(Val: op.getLhs().getType());
6393 mlir::Type complexElemTy =
6394 getTypeConverter()->convertType(t: complexType.getElementType());
6395 auto lhsReal = mlir::LLVM::ExtractValueOp::create(
6396 builder&: rewriter, location: loc, res: complexElemTy, container: lhs, position: ArrayRef(int64_t{0}));
6397 auto lhsImag = mlir::LLVM::ExtractValueOp::create(
6398 builder&: rewriter, location: loc, res: complexElemTy, container: lhs, position: ArrayRef(int64_t{1}));
6399 auto rhsReal = mlir::LLVM::ExtractValueOp::create(
6400 builder&: rewriter, location: loc, res: complexElemTy, container: rhs, position: ArrayRef(int64_t{0}));
6401 auto rhsImag = mlir::LLVM::ExtractValueOp::create(
6402 builder&: rewriter, location: loc, res: complexElemTy, container: rhs, position: ArrayRef(int64_t{1}));
6403
6404 assert(!cir::MissingFeatures::fastMathFlags());
6405 mlir::Value newReal;
6406 mlir::Value newImag;
6407 if (cir::FenvAttr fenv = op.getFenvAttr()) {
6408 newReal = createConstrainedFPIntrinsicCall(
6409 rewriter, loc, operands: {lhsReal, rhsReal}, llvmResTy: complexElemTy, fenv,
6410 /*constrainedMnemonic=*/"fadd",
6411 /*hasRoundingMode=*/true);
6412
6413 newImag = createConstrainedFPIntrinsicCall(
6414 rewriter, loc, operands: {lhsImag, rhsImag}, llvmResTy: complexElemTy, fenv,
6415 /*constrainedMnemonic=*/"fadd",
6416 /*hasRoundingMode=*/true);
6417 } else {
6418 newReal = mlir::LLVM::FAddOp::create(builder&: rewriter, location: loc, res: complexElemTy, lhs: lhsReal,
6419 rhs: rhsReal);
6420 newImag = mlir::LLVM::FAddOp::create(builder&: rewriter, location: loc, res: complexElemTy, lhs: lhsImag,
6421 rhs: rhsImag);
6422 }
6423
6424 mlir::Type complexLLVMTy =
6425 getTypeConverter()->convertType(t: op.getResult().getType());
6426 auto initialComplex =
6427 mlir::LLVM::PoisonOp::create(builder&: rewriter, location: op->getLoc(), res: complexLLVMTy);
6428
6429 auto realComplex = mlir::LLVM::InsertValueOp::create(
6430 builder&: rewriter, location: op->getLoc(), container: initialComplex, value: newReal, position: ArrayRef(int64_t{0}));
6431
6432 rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>(
6433 op, args&: realComplex, args&: newImag, args: ArrayRef(int64_t{1}));
6434
6435 return mlir::success();
6436}
6437
6438mlir::LogicalResult CIRToLLVMComplexFSubOpLowering::matchAndRewrite(
6439 cir::ComplexFSubOp op, OpAdaptor adaptor,
6440 mlir::ConversionPatternRewriter &rewriter) const {
6441 mlir::Value lhs = adaptor.getLhs();
6442 mlir::Value rhs = adaptor.getRhs();
6443 mlir::Location loc = op.getLoc();
6444
6445 auto complexType = mlir::cast<cir::ComplexType>(Val: op.getLhs().getType());
6446 mlir::Type complexElemTy =
6447 getTypeConverter()->convertType(t: complexType.getElementType());
6448 auto lhsReal = mlir::LLVM::ExtractValueOp::create(
6449 builder&: rewriter, location: loc, res: complexElemTy, container: lhs, position: ArrayRef(int64_t{0}));
6450 auto lhsImag = mlir::LLVM::ExtractValueOp::create(
6451 builder&: rewriter, location: loc, res: complexElemTy, container: lhs, position: ArrayRef(int64_t{1}));
6452 auto rhsReal = mlir::LLVM::ExtractValueOp::create(
6453 builder&: rewriter, location: loc, res: complexElemTy, container: rhs, position: ArrayRef(int64_t{0}));
6454 auto rhsImag = mlir::LLVM::ExtractValueOp::create(
6455 builder&: rewriter, location: loc, res: complexElemTy, container: rhs, position: ArrayRef(int64_t{1}));
6456
6457 assert(!cir::MissingFeatures::fastMathFlags());
6458 mlir::Value newReal;
6459 mlir::Value newImag;
6460 if (cir::FenvAttr fenv = op.getFenvAttr()) {
6461 newReal = createConstrainedFPIntrinsicCall(
6462 rewriter, loc, operands: {lhsReal, rhsReal}, llvmResTy: complexElemTy, fenv,
6463 /*constrainedMnemonic=*/"fsub",
6464 /*hasRoundingMode=*/true);
6465
6466 newImag = createConstrainedFPIntrinsicCall(
6467 rewriter, loc, operands: {lhsImag, rhsImag}, llvmResTy: complexElemTy, fenv,
6468 /*constrainedMnemonic=*/"fsub",
6469 /*hasRoundingMode=*/true);
6470 } else {
6471 newReal = mlir::LLVM::FSubOp::create(builder&: rewriter, location: loc, res: complexElemTy, lhs: lhsReal,
6472 rhs: rhsReal);
6473 newImag = mlir::LLVM::FSubOp::create(builder&: rewriter, location: loc, res: complexElemTy, lhs: lhsImag,
6474 rhs: rhsImag);
6475 }
6476
6477 mlir::Type complexLLVMTy =
6478 getTypeConverter()->convertType(t: op.getResult().getType());
6479 auto initialComplex =
6480 mlir::LLVM::PoisonOp::create(builder&: rewriter, location: op->getLoc(), res: complexLLVMTy);
6481
6482 auto realComplex = mlir::LLVM::InsertValueOp::create(
6483 builder&: rewriter, location: op->getLoc(), container: initialComplex, value: newReal, position: ArrayRef(int64_t{0}));
6484
6485 rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>(
6486 op, args&: realComplex, args&: newImag, args: ArrayRef(int64_t{1}));
6487
6488 return mlir::success();
6489}
6490
6491mlir::IntegerType computeBitfieldIntType(mlir::Type storageType,
6492 mlir::MLIRContext *context,
6493 unsigned &storageSize) {
6494 return TypeSwitch<mlir::Type, mlir::IntegerType>(storageType)
6495 .Case<cir::ArrayType>(caseFn: [&](cir::ArrayType atTy) {
6496 storageSize = atTy.getSize() * 8;
6497 return mlir::IntegerType::get(context, width: storageSize);
6498 })
6499 .Case<cir::IntType>(caseFn: [&](cir::IntType intTy) {
6500 storageSize = intTy.getWidth();
6501 return mlir::IntegerType::get(context, width: storageSize);
6502 })
6503 .Default(defaultFn: [](mlir::Type) -> mlir::IntegerType {
6504 llvm_unreachable(
6505 "Either ArrayType or IntType expected for bitfields storage");
6506 });
6507}
6508
6509mlir::LogicalResult CIRToLLVMSetBitfieldOpLowering::matchAndRewrite(
6510 cir::SetBitfieldOp op, OpAdaptor adaptor,
6511 mlir::ConversionPatternRewriter &rewriter) const {
6512 mlir::OpBuilder::InsertionGuard guard(rewriter);
6513 rewriter.setInsertionPoint(op);
6514
6515 cir::BitfieldInfoAttr info = op.getBitfieldInfo();
6516 uint64_t size = info.getSize();
6517 uint64_t offset = info.getOffset();
6518 mlir::Type storageType = info.getStorageType();
6519 mlir::MLIRContext *context = storageType.getContext();
6520
6521 unsigned storageSize = 0;
6522
6523 mlir::IntegerType intType =
6524 computeBitfieldIntType(storageType, context, storageSize);
6525
6526 mlir::Value srcVal = createIntCast(bld&: rewriter, src: adaptor.getSrc(), dstTy: intType);
6527 unsigned srcWidth = storageSize;
6528 mlir::Value resultVal = srcVal;
6529
6530 if (storageSize != size) {
6531 assert(storageSize > size && "Invalid bitfield size.");
6532
6533 mlir::Value val = mlir::LLVM::LoadOp::create(
6534 builder&: rewriter, location: op.getLoc(), type: intType, addr: adaptor.getAddr(), alignment: op.getAlignment(),
6535 isVolatile: op.getIsVolatile());
6536
6537 srcVal =
6538 createAnd(bld&: rewriter, lhs: srcVal, rhs: llvm::APInt::getLowBitsSet(numBits: srcWidth, loBitsSet: size));
6539 resultVal = srcVal;
6540 srcVal = createShL(bld&: rewriter, lhs: srcVal, rhs: offset);
6541
6542 // Mask out the original value.
6543 val = createAnd(bld&: rewriter, lhs: val,
6544 rhs: ~llvm::APInt::getBitsSet(numBits: srcWidth, loBit: offset, hiBit: offset + size));
6545
6546 // Or together the unchanged values and the source value.
6547 srcVal = mlir::LLVM::OrOp::create(builder&: rewriter, location: op.getLoc(), lhs: val, rhs: srcVal);
6548 }
6549
6550 mlir::LLVM::StoreOp::create(builder&: rewriter, location: op.getLoc(), value: srcVal, addr: adaptor.getAddr(),
6551 alignment: op.getAlignment(), isVolatile: op.getIsVolatile());
6552
6553 mlir::Type resultTy = getTypeConverter()->convertType(t: op.getType());
6554
6555 if (info.getIsSigned()) {
6556 assert(size <= storageSize);
6557 unsigned highBits = storageSize - size;
6558
6559 if (highBits) {
6560 resultVal = createShL(bld&: rewriter, lhs: resultVal, rhs: highBits);
6561 resultVal = createAShR(bld&: rewriter, lhs: resultVal, rhs: highBits);
6562 }
6563 }
6564
6565 resultVal = createIntCast(bld&: rewriter, src: resultVal,
6566 dstTy: mlir::cast<mlir::IntegerType>(Val&: resultTy),
6567 isSigned: info.getIsSigned());
6568
6569 rewriter.replaceOp(op, newValues: resultVal);
6570 return mlir::success();
6571}
6572
6573mlir::LogicalResult CIRToLLVMComplexImagPtrOpLowering::matchAndRewrite(
6574 cir::ComplexImagPtrOp op, OpAdaptor adaptor,
6575 mlir::ConversionPatternRewriter &rewriter) const {
6576 cir::PointerType operandTy = op.getOperand().getType();
6577 mlir::Type resultLLVMTy = getTypeConverter()->convertType(t: op.getType());
6578 if (!resultLLVMTy)
6579 return emitTypeConversionError(op, type: op.getType());
6580 mlir::Type elementLLVMTy =
6581 getTypeConverter()->convertType(t: operandTy.getPointee());
6582
6583 mlir::LLVM::GEPArg gepIndices[2] = {{0}, {1}};
6584 mlir::LLVM::GEPNoWrapFlags inboundsNuw =
6585 mlir::LLVM::GEPNoWrapFlags::inbounds | mlir::LLVM::GEPNoWrapFlags::nuw;
6586 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
6587 op, args&: resultLLVMTy, args&: elementLLVMTy, args: adaptor.getOperand(), args&: gepIndices,
6588 args&: inboundsNuw);
6589 return mlir::success();
6590}
6591
6592mlir::LogicalResult CIRToLLVMComplexRealPtrOpLowering::matchAndRewrite(
6593 cir::ComplexRealPtrOp op, OpAdaptor adaptor,
6594 mlir::ConversionPatternRewriter &rewriter) const {
6595 cir::PointerType operandTy = op.getOperand().getType();
6596 mlir::Type resultLLVMTy = getTypeConverter()->convertType(t: op.getType());
6597 if (!resultLLVMTy)
6598 return emitTypeConversionError(op, type: op.getType());
6599 mlir::Type elementLLVMTy =
6600 getTypeConverter()->convertType(t: operandTy.getPointee());
6601
6602 mlir::LLVM::GEPArg gepIndices[2] = {0, 0};
6603 mlir::LLVM::GEPNoWrapFlags inboundsNuw =
6604 mlir::LLVM::GEPNoWrapFlags::inbounds | mlir::LLVM::GEPNoWrapFlags::nuw;
6605 rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
6606 op, args&: resultLLVMTy, args&: elementLLVMTy, args: adaptor.getOperand(), args&: gepIndices,
6607 args&: inboundsNuw);
6608 return mlir::success();
6609}
6610
6611mlir::LogicalResult CIRToLLVMGetBitfieldOpLowering::matchAndRewrite(
6612 cir::GetBitfieldOp op, OpAdaptor adaptor,
6613 mlir::ConversionPatternRewriter &rewriter) const {
6614
6615 mlir::OpBuilder::InsertionGuard guard(rewriter);
6616 rewriter.setInsertionPoint(op);
6617
6618 cir::BitfieldInfoAttr info = op.getBitfieldInfo();
6619 uint64_t size = info.getSize();
6620 uint64_t offset = info.getOffset();
6621 mlir::Type storageType = info.getStorageType();
6622 mlir::MLIRContext *context = storageType.getContext();
6623 unsigned storageSize = 0;
6624
6625 mlir::IntegerType intType =
6626 computeBitfieldIntType(storageType, context, storageSize);
6627
6628 mlir::Value val = mlir::LLVM::LoadOp::create(
6629 builder&: rewriter, location: op.getLoc(), type: intType, addr: adaptor.getAddr(), alignment: op.getAlignment(),
6630 isVolatile: op.getIsVolatile());
6631 val = mlir::LLVM::BitcastOp::create(builder&: rewriter, location: op.getLoc(), res: intType, arg: val);
6632
6633 if (info.getIsSigned()) {
6634 assert(static_cast<unsigned>(offset + size) <= storageSize);
6635 unsigned highBits = storageSize - offset - size;
6636 val = createShL(bld&: rewriter, lhs: val, rhs: highBits);
6637 val = createAShR(bld&: rewriter, lhs: val, rhs: offset + highBits);
6638 } else {
6639 val = createLShR(bld&: rewriter, lhs: val, rhs: offset);
6640
6641 if (static_cast<unsigned>(offset) + size < storageSize)
6642 val = createAnd(bld&: rewriter, lhs: val,
6643 rhs: llvm::APInt::getLowBitsSet(numBits: storageSize, loBitsSet: size));
6644 }
6645
6646 mlir::Type resTy = getTypeConverter()->convertType(t: op.getType());
6647 mlir::Value newOp = createIntCast(
6648 bld&: rewriter, src: val, dstTy: mlir::cast<mlir::IntegerType>(Val&: resTy), isSigned: info.getIsSigned());
6649 rewriter.replaceOp(op, newValues: newOp);
6650 return mlir::success();
6651}
6652
6653mlir::LogicalResult CIRToLLVMInlineAsmOpLowering::matchAndRewrite(
6654 cir::InlineAsmOp op, OpAdaptor adaptor,
6655 mlir::ConversionPatternRewriter &rewriter) const {
6656 mlir::Type llResTy;
6657 if (op.getNumResults()) {
6658 llResTy = getTypeConverter()->convertType(t: op.getType(i: 0));
6659 if (!llResTy)
6660 return emitTypeConversionError(op, type: op.getType(i: 0));
6661 }
6662
6663 cir::AsmFlavor dialect = op.getAsmFlavor();
6664 mlir::LLVM::AsmDialect llDialect = dialect == cir::AsmFlavor::x86_att
6665 ? mlir::LLVM::AsmDialect::AD_ATT
6666 : mlir::LLVM::AsmDialect::AD_Intel;
6667
6668 SmallVector<mlir::Attribute> opAttrs;
6669 StringRef llvmAttrName = mlir::LLVM::InlineAsmOp::getElementTypeAttrName();
6670
6671 // this is for the lowering to LLVM from LLVM dialect. Otherwise, if we
6672 // don't have the result (i.e. void type as a result of operation), the
6673 // element type attribute will be attached to the whole instruction, but not
6674 // to the operand
6675 if (!op.getNumResults())
6676 opAttrs.push_back(Elt: mlir::Attribute());
6677
6678 // so far we infer the llvm dialect element type attr from
6679 // CIR operand type.
6680 for (auto [index, cirOp] : llvm::enumerate(First: op->getOperands())) {
6681 if (!op.isMemoryOperand(index)) {
6682 opAttrs.push_back(Elt: mlir::Attribute());
6683 continue;
6684 }
6685
6686 llvm::SmallVector<mlir::NamedAttribute, 1> attrs;
6687 cir::PointerType typ = mlir::cast<cir::PointerType>(Val: cirOp.getType());
6688 mlir::Type elementTy =
6689 convertTypeForMemory(converter: *getTypeConverter(), dataLayout, type: typ.getPointee());
6690 if (!elementTy)
6691 return emitTypeConversionError(op, type: typ.getPointee());
6692 mlir::TypeAttr typAttr = mlir::TypeAttr::get(type: elementTy);
6693
6694 attrs.push_back(Elt: rewriter.getNamedAttr(name: llvmAttrName, val: typAttr));
6695 mlir::DictionaryAttr newDict = rewriter.getDictionaryAttr(value: attrs);
6696 opAttrs.push_back(Elt: newDict);
6697 }
6698
6699 rewriter.replaceOpWithNewOp<mlir::LLVM::InlineAsmOp>(
6700 op, args&: llResTy, args: adaptor.getOperands(), args: op.getAsmStringAttr(),
6701 args: op.getConstraintsAttr(), args: op.getSideEffectsAttr(),
6702 /*is_align_stack*/ args: mlir::UnitAttr(),
6703 /*tail_call_kind*/
6704 args: mlir::LLVM::TailCallKindAttr::get(
6705 context: getContext(), tailCallKind: mlir::LLVM::tailcallkind::TailCallKind::None),
6706 args: op.getConvergentAttr(),
6707 args: mlir::LLVM::AsmDialectAttr::get(context: getContext(), val: llDialect),
6708 args: rewriter.getArrayAttr(value: opAttrs));
6709
6710 return mlir::success();
6711}
6712
6713mlir::LogicalResult CIRToLLVMVAStartOpLowering::matchAndRewrite(
6714 cir::VAStartOp op, OpAdaptor adaptor,
6715 mlir::ConversionPatternRewriter &rewriter) const {
6716 rewriter.replaceOpWithNewOp<mlir::LLVM::VaStartOp>(op, args: adaptor.getArgList());
6717 return mlir::success();
6718}
6719
6720mlir::LogicalResult CIRToLLVMVAEndOpLowering::matchAndRewrite(
6721 cir::VAEndOp op, OpAdaptor adaptor,
6722 mlir::ConversionPatternRewriter &rewriter) const {
6723 rewriter.replaceOpWithNewOp<mlir::LLVM::VaEndOp>(op, args: adaptor.getArgList());
6724 return mlir::success();
6725}
6726
6727mlir::LogicalResult CIRToLLVMVACopyOpLowering::matchAndRewrite(
6728 cir::VACopyOp op, OpAdaptor adaptor,
6729 mlir::ConversionPatternRewriter &rewriter) const {
6730 rewriter.replaceOpWithNewOp<mlir::LLVM::VaCopyOp>(op, args: adaptor.getDstList(),
6731 args: adaptor.getSrcList());
6732 return mlir::success();
6733}
6734
6735mlir::LogicalResult CIRToLLVMVAArgOpLowering::matchAndRewrite(
6736 cir::VAArgOp op, OpAdaptor adaptor,
6737 mlir::ConversionPatternRewriter &rewriter) const {
6738 assert(!cir::MissingFeatures::vaArgABILowering());
6739 mlir::Type llvmType =
6740 getTypeConverter()->convertType(t: op->getResultTypes().front());
6741 if (!llvmType)
6742 return mlir::failure();
6743
6744 rewriter.replaceOpWithNewOp<mlir::LLVM::VaArgOp>(op, args&: llvmType,
6745 args: adaptor.getArgList());
6746 return mlir::success();
6747}
6748
6749mlir::LogicalResult CIRToLLVMLabelOpLowering::matchAndRewrite(
6750 cir::LabelOp op, OpAdaptor adaptor,
6751 mlir::ConversionPatternRewriter &rewriter) const {
6752 mlir::MLIRContext *ctx = rewriter.getContext();
6753 mlir::Block *block = op->getBlock();
6754 // A BlockTagOp cannot reside in the entry block. The address of the entry
6755 // block cannot be taken
6756 if (block->isEntryBlock()) {
6757 mlir::Block *newBlock =
6758 rewriter.splitBlock(block: op->getBlock(), before: mlir::Block::iterator(op));
6759 rewriter.setInsertionPointToEnd(block);
6760 mlir::LLVM::BrOp::create(builder&: rewriter, location: op.getLoc(), dest: newBlock);
6761 }
6762 auto tagAttr =
6763 mlir::LLVM::BlockTagAttr::get(context: ctx, id: blockInfoAddr.getTagIndex());
6764 rewriter.setInsertionPoint(op);
6765
6766 auto blockTagOp =
6767 mlir::LLVM::BlockTagOp::create(builder&: rewriter, location: op->getLoc(), tag: tagAttr);
6768 mlir::LLVM::LLVMFuncOp func = op->getParentOfType<mlir::LLVM::LLVMFuncOp>();
6769 auto blockInfoAttr =
6770 cir::BlockAddrInfoAttr::get(context: ctx, func_name: func.getSymName(), label_name: op.getLabel());
6771 blockInfoAddr.mapBlockTag(info: blockInfoAttr, tagOp: blockTagOp);
6772 rewriter.eraseOp(op);
6773
6774 return mlir::success();
6775}
6776
6777mlir::LogicalResult CIRToLLVMBlockAddressOpLowering::matchAndRewrite(
6778 cir::BlockAddressOp op, OpAdaptor adaptor,
6779 mlir::ConversionPatternRewriter &rewriter) const {
6780 mlir::MLIRContext *ctx = rewriter.getContext();
6781
6782 mlir::LLVM::BlockTagOp matchLabel =
6783 blockInfoAddr.lookupBlockTag(info: op.getBlockAddrInfoAttr());
6784 mlir::LLVM::BlockTagAttr tagAttr;
6785 if (!matchLabel)
6786 // If the BlockTagOp has not been emitted yet, use a placeholder.
6787 // This will later be replaced with the correct tag index during
6788 // `resolveBlockAddressOp`.
6789 tagAttr = {};
6790 else
6791 tagAttr = matchLabel.getTag();
6792
6793 auto blkAddr = mlir::LLVM::BlockAddressAttr::get(
6794 context: rewriter.getContext(), function: op.getBlockAddrInfoAttr().getFunc(), tag: tagAttr);
6795 rewriter.setInsertionPoint(op);
6796 auto newOp = mlir::LLVM::BlockAddressOp::create(
6797 builder&: rewriter, location: op.getLoc(), res: mlir::LLVM::LLVMPointerType::get(context: ctx), block_addr: blkAddr);
6798 if (!matchLabel)
6799 blockInfoAddr.addUnresolvedBlockAddress(op: newOp, info: op.getBlockAddrInfoAttr());
6800 rewriter.replaceOp(op, newOp);
6801 return mlir::success();
6802}
6803
6804mlir::LogicalResult CIRToLLVMIndirectBrOpLowering::matchAndRewrite(
6805 cir::IndirectBrOp op, OpAdaptor adaptor,
6806 mlir::ConversionPatternRewriter &rewriter) const {
6807
6808 mlir::Value targetAddr = adaptor.getAddr();
6809
6810 // If the poison attribute is set, use llvm.mlir.poison as the address.
6811 // This happens when the block has no predecessors and is essentially
6812 // unreachable. Do NOT erase the block argument directly, as that violates
6813 // the MLIR dialect conversion framework contract (the framework tracks block
6814 // arguments and will clean them up). A block with no predecessors simply
6815 // produces no PHI node.
6816 if (op.getPoison()) {
6817 auto llvmPtrType = mlir::LLVM::LLVMPointerType::get(context: rewriter.getContext());
6818 targetAddr =
6819 mlir::LLVM::PoisonOp::create(builder&: rewriter, location: op->getLoc(), res: llvmPtrType);
6820 }
6821
6822 rewriter.replaceOpWithNewOp<mlir::LLVM::IndirectBrOp>(
6823 op, args&: targetAddr, args: adaptor.getSuccOperands(), args: op.getSuccessors());
6824 return mlir::success();
6825}
6826
6827mlir::LogicalResult CIRToLLVMTokenNoneOpLowering::matchAndRewrite(
6828 cir::TokenNoneOp op, OpAdaptor adaptor,
6829 mlir::ConversionPatternRewriter &rewriter) const {
6830 rewriter.replaceOpWithNewOp<mlir::LLVM::NoneTokenOp>(
6831 op, args: mlir::TokenType::get(ctx: rewriter.getContext()));
6832 return mlir::success();
6833}
6834
6835mlir::LogicalResult CIRToLLVMCpuIdOpLowering::matchAndRewrite(
6836 cir::CpuIdOp op, OpAdaptor adaptor,
6837 mlir::ConversionPatternRewriter &rewriter) const {
6838 mlir::ModuleOp moduleOp = op->getParentOfType<mlir::ModuleOp>();
6839 std::optional<llvm::Triple> triple = cir::getTripleFromModule(mod: moduleOp);
6840 if (!triple)
6841 return op.emitOpError()
6842 << "requires a '" << cir::CIRDialect::getTripleAttrName()
6843 << "' module attribute";
6844
6845 mlir::Type i32Ty = rewriter.getI32Type();
6846 mlir::Type i64Ty = rewriter.getI64Type();
6847 mlir::Type i32PtrTy = mlir::LLVM::LLVMPointerType::get(context: i32Ty.getContext(), addressSpace: 0);
6848
6849 mlir::Type cpuidRetTy = mlir::LLVM::LLVMStructType::getLiteral(
6850 context: rewriter.getContext(), types: {i32Ty, i32Ty, i32Ty, i32Ty});
6851
6852 mlir::Value functionId = adaptor.getFunctionId();
6853 mlir::Value subFunctionId = adaptor.getSubFunctionId();
6854
6855 StringRef asmString, constraints;
6856 if (triple->getArch() == llvm::Triple::x86) {
6857 asmString = "cpuid";
6858 constraints = "={ax},={bx},={cx},={dx},{ax},{cx}";
6859 } else {
6860 // x86-64 uses %rbx as the base register, so preserve it.
6861 asmString = "xchgq %rbx, ${1:q}\n"
6862 "cpuid\n"
6863 "xchgq %rbx, ${1:q}";
6864 constraints = "={ax},=r,={cx},={dx},0,2";
6865 }
6866
6867 mlir::Value inlineAsm =
6868 mlir::LLVM::InlineAsmOp::create(
6869 builder&: rewriter, location: op.getLoc(), res: cpuidRetTy, operands: {functionId, subFunctionId},
6870 asm_string: rewriter.getStringAttr(bytes: asmString),
6871 constraints: rewriter.getStringAttr(bytes: constraints),
6872 /*has_side_effects=*/mlir::UnitAttr{},
6873 /*is_align_stack=*/mlir::UnitAttr{},
6874 /*tail_call_kind=*/mlir::LLVM::TailCallKindAttr{},
6875 /*convergent=*/{},
6876 /*asm_dialect=*/mlir::LLVM::AsmDialectAttr{},
6877 /*operand_attrs=*/mlir::ArrayAttr{})
6878 .getResult(i: 0);
6879
6880 mlir::Value basePtr = adaptor.getCpuInfo();
6881
6882 mlir::DataLayout layout(op->getParentOfType<mlir::ModuleOp>());
6883 unsigned alignment = layout.getTypeABIAlignment(t: i32Ty);
6884 for (unsigned i = 0; i < 4; i++) {
6885 mlir::Value extracted =
6886 mlir::LLVM::ExtractValueOp::create(builder&: rewriter, location: op.getLoc(), container: inlineAsm, position: i)
6887 .getResult();
6888 mlir::Value index = mlir::LLVM::ConstantOp::create(
6889 builder&: rewriter, location: op.getLoc(), res: i64Ty, value: rewriter.getI64IntegerAttr(value: i));
6890 llvm::SmallVector<mlir::Value, 1> gepIndices = {index};
6891 mlir::Value storePtr = mlir::LLVM::GEPOp::create(
6892 builder&: rewriter, location: op.getLoc(), resultType: i32PtrTy, elementType: i32Ty, basePtr,
6893 indices: gepIndices, noWrapFlags: mlir::LLVM::GEPNoWrapFlags::none)
6894 .getResult();
6895 mlir::LLVM::StoreOp::create(builder&: rewriter, location: op.getLoc(), value: extracted, addr: storePtr,
6896 alignment);
6897 }
6898
6899 rewriter.eraseOp(op);
6900 return mlir::success();
6901}
6902
6903mlir::LogicalResult CIRToLLVMMemChrOpLowering::matchAndRewrite(
6904 cir::MemChrOp op, OpAdaptor adaptor,
6905 mlir::ConversionPatternRewriter &rewriter) const {
6906 auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(context: rewriter.getContext());
6907 mlir::Type srcTy = getTypeConverter()->convertType(t: op.getSrc().getType());
6908 mlir::Type patternTy =
6909 getTypeConverter()->convertType(t: op.getPattern().getType());
6910 mlir::Type lenTy = getTypeConverter()->convertType(t: op.getLen().getType());
6911 auto fnTy =
6912 mlir::LLVM::LLVMFunctionType::get(result: llvmPtrTy, arguments: {srcTy, patternTy, lenTy},
6913 /*isVarArg=*/false);
6914 llvm::StringRef fnName = "memchr";
6915
6916 mlir::Builder b(rewriter.getContext());
6917 mlir::NamedAttribute noundefAttr =
6918 b.getNamedAttr(name: "llvm.noundef", val: b.getUnitAttr());
6919 mlir::DictionaryAttr noundefDict = mlir::DictionaryAttr::get(
6920 context: rewriter.getContext(), value: llvm::ArrayRef(noundefAttr));
6921 SmallVector<mlir::Attribute> argAttrVec(3, noundefDict);
6922 mlir::ArrayAttr argAttrs =
6923 mlir::ArrayAttr::get(context: rewriter.getContext(), value: argAttrVec);
6924
6925 createLLVMFuncOpIfNotExist(rewriter, symbolTables, srcOp: op, fnName, fnTy,
6926 argAttrs);
6927
6928 mlir::LLVM::CallOp newCall = rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
6929 op, args: mlir::TypeRange{llvmPtrTy}, args&: fnName,
6930 args: mlir::ValueRange{adaptor.getSrc(), adaptor.getPattern(),
6931 adaptor.getLen()});
6932 newCall.setArgAttrsAttr(argAttrs);
6933 return mlir::success();
6934}
6935
6936// Function to do the clear-padding operation. This is a faithful translation of
6937// CGBuiltin.cpp's ClearPadding function.
6938static void clearPadding(mlir::ConversionPatternRewriter &rewriter,
6939 mlir::Location loc, mlir::Value inputPtr,
6940 uint64_t baseAlignment,
6941 cir::OffsetPairAttr paddingAttr) {
6942 // FIXME(cir): Classic-codegen pulls this from the data layout, but MLIR
6943 // DataLayout just assumes it is 8 everywhere (as does all our lowering).
6944 // Clang doesn't support any non-8-bit-CHAR_BIT architectures, but keeping
6945 // this separate so we can use it next time.
6946 uint64_t charWidth = 8;
6947 mlir::Type i8Ty = rewriter.getI8Type();
6948 mlir::Type ptrTy = mlir::LLVM::LLVMPointerType::get(context: i8Ty.getContext());
6949
6950 auto startByte = paddingAttr.getStart() / charWidth;
6951 auto startBit = paddingAttr.getStart() % charWidth;
6952 auto endByte = paddingAttr.getEnd() / charWidth;
6953 auto endBit = paddingAttr.getEnd() % charWidth;
6954
6955 if (startByte == endByte) {
6956 // Interval is within a single byte
6957 auto index = mlir::LLVM::ConstantOp::create(
6958 builder&: rewriter, location: loc, type: rewriter.getI32Type(), value: startByte);
6959 auto element = mlir::LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: ptrTy, elementType: i8Ty,
6960 basePtr: inputPtr, indices: {index});
6961
6962 uint64_t adjustedAlignment = llvm::MinAlign(A: baseAlignment, B: startByte);
6963
6964 auto value = mlir::LLVM::LoadOp::create(builder&: rewriter, location: loc, type: i8Ty, addr: element,
6965 alignment: adjustedAlignment);
6966
6967 // Create mask to clear bits within the byte
6968 // We want to clear bits from StartBit to EndBit-1
6969 uint8_t bitsToClear = ((1 << endBit) - 1) & ~((1 << startBit) - 1);
6970 uint8_t bitsToKeep = ~bitsToClear;
6971 auto maskValue =
6972 mlir::LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i8Ty, value: bitsToKeep);
6973 auto newValue = mlir::LLVM::AndOp::create(builder&: rewriter, location: loc, lhs: value, rhs: maskValue);
6974
6975 mlir::LLVM::StoreOp::create(builder&: rewriter, location: loc, value: newValue, addr: element,
6976 alignment: adjustedAlignment);
6977 } else {
6978 // Handle the start byte
6979 if (startBit != 0) {
6980 auto index = mlir::LLVM::ConstantOp::create(
6981 builder&: rewriter, location: loc, type: rewriter.getI32Type(), value: startByte);
6982 auto element = mlir::LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: ptrTy, elementType: i8Ty,
6983 basePtr: inputPtr, indices: {index});
6984 uint64_t adjustedAlignment = llvm::MinAlign(A: baseAlignment, B: startByte);
6985
6986 auto value = mlir::LLVM::LoadOp::create(builder&: rewriter, location: loc, type: i8Ty, addr: element,
6987 alignment: adjustedAlignment);
6988
6989 uint8_t bitsToClear = ((1 << (charWidth - startBit)) - 1) << startBit;
6990 uint8_t bitsToKeep = ~bitsToClear;
6991 auto maskValue =
6992 mlir::LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i8Ty, value: bitsToKeep);
6993 auto newValue =
6994 mlir::LLVM::AndOp::create(builder&: rewriter, location: loc, lhs: value, rhs: maskValue);
6995
6996 mlir::LLVM::StoreOp::create(builder&: rewriter, location: loc, value: newValue, addr: element,
6997 alignment: adjustedAlignment);
6998 ++startByte;
6999 }
7000
7001 // Handle full bytes in the middle
7002 for (auto offset = startByte; offset < endByte; ++offset) {
7003 auto index = mlir::LLVM::ConstantOp::create(
7004 builder&: rewriter, location: loc, type: rewriter.getI32Type(), value: offset);
7005 auto element = mlir::LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: ptrTy, elementType: i8Ty,
7006 basePtr: inputPtr, indices: {index});
7007 uint64_t adjustedAlignment = llvm::MinAlign(A: baseAlignment, B: offset);
7008
7009 auto zero = mlir::LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i8Ty, value: 0);
7010 mlir::LLVM::StoreOp::create(builder&: rewriter, location: loc, value: zero, addr: element,
7011 alignment: adjustedAlignment);
7012 }
7013
7014 // Handle the end byte
7015 if (endBit != 0) {
7016 auto index = mlir::LLVM::ConstantOp::create(
7017 builder&: rewriter, location: loc, type: rewriter.getI32Type(), value: endByte);
7018 auto element = mlir::LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: ptrTy, elementType: i8Ty,
7019 basePtr: inputPtr, indices: {index});
7020 uint64_t adjustedAlignment = llvm::MinAlign(A: baseAlignment, B: endByte);
7021
7022 auto value = mlir::LLVM::LoadOp::create(builder&: rewriter, location: loc, type: i8Ty, addr: element,
7023 alignment: adjustedAlignment);
7024
7025 uint8_t bitsToClear = (1 << endBit) - 1;
7026 uint8_t bitsToKeep = ~bitsToClear;
7027 auto maskValue =
7028 mlir::LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i8Ty, value: bitsToKeep);
7029 auto newValue =
7030 mlir::LLVM::AndOp::create(builder&: rewriter, location: loc, lhs: value, rhs: maskValue);
7031
7032 mlir::LLVM::StoreOp::create(builder&: rewriter, location: loc, value: newValue, addr: element,
7033 alignment: adjustedAlignment);
7034 }
7035 }
7036}
7037
7038mlir::LogicalResult CIRToLLVMClearPaddingOpLowering::matchAndRewrite(
7039 cir::ClearPaddingOp op, OpAdaptor adaptor,
7040 mlir::ConversionPatternRewriter &rewriter) const {
7041
7042 mlir::Value inputPtr = adaptor.getArg();
7043 for (mlir::Attribute attr : op.getPadding())
7044 clearPadding(rewriter, loc: op.getLoc(), inputPtr, baseAlignment: op.getAlignment(),
7045 paddingAttr: cast<cir::OffsetPairAttr>(Val&: attr));
7046
7047 rewriter.eraseOp(op);
7048 return mlir::success();
7049}
7050
7051std::unique_ptr<mlir::Pass> createConvertCIRToLLVMPass() {
7052 return std::make_unique<ConvertCIRToLLVMPass>();
7053}
7054
7055void populateCIRToLLVMPasses(mlir::OpPassManager &pm, bool enableOpenMP,
7056 bool enableOptimizations) {
7057 mlir::populateCIRPreLoweringPasses(pm);
7058 if (enableOpenMP)
7059 pm.addPass(pass: mlir::omp::createMarkDeclareTargetPass());
7060 pm.addPass(pass: createConvertCIRToLLVMPass());
7061 if (enableOpenMP) {
7062 pm.addPass(pass: mlir::omp::createHostOpFilteringPass());
7063 pm.nest<mlir::LLVM::LLVMFuncOp>().addPass(
7064 pass: mlir::omp::createStackToSharedPass());
7065 }
7066 if (enableOptimizations)
7067 pm.nest<mlir::LLVM::LLVMFuncOp>().addPass(
7068 pass: mlir::LLVM::createLLVMSimplifyCFGPass());
7069}
7070
7071std::unique_ptr<llvm::Module>
7072lowerDirectlyFromCIRToLLVMIR(mlir::ModuleOp mlirModule, LLVMContext &llvmCtx,
7073 bool enableOpenMP, StringRef mlirSaveTempsOutFile,
7074 llvm::vfs::FileSystem *fs,
7075 bool enableOptimizations) {
7076 llvm::TimeTraceScope scope("lower from CIR to LLVM directly");
7077
7078 mlir::MLIRContext *mlirCtx = mlirModule.getContext();
7079
7080 mlir::PassManager pm(mlirCtx);
7081 populateCIRToLLVMPasses(pm, enableOpenMP, enableOptimizations);
7082
7083 (void)mlir::applyPassManagerCLOptions(pm);
7084
7085 // The failing pass has already reported its error through the MLIR
7086 // diagnostic engine.
7087 if (mlir::failed(Result: pm.run(op: mlirModule)))
7088 return nullptr;
7089
7090 if (!mlirSaveTempsOutFile.empty()) {
7091 std::error_code ec;
7092 llvm::raw_fd_ostream out(mlirSaveTempsOutFile, ec);
7093 if (!ec)
7094 mlirModule->print(os&: out);
7095 }
7096
7097 mlir::registerBuiltinDialectTranslation(context&: *mlirCtx);
7098 mlir::registerLLVMDialectTranslation(context&: *mlirCtx);
7099 mlir::registerOpenMPDialectTranslation(context&: *mlirCtx);
7100 mlir::registerCIRDialectTranslation(context&: *mlirCtx);
7101
7102 llvm::TimeTraceScope translateScope("translateModuleToLLVMIR");
7103
7104 StringRef moduleName = mlirModule.getName().value_or(u: "CIRToLLVMModule");
7105 std::unique_ptr<llvm::Module> llvmModule = mlir::translateModuleToLLVMIR(
7106 module: mlirModule, llvmContext&: llvmCtx, name: moduleName, /*disableVerification=*/false, fs);
7107
7108 return llvmModule;
7109}
7110} // namespace direct
7111} // namespace cir
7112