1//===---- LowerItaniumCXXABI.cpp - Emit CIR code Itanium-specific code ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides CIR lowering logic targeting the Itanium C++ ABI. The class in
10// this file generates records that follow the Itanium C++ ABI, which is
11// documented at:
12// https://itanium-cxx-abi.github.io/cxx-abi/abi.html
13// https://itanium-cxx-abi.github.io/cxx-abi/abi-eh.html
14//
15// It also supports the closely-related ARM ABI, documented at:
16// https://developer.arm.com/documentation/ihi0041/g/
17//
18// This file partially mimics clang/lib/CodeGen/ItaniumCXXABI.cpp. The queries
19// are adapted to operate on the CIR dialect, however.
20//
21//===----------------------------------------------------------------------===//
22
23#include "CIRCXXABI.h"
24#include "LowerModule.h"
25#include "mlir/IR/ImplicitLocOpBuilder.h"
26#include "clang/CIR/Dialect/IR/CIRDialect.h"
27#include "llvm/Support/ErrorHandling.h"
28
29namespace cir {
30
31namespace {
32
33class LowerItaniumCXXABI : public CIRCXXABI {
34protected:
35 bool useARMMethodPtrABI;
36 bool use32BitVTableOffsetABI;
37
38public:
39 LowerItaniumCXXABI(LowerModule &lm, bool useARMMethodPtrABI = false,
40 bool use32BitVTableOffsetABI = false)
41 : CIRCXXABI(lm), useARMMethodPtrABI(useARMMethodPtrABI),
42 use32BitVTableOffsetABI(use32BitVTableOffsetABI) {}
43
44 /// Lower the given data member pointer type to its ABI type. The returned
45 /// type is also a CIR type.
46 virtual mlir::Type
47 lowerDataMemberType(cir::DataMemberType type,
48 const mlir::TypeConverter &typeConverter) const override;
49
50 mlir::Type
51 lowerMethodType(cir::MethodType type,
52 const mlir::TypeConverter &typeConverter) const override;
53
54 mlir::TypedAttr lowerDataMemberConstant(
55 cir::DataMemberAttr attr, const mlir::DataLayout &layout,
56 const mlir::TypeConverter &typeConverter) const override;
57
58 mlir::TypedAttr lowerDataMemberOffsetConstant(
59 cir::DataMemberOffsetAttr attr, const mlir::DataLayout &layout,
60 const mlir::TypeConverter &typeConverter) const override;
61
62 mlir::TypedAttr
63 lowerMethodConstant(cir::MethodAttr attr, const mlir::DataLayout &layout,
64 const mlir::TypeConverter &typeConverter) const override;
65
66 mlir::Operation *
67 lowerGetRuntimeMember(cir::GetRuntimeMemberOp op, mlir::Type loweredResultTy,
68 mlir::Value loweredAddr, mlir::Value loweredMember,
69 mlir::OpBuilder &builder) const override;
70
71 void lowerGetMethod(cir::GetMethodOp op, mlir::Value &callee,
72 mlir::Value &thisArg, mlir::Value loweredMethod,
73 mlir::Value loweredObjectPtr,
74 mlir::ConversionPatternRewriter &rewriter) const override;
75
76 mlir::Value lowerBaseDataMember(cir::BaseDataMemberOp op,
77 mlir::Value loweredSrc,
78 mlir::OpBuilder &builder) const override;
79
80 mlir::Value lowerDerivedDataMember(cir::DerivedDataMemberOp op,
81 mlir::Value loweredSrc,
82 mlir::OpBuilder &builder) const override;
83
84 mlir::Value lowerBaseMethod(cir::BaseMethodOp op, mlir::Value loweredSrc,
85 mlir::OpBuilder &builder) const override;
86
87 mlir::Value lowerDerivedMethod(cir::DerivedMethodOp op,
88 mlir::Value loweredSrc,
89 mlir::OpBuilder &builder) const override;
90
91 mlir::Value lowerDataMemberCmp(cir::CmpOp op, mlir::Value loweredLhs,
92 mlir::Value loweredRhs,
93 mlir::OpBuilder &builder) const override;
94
95 mlir::Value lowerMethodCmp(cir::CmpOp op, mlir::Value loweredLhs,
96 mlir::Value loweredRhs,
97 mlir::OpBuilder &builder) const override;
98
99 mlir::Value lowerDataMemberBitcast(cir::CastOp op, mlir::Type loweredDstTy,
100 mlir::Value loweredSrc,
101 mlir::OpBuilder &builder) const override;
102
103 mlir::Value
104 lowerDataMemberToBoolCast(cir::CastOp op, mlir::Value loweredSrc,
105 mlir::OpBuilder &builder) const override;
106
107 mlir::Value lowerMethodBitcast(cir::CastOp op, mlir::Type loweredDstTy,
108 mlir::Value loweredSrc,
109 mlir::OpBuilder &builder) const override;
110
111 mlir::Value lowerMethodToBoolCast(cir::CastOp op, mlir::Value loweredSrc,
112 mlir::OpBuilder &builder) const override;
113
114 mlir::Value lowerDynamicCast(cir::DynamicCastOp op,
115 mlir::OpBuilder &builder) const override;
116 mlir::Value lowerVTableGetTypeInfo(cir::VTableGetTypeInfoOp op,
117 mlir::OpBuilder &builder) const override;
118
119 clang::CharUnits
120 getArrayCookieSizeImpl(clang::CharUnits elementAlign) const override;
121
122 mlir::Value readArrayCookieImpl(mlir::Location loc, mlir::Value allocPtr,
123 clang::CharUnits cookieSize,
124 clang::CharUnits cookieAlignment,
125 const mlir::DataLayout &dataLayout,
126 CIRBaseBuilderTy &builder) const override;
127};
128
129} // namespace
130
131std::unique_ptr<CIRCXXABI> createItaniumCXXABI(LowerModule &lm) {
132 switch (lm.getCXXABIKind()) {
133 // Note that AArch64 uses the generic ItaniumCXXABI class since it doesn't
134 // include the other 32-bit ARM oddities: constructor/destructor return values
135 // and array cookies.
136 case clang::TargetCXXABI::GenericAArch64:
137 return std::make_unique<LowerItaniumCXXABI>(
138 args&: lm,
139 /*useARMMethodPtrABI=*/args: true,
140 /*use32BitVTableOffsetABI=*/args: false);
141 case clang::TargetCXXABI::AppleARM64:
142 // TODO: this isn't quite right, clang uses AppleARM64CXXABI which inherits
143 // from ARMCXXABI. We'll have to follow suit.
144 assert(!cir::MissingFeatures::appleArm64CXXABI());
145 return std::make_unique<LowerItaniumCXXABI>(
146 args&: lm,
147 /*useARMMethodPtrABI=*/args: true,
148 /*use32BitVTableOffsetABI=*/args: true);
149
150 case clang::TargetCXXABI::GenericItanium:
151 return std::make_unique<LowerItaniumCXXABI>(args&: lm);
152
153 case clang::TargetCXXABI::GenericARM:
154 case clang::TargetCXXABI::iOS:
155 case clang::TargetCXXABI::WatchOS:
156 case clang::TargetCXXABI::Fuchsia:
157 case clang::TargetCXXABI::GenericMIPS:
158 case clang::TargetCXXABI::WebAssembly:
159 case clang::TargetCXXABI::XL:
160 // FIXME: these ABIS are not yet implemented by CIR. The error is reported
161 // by consumers of LowerModule/CIRCXXABI when missing CIRCXXABI is
162 // encountered.
163 return nullptr;
164 case clang::TargetCXXABI::Microsoft:
165 llvm_unreachable("Microsoft ABI is not Itanium-based");
166 }
167 llvm_unreachable("bad ABI kind");
168}
169
170static cir::IntType getPtrDiffCIRTy(LowerModule &lm) {
171 const clang::TargetInfo &target = lm.getTarget();
172 clang::TargetInfo::IntType ptrdiffTy =
173 target.getPtrDiffType(AddrSpace: clang::LangAS::Default);
174 return cir::IntType::get(context: lm.getMLIRContext(), width: target.getTypeWidth(T: ptrdiffTy),
175 is_signed: target.isTypeSigned(T: ptrdiffTy));
176}
177
178mlir::Type LowerItaniumCXXABI::lowerDataMemberType(
179 cir::DataMemberType type, const mlir::TypeConverter &typeConverter) const {
180 // Itanium C++ ABI 2.3.1:
181 // A data member pointer is represented as the data member's offset in bytes
182 // from the address point of an object of the base type, as a ptrdiff_t.
183 return getPtrDiffCIRTy(lm);
184}
185
186mlir::Type LowerItaniumCXXABI::lowerMethodType(
187 cir::MethodType type, const mlir::TypeConverter &typeConverter) const {
188 // Itanium C++ ABI 2.3.2:
189 // In all representations, the basic ABI properties of member function
190 // pointer types are those of the following class, where fnptr_t is the
191 // appropriate function-pointer type for a member function of this type:
192 //
193 // struct {
194 // fnptr_t ptr;
195 // ptrdiff_t adj;
196 // };
197
198 cir::IntType ptrdiffCIRTy = getPtrDiffCIRTy(lm);
199
200 // Note that clang CodeGen emits struct{ptrdiff_t, ptrdiff_t} for member
201 // function pointers. Let's follow this approach.
202 mlir::Type members[] = {ptrdiffCIRTy, ptrdiffCIRTy};
203 return cir::StructType::get(context: type.getContext(), members, /*packed=*/false,
204 /*is_class=*/false,
205 member_kinds: cir::RecordType::getAllDataKinds(members));
206}
207
208mlir::TypedAttr LowerItaniumCXXABI::lowerDataMemberConstant(
209 cir::DataMemberAttr attr, const mlir::DataLayout &layout,
210 const mlir::TypeConverter &typeConverter) const {
211 int64_t memberOffset;
212 if (attr.isNullPtr()) {
213 // Itanium C++ ABI 2.3:
214 // A NULL pointer is represented as -1.
215 memberOffset = -1;
216 } else {
217 // Itanium C++ ABI 2.3:
218 // A pointer to data member is an offset from the base address of
219 // the class object containing it, represented as a ptrdiff_t.
220 // Walk the GEP-style path, accumulating the byte offset at each step.
221 memberOffset = 0;
222 mlir::Type currentTy = attr.getType().getClassTy();
223 for (int32_t idx : attr.getPath()) {
224 auto recTy = mlir::cast<cir::RecordType>(Val&: currentTy);
225 memberOffset += static_cast<int64_t>(recTy.getElementOffset(dataLayout: layout, idx));
226 currentTy = recTy.getMembers()[idx];
227 }
228 }
229
230 mlir::Type abiTy = lowerDataMemberType(type: attr.getType(), typeConverter);
231 return cir::IntAttr::get(type: abiTy, value: memberOffset);
232}
233
234mlir::TypedAttr LowerItaniumCXXABI::lowerDataMemberOffsetConstant(
235 cir::DataMemberOffsetAttr attr, const mlir::DataLayout &layout,
236 const mlir::TypeConverter &typeConverter) const {
237 // Itanium C++ ABI 2.3:
238 // A pointer to data member is an offset from the base address of the class
239 // object containing it, represented as a ptrdiff_t.
240 // The offset is already known (the member has no CIR field index).
241 mlir::Type abiTy = lowerDataMemberType(type: attr.getType(), typeConverter);
242 return cir::IntAttr::get(type: abiTy, value: static_cast<int64_t>(attr.getOffset()));
243}
244
245mlir::TypedAttr LowerItaniumCXXABI::lowerMethodConstant(
246 cir::MethodAttr attr, const mlir::DataLayout &layout,
247 const mlir::TypeConverter &typeConverter) const {
248 cir::IntType ptrdiffCIRTy = getPtrDiffCIRTy(lm);
249
250 auto loweredMethodTy = mlir::cast<cir::StructType>(
251 Val: lowerMethodType(type: attr.getType(), typeConverter));
252
253 auto zero = cir::IntAttr::get(type: ptrdiffCIRTy, value: 0);
254
255 // Itanium C++ ABI 2.3.2:
256 // In all representations, the basic ABI properties of member function
257 // pointer types are those of the following class, where fnptr_t is the
258 // appropriate function-pointer type for a member function of this type:
259 //
260 // struct {
261 // fnptr_t ptr;
262 // ptrdiff_t adj;
263 // };
264
265 if (attr.isNull()) {
266 // Itanium C++ ABI 2.3.2:
267 //
268 // In the standard representation, a null member function pointer is
269 // represented with ptr set to a null pointer. The value of adj is
270 // unspecified for null member function pointers.
271 //
272 // clang CodeGen emits struct{null, null} for null member function pointers.
273 // Let's do the same here.
274 return cir::ConstRecordAttr::get(
275 type: loweredMethodTy, members: mlir::ArrayAttr::get(context: attr.getContext(), value: {zero, zero}));
276 }
277
278 if (attr.isVirtual()) {
279 // The member pointer holds the offset in bytes of the vtable entry from the
280 // address point.
281 uint64_t vtableOffset = attr.getVtableIndex().value() *
282 cir::getVTableEntrySize(mod: lm.getModule(), dataLayout: layout);
283
284 if (useARMMethodPtrABI) {
285 // ARM C++ ABI 3.2.1:
286 // This ABI specifies that adj contains twice the this
287 // adjustment, plus 1 if the member function is virtual. The
288 // least significant bit of adj then makes exactly the same
289 // discrimination as the least significant bit of ptr does for
290 // Itanium.
291 assert(!cir::MissingFeatures::pointerAuthentication());
292 auto ptr = cir::IntAttr::get(type: ptrdiffCIRTy, value: vtableOffset);
293 auto one = cir::IntAttr::get(type: ptrdiffCIRTy, value: 1);
294 return cir::ConstRecordAttr::get(
295 type: loweredMethodTy, members: mlir::ArrayAttr::get(context: attr.getContext(), value: {ptr, one}));
296 }
297
298 // Itanium C++ ABI 2.3.2:
299 //
300 // In the standard representation, a member function pointer for a
301 // virtual function is represented with ptr set to 1 plus the function's
302 // v-table entry offset (in bytes), converted to a function pointer as if
303 // by reinterpret_cast<fnptr_t>(uintfnptr_t(1 + offset)), where
304 // uintfnptr_t is an unsigned integer of the same size as fnptr_t.
305 auto ptr = cir::IntAttr::get(type: ptrdiffCIRTy, value: 1 + vtableOffset);
306 return cir::ConstRecordAttr::get(
307 type: loweredMethodTy, members: mlir::ArrayAttr::get(context: attr.getContext(), value: {ptr, zero}));
308 }
309
310 // Itanium C++ ABI 2.3.2:
311 //
312 // A member function pointer for a non-virtual member function is
313 // represented with ptr set to a pointer to the function, using the base
314 // ABI's representation of function pointers.
315 auto ptr = cir::GlobalViewAttr::get(type: ptrdiffCIRTy, symbol: attr.getSymbol().value());
316 return cir::ConstRecordAttr::get(
317 type: loweredMethodTy, members: mlir::ArrayAttr::get(context: attr.getContext(), value: {ptr, zero}));
318}
319
320mlir::Operation *LowerItaniumCXXABI::lowerGetRuntimeMember(
321 cir::GetRuntimeMemberOp op, mlir::Type loweredResultTy,
322 mlir::Value loweredAddr, mlir::Value loweredMember,
323 mlir::OpBuilder &builder) const {
324 auto byteTy = cir::IntType::get(context: op.getContext(), width: 8, is_signed: true);
325 auto bytePtrTy = cir::PointerType::get(
326 pointee: byteTy,
327 addrSpace: mlir::cast<cir::PointerType>(Val: op.getAddr().getType()).getAddrSpace());
328 auto objectBytesPtr = cir::CastOp::create(
329 builder, location: op.getLoc(), result: bytePtrTy, kind: cir::CastKind::bitcast, src: op.getAddr());
330 auto memberBytesPtr = cir::PtrStrideOp::create(
331 builder, location: op.getLoc(), result: bytePtrTy, base: objectBytesPtr, stride: loweredMember);
332 return cir::CastOp::create(builder, location: op.getLoc(), result: op.getType(),
333 kind: cir::CastKind::bitcast, src: memberBytesPtr);
334}
335
336void LowerItaniumCXXABI::lowerGetMethod(
337 cir::GetMethodOp op, mlir::Value &callee, mlir::Value &thisArg,
338 mlir::Value loweredMethod, mlir::Value loweredObjectPtr,
339 mlir::ConversionPatternRewriter &rewriter) const {
340 // In the Itanium and ARM ABIs, method pointers have the form:
341 // struct { ptrdiff_t ptr; ptrdiff_t adj; } memptr;
342 //
343 // In the Itanium ABI:
344 // - method pointers are virtual if (memptr.ptr & 1) is nonzero
345 // - the this-adjustment is (memptr.adj)
346 // - the virtual offset is (memptr.ptr - 1)
347 //
348 // In the ARM ABI:
349 // - method pointers are virtual if (memptr.adj & 1) is nonzero
350 // - the this-adjustment is (memptr.adj >> 1)
351 // - the virtual offset is (memptr.ptr)
352 // ARM uses 'adj' for the virtual flag because Thumb functions
353 // may be only single-byte aligned.
354 //
355 // If the member is virtual, the adjusted 'this' pointer points
356 // to a vtable pointer from which the virtual offset is applied.
357 //
358 // If the member is non-virtual, memptr.ptr is the address of
359 // the function to call.
360
361 mlir::ImplicitLocOpBuilder locBuilder(op.getLoc(), rewriter);
362 mlir::Type calleePtrTy = op.getCallee().getType();
363
364 cir::IntType ptrdiffCIRTy = getPtrDiffCIRTy(lm);
365 mlir::Value ptrdiffOne =
366 cir::ConstantOp::create(builder&: locBuilder, value: cir::IntAttr::get(type: ptrdiffCIRTy, value: 1));
367
368 mlir::Value rawAdj =
369 cir::ExtractMemberOp::create(builder&: locBuilder, result: ptrdiffCIRTy, record: loweredMethod, index: 1);
370 mlir::Value adj = rawAdj;
371 if (useARMMethodPtrABI)
372 adj = cir::ShiftOp::create(builder&: locBuilder, result: ptrdiffCIRTy, value: adj, amount: ptrdiffOne,
373 /*isLeftShift=*/isShiftleft: false);
374
375 // Apply the adjustment to the 'this' pointer.
376 mlir::Type thisVoidPtrTy =
377 cir::PointerType::get(pointee: cir::VoidType::get(ctx: locBuilder.getContext()),
378 addrSpace: op.getObject().getType().getAddrSpace());
379 mlir::Value thisVoidPtr = cir::CastOp::create(
380 builder&: locBuilder, result: thisVoidPtrTy, kind: cir::CastKind::bitcast, src: loweredObjectPtr);
381 thisArg =
382 cir::PtrStrideOp::create(builder&: locBuilder, result: thisVoidPtrTy, base: thisVoidPtr, stride: adj);
383
384 // Load the "ptr" field of the member function pointer and determine if it
385 // points to a virtual function.
386 mlir::Value methodPtrField =
387 cir::ExtractMemberOp::create(builder&: locBuilder, result: ptrdiffCIRTy, record: loweredMethod, index: 0);
388 mlir::Value virtualBit;
389 if (useARMMethodPtrABI)
390 virtualBit = cir::AndOp::create(builder&: locBuilder, lhs: rawAdj, rhs: ptrdiffOne);
391 else
392 virtualBit = cir::AndOp::create(builder&: locBuilder, lhs: methodPtrField, rhs: ptrdiffOne);
393 mlir::Value isVirtual = cir::CmpOp::create(builder&: locBuilder, kind: cir::CmpOpKind::eq,
394 lhs: virtualBit, rhs: ptrdiffOne);
395
396 assert(!cir::MissingFeatures::emitCFICheck());
397 assert(!cir::MissingFeatures::emitVFEInfo());
398 assert(!cir::MissingFeatures::emitWPDInfo());
399
400 auto buildVirtualCallee = [&](mlir::OpBuilder &b, mlir::Location loc) {
401 // Load vtable pointer.
402 auto vtablePtrPtr = cir::VTableGetVPtrOp::create(builder&: b, location: loc, src: thisArg);
403 assert(!cir::MissingFeatures::opTBAA());
404 mlir::Value vtablePtr =
405 cir::LoadOp::create(builder&: b, location: loc, addr: vtablePtrPtr, /*isDeref=*/false,
406 /*isVolatile=*/is_volatile: false,
407 /*isNontemporal=*/is_nontemporal: false,
408 /*alignment=*/mlir::IntegerAttr(),
409 /*sync_scope=*/cir::SyncScopeKindAttr{},
410 /*mem_order=*/cir::MemOrderAttr(),
411 /*invariant=*/false);
412
413 // Apply the offset.
414 // On ARM64, to reserve extra space in virtual member function pointers,
415 // we only pay attention to the low 32 bits of the offset.
416 mlir::Value vtableOffset = methodPtrField;
417 if (!useARMMethodPtrABI)
418 vtableOffset = cir::SubOp::create(builder&: b, location: loc, result: vtableOffset.getType(),
419 lhs: vtableOffset, rhs: ptrdiffOne);
420 if (use32BitVTableOffsetABI)
421 llvm_unreachable("AppleARM64 method ptr abi NYI");
422
423 assert(!cir::MissingFeatures::emitCFICheck());
424 assert(!cir::MissingFeatures::emitVFEInfo());
425 assert(!cir::MissingFeatures::emitWPDInfo());
426
427 // Get the pointer to the target virtual function from the entry at that
428 // offset of the vtable.
429 mlir::Value fnPtr = cir::VTableGetVirtualFnByOffsetOp::create(
430 builder&: b, location: loc, result: calleePtrTy, vptr: vtablePtr, offset: vtableOffset);
431
432 cir::YieldOp::create(builder&: b, location: loc, args: fnPtr);
433 assert(!cir::MissingFeatures::emitCFICheck());
434 };
435
436 callee = cir::TernaryOp::create(
437 builder&: locBuilder, cond: isVirtual, /*thenBuilder=*/trueBuilder: buildVirtualCallee,
438 /*elseBuilder=*/
439 falseBuilder: [&](mlir::OpBuilder &b, mlir::Location loc) {
440 auto fnPtr = cir::CastOp::create(builder&: b, location: loc, result: calleePtrTy,
441 kind: cir::CastKind::int_to_ptr,
442 src: methodPtrField);
443 cir::YieldOp::create(builder&: b, location: loc, args: fnPtr.getResult());
444 })
445 .getResult();
446}
447
448static mlir::Value lowerDataMemberCast(mlir::Operation *op,
449 mlir::Value loweredSrc,
450 std::int64_t offset,
451 bool isDerivedToBase,
452 mlir::OpBuilder &builder) {
453 if (offset == 0)
454 return loweredSrc;
455 mlir::Location loc = op->getLoc();
456 mlir::Type ty = loweredSrc.getType();
457
458 auto getConstantInt = [&](int64_t value) -> cir::ConstantOp {
459 return cir::ConstantOp::create(builder, location: loc, value: cir::IntAttr::get(type: ty, value));
460 };
461
462 cir::ConstantOp nullValue = getConstantInt(-1);
463 auto isNull = cir::CmpOp::create(builder, location: loc, kind: cir::CmpOpKind::eq, lhs: loweredSrc,
464 rhs: nullValue);
465
466 cir::ConstantOp offsetValue = getConstantInt(offset);
467 mlir::Value adjustedPtr;
468 if (isDerivedToBase)
469 adjustedPtr = cir::SubOp::create(builder, location: loc, lhs: loweredSrc, rhs: offsetValue,
470 overflow_flags: cir::OverflowFlags::NoSignedWrap);
471 else
472 adjustedPtr = cir::AddOp::create(builder, location: loc, lhs: loweredSrc, rhs: offsetValue,
473 overflow_flags: cir::OverflowFlags::NoSignedWrap);
474
475 return cir::SelectOp::create(builder, location: loc, result: ty, condition: isNull, true_value: loweredSrc,
476 false_value: adjustedPtr);
477}
478
479mlir::Value
480LowerItaniumCXXABI::lowerBaseDataMember(cir::BaseDataMemberOp op,
481 mlir::Value loweredSrc,
482 mlir::OpBuilder &builder) const {
483 return lowerDataMemberCast(op, loweredSrc, offset: op.getOffset().getSExtValue(),
484 /*isDerivedToBase=*/true, builder);
485}
486
487mlir::Value
488LowerItaniumCXXABI::lowerDerivedDataMember(cir::DerivedDataMemberOp op,
489 mlir::Value loweredSrc,
490 mlir::OpBuilder &builder) const {
491 return lowerDataMemberCast(op, loweredSrc, offset: op.getOffset().getSExtValue(),
492 /*isDerivedToBase=*/false, builder);
493}
494
495static mlir::Value lowerMethodCast(mlir::Operation *op, mlir::Value loweredSrc,
496 std::int64_t offset, bool isDerivedToBase,
497 bool useARMMethodPtrABI,
498 LowerModule &lowerMod,
499 mlir::OpBuilder &builder) {
500 if (offset == 0)
501 return loweredSrc;
502
503 // The this-adjustment is left-shifted by 1 on ARM, since the low bit of the
504 // adjustment field is used to encode whether the member function is virtual.
505 if (useARMMethodPtrABI)
506 offset <<= 1;
507
508 cir::IntType ptrdiffCIRTy = getPtrDiffCIRTy(lm&: lowerMod);
509 auto adjField = cir::ExtractMemberOp::create(builder, location: op->getLoc(),
510 result: ptrdiffCIRTy, record: loweredSrc, index: 1);
511
512 auto offsetValue = cir::ConstantOp::create(
513 builder, location: op->getLoc(), value: cir::IntAttr::get(type: ptrdiffCIRTy, value: offset));
514 mlir::Value adjustedAdjField;
515 if (isDerivedToBase)
516 adjustedAdjField =
517 cir::SubOp::create(builder, location: op->getLoc(), lhs: adjField, rhs: offsetValue,
518 overflow_flags: cir::OverflowFlags::NoSignedWrap);
519 else
520 adjustedAdjField =
521 cir::AddOp::create(builder, location: op->getLoc(), lhs: adjField, rhs: offsetValue,
522 overflow_flags: cir::OverflowFlags::NoSignedWrap);
523
524 return cir::InsertMemberOp::create(builder, location: op->getLoc(), record: loweredSrc, index: 1,
525 value: adjustedAdjField);
526}
527
528mlir::Value
529LowerItaniumCXXABI::lowerBaseMethod(cir::BaseMethodOp op,
530 mlir::Value loweredSrc,
531 mlir::OpBuilder &builder) const {
532 return lowerMethodCast(op, loweredSrc, offset: op.getOffset().getSExtValue(),
533 /*isDerivedToBase=*/true, useARMMethodPtrABI, lowerMod&: lm,
534 builder);
535}
536
537mlir::Value
538LowerItaniumCXXABI::lowerDerivedMethod(cir::DerivedMethodOp op,
539 mlir::Value loweredSrc,
540 mlir::OpBuilder &builder) const {
541 return lowerMethodCast(op, loweredSrc, offset: op.getOffset().getSExtValue(),
542 /*isDerivedToBase=*/false, useARMMethodPtrABI, lowerMod&: lm,
543 builder);
544}
545
546mlir::Value
547LowerItaniumCXXABI::lowerDataMemberCmp(cir::CmpOp op, mlir::Value loweredLhs,
548 mlir::Value loweredRhs,
549 mlir::OpBuilder &builder) const {
550 return cir::CmpOp::create(builder, location: op.getLoc(), kind: op.getKind(), lhs: loweredLhs,
551 rhs: loweredRhs);
552}
553
554mlir::Value LowerItaniumCXXABI::lowerMethodCmp(cir::CmpOp op,
555 mlir::Value loweredLhs,
556 mlir::Value loweredRhs,
557 mlir::OpBuilder &builder) const {
558 assert(op.getKind() == cir::CmpOpKind::eq ||
559 op.getKind() == cir::CmpOpKind::ne);
560
561 mlir::ImplicitLocOpBuilder locBuilder(op.getLoc(), builder);
562 cir::IntType ptrdiffCIRTy = getPtrDiffCIRTy(lm);
563 mlir::Value ptrdiffZero =
564 cir::ConstantOp::create(builder&: locBuilder, value: cir::IntAttr::get(type: ptrdiffCIRTy, value: 0));
565
566 mlir::Value lhsPtrField =
567 cir::ExtractMemberOp::create(builder&: locBuilder, result: ptrdiffCIRTy, record: loweredLhs, index: 0);
568 mlir::Value rhsPtrField =
569 cir::ExtractMemberOp::create(builder&: locBuilder, result: ptrdiffCIRTy, record: loweredRhs, index: 0);
570 mlir::Value ptrCmp =
571 cir::CmpOp::create(builder&: locBuilder, kind: op.getKind(), lhs: lhsPtrField, rhs: rhsPtrField);
572 mlir::Value ptrCmpToNull =
573 cir::CmpOp::create(builder&: locBuilder, kind: op.getKind(), lhs: lhsPtrField, rhs: ptrdiffZero);
574
575 mlir::Value lhsAdjField =
576 cir::ExtractMemberOp::create(builder&: locBuilder, result: ptrdiffCIRTy, record: loweredLhs, index: 1);
577 mlir::Value rhsAdjField =
578 cir::ExtractMemberOp::create(builder&: locBuilder, result: ptrdiffCIRTy, record: loweredRhs, index: 1);
579 mlir::Value adjCmp =
580 cir::CmpOp::create(builder&: locBuilder, kind: op.getKind(), lhs: lhsAdjField, rhs: rhsAdjField);
581
582 auto create_and = [&](mlir::Value lhs, mlir::Value rhs) {
583 return cir::AndOp::create(builder&: locBuilder, result: lhs.getType(), lhs, rhs);
584 };
585 auto create_or = [&](mlir::Value lhs, mlir::Value rhs) {
586 return cir::OrOp::create(builder&: locBuilder, result: lhs.getType(), lhs, rhs);
587 };
588
589 // Null member function pointers on ARM clear the low bit of Adj,
590 // so the zero condition has to check that neither low bit is set.
591 if (useARMMethodPtrABI) {
592 mlir::Value one =
593 cir::ConstantOp::create(builder&: locBuilder, value: cir::IntAttr::get(type: ptrdiffCIRTy, value: 1));
594
595 // The low bit of the adjustment field is used to encode whether the member
596 // function is virtual, but the ARM ABI specifies that for null pointers
597 // this bit must be clear. Therefore, to test whether the member pointer is
598 // null, we need to check that bit.
599 //
600 // If we are performing an equality check, ptrCmpToNull indicates that both
601 // pointers are null (if they are equal -- we only actually test lhs).
602 // If we are performing an inequality check, ptrCmpToNull indicates that
603 // one of the pointers is not null.
604 //
605 // To apply the ARM-specific logic, if either virtual bit is set, they
606 // cannot both be null (equality case -- ptrCmpToNull &= orAdjAnd1CmpZero),
607 // and if either virtual bit is set, one of the pointers is not null
608 // (inequality case -- ptrCmpToNull |= orAdjAnd1CmpZero).
609 mlir::Value orAdj = create_or(lhsAdjField, rhsAdjField);
610 mlir::Value orAdjAnd1 = create_and(orAdj, one);
611 mlir::Value orAdjAnd1CmpZero =
612 cir::CmpOp::create(builder&: locBuilder, kind: op.getKind(), lhs: orAdjAnd1, rhs: ptrdiffZero);
613
614 if (op.getKind() == cir::CmpOpKind::eq)
615 ptrCmpToNull = create_and(ptrCmpToNull, orAdjAnd1CmpZero);
616 else
617 ptrCmpToNull = create_or(ptrCmpToNull, orAdjAnd1CmpZero);
618 }
619
620 mlir::Value result;
621 if (op.getKind() == cir::CmpOpKind::eq) {
622 // (lhs.ptr == null || lhs.adj == rhs.adj) && lhs.ptr == rhs.ptr
623 result = create_and(ptrCmp, create_or(ptrCmpToNull, adjCmp));
624 } else {
625 // lhs.ptr == rhs.ptr && (lhs.ptr == null || lhs.adj == rhs.adj)
626 result = create_or(ptrCmp, create_and(ptrCmpToNull, adjCmp));
627 }
628
629 return result;
630}
631
632mlir::Value LowerItaniumCXXABI::lowerDataMemberBitcast(
633 cir::CastOp op, mlir::Type loweredDstTy, mlir::Value loweredSrc,
634 mlir::OpBuilder &builder) const {
635 if (loweredSrc.getType() == loweredDstTy)
636 return loweredSrc;
637
638 return cir::CastOp::create(builder, location: op.getLoc(), result: loweredDstTy,
639 kind: cir::CastKind::bitcast, src: loweredSrc);
640}
641
642mlir::Value LowerItaniumCXXABI::lowerDataMemberToBoolCast(
643 cir::CastOp op, mlir::Value loweredSrc, mlir::OpBuilder &builder) const {
644 // Itanium C++ ABI 2.3:
645 // A NULL pointer is represented as -1.
646 auto nullAttr = cir::IntAttr::get(type: getPtrDiffCIRTy(lm), value: -1);
647 auto nullValue = cir::ConstantOp::create(builder, location: op.getLoc(), value: nullAttr);
648 return cir::CmpOp::create(builder, location: op.getLoc(), kind: cir::CmpOpKind::ne,
649 lhs: loweredSrc, rhs: nullValue);
650}
651
652mlir::Value
653LowerItaniumCXXABI::lowerMethodBitcast(cir::CastOp op, mlir::Type loweredDstTy,
654 mlir::Value loweredSrc,
655 mlir::OpBuilder &builder) const {
656 if (loweredSrc.getType() == loweredDstTy)
657 return loweredSrc;
658
659 return loweredSrc;
660}
661
662mlir::Value LowerItaniumCXXABI::lowerMethodToBoolCast(
663 cir::CastOp op, mlir::Value loweredSrc, mlir::OpBuilder &builder) const {
664 mlir::ImplicitLocOpBuilder locBuilder(op.getLoc(), builder);
665
666 // Itanium C++ ABI 2.3.2:
667 //
668 // In the standard representation, a null member function pointer is
669 // represented with ptr set to a null pointer. The value of adj is
670 // unspecified for null member function pointers.
671 cir::IntType ptrdiffCIRTy = getPtrDiffCIRTy(lm);
672 mlir::Value ptrdiffZero =
673 cir::ConstantOp::create(builder&: locBuilder, value: cir::IntAttr::get(type: ptrdiffCIRTy, value: 0));
674 mlir::Value ptrField =
675 cir::ExtractMemberOp::create(builder&: locBuilder, result: ptrdiffCIRTy, record: loweredSrc, index: 0);
676
677 mlir::Value result =
678 cir::CmpOp::create(builder&: locBuilder, kind: cir::CmpOpKind::ne, lhs: ptrField, rhs: ptrdiffZero);
679
680 // On ARM, a member function pointer is also non-null if the low bit of 'adj'
681 // (the virtual bit) is set.
682 if (useARMMethodPtrABI) {
683 mlir::Value one =
684 cir::ConstantOp::create(builder&: locBuilder, value: cir::IntAttr::get(type: ptrdiffCIRTy, value: 1));
685 mlir::Value adj =
686 cir::ExtractMemberOp::create(builder&: locBuilder, result: ptrdiffCIRTy, record: loweredSrc, index: 1);
687 mlir::Value virtualBit =
688 cir::AndOp::create(builder&: locBuilder, result: ptrdiffCIRTy, lhs: adj, rhs: one);
689 mlir::Value isVirtual = cir::CmpOp::create(builder&: locBuilder, kind: cir::CmpOpKind::ne,
690 lhs: virtualBit, rhs: ptrdiffZero);
691 result = cir::OrOp::create(builder&: locBuilder, lhs: result, rhs: isVirtual);
692 }
693
694 return result;
695}
696
697static void buildBadCastCall(mlir::OpBuilder &builder, mlir::Location loc,
698 mlir::FlatSymbolRefAttr badCastFuncRef) {
699 // TODO(cir): set the runtime calling convention to this call.
700 assert(!cir::MissingFeatures::opFuncCallingConv());
701 auto callOp = cir::CallOp::create(builder, location: loc, callee: badCastFuncRef,
702 /*resType=*/cir::VoidType(),
703 /*operands=*/mlir::ValueRange{});
704 callOp->setAttr(name: cir::CIRDialect::getNoReturnAttrName(),
705 value: builder.getUnitAttr());
706
707 cir::UnreachableOp::create(builder, location: loc);
708 builder.clearInsertionPoint();
709}
710
711static mlir::Value buildDynamicCastAfterNullCheck(cir::DynamicCastOp op,
712 mlir::OpBuilder &builder) {
713 mlir::Location loc = op->getLoc();
714 mlir::Value srcValue = op.getSrc();
715 cir::DynamicCastInfoAttr castInfo = op.getInfo().value();
716
717 // TODO(cir): consider address space
718 assert(!cir::MissingFeatures::addressSpace());
719
720 auto voidPtrTy =
721 cir::PointerType::get(pointee: cir::VoidType::get(ctx: builder.getContext()));
722
723 mlir::Value srcPtr = cir::CastOp::create(builder, location: loc, result: voidPtrTy,
724 kind: cir::CastKind::bitcast, src: srcValue);
725 mlir::Value srcRtti =
726 cir::ConstantOp::create(builder, location: loc, value: castInfo.getSrcRtti());
727 mlir::Value destRtti =
728 cir::ConstantOp::create(builder, location: loc, value: castInfo.getDestRtti());
729 mlir::Value offsetHint =
730 cir::ConstantOp::create(builder, location: loc, value: castInfo.getOffsetHint());
731
732 mlir::FlatSymbolRefAttr dynCastFuncRef = castInfo.getRuntimeFunc();
733 mlir::Value dynCastFuncArgs[4] = {srcPtr, srcRtti, destRtti, offsetHint};
734
735 // TODO(cir): set the runtime calling convention to this call.
736 assert(!cir::MissingFeatures::opFuncCallingConv());
737
738 mlir::Value castedPtr = cir::CallOp::create(builder, location: loc, callee: dynCastFuncRef,
739 resType: voidPtrTy, operands: dynCastFuncArgs)
740 .getResult();
741
742 assert(mlir::isa<cir::PointerType>(castedPtr.getType()) &&
743 "the return value of __dynamic_cast should be a ptr");
744
745 /// C++ [expr.dynamic.cast]p9:
746 /// A failed cast to reference type throws std::bad_cast
747 if (op.isRefCast()) {
748 // Emit a cir.if that checks the casted value.
749 mlir::Value null = cir::ConstantOp::create(
750 builder, location: loc,
751 value: cir::ConstPtrAttr::get(type: castedPtr.getType(),
752 value: builder.getI64IntegerAttr(value: 0)));
753 mlir::Value castedPtrIsNull =
754 cir::CmpOp::create(builder, location: loc, kind: cir::CmpOpKind::eq, lhs: castedPtr, rhs: null);
755 cir::IfOp::create(builder, location: loc, cond: castedPtrIsNull, withElseRegion: false,
756 thenBuilder: [&](mlir::OpBuilder &, mlir::Location) {
757 buildBadCastCall(builder, loc,
758 badCastFuncRef: castInfo.getBadCastFunc());
759 });
760 }
761
762 // Note that castedPtr is a void*. Cast it to a pointer to the destination
763 // type before return.
764 return cir::CastOp::create(builder, location: loc, result: op.getType(), kind: cir::CastKind::bitcast,
765 src: castedPtr);
766}
767
768static mlir::Value buildDynamicCastToVoidAfterNullCheck(
769 cir::DynamicCastOp op, cir::LowerModule &lm, mlir::OpBuilder &builder) {
770 mlir::Location loc = op.getLoc();
771
772 // TODO(cir): consider address space of the vtable pointer.
773 assert(!cir::MissingFeatures::addressSpace());
774
775 // Access vtable to get the offset from the given object to its containing
776 // complete object.
777 uint64_t vptrAlign = llvm::divideCeil(
778 Numerator: lm.getTarget().getPointerAlign(AddrSpace: clang::LangAS::Default), Denominator: 8);
779 auto vptrPtr = cir::VTableGetVPtrOp::create(builder, location: loc, src: op.getSrc());
780 mlir::Value vptr =
781 cir::LoadOp::create(builder, location: loc, addr: vptrPtr,
782 /*isDeref=*/false,
783 /*is_volatile=*/false,
784 /*isNontemporal=*/is_nontemporal: false,
785 /*alignment=*/builder.getI64IntegerAttr(value: vptrAlign),
786 /*sync_scope=*/cir::SyncScopeKindAttr(),
787 /*mem_order=*/cir::MemOrderAttr(),
788 /*invariant=*/false);
789 // The Itanium ABI always places an offset to the complete object at entry -2
790 // in the vtable.
791 mlir::Value offsetToTop = cir::VTableGetOffsetOp::create(
792 builder, location: loc, result: getPtrDiffCIRTy(lm), vptr, /*index=*/-2);
793
794 // Add the offset to the given pointer to get the cast result.
795 // Cast the input pointer to a uint8_t* to allow pointer arithmetic.
796 mlir::Type u8PtrTy =
797 cir::PointerType::get(pointee: cir::IntType::get(context: builder.getContext(), /*width=*/8,
798 /*isSigned=*/is_signed: false),
799 addrSpace: op.getSrc().getType().getAddrSpace());
800 mlir::Value srcBytePtr = cir::CastOp::create(
801 builder, location: loc, result: u8PtrTy, kind: cir::CastKind::bitcast, src: op.getSrc());
802 auto dstBytePtr =
803 cir::PtrStrideOp::create(builder, location: loc, result: u8PtrTy, base: srcBytePtr, stride: offsetToTop);
804 // Cast the result to a void*.
805 return cir::CastOp::create(builder, location: loc, result: op.getType(), kind: cir::CastKind::bitcast,
806 src: dstBytePtr);
807}
808
809mlir::Value
810LowerItaniumCXXABI::lowerDynamicCast(cir::DynamicCastOp op,
811 mlir::OpBuilder &builder) const {
812 mlir::Location loc = op->getLoc();
813 mlir::Value srcValue = op.getSrc();
814
815 assert(!cir::MissingFeatures::emitTypeCheck());
816
817 if (op.isRefCast())
818 return buildDynamicCastAfterNullCheck(op, builder);
819
820 mlir::Value srcValueIsNotNull = cir::CastOp::create(
821 builder, location: loc, result: cir::BoolType::get(ctx: builder.getContext()),
822 kind: cir::CastKind::ptr_to_bool, src: srcValue);
823 return cir::TernaryOp::create(
824 builder, location: loc, cond: srcValueIsNotNull,
825 trueBuilder: [&](mlir::OpBuilder &, mlir::Location) {
826 mlir::Value castedValue =
827 op.isCastToVoid()
828 ? buildDynamicCastToVoidAfterNullCheck(op, lm, builder)
829 : buildDynamicCastAfterNullCheck(op, builder);
830 cir::YieldOp::create(builder, location: loc, args: castedValue);
831 },
832 falseBuilder: [&](mlir::OpBuilder &, mlir::Location) {
833 mlir::Value null = cir::ConstantOp::create(
834 builder, location: loc,
835 value: cir::ConstPtrAttr::get(type: op.getType(),
836 value: builder.getI64IntegerAttr(value: 0)));
837 cir::YieldOp::create(builder, location: loc, args: null);
838 })
839 .getResult();
840}
841mlir::Value
842LowerItaniumCXXABI::lowerVTableGetTypeInfo(cir::VTableGetTypeInfoOp op,
843 mlir::OpBuilder &builder) const {
844 // The Itanium ABI always places the pointer to the type_info at entry -1 in
845 // the vtable.
846 return cir::VTableGetEntryOp::create(builder, location: op->getLoc(), result: op.getType(),
847 vptr: op.getVptr(), /*index=*/-1);
848}
849
850clang::CharUnits LowerItaniumCXXABI::getArrayCookieSizeImpl(
851 clang::CharUnits elementAlign) const {
852 // The array cookie is a size_t; pad that up to the element alignment.
853 // The cookie is actually right-justified in that space.
854 clang::CharUnits sizeOfSizeT =
855 clang::CharUnits::fromQuantity(Quantity: getPtrSizeInBits() / 8);
856 return std::max(a: sizeOfSizeT, b: elementAlign);
857}
858
859mlir::Value LowerItaniumCXXABI::readArrayCookieImpl(
860 mlir::Location loc, mlir::Value allocPtr, clang::CharUnits cookieSize,
861 clang::CharUnits cookieAlignment, const mlir::DataLayout &dataLayout,
862 CIRBaseBuilderTy &builder) const {
863 unsigned ptrSizeInBits = getPtrSizeInBits();
864 mlir::ptr::MemorySpaceAttrInterface addrSpace =
865 mlir::cast<cir::PointerType>(Val: allocPtr.getType()).getAddrSpace();
866 auto u8PtrTy = builder.getPointerTo(ty: builder.getUIntNTy(n: 8), as: addrSpace);
867 auto ptrDiffTy = builder.getSIntNTy(n: ptrSizeInBits);
868 auto sizeTy = builder.getUIntNTy(n: ptrSizeInBits);
869
870 // The element count is right-justified in the cookie.
871 clang::CharUnits sizeOfSizeT =
872 clang::CharUnits::fromQuantity(Quantity: ptrSizeInBits / 8);
873 clang::CharUnits countOffset = cookieSize - sizeOfSizeT;
874
875 mlir::Value countBytePtr = allocPtr;
876 clang::CharUnits countAlignment = cookieAlignment;
877 if (!countOffset.isZero()) {
878 mlir::Value offsetVal = cir::ConstantOp::create(
879 builder, location: loc, value: cir::IntAttr::get(type: ptrDiffTy, value: countOffset.getQuantity()));
880 countBytePtr =
881 cir::PtrStrideOp::create(builder, location: loc, result: u8PtrTy, base: allocPtr, stride: offsetVal);
882 countAlignment = cookieAlignment.alignmentAtOffset(offset: countOffset);
883 }
884
885 auto countPtrTy = cir::PointerType::get(pointee: sizeTy, addrSpace);
886 mlir::Value countPtr = cir::CastOp::create(
887 builder, location: loc, result: countPtrTy, kind: cir::CastKind::bitcast, src: countBytePtr);
888 return cir::LoadOp::create(
889 builder, location: loc, addr: countPtr, /*isDeref=*/false, /*isVolatile=*/is_volatile: false,
890 /*isNontemporal=*/is_nontemporal: false,
891 alignment: builder.getI64IntegerAttr(value: countAlignment.getQuantity()),
892 sync_scope: cir::SyncScopeKindAttr(), mem_order: cir::MemOrderAttr(),
893 /*invariant=*/false);
894}
895
896} // namespace cir
897