1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Expr nodes as CIR code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "Address.h"
14#include "CIRGenConstantEmitter.h"
15#include "CIRGenFunction.h"
16#include "CIRGenModule.h"
17#include "CIRGenValue.h"
18#include "TargetInfo.h"
19#include "mlir/Dialect/Ptr/IR/MemorySpaceInterfaces.h"
20#include "mlir/IR/BuiltinAttributes.h"
21#include "mlir/IR/Value.h"
22#include "clang/AST/ASTLambda.h"
23#include "clang/AST/Attr.h"
24#include "clang/AST/CharUnits.h"
25#include "clang/AST/Decl.h"
26#include "clang/AST/Expr.h"
27#include "clang/AST/ExprCXX.h"
28#include "clang/AST/MatrixUtils.h"
29#include "clang/Basic/AddressSpaces.h"
30#include "clang/Basic/TargetInfo.h"
31#include "clang/CIR/Dialect/IR/CIRAttrs.h"
32#include "clang/CIR/Dialect/IR/CIRDialect.h"
33#include "clang/CIR/Dialect/IR/CIRTypes.h"
34#include "clang/CIR/MissingFeatures.h"
35#include "clang/CodeGenUtils/ExprUtils.h"
36#include "clang/CodeGenUtils/RecordLayoutUtils.h"
37#include "clang/CodeGenUtils/TargetUtils.h"
38#include <optional>
39
40using namespace clang;
41using namespace clang::CIRGen;
42using namespace cir;
43
44/// Get the address of a zero-sized field within a record. Zero-sized fields
45/// (e.g. empty bases with [[no_unique_address]]) don't appear in the CIR
46/// record layout, so we compute their address using the ASTContext field
47/// offset and byte-level pointer arithmetic instead of cir.get_member.
48static Address emitAddrOfZeroSizeField(CIRGenFunction &cgf, Address base,
49 const FieldDecl *field) {
50 CIRGenBuilderTy &builder = cgf.getBuilder();
51 CharUnits offset = cgf.getContext().toCharUnitsFromBits(
52 BitSize: cgf.getContext().getFieldOffset(FD: field));
53 mlir::Type fieldType = cgf.convertType(t: field->getType());
54
55 if (offset.isZero()) {
56 return Address(builder.createPtrBitcast(src: base.getPointer(), newPointeeTy: fieldType),
57 base.getAlignment());
58 }
59
60 // Cast to byte pointer, stride by the field offset, then cast to the
61 // field pointer type (CIR pointers are typed, so we need explicit casts
62 // unlike OG's opaque-pointer GEP).
63 mlir::Location loc = cgf.getLoc(srcLoc: field->getLocation());
64 mlir::Value addr =
65 builder.createPtrBitcast(src: base.getPointer(), newPointeeTy: builder.getUInt8Ty());
66 addr = builder.createPtrStride(loc, base: addr,
67 stride: builder.getUInt64(c: offset.getQuantity(), loc));
68 addr = builder.createPtrBitcast(src: addr, newPointeeTy: fieldType);
69 return Address(addr, base.getAlignment().alignmentAtOffset(offset));
70}
71
72Address CIRGenFunction::emitAddrOfFieldStorage(Address base,
73 const FieldDecl *field,
74 llvm::StringRef fieldName,
75 unsigned fieldIndex) {
76 // Retrieve layout information for both type resolution and alignment.
77 const RecordDecl *rec = field->getParent();
78 const CIRGenRecordLayout &layout = cgm.getTypes().getCIRGenRecordLayout(rd: rec);
79
80 bool addressedByFieldIndex =
81 field->isPotentiallyOverlapping()
82 ? layout.hasCIRField(fd: field)
83 : !CodeGenUtils::isEmptyFieldForLayout(Ctx: getContext(), FD: field);
84 if (!addressedByFieldIndex)
85 return emitAddrOfZeroSizeField(cgf&: *this, base, field);
86
87 mlir::Location loc = getLoc(srcLoc: field->getLocation());
88 unsigned idx = layout.getCIRFieldNo(fd: field);
89
90 // For potentially-overlapping fields (e.g. [[no_unique_address]]), the
91 // record stores the base subobject type (without tail padding) rather than
92 // the complete object type. Use the record's member type for get_member,
93 // then bitcast to the complete type for downstream use.
94 //
95 // For unions, all fields map to index 0, so we use the field's declared type
96 // directly instead of looking up the member type from the layout.
97 mlir::Type fieldType = convertType(t: field->getType());
98 // A member lives in the same address space as its record.
99 mlir::ptr::MemorySpaceAttrInterface addrSpace = base.getAddressSpace();
100 auto fieldPtr = cir::PointerType::get(pointee: fieldType, addrSpace);
101 bool needsBitcast = false;
102
103 if (!rec->isUnion() && field->isPotentiallyOverlapping()) {
104 mlir::Type memberType = layout.getCIRType().getMembers()[idx];
105 fieldPtr = cir::PointerType::get(pointee: memberType, addrSpace);
106 needsBitcast = true;
107 }
108
109 // For most cases fieldName is the same as field->getName() but for lambdas,
110 // which do not currently carry the name, so it can be passed down from the
111 // CaptureStmt.
112 mlir::Value addr = builder.createGetMember(loc, resultTy: fieldPtr, base: base.getPointer(),
113 name: fieldName, index: fieldIndex);
114
115 // If the field is potentially overlapping, the record member uses the base
116 // subobject type. Cast to the complete object pointer type expected by
117 // callers (analogous to OG's opaque pointer behavior).
118 if (needsBitcast)
119 addr = builder.createPtrBitcast(src: addr, newPointeeTy: fieldType);
120
121 CharUnits offset = CharUnits::fromQuantity(
122 Quantity: layout.getCIRType().getElementOffset(dataLayout: cgm.getDataLayout().layout, idx));
123 return Address(addr, base.getAlignment().alignmentAtOffset(offset));
124}
125
126/// Given an expression of pointer type, try to
127/// derive a more accurate bound on the alignment of the pointer.
128Address CIRGenFunction::emitPointerWithAlignment(const Expr *expr,
129 LValueBaseInfo *baseInfo) {
130 // We allow this with ObjC object pointers because of fragile ABIs.
131 assert(expr->getType()->isPointerType() ||
132 expr->getType()->isObjCObjectPointerType());
133 expr = expr->IgnoreParens();
134
135 // Casts:
136 if (auto const *ce = dyn_cast<CastExpr>(Val: expr)) {
137 if (const auto *ece = dyn_cast<ExplicitCastExpr>(Val: ce))
138 cgm.emitExplicitCastExprType(e: ece);
139
140 switch (ce->getCastKind()) {
141 // Non-converting casts (but not C's implicit conversion from void*).
142 case CK_BitCast:
143 case CK_NoOp:
144 case CK_AddressSpaceConversion: {
145 if (const auto *ptrTy =
146 ce->getSubExpr()->getType()->getAs<PointerType>()) {
147 if (ptrTy->getPointeeType()->isVoidType())
148 break;
149
150 LValueBaseInfo innerBaseInfo;
151 assert(!cir::MissingFeatures::opTBAA());
152 Address addr =
153 emitPointerWithAlignment(expr: ce->getSubExpr(), baseInfo: &innerBaseInfo);
154 if (baseInfo)
155 *baseInfo = innerBaseInfo;
156
157 if (isa<ExplicitCastExpr>(Val: ce)) {
158 LValueBaseInfo targetTypeBaseInfo;
159
160 const QualType pointeeType = expr->getType()->getPointeeType();
161 const CharUnits align =
162 cgm.getNaturalTypeAlignment(t: pointeeType, baseInfo: &targetTypeBaseInfo);
163
164 // If the source l-value is opaque, honor the alignment of the
165 // casted-to type.
166 if (innerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) {
167 if (baseInfo)
168 baseInfo->mergeForCast(info: targetTypeBaseInfo);
169 addr = Address(addr.getPointer(), addr.getElementType(), align);
170 }
171 }
172
173 assert(!cir::MissingFeatures::sanitizers());
174
175 const mlir::Type eltTy =
176 convertTypeForMem(t: expr->getType()->getPointeeType());
177 addr = getBuilder().createElementBitCast(loc: getLoc(srcLoc: expr->getSourceRange()),
178 addr, destType: eltTy);
179 if (ce->getCastKind() == CK_AddressSpaceConversion)
180 addr = addr.withPointer(newPtr: performAddrSpaceCast(
181 v: addr.getPointer(), destTy: convertType(t: expr->getType())));
182
183 return addr;
184 }
185 break;
186 }
187
188 // Array-to-pointer decay. TODO(cir): BaseInfo and TBAAInfo.
189 case CK_ArrayToPointerDecay:
190 return emitArrayToPointerDecay(e: ce->getSubExpr(), baseInfo);
191
192 case CK_UncheckedDerivedToBase:
193 case CK_DerivedToBase: {
194 assert(!cir::MissingFeatures::opTBAA());
195 assert(!cir::MissingFeatures::addressIsKnownNonNull());
196 Address addr = emitPointerWithAlignment(expr: ce->getSubExpr(), baseInfo);
197 const CXXRecordDecl *derived =
198 ce->getSubExpr()->getType()->getPointeeCXXRecordDecl();
199 return getAddressOfBaseClass(value: addr, derived, path: ce->path(),
200 nullCheckValue: shouldNullCheckClassCastValue(ce),
201 loc: ce->getExprLoc());
202 }
203
204 case CK_AnyPointerToBlockPointerCast:
205 case CK_BaseToDerived:
206 case CK_BaseToDerivedMemberPointer:
207 case CK_BlockPointerToObjCPointerCast:
208 case CK_BuiltinFnToFnPtr:
209 case CK_CPointerToObjCPointerCast:
210 case CK_DerivedToBaseMemberPointer:
211 case CK_Dynamic:
212 case CK_FunctionToPointerDecay:
213 case CK_IntegralToPointer:
214 case CK_LValueToRValue:
215 case CK_LValueToRValueBitCast:
216 case CK_NullToMemberPointer:
217 case CK_NullToPointer:
218 case CK_ReinterpretMemberPointer:
219 case CK_UserDefinedConversion:
220 // Common pointer conversions, nothing to do here.
221 // TODO: Is there any reason to treat base-to-derived conversions
222 // specially?
223 break;
224
225 case CK_ARCConsumeObject:
226 case CK_ARCExtendBlockObject:
227 case CK_ARCProduceObject:
228 case CK_ARCReclaimReturnedObject:
229 case CK_AtomicToNonAtomic:
230 case CK_BooleanToSignedIntegral:
231 case CK_ConstructorConversion:
232 case CK_CopyAndAutoreleaseBlockObject:
233 case CK_Dependent:
234 case CK_FixedPointCast:
235 case CK_FixedPointToBoolean:
236 case CK_FixedPointToFloating:
237 case CK_FixedPointToIntegral:
238 case CK_FloatingCast:
239 case CK_FloatingComplexCast:
240 case CK_FloatingComplexToBoolean:
241 case CK_FloatingComplexToIntegralComplex:
242 case CK_FloatingComplexToReal:
243 case CK_FloatingRealToComplex:
244 case CK_FloatingToBoolean:
245 case CK_FloatingToFixedPoint:
246 case CK_FloatingToIntegral:
247 case CK_HLSLAggregateSplatCast:
248 case CK_HLSLArrayRValue:
249 case CK_HLSLElementwiseCast:
250 case CK_HLSLVectorTruncation:
251 case CK_HLSLMatrixTruncation:
252 case CK_IntToOCLSampler:
253 case CK_IntegralCast:
254 case CK_IntegralComplexCast:
255 case CK_IntegralComplexToBoolean:
256 case CK_IntegralComplexToFloatingComplex:
257 case CK_IntegralComplexToReal:
258 case CK_IntegralRealToComplex:
259 case CK_IntegralToBoolean:
260 case CK_IntegralToFixedPoint:
261 case CK_IntegralToFloating:
262 case CK_LValueBitCast:
263 case CK_MatrixCast:
264 case CK_MemberPointerToBoolean:
265 case CK_NonAtomicToAtomic:
266 case CK_ObjCObjectLValueCast:
267 case CK_PointerToBoolean:
268 case CK_PointerToIntegral:
269 case CK_ToUnion:
270 case CK_ToVoid:
271 case CK_VectorSplat:
272 case CK_ZeroToOCLOpaqueType:
273 // Classic codegen has a default that does nothing. In CIR, we are issuing
274 // a diagnostic so we can examine casts that are reached here to be sure
275 // no action is needed. If nothing is needed, the cast can be moved to the
276 // group above that does nothing.
277 cgm.errorNYI(loc: ce->getSourceRange(),
278 feature: "unexpected cast for emitPointerWithAlignment: ",
279 name: ce->getCastKindName());
280 break;
281 }
282 }
283
284 // Unary &
285 if (const UnaryOperator *uo = dyn_cast<UnaryOperator>(Val: expr)) {
286 // TODO(cir): maybe we should use a CIR unary op for pointers here instead.
287 if (uo->getOpcode() == UO_AddrOf) {
288 LValue lv = emitLValue(e: uo->getSubExpr());
289 if (baseInfo)
290 *baseInfo = lv.getBaseInfo();
291 assert(!cir::MissingFeatures::opTBAA());
292 return lv.getAddress();
293 }
294 }
295
296 // std::addressof and variants.
297 if (auto const *call = dyn_cast<CallExpr>(Val: expr)) {
298 switch (call->getBuiltinCallee()) {
299 default:
300 break;
301 case Builtin::BIaddressof:
302 case Builtin::BI__addressof:
303 case Builtin::BI__builtin_addressof: {
304 LValue lv = emitLValue(e: call->getArg(Arg: 0));
305 if (baseInfo)
306 *baseInfo = lv.getBaseInfo();
307 assert(!cir::MissingFeatures::opTBAA());
308 return lv.getAddress();
309 }
310 }
311 }
312
313 // Otherwise, use the alignment of the type.
314 return makeNaturalAddressForPointer(
315 ptr: emitScalarExpr(e: expr), t: expr->getType()->getPointeeType(), alignment: CharUnits(),
316 /*forPointeeType=*/true, baseInfo);
317}
318
319void CIRGenFunction::emitStoreThroughExtVectorComponentLValue(RValue src,
320 LValue dst) {
321 auto getScalarSizeInBits = [&](mlir::Type ty) -> unsigned {
322 mlir::Type scalarTy = mlir::isa<cir::VectorType>(Val: ty)
323 ? mlir::cast<cir::VectorType>(Val&: ty).getElementType()
324 : ty;
325 const cir::CIRDataLayout &dl = cgm.getDataLayout();
326 return dl.getTypeSizeInBits(ty: scalarTy).getFixedValue();
327 };
328
329 mlir::Value srcVal = src.getValue();
330 Address dstAddr = dst.getExtVectorAddress();
331 if (getScalarSizeInBits(dstAddr.getElementType()) >
332 getScalarSizeInBits(srcVal.getType())) {
333 cgm.errorNYI(
334 loc: dst.getPointer().getLoc(),
335 feature: "emitStoreThroughExtVectorComponentLValue: dstTySize > srcTysize");
336 return;
337 }
338
339 if (getLangOpts().HLSL) {
340 cgm.errorNYI(loc: dst.getPointer().getLoc(),
341 feature: "emitStoreThroughExtVectorComponentLValue: HLSL");
342 return;
343 }
344
345 // This access turns into a read/modify/write of the vector. Load the input
346 // value now.
347 mlir::Location loc = dst.getExtVectorPointer().getLoc();
348
349 mlir::ArrayAttr elts = dst.getExtVectorElts();
350
351 mlir::Value vec = builder.createLoad(loc, addr: dstAddr, isVolatile: dst.isVolatile());
352 if (const auto *vecTy = dst.getType()->getAs<clang::VectorType>()) {
353 unsigned numSrcElts = vecTy->getNumElements();
354 unsigned numDstElts = cast<cir::VectorType>(Val: vec.getType()).getFixedSize();
355 if (numDstElts == numSrcElts) {
356 // Use shuffle vector is the src and destination are the same number of
357 // elements and restore the vector mask since it is on the side it will be
358 // stored.
359 SmallVector<int64_t> mask(numDstElts);
360 for (unsigned i = 0; i != numDstElts; ++i)
361 mask[getAccessedFieldNo(idx: i, elts)] = i;
362
363 vec = builder.createVecShuffle(loc, vec1: srcVal, mask);
364 } else if (numDstElts > numSrcElts) {
365 // Extended the source vector to the same length and then shuffle it
366 // into the destination.
367 // FIXME: since we're shuffling with undef, can we just use the indices
368 // into that? This could be simpler.
369 SmallVector<int64_t> extMask(numDstElts, -1);
370 std::iota(first: extMask.begin(), last: extMask.begin() + numSrcElts, value: 0);
371
372 mlir::Value extSrcVal = builder.createVecShuffle(loc, vec1: srcVal, mask: extMask);
373
374 // build identity
375 SmallVector<int64_t> mask(numDstElts);
376 std::iota(first: mask.begin(), last: mask.begin() + numDstElts, value: 0);
377
378 // When the vector size is odd and .odd or .hi is used, the last element
379 // of the Elts constant array will be one past the size of the vector.
380 // Ignore the last element here, if it is greater than the mask size.
381 if ((unsigned)getAccessedFieldNo(idx: numSrcElts - 1, elts) == mask.size())
382 numSrcElts--;
383
384 // modify when what gets shuffled in
385 for (unsigned i = 0; i != numSrcElts; ++i)
386 mask[getAccessedFieldNo(idx: i, elts)] = i + numDstElts;
387
388 vec = builder.createVecShuffle(loc, vec1: vec, vec2: extSrcVal, mask);
389 } else {
390 // We should never shorten the vector
391 llvm_unreachable("unexpected shorten vector length");
392 }
393 } else {
394 // If the Src is a scalar (not a vector), and the target is a vector it
395 // must be updating one element.
396 unsigned inIdx = getAccessedFieldNo(idx: 0, elts);
397 cir::ConstantOp elt = builder.getSInt64(c: inIdx, loc);
398 vec = cir::VecInsertOp::create(builder, location: loc, vec, value: srcVal, index: elt);
399 }
400
401 builder.createStore(loc, val: vec, dst: dst.getExtVectorAddress(),
402 isVolatile: dst.isVolatileQualified());
403}
404
405void CIRGenFunction::emitStoreThroughLValue(RValue src, LValue dst,
406 bool isInit) {
407 if (!dst.isSimple()) {
408 if (dst.isVectorElt()) {
409 // Read/modify/write the vector, inserting the new element
410 const mlir::Location loc = dst.getVectorPointer().getLoc();
411 const mlir::Value vector =
412 builder.createLoad(loc, addr: dst.getVectorAddress());
413 const mlir::Value newVector = cir::VecInsertOp::create(
414 builder, location: loc, vec: vector, value: src.getValue(), index: dst.getVectorIdx());
415 builder.createStore(loc, val: newVector, dst: dst.getVectorAddress());
416 return;
417 }
418
419 if (dst.isExtVectorElt())
420 return emitStoreThroughExtVectorComponentLValue(src, dst);
421
422 if (dst.isMatrixElt()) {
423 // Read/modify/write the matrix, inserting the new element.
424 const mlir::Location loc = dst.getPointer().getLoc();
425
426 // In Opt-level >1, we emit an assume here.
427 assert(!MissingFeatures::matrixTypeIndexAssumption());
428
429 const mlir::Value matrix =
430 builder.createLoad(loc, addr: dst.getMatrixAddress());
431 const mlir::Value newMatrix = builder.createMatrixInsert(
432 loc, value: src.getValue(), matrix, rowIdx: dst.getMatrixRowIdx(),
433 columnIdx: dst.getMatrixColumnIdx());
434 builder.createStore(loc, val: newMatrix, dst: dst.getMatrixAddress(),
435 isVolatile: dst.isVolatileQualified());
436 return;
437 }
438
439 if (dst.isMatrixRow()) {
440 cgm.errorNYI(feature: "emitStoreThroughLValue: !dst.isSimple() && isMatrixRow");
441 return;
442 }
443
444 assert(dst.isBitField() && "Unknown LValue type");
445 emitStoreThroughBitfieldLValue(src, dstresult: dst);
446 return;
447 }
448
449 assert(!cir::MissingFeatures::objCLifetime());
450 assert(!cir::MissingFeatures::opLoadStoreObjC());
451
452 assert(src.isScalar() && "Can't emit an aggregate store with this method");
453 emitStoreOfScalar(value: src.getValue(), lvalue: dst, isInit);
454}
455
456static std::optional<LValue> emitDeclTargetVarDeclLValue(CIRGenFunction &cgf,
457 const Expr *e,
458 const VarDecl *vd) {
459 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> res =
460 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD: vd);
461 if (!res)
462 return std::nullopt;
463 // MT_Local uses direct access.
464 if (*res == OMPDeclareTargetDeclAttr::MT_Local)
465 return std::nullopt;
466 // MT_To/MT_Enter without unified memory use direct access.
467 if ((*res == OMPDeclareTargetDeclAttr::MT_To ||
468 *res == OMPDeclareTargetDeclAttr::MT_Enter) &&
469 !cgf.cgm.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())
470 return std::nullopt;
471 cgf.cgm.errorNYI(e->getSourceRange(),
472 "emitDeclTargetVarDeclLValue: indirect access");
473 return std::nullopt;
474}
475
476static LValue emitGlobalVarDeclLValue(CIRGenFunction &cgf, const Expr *e,
477 const VarDecl *vd) {
478 QualType t = e->getType();
479
480 // In classic codegen, thread-locals get a wrapper function here. Rather than
481 // doing that, we instead treat this as a normal 'global', and leave it to
482 // lowerng-prepare to correctly generate the wrapper/etc.
483
484 // Check if the variable is marked as declare target with link clause in
485 // device codegen.
486 if (cgf.getLangOpts().OpenMPIsTargetDevice)
487 if (std::optional<LValue> lv = emitDeclTargetVarDeclLValue(cgf, e, vd))
488 return *lv;
489
490 // Traditional LLVM codegen handles thread local separately, CIR handles
491 // as part of getAddrOfGlobalVar.
492 mlir::Value v = cgf.cgm.getAddrOfGlobalVar(d: vd);
493
494 mlir::Type realVarTy = cgf.convertTypeForMem(t: vd->getType());
495 cir::PointerType realPtrTy = cir::PointerType::get(
496 pointee: realVarTy, addrSpace: mlir::cast<cir::PointerType>(Val: v.getType()).getAddrSpace());
497 if (realPtrTy != v.getType())
498 v = cgf.getBuilder().createBitcast(loc: v.getLoc(), src: v, newTy: realPtrTy);
499
500 CharUnits alignment = cgf.getContext().getDeclAlign(D: vd);
501 Address addr(v, realVarTy, alignment);
502 LValue lv;
503 if (vd->getType()->isReferenceType())
504 lv = cgf.emitLoadOfReferenceLValue(refAddr: addr, loc: cgf.getLoc(srcLoc: e->getSourceRange()),
505 refTy: vd->getType(), source: AlignmentSource::Decl);
506 else
507 lv = cgf.makeAddrLValue(addr, ty: t, source: AlignmentSource::Decl);
508 assert(!cir::MissingFeatures::setObjCGCLValueClass());
509 return lv;
510}
511
512void CIRGenFunction::emitStoreOfScalar(mlir::Value value, Address addr,
513 bool isVolatile, QualType ty,
514 LValueBaseInfo baseInfo, bool isInit,
515 bool isNontemporal) {
516
517 if (const auto *clangVecTy = ty->getAs<clang::VectorType>()) {
518 // Handle vectors of size 3 like size 4 for better performance.
519 // TODO(CIR): Use `ABIInfo::getOptimalVectorMemoryType` once it upstreamed
520 assert(!cir::MissingFeatures::cirgenABIInfo());
521 if (clangVecTy->getNumElements() == 3 && !getLangOpts().PreserveVec3Type)
522 cgm.errorNYI(loc: addr.getPointer().getLoc(),
523 feature: "emitStoreOfScalar Vec3 & PreserveVec3Type disabled");
524 }
525
526 value = emitToMemory(value, ty);
527
528 assert(!cir::MissingFeatures::opLoadStoreTbaa());
529 LValue atomicLValue = LValue::makeAddr(address: addr, t: ty, baseInfo);
530 if (ty->isAtomicType() ||
531 (!isInit && isLValueSuitableForInlineAtomic(lv: atomicLValue))) {
532 emitAtomicStore(rvalue: RValue::get(v: value), dest: atomicLValue, isInit);
533 return;
534 }
535
536 // Update the alloca with more info on initialization.
537 assert(addr.getPointer() && "expected pointer to exist");
538 cir::AllocaOp srcAlloca = addr.getUnderlyingAllocaOp();
539 if (currVarDecl && srcAlloca) {
540 const VarDecl *vd = currVarDecl;
541 assert(vd && "VarDecl expected");
542 if (vd->hasInit())
543 srcAlloca.setInitAttr(mlir::UnitAttr::get(context: &getMLIRContext()));
544 }
545
546 assert(currSrcLoc && "must pass in source location");
547 builder.createStore(loc: getLoc(srcLoc: *currSrcLoc), val: value, dst: addr, isVolatile,
548 isNontemporal);
549
550 assert(!cir::MissingFeatures::opTBAA());
551}
552
553mlir::Value CIRGenFunction::emitStoreThroughBitfieldLValue(RValue src,
554 LValue dst) {
555
556 const CIRGenBitFieldInfo &info = dst.getBitFieldInfo();
557 mlir::Type resLTy = convertTypeForMem(t: dst.getType());
558 Address ptr = dst.getBitFieldAddress();
559
560 bool useVoaltile =
561 cgm.getCodeGenOpts().AAPCSBitfieldWidth && dst.isVolatileQualified() &&
562 info.volatileStorageSize != 0 && CodeGenUtils::isAAPCS(TargetInfo: cgm.getTarget());
563
564 assert(currSrcLoc && "must pass in source location");
565
566 return builder.createSetBitfield(loc: getLoc(srcLoc: *currSrcLoc), resultType: resLTy, dstAddr: ptr,
567 storageType: ptr.getElementType(), src: src.getValue(), info,
568 isLvalueVolatile: dst.isVolatileQualified(), useVolatile: useVoaltile);
569}
570
571RValue CIRGenFunction::emitLoadOfBitfieldLValue(LValue lv, SourceLocation loc) {
572 const CIRGenBitFieldInfo &info = lv.getBitFieldInfo();
573
574 // Get the output type.
575 mlir::Type resLTy = convertType(t: lv.getType());
576 Address ptr = lv.getBitFieldAddress();
577
578 bool useVoaltile = lv.isVolatileQualified() && info.volatileOffset != 0 &&
579 CodeGenUtils::isAAPCS(TargetInfo: cgm.getTarget());
580
581 mlir::Value field =
582 builder.createGetBitfield(loc: getLoc(srcLoc: loc), resultType: resLTy, addr: ptr, storageType: ptr.getElementType(),
583 info, isLvalueVolatile: lv.isVolatile(), useVolatile: useVoaltile);
584 assert(!cir::MissingFeatures::opLoadEmitScalarRangeCheck() && "NYI");
585 return RValue::get(v: field);
586}
587
588Address CIRGenFunction::getAddrOfBitFieldStorage(LValue base,
589 const FieldDecl *field,
590 mlir::Type fieldType,
591 unsigned index) {
592 mlir::Location loc = getLoc(srcLoc: field->getLocation());
593 cir::PointerType fieldPtr =
594 cir::PointerType::get(pointee: fieldType, addrSpace: base.getAddress().getAddressSpace());
595 auto rec = cast<cir::RecordType>(Val: base.getAddress().getElementType());
596 cir::GetMemberOp sea = getBuilder().createGetMember(
597 loc, resultTy: fieldPtr, base: base.getPointer(), name: field->getName(),
598 index: mlir::isa<cir::UnionType>(Val: rec) ? field->getFieldIndex() : index);
599 CharUnits offset = CharUnits::fromQuantity(
600 Quantity: rec.getElementOffset(dataLayout: cgm.getDataLayout().layout, idx: index));
601 return Address(sea, base.getAlignment().alignmentAtOffset(offset));
602}
603
604LValue CIRGenFunction::emitLValueForBitField(LValue base,
605 const FieldDecl *field) {
606 LValueBaseInfo baseInfo = base.getBaseInfo();
607 const CIRGenRecordLayout &layout =
608 cgm.getTypes().getCIRGenRecordLayout(rd: field->getParent());
609 const CIRGenBitFieldInfo &info = layout.getBitFieldInfo(fd: field);
610
611 assert(!cir::MissingFeatures::preservedAccessIndexRegion());
612
613 unsigned idx = layout.getCIRFieldNo(fd: field);
614 Address addr = getAddrOfBitFieldStorage(base, field, fieldType: info.storageType, index: idx);
615
616 mlir::Location loc = getLoc(srcLoc: field->getLocation());
617 if (addr.getElementType() != info.storageType)
618 addr = builder.createElementBitCast(loc, addr, destType: info.storageType);
619
620 QualType fieldType =
621 field->getType().withCVRQualifiers(CVR: base.getVRQualifiers());
622 // TODO(cir): Support TBAA for bit fields.
623 assert(!cir::MissingFeatures::opTBAA());
624 LValueBaseInfo fieldBaseInfo(baseInfo.getAlignmentSource());
625 return LValue::makeBitfield(addr, info, type: fieldType, baseInfo: fieldBaseInfo);
626}
627
628LValue CIRGenFunction::emitLValueForField(LValue base, const FieldDecl *field) {
629 LValueBaseInfo baseInfo = base.getBaseInfo();
630
631 if (field->isBitField())
632 return emitLValueForBitField(base, field);
633
634 QualType fieldType = field->getType();
635 const RecordDecl *rec = field->getParent();
636 AlignmentSource baseAlignSource = baseInfo.getAlignmentSource();
637 LValueBaseInfo fieldBaseInfo(getFieldAlignmentSource(source: baseAlignSource));
638 assert(!cir::MissingFeatures::opTBAA());
639
640 Address addr = base.getAddress();
641 if (auto *classDecl = dyn_cast<CXXRecordDecl>(Val: rec)) {
642 if (cgm.getCodeGenOpts().StrictVTablePointers &&
643 classDecl->isDynamicClass()) {
644 cgm.errorNYI(field->getSourceRange(),
645 "emitLValueForField: strict vtable for dynamic class");
646 }
647 }
648
649 unsigned recordCVR = base.getVRQualifiers();
650
651 llvm::StringRef fieldName = field->getName();
652 unsigned fieldIndex;
653 if (cgm.lambdaFieldToName.count(Val: field))
654 fieldName = cgm.lambdaFieldToName[field];
655
656 // Fields with no CIR field index (e.g. a [[no_unique_address]] field that
657 // is empty for both layout and the ABI) aren't in the record layout, so
658 // handle them separately: use the base address directly with the right
659 // type instead of get_member.
660 const CIRGenRecordLayout &layout =
661 cgm.getTypes().getCIRGenRecordLayout(rd: field->getParent());
662 if (!rec->isUnion() && !layout.hasCIRField(fd: field)) {
663 addr = emitAddrOfZeroSizeField(cgf&: *this, base: addr, field);
664 LValue lv = makeAddrLValue(addr, ty: fieldType, baseInfo: fieldBaseInfo);
665 lv.getQuals().addCVRQualifiers(mask: recordCVR);
666 return lv;
667 }
668
669 if (rec->isUnion())
670 fieldIndex = field->getFieldIndex();
671 else
672 fieldIndex = layout.getCIRFieldNo(fd: field);
673
674 addr = emitAddrOfFieldStorage(base: addr, field, fieldName, fieldIndex);
675 assert(!cir::MissingFeatures::preservedAccessIndexRegion());
676
677 // If this is a reference field, load the reference right now.
678 if (fieldType->isReferenceType()) {
679 assert(!cir::MissingFeatures::opTBAA());
680 LValue refLVal = makeAddrLValue(addr, ty: fieldType, baseInfo: fieldBaseInfo);
681 if (recordCVR & Qualifiers::Volatile)
682 refLVal.getQuals().addVolatile();
683 addr = emitLoadOfReference(refLVal, loc: getLoc(srcLoc: field->getSourceRange()),
684 pointeeBaseInfo: &fieldBaseInfo);
685
686 // Qualifiers on the struct don't apply to the referencee.
687 recordCVR = 0;
688 fieldType = fieldType->getPointeeType();
689 }
690
691 if (field->hasAttr<AnnotateAttr>())
692 addr = emitFieldAnnotations(field, addr);
693
694 LValue lv = makeAddrLValue(addr, ty: fieldType, baseInfo: fieldBaseInfo);
695 lv.getQuals().addCVRQualifiers(mask: recordCVR);
696
697 // __weak attribute on a field is ignored.
698 if (lv.getQuals().getObjCGCAttr() == Qualifiers::Weak) {
699 cgm.errorNYI(field->getSourceRange(),
700 "emitLValueForField: __weak attribute");
701 return LValue();
702 }
703
704 return lv;
705}
706
707LValue CIRGenFunction::emitLValueForFieldInitialization(
708 LValue base, const clang::FieldDecl *field, llvm::StringRef fieldName) {
709 QualType fieldType = field->getType();
710
711 if (!fieldType->isReferenceType())
712 return emitLValueForField(base, field);
713
714 Address v = base.getAddress();
715 const CIRGenRecordLayout &layout =
716 cgm.getTypes().getCIRGenRecordLayout(rd: field->getParent());
717 if (!layout.hasCIRField(fd: field)) {
718 v = emitAddrOfZeroSizeField(cgf&: *this, base: v, field);
719 } else {
720 unsigned fieldIndex = layout.getCIRFieldNo(fd: field);
721 v = emitAddrOfFieldStorage(base: v, field, fieldName, fieldIndex);
722 }
723
724 // Make sure that the address is pointing to the right type.
725 mlir::Type memTy = convertTypeForMem(t: fieldType);
726 v = builder.createElementBitCast(loc: getLoc(srcLoc: field->getSourceRange()), addr: v, destType: memTy);
727
728 // TODO: Generate TBAA information that describes this access as a structure
729 // member access and not just an access to an object of the field's type. This
730 // should be similar to what we do in EmitLValueForField().
731 LValueBaseInfo baseInfo = base.getBaseInfo();
732 AlignmentSource fieldAlignSource = baseInfo.getAlignmentSource();
733 LValueBaseInfo fieldBaseInfo(getFieldAlignmentSource(source: fieldAlignSource));
734 assert(!cir::MissingFeatures::opTBAA());
735 return makeAddrLValue(addr: v, ty: fieldType, baseInfo: fieldBaseInfo);
736}
737
738/// Converts a scalar value from its primary IR type (as returned
739/// by ConvertType) to its load/store type.
740mlir::Value CIRGenFunction::emitToMemory(mlir::Value value, QualType ty) {
741 if (auto *atomicTy = ty->getAs<AtomicType>())
742 ty = atomicTy->getValueType();
743
744 if (ty->isConstantMatrixBoolType()) {
745 cgm.errorNYI(feature: "emitToMemory: ConstantMatrixBoolType");
746 return {};
747 }
748
749 // Unlike in classic codegen CIR, bools are kept as `cir.bool` and BitInts are
750 // kept as `cir.int<N>` until further lowering
751 return value;
752}
753
754mlir::Value CIRGenFunction::emitFromMemory(mlir::Value value, QualType ty) {
755 if (auto *atomicTy = ty->getAs<AtomicType>())
756 ty = atomicTy->getValueType();
757
758 if (ty->isPackedVectorBoolType(ctx: getContext())) {
759 cgm.errorNYI(feature: "emitFromMemory: PackedVectorBoolType");
760 }
761
762 return value;
763}
764
765void CIRGenFunction::emitStoreOfScalar(mlir::Value value, LValue lvalue,
766 bool isInit) {
767 emitStoreOfScalar(value, addr: lvalue.getAddress(), isVolatile: lvalue.isVolatile(),
768 ty: lvalue.getType(), baseInfo: lvalue.getBaseInfo(), isInit,
769 isNontemporal: lvalue.isNontemporal());
770}
771
772mlir::Value CIRGenFunction::emitLoadOfScalar(Address addr, bool isVolatile,
773 QualType ty, SourceLocation loc,
774 LValueBaseInfo baseInfo,
775 bool isNontemporal) {
776 // Traditional LLVM codegen handles thread local separately, CIR handles
777 // as part of getAddrOfGlobalVar (GetGlobalOp).
778 mlir::Type eltTy = addr.getElementType();
779 if (const auto *clangVecTy = ty->getAs<clang::VectorType>()) {
780 // Handle vectors of size 3 like size 4 for better performance.
781 assert(!cir::MissingFeatures::cirgenABIInfo());
782 if (clangVecTy->getNumElements() == 3 && !getLangOpts().PreserveVec3Type)
783 cgm.errorNYI(loc: addr.getPointer().getLoc(),
784 feature: "emitLoadOfScalar Vec3 & PreserveVec3Type disabled");
785 }
786
787 assert(!cir::MissingFeatures::opLoadStoreTbaa());
788 LValue atomicLValue = LValue::makeAddr(address: addr, t: ty, baseInfo);
789 if (ty->isAtomicType() || isLValueSuitableForInlineAtomic(lv: atomicLValue))
790 return emitAtomicLoad(lvalue: atomicLValue, loc).getValue();
791
792 if (mlir::isa<cir::VoidType>(Val: eltTy))
793 cgm.errorNYI(loc, "emitLoadOfScalar: void type");
794
795 assert(!cir::MissingFeatures::opLoadEmitScalarRangeCheck());
796
797 mlir::Value loadOp =
798 builder.createLoad(loc: getLoc(srcLoc: loc), addr, isVolatile, isNontemporal);
799
800 // Types with a boolean representation that are not the builtin bool (an enum
801 // whose underlying type is bool, or a _BitInt(1)) need no register/memory
802 // conversion here: like bool and _BitInt(N), CIR keeps them in their literal
803 // type until LowerToLLVM widens them to the in-memory integer type (see
804 // emitToMemory).
805 return loadOp;
806}
807
808mlir::Value CIRGenFunction::emitLoadOfScalar(LValue lvalue,
809 SourceLocation loc) {
810 assert(!cir::MissingFeatures::opLoadStoreTbaa());
811 return emitLoadOfScalar(addr: lvalue.getAddress(), isVolatile: lvalue.isVolatile(),
812 ty: lvalue.getType(), loc, baseInfo: lvalue.getBaseInfo(),
813 isNontemporal: lvalue.isNontemporal());
814}
815
816static RValue emitLoadOfMatrixLValue(LValue lv, SourceLocation loc,
817 CIRGenFunction &cgf) {
818 if (cgf.getLangOpts().HLSL &&
819 lv.getType().getAddressSpace() == LangAS::hlsl_constant) {
820 cgf.cgm.errorNYI(
821 loc, "emitLoadOfMatrixLValue: HLSL & hlsl_constant address space");
822 return {};
823 }
824
825 return RValue::get(v: cgf.emitLoadOfScalar(lvalue: lv, loc));
826}
827
828/// Given an expression that represents a value lvalue, this
829/// method emits the address of the lvalue, then loads the result as an rvalue,
830/// returning the rvalue.
831RValue CIRGenFunction::emitLoadOfLValue(LValue lv, SourceLocation loc) {
832 assert(!lv.getType()->isFunctionType());
833
834 if (lv.isBitField())
835 return emitLoadOfBitfieldLValue(lv, loc);
836
837 if (lv.isSimple()) {
838 if (lv.getType()->isConstantMatrixType())
839 return emitLoadOfMatrixLValue(lv, loc, cgf&: *this);
840
841 return RValue::get(v: emitLoadOfScalar(lvalue: lv, loc));
842 }
843
844 if (lv.isVectorElt()) {
845 const mlir::Value load =
846 builder.createLoad(loc: getLoc(srcLoc: loc), addr: lv.getVectorAddress());
847 return RValue::get(v: cir::VecExtractOp::create(builder, location: getLoc(srcLoc: loc), vec: load,
848 index: lv.getVectorIdx()));
849 }
850
851 if (lv.isExtVectorElt())
852 return emitLoadOfExtVectorElementLValue(lv);
853
854 if (lv.isMatrixElt()) {
855 // In Opt-level >1, we emit an assume here.
856 assert(!MissingFeatures::matrixTypeIndexAssumption());
857 const mlir::Location mlirLoc = getLoc(srcLoc: loc);
858 const mlir::Value matrix =
859 builder.createLoad(loc: mlirLoc, addr: lv.getMatrixAddress(), isVolatile: lv.isVolatile());
860 return RValue::get(v: builder.createMatrixExtract(
861 loc: mlirLoc, matrix, rowIdx: lv.getMatrixRowIdx(), columnIdx: lv.getMatrixColumnIdx()));
862 }
863
864 cgm.errorNYI(loc, "emitLoadOfLValue");
865 return RValue::get(v: nullptr);
866}
867
868int64_t CIRGenFunction::getAccessedFieldNo(unsigned int idx,
869 const mlir::ArrayAttr elts) {
870 auto elt = mlir::cast<mlir::IntegerAttr>(Val: elts[idx]);
871 return elt.getInt();
872}
873
874// If this is a reference to a subset of the elements of a vector, create an
875// appropriate shufflevector.
876RValue CIRGenFunction::emitLoadOfExtVectorElementLValue(LValue lv) {
877 mlir::Location loc = lv.getExtVectorPointer().getLoc();
878 mlir::Value vec = builder.createLoad(loc, addr: lv.getExtVectorAddress());
879
880 // HLSL allows treating scalars as one-element vectors. Converting the scalar
881 // IR value to a vector here allows the rest of codegen to behave as normal.
882 if (getLangOpts().HLSL &&
883 !mlir::isa<cir::VectorType, cir::MatrixType>(Val: vec.getType())) {
884 auto dstTy = cir::VectorType::get(
885 element_type: vec.getType(), size: llvm::ElementCount::get(MinVal: 1, /*Scalable=*/false));
886 cir::ConstantOp poison =
887 builder.getConstant(loc, attr: cir::PoisonAttr::get(type: dstTy));
888 vec = builder.createInsertElement(loc, vec: poison, newElt: vec, /*idx=*/0);
889 }
890
891 assert((mlir::isa<cir::VectorType, cir::MatrixType>(vec.getType())) &&
892 "ExtVecElement with non vector or matrix type on non-HLSL mode");
893
894 const mlir::ArrayAttr elts = lv.getExtVectorElts();
895
896 // If the result of the expression is a non-vector type, we must be extracting
897 // a single element. Just codegen as an extractelement.
898 const auto *exprVecTy = lv.getType()->getAs<clang::VectorType>();
899 if (!exprVecTy) {
900 int64_t indexValue = getAccessedFieldNo(idx: 0, elts);
901 cir::ConstantOp index =
902 builder.getConstInt(loc, t: builder.getSInt64Ty(), c: indexValue);
903 if (mlir::isa<cir::VectorType>(Val: vec.getType()))
904 return RValue::get(v: cir::VecExtractOp::create(builder, location: loc, vec, index));
905
906 cir::ConstantOp columnIdx =
907 builder.getConstInt(loc, t: builder.getSInt64Ty(), c: 0);
908 mlir::Value element =
909 cir::MatrixExtractOp::create(builder, location: loc, matrix: vec, rowIdx: index, columnIdx);
910 return RValue::get(v: element);
911 }
912
913 // Always use shuffle vector to try to retain the original program structure
914 SmallVector<int64_t> mask;
915 for (auto i : llvm::seq<unsigned>(Begin: 0, End: exprVecTy->getNumElements()))
916 mask.push_back(Elt: getAccessedFieldNo(idx: i, elts));
917
918 cir::VecShuffleOp resultVec = builder.createVecShuffle(loc, vec1: vec, mask);
919 if (lv.getType()->isExtVectorBoolType()) {
920 cgm.errorNYI(loc, feature: "emitLoadOfExtVectorElementLValue: ExtVectorBoolType");
921 return {};
922 }
923
924 return RValue::get(v: resultVec);
925}
926
927LValue
928CIRGenFunction::emitPointerToDataMemberBinaryExpr(const BinaryOperator *e) {
929 assert((e->getOpcode() == BO_PtrMemD || e->getOpcode() == BO_PtrMemI) &&
930 "unexpected binary operator opcode");
931
932 Address baseAddr = Address::invalid();
933 if (e->getOpcode() == BO_PtrMemD)
934 baseAddr = emitLValue(e: e->getLHS()).getAddress();
935 else
936 baseAddr = emitPointerWithAlignment(expr: e->getLHS());
937
938 const auto *memberPtrTy = e->getRHS()->getType()->castAs<MemberPointerType>();
939
940 mlir::Value memberPtr = emitScalarExpr(e: e->getRHS());
941
942 LValueBaseInfo baseInfo;
943 assert(!cir::MissingFeatures::opTBAA());
944 Address memberAddr = emitCXXMemberDataPointerAddress(e, base: baseAddr, memberPtr,
945 memberPtrType: memberPtrTy, baseInfo: &baseInfo);
946
947 return makeAddrLValue(addr: memberAddr, ty: memberPtrTy->getPointeeType(), baseInfo);
948}
949
950/// Generates lvalue for partial ext_vector access.
951Address CIRGenFunction::emitExtVectorElementLValue(LValue lv,
952 mlir::Location loc) {
953 Address vectorAddress = lv.getExtVectorAddress();
954 QualType elementTy = lv.getType()->castAs<VectorType>()->getElementType();
955 mlir::Type vectorElementTy = cgm.getTypes().convertType(type: elementTy);
956 Address castToPointerElement =
957 vectorAddress.withElementType(builder, ElemTy: vectorElementTy);
958
959 mlir::ArrayAttr extVecElts = lv.getExtVectorElts();
960 unsigned idx = getAccessedFieldNo(idx: 0, elts: extVecElts);
961 mlir::Value idxValue =
962 builder.getConstInt(loc, t: mlir::cast<cir::IntType>(Val&: ptrDiffTy), c: idx);
963
964 mlir::Value elementValue = builder.getArrayElement(
965 arrayLocBegin: loc, arrayLocEnd: loc, arrayPtr: castToPointerElement.getPointer(), eltTy: vectorElementTy, idx: idxValue,
966 /*shouldDecay=*/false);
967
968 const CharUnits eltSize = getContext().getTypeSizeInChars(T: elementTy);
969 const CharUnits alignment =
970 castToPointerElement.getAlignment().alignmentAtOffset(offset: idx * eltSize);
971 return Address(elementValue, vectorElementTy, alignment);
972}
973
974cir::FuncOp CIRGenModule::getRawFunctionPointer(GlobalDecl gd) {
975 const auto *fd = cast<FunctionDecl>(Val: gd.getDecl());
976 if (fd->hasAttr<WeakRefAttr>())
977 return cast_or_null<cir::FuncOp>(Val: getWeakRefReference(vd: fd));
978 return getAddrOfFunction(gd);
979}
980
981static LValue emitCapturedFieldLValue(CIRGenFunction &cgf, const FieldDecl *fd,
982 mlir::Value thisValue) {
983 return cgf.emitLValueForLambdaField(field: fd, thisValue);
984}
985
986/// Given that we are currently emitting a lambda, emit an l-value for
987/// one of its members.
988///
989LValue CIRGenFunction::emitLValueForLambdaField(const FieldDecl *field,
990 mlir::Value thisValue) {
991 bool hasExplicitObjectParameter = false;
992 const auto *md = dyn_cast_if_present<CXXMethodDecl>(Val: curCodeDecl);
993 LValue lambdaLV;
994 if (md) {
995 hasExplicitObjectParameter = md->isExplicitObjectMemberFunction();
996 assert(md->getParent()->isLambda());
997 assert(md->getParent() == field->getParent());
998 }
999
1000 if (hasExplicitObjectParameter) {
1001 const VarDecl *d = cast<CXXMethodDecl>(Val: curCodeDecl)->getParamDecl(i: 0);
1002 auto it = localDeclMap.find(Val: d);
1003 assert(it != localDeclMap.end() && "explicit parameter not loaded?");
1004 Address addrOfExplicitObject = it->second;
1005 if (d->getType()->isReferenceType())
1006 lambdaLV = emitLoadOfReferenceLValue(refAddr: addrOfExplicitObject,
1007 loc: getLoc(srcLoc: field->getSourceRange()),
1008 refTy: d->getType(), source: AlignmentSource::Decl);
1009 else
1010 lambdaLV = makeAddrLValue(addr: addrOfExplicitObject,
1011 ty: d->getType().getNonReferenceType());
1012
1013 // Make sure we have an lvalue to the lambda itself and not a derived class.
1014 auto *thisTy = d->getType().getNonReferenceType()->getAsCXXRecordDecl();
1015 auto *lambdaTy = cast<CXXRecordDecl>(Val: field->getParent());
1016 if (thisTy != lambdaTy) {
1017 const CXXCastPath &basePathArray = getContext().LambdaCastPaths.at(Val: md);
1018 Address base = getAddressOfBaseClass(
1019 value: lambdaLV.getAddress(), derived: thisTy,
1020 path: llvm::make_range(x: basePathArray.begin(), y: basePathArray.end()),
1021 /*nullCheckValue=*/false, loc: SourceLocation());
1022 CanQualType t = getContext().getCanonicalTagType(TD: lambdaTy);
1023 lambdaLV = makeAddrLValue(addr: base, ty: t);
1024 }
1025 } else {
1026 CanQualType lambdaTagType =
1027 getContext().getCanonicalTagType(TD: field->getParent());
1028 lambdaLV = makeNaturalAlignAddrLValue(val: thisValue, ty: lambdaTagType);
1029 }
1030 return emitLValueForField(base: lambdaLV, field);
1031}
1032
1033LValue CIRGenFunction::emitLValueForLambdaField(const FieldDecl *field) {
1034 return emitLValueForLambdaField(field, thisValue: cxxabiThisValue);
1035}
1036
1037static LValue emitFunctionDeclLValue(CIRGenFunction &cgf, const Expr *e,
1038 GlobalDecl gd) {
1039 const FunctionDecl *fd = cast<FunctionDecl>(Val: gd.getDecl());
1040 cir::FuncOp funcOp = cgf.cgm.getFunctionPointer(GD: gd);
1041 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
1042 CharUnits align = cgf.getContext().getDeclAlign(D: fd);
1043
1044 assert(!cir::MissingFeatures::sanitizers());
1045
1046 mlir::Type fnTy = funcOp.getFunctionType();
1047 mlir::ptr::MemorySpaceAttrInterface fnAddrSpace =
1048 cgf.cgm.getTypes().getPointerAddressSpace(pointeeTy: fd->getType());
1049 mlir::Type ptrTy = cir::PointerType::get(pointee: fnTy, addrSpace: fnAddrSpace);
1050 mlir::Value addr;
1051
1052 // On the HIP host, a reference to a __global__ kernel must resolve to the
1053 // address of the kernel handle registered with the offload runtime, not
1054 // the device stub's own address. CUDA uses the device stub itself as the
1055 // kernel handle.
1056 if (cgf.cgm.getLangOpts().HIP && !cgf.cgm.getLangOpts().CUDAIsDevice &&
1057 fd->hasAttr<CUDAGlobalAttr>()) {
1058 auto handle = mlir::cast<cir::GlobalOp>(
1059 Val: cgf.cgm.getCUDARuntime().getKernelHandle(fn: funcOp, gd));
1060 cir::PointerType handlePtrTy = cir::PointerType::get(pointee: handle.getSymType());
1061 mlir::Value handleAddr = cir::GetGlobalOp::create(
1062 builder&: cgf.getBuilder(), location: loc, addr: handlePtrTy, name: handle.getSymName());
1063 addr = cir::CastOp::create(builder&: cgf.getBuilder(), location: loc, result: ptrTy,
1064 kind: cir::CastKind::bitcast, src: handleAddr);
1065 }
1066
1067 if (!addr)
1068 addr = cir::GetGlobalOp::create(builder&: cgf.getBuilder(), location: loc, addr: ptrTy,
1069 name: funcOp.getSymName());
1070
1071 if (funcOp.getFunctionType() != cgf.convertType(t: fd->getType())) {
1072 fnTy = cgf.convertType(t: fd->getType());
1073 ptrTy = cir::PointerType::get(pointee: fnTy, addrSpace: fnAddrSpace);
1074
1075 addr = cir::CastOp::create(builder&: cgf.getBuilder(), location: addr.getLoc(), result: ptrTy,
1076 kind: cir::CastKind::bitcast, src: addr);
1077 }
1078
1079 return cgf.makeAddrLValue(addr: Address(addr, fnTy, align), ty: e->getType(),
1080 source: AlignmentSource::Decl);
1081}
1082
1083/// Determine whether we can emit a reference to \p vd from the current
1084/// context, despite not necessarily having seen an odr-use of the variable in
1085/// this context.
1086/// TODO(cir): This could be shared with classic codegen.
1087static bool canEmitSpuriousReferenceToVariable(CIRGenFunction &cgf,
1088 const DeclRefExpr *e,
1089 const VarDecl *vd) {
1090 // For a variable declared in an enclosing scope, do not emit a spurious
1091 // reference even if we have a capture, as that will emit an unwarranted
1092 // reference to our capture state, and will likely generate worse code than
1093 // emitting a local copy.
1094 if (e->refersToEnclosingVariableOrCapture())
1095 return false;
1096
1097 // For a local declaration declared in this function, we can always reference
1098 // it even if we don't have an odr-use.
1099 if (vd->hasLocalStorage()) {
1100 return vd->getDeclContext() ==
1101 dyn_cast_or_null<DeclContext>(Val: cgf.curCodeDecl);
1102 }
1103
1104 // For a global declaration, we can emit a reference to it if we know
1105 // for sure that we are able to emit a definition of it.
1106 vd = vd->getDefinition(C&: cgf.getContext());
1107 if (!vd)
1108 return false;
1109
1110 // Don't emit a spurious reference if it might be to a variable that only
1111 // exists on a different device / target.
1112 // FIXME: This is unnecessarily broad. Check whether this would actually be a
1113 // cross-target reference.
1114 if (cgf.getLangOpts().OpenMP || cgf.getLangOpts().CUDA ||
1115 cgf.getLangOpts().OpenCL) {
1116 return false;
1117 }
1118
1119 // We can emit a spurious reference only if the linkage implies that we'll
1120 // be emitting a non-interposable symbol that will be retained until link
1121 // time.
1122 switch (cgf.cgm.getCIRLinkageVarDefinition(vd)) {
1123 case cir::GlobalLinkageKind::ExternalLinkage:
1124 case cir::GlobalLinkageKind::LinkOnceODRLinkage:
1125 case cir::GlobalLinkageKind::WeakODRLinkage:
1126 case cir::GlobalLinkageKind::InternalLinkage:
1127 case cir::GlobalLinkageKind::PrivateLinkage:
1128 return true;
1129 default:
1130 return false;
1131 }
1132}
1133
1134LValue CIRGenFunction::emitDeclRefLValue(const DeclRefExpr *e) {
1135 const NamedDecl *nd = e->getDecl();
1136 QualType ty = e->getType();
1137
1138 assert(e->isNonOdrUse() != NOUR_Unevaluated &&
1139 "should not emit an unevaluated operand");
1140
1141 if (const auto *vd = dyn_cast<VarDecl>(Val: nd)) {
1142 // Global Named registers access via intrinsics only
1143 if (vd->getStorageClass() == SC_Register && vd->hasAttr<AsmLabelAttr>() &&
1144 !vd->isLocalVarDecl()) {
1145 cgm.errorNYI(e->getSourceRange(),
1146 "emitDeclRefLValue: Global Named registers access");
1147 return LValue();
1148 }
1149
1150 if (e->isNonOdrUse() == NOUR_Constant &&
1151 (vd->getType()->isReferenceType() ||
1152 !canEmitSpuriousReferenceToVariable(cgf&: *this, e, vd))) {
1153 vd->getAnyInitializer(D&: vd);
1154 mlir::Attribute val = ConstantEmitter(*this).emitAbstract(
1155 loc: e->getLocation(), value: *vd->evaluateValue(), destType: vd->getType());
1156 assert(val && "failed to emit constant expression");
1157
1158 Address addr = Address::invalid();
1159 if (!vd->getType()->isReferenceType()) {
1160 // Spill the constant value to a global.
1161 addr = cgm.createUnnamedGlobalFrom(d: *vd, constAttr: val,
1162 align: getContext().getDeclAlign(D: vd));
1163 mlir::Type varTy = getTypes().convertTypeForMem(vd->getType());
1164 auto ptrTy = mlir::cast<cir::PointerType>(Val: addr.getPointer().getType());
1165 if (ptrTy.getPointee() != varTy) {
1166 addr = addr.withElementType(builder, ElemTy: varTy);
1167 }
1168 } else {
1169 // Should we be using the alignment of the constant pointer we emitted?
1170 CharUnits alignment = cgm.getNaturalTypeAlignment(
1171 t: e->getType(), /*baseInfo=*/nullptr, /*forPointeeType=*/true);
1172 // Classic codegen passes TBAA as null-ptr to the above function, so it
1173 // probably needs to deal with that.
1174 assert(!cir::MissingFeatures::opTBAA());
1175 mlir::Value ptrVal = getBuilder().getConstant(
1176 loc: getLoc(srcLoc: e->getSourceRange()), attr: mlir::cast<mlir::TypedAttr>(Val&: val));
1177 addr = makeNaturalAddressForPointer(ptr: ptrVal, t: ty, alignment);
1178 }
1179 return makeAddrLValue(addr, ty, source: AlignmentSource::Decl);
1180 }
1181
1182 // Check for captured variables.
1183 if (e->refersToEnclosingVariableOrCapture()) {
1184 vd = vd->getCanonicalDecl();
1185 if (FieldDecl *fd = lambdaCaptureFields.lookup(Val: vd))
1186 return emitCapturedFieldLValue(cgf&: *this, fd, thisValue: cxxabiThisValue);
1187 assert(!cir::MissingFeatures::cgCapturedStmtInfo());
1188 assert(!cir::MissingFeatures::openMP());
1189 }
1190 }
1191
1192 // FIXME: We should be able to assert this for FunctionDecls as well!
1193 // FIXME: We should be able to assert this for all DeclRefExprs, not just
1194 // those with a valid source location.
1195 assert((nd->isUsed(false) || !isa<VarDecl>(nd) || e->isNonOdrUse() ||
1196 !e->getLocation().isValid()) &&
1197 "Should not use decl without marking it used!");
1198
1199 // A weakref variable refers to its alias target. Weakref functions are
1200 // handled by getRawFunctionPointer.
1201 if (const auto *vd = dyn_cast<VarDecl>(Val: nd);
1202 vd && vd->hasAttr<WeakRefAttr>()) {
1203 auto aliasee =
1204 mlir::cast_or_null<cir::GlobalOp>(Val: cgm.getWeakRefReference(vd));
1205 if (!aliasee)
1206 return LValue();
1207 mlir::Value v =
1208 builder.createGetGlobal(loc: getLoc(srcLoc: e->getSourceRange()), global: aliasee);
1209 Address addr(v, convertTypeForMem(t: vd->getType()),
1210 getContext().getDeclAlign(D: vd));
1211 return makeAddrLValue(addr, ty, source: AlignmentSource::Decl);
1212 }
1213
1214 if (const auto *vd = dyn_cast<VarDecl>(Val: nd)) {
1215 // Checks for omitted feature handling
1216 assert(!cir::MissingFeatures::opAllocaStaticLocal());
1217 assert(!cir::MissingFeatures::opAllocaNonGC());
1218 assert(!cir::MissingFeatures::opAllocaImpreciseLifetime());
1219 assert(!cir::MissingFeatures::opAllocaTLS());
1220 assert(!cir::MissingFeatures::opAllocaOpenMPThreadPrivate());
1221 assert(!cir::MissingFeatures::opAllocaEscapeByReference());
1222
1223 // Check if this is a global variable
1224 if (vd->hasLinkage() || vd->isStaticDataMember())
1225 return emitGlobalVarDeclLValue(cgf&: *this, e, vd);
1226
1227 Address addr = Address::invalid();
1228
1229 // The variable should generally be present in the local decl map.
1230 auto iter = localDeclMap.find(Val: vd);
1231 if (iter != localDeclMap.end()) {
1232 addr = iter->second;
1233
1234 } else if (vd->isStaticLocal()) {
1235 // Otherwise, it might be static local we haven't emitted yet for some
1236 // reason; most likely, because it's in an outer function.
1237 cir::GlobalOp var =
1238 cgm.getOrCreateStaticVarDecl(d: *vd, linkage: cgm.getCIRLinkageVarDefinition(vd));
1239 mlir::Value getGlobVal = builder.createGetGlobal(global: var);
1240 auto getGlob = getGlobVal.getDefiningOp<cir::GetGlobalOp>();
1241 getGlob.setStaticLocal(var.getDynamicInitGuard().has_value());
1242 getGlob.setTls(vd->getTLSKind() != VarDecl::TLS_None);
1243 addr = Address(cgm.castGlobalToDeclAddrSpace(addr: getGlob, vd: *vd),
1244 convertTypeForMem(t: vd->getType()),
1245 getContext().getDeclAlign(D: vd));
1246 } else {
1247 llvm_unreachable("DeclRefExpr for Decl not entered in localDeclMap?");
1248 }
1249
1250 // Drill into reference types.
1251 LValue lv =
1252 vd->getType()->isReferenceType()
1253 ? emitLoadOfReferenceLValue(refAddr: addr, loc: getLoc(srcLoc: e->getSourceRange()),
1254 refTy: vd->getType(), source: AlignmentSource::Decl)
1255 : makeAddrLValue(addr, ty, source: AlignmentSource::Decl);
1256
1257 // Statics are defined as globals, so they are not include in the function's
1258 // symbol table.
1259 assert((vd->isStaticLocal() || symbolTable.count(vd)) &&
1260 "non-static locals should be already mapped");
1261
1262 return lv;
1263 }
1264
1265 if (const auto *bd = dyn_cast<BindingDecl>(Val: nd)) {
1266 if (e->refersToEnclosingVariableOrCapture()) {
1267 FieldDecl *fd = lambdaCaptureFields.lookup(Val: bd);
1268 return emitCapturedFieldLValue(cgf&: *this, fd, thisValue: cxxabiThisValue);
1269 }
1270 return emitLValue(e: bd->getBinding());
1271 }
1272
1273 if (const auto *fd = dyn_cast<FunctionDecl>(Val: nd)) {
1274 LValue lv = emitFunctionDeclLValue(cgf&: *this, e, gd: fd);
1275
1276 // Emit debuginfo for the function declaration if the target wants to.
1277 if (getContext().getTargetInfo().allowDebugInfoForExternalRef())
1278 assert(!cir::MissingFeatures::generateDebugInfo());
1279
1280 return lv;
1281 }
1282 if (isa<MSGuidDecl>(Val: nd))
1283 cgm.errorNYI(e->getSourceRange(),
1284 "emitDeclRefLValue: unhandled MS Guid Decl");
1285
1286 if (const auto *tpo = dyn_cast<TemplateParamObjectDecl>(Val: nd)) {
1287 CharUnits alignment = cgm.getNaturalTypeAlignment(t: tpo->getType());
1288 cir::GetGlobalOp atpo =
1289 builder.createGetGlobal(global: cgm.getAddrOfTemplateParamObject(tpo));
1290 assert(!MissingFeatures::addressSpace() &&
1291 "Do an address space conversion if necessary");
1292
1293 return makeAddrLValue(
1294 addr: Address(atpo, convertTypeForMem(t: tpo->getType()), alignment), ty,
1295 source: AlignmentSource::Decl);
1296 }
1297
1298 llvm_unreachable("Unhandled DeclRefExpr");
1299}
1300
1301mlir::Value CIRGenFunction::evaluateExprAsBool(const Expr *e) {
1302 QualType boolTy = getContext().BoolTy;
1303 SourceLocation loc = e->getExprLoc();
1304
1305 assert(!cir::MissingFeatures::pgoUse());
1306 if (e->getType()->getAs<MemberPointerType>()) {
1307 cgm.errorNYI(e->getSourceRange(),
1308 "evaluateExprAsBool: member pointer type");
1309 return createDummyValue(loc: getLoc(srcLoc: loc), qt: boolTy);
1310 }
1311
1312 CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
1313 if (!e->getType()->isAnyComplexType())
1314 return emitScalarConversion(src: emitScalarExpr(e), srcType: e->getType(), dstType: boolTy, loc);
1315
1316 return emitComplexToScalarConversion(src: emitComplexExpr(e), srcTy: e->getType(), dstTy: boolTy,
1317 loc);
1318}
1319
1320LValue CIRGenFunction::emitUnaryOpLValue(const UnaryOperator *e) {
1321 UnaryOperatorKind op = e->getOpcode();
1322
1323 // __extension__ doesn't affect lvalue-ness.
1324 if (op == UO_Extension)
1325 return emitLValue(e: e->getSubExpr());
1326
1327 switch (op) {
1328 case UO_Deref: {
1329 QualType t = e->getSubExpr()->getType()->getPointeeType();
1330 assert(!t.isNull() && "CodeGenFunction::EmitUnaryOpLValue: Illegal type");
1331
1332 assert(!cir::MissingFeatures::opTBAA());
1333 LValueBaseInfo baseInfo;
1334 Address addr = emitPointerWithAlignment(expr: e->getSubExpr(), baseInfo: &baseInfo);
1335
1336 // Tag 'load' with deref attribute.
1337 // FIXME: This misses some derefence cases and has problematic interactions
1338 // with other operators.
1339 if (auto loadOp = addr.getDefiningOp<cir::LoadOp>())
1340 loadOp.setIsDerefAttr(mlir::UnitAttr::get(context: &getMLIRContext()));
1341
1342 LValue lv = makeAddrLValue(addr, ty: t, baseInfo);
1343 assert(!cir::MissingFeatures::addressSpace());
1344 assert(!cir::MissingFeatures::setNonGC());
1345 return lv;
1346 }
1347 case UO_Real:
1348 case UO_Imag: {
1349 LValue lv = emitLValue(e: e->getSubExpr());
1350 assert(lv.isSimple() && "real/imag on non-ordinary l-value");
1351
1352 // __real is valid on scalars. This is a faster way of testing that.
1353 // __imag can only produce an rvalue on scalars.
1354 if (e->getOpcode() == UO_Real &&
1355 !mlir::isa<cir::ComplexType>(Val: lv.getAddress().getElementType())) {
1356 assert(e->getSubExpr()->getType()->isArithmeticType());
1357 return lv;
1358 }
1359
1360 QualType exprTy = getContext().getCanonicalType(T: e->getSubExpr()->getType());
1361 QualType elemTy = exprTy->castAs<clang::ComplexType>()->getElementType();
1362 mlir::Location loc = getLoc(srcLoc: e->getExprLoc());
1363 Address component =
1364 e->getOpcode() == UO_Real
1365 ? builder.createComplexRealPtr(loc, addr: lv.getAddress())
1366 : builder.createComplexImagPtr(loc, addr: lv.getAddress());
1367 assert(!cir::MissingFeatures::opTBAA());
1368 LValue elemLV = makeAddrLValue(addr: component, ty: elemTy);
1369 elemLV.getQuals().addQualifiers(Q: lv.getQuals());
1370 return elemLV;
1371 }
1372 case UO_PreInc:
1373 case UO_PreDec: {
1374 LValue lv = emitLValue(e: e->getSubExpr());
1375
1376 assert(e->isPrefix() && "Prefix operator in unexpected state!");
1377
1378 if (e->getType()->isAnyComplexType())
1379 emitComplexPrePostIncDec(e, lv);
1380 else
1381 emitScalarPrePostIncDec(e, lv);
1382
1383 return lv;
1384 }
1385 case UO_Extension:
1386 llvm_unreachable("UnaryOperator extension should be handled above!");
1387 case UO_Plus:
1388 case UO_Minus:
1389 case UO_Not:
1390 case UO_LNot:
1391 case UO_AddrOf:
1392 case UO_PostInc:
1393 case UO_PostDec:
1394 case UO_Coawait:
1395 llvm_unreachable("UnaryOperator of non-lvalue kind!");
1396 }
1397 llvm_unreachable("Unknown unary operator kind!");
1398}
1399
1400/// If the specified expr is a simple decay from an array to pointer,
1401/// return the array subexpression.
1402/// FIXME: this could be abstracted into a common AST helper.
1403static const Expr *getSimpleArrayDecayOperand(const Expr *e) {
1404 // If this isn't just an array->pointer decay, bail out.
1405 const auto *castExpr = dyn_cast<CastExpr>(Val: e);
1406 if (!castExpr || castExpr->getCastKind() != CK_ArrayToPointerDecay)
1407 return nullptr;
1408
1409 // If this is a decay from variable width array, bail out.
1410 const Expr *subExpr = castExpr->getSubExpr();
1411 if (subExpr->getType()->isVariableArrayType())
1412 return nullptr;
1413
1414 return subExpr;
1415}
1416
1417static cir::IntAttr getConstantIndexOrNull(mlir::Value idx) {
1418 // TODO(cir): should we consider using MLIRs IndexType instead of IntegerAttr?
1419 if (auto constantOp = idx.getDefiningOp<cir::ConstantOp>())
1420 return constantOp.getValueAttr<cir::IntAttr>();
1421 return {};
1422}
1423
1424static CharUnits getArrayElementAlign(CharUnits arrayAlign, mlir::Value idx,
1425 CharUnits eltSize) {
1426 // If we have a constant index, we can use the exact offset of the
1427 // element we're accessing.
1428 if (const cir::IntAttr constantIdx = getConstantIndexOrNull(idx)) {
1429 const CharUnits offset = constantIdx.getValue().getZExtValue() * eltSize;
1430 return arrayAlign.alignmentAtOffset(offset);
1431 }
1432 // Otherwise, use the worst-case alignment for any element.
1433 return arrayAlign.alignmentOfArrayElement(elementSize: eltSize);
1434}
1435
1436/// Return true if \p ptr is derived from a cir.alloca through cir.get_element,
1437/// cir.ptr_stride, or a cir.cast that is array_to_ptrdecay or for which
1438/// isAllocaPreservingCast() holds, so the alloca is its underlying object.
1439static bool isBasedOnAlloca(mlir::Value ptr) {
1440 while (mlir::Operation *op = ptr.getDefiningOp()) {
1441 if (mlir::isa<cir::AllocaOp>(Val: op))
1442 return true;
1443 if (auto elementOp = mlir::dyn_cast<cir::GetElementOp>(Val: op))
1444 ptr = elementOp.getBase();
1445 else if (auto strideOp = mlir::dyn_cast<cir::PtrStrideOp>(Val: op))
1446 ptr = strideOp.getBase();
1447 else if (auto castOp = mlir::dyn_cast<cir::CastOp>(Val: op);
1448 castOp && (castOp.isAllocaPreservingCast() ||
1449 castOp.getKind() == cir::CastKind::array_to_ptrdecay))
1450 ptr = castOp.getSrc();
1451 else
1452 return false;
1453 }
1454 return false;
1455}
1456
1457/// Return the no-wrap flags for an array subscript of \p ptr: none when
1458/// pointer overflow is defined, otherwise inbounds unless \p ptr may come from
1459/// an OpenMP section map, plus nuw unless \p signedIndices.
1460static cir::PtrOffsetFlags getArraySubscriptFlags(CIRGenFunction &cgf,
1461 mlir::Value ptr,
1462 bool signedIndices) {
1463 if (cgf.getLangOpts().PointerOverflowDefined)
1464 return cir::PtrOffsetFlags::None;
1465
1466 // An OpenMP section map passes `section - host_offset`, which can point
1467 // outside the mapped allocation, so a subscript of an array of arrays based
1468 // on it cannot be marked inbounds. A base derived from a cir.alloca is
1469 // never such a pointer.
1470 auto arrayTy = mlir::dyn_cast<cir::ArrayType>(
1471 Val: mlir::cast<cir::PointerType>(Val: ptr.getType()).getPointee());
1472 bool isSectionSubscript =
1473 cgf.getLangOpts().OpenMPIsTargetDevice && arrayTy &&
1474 mlir::isa<cir::ArrayType>(Val: arrayTy.getElementType()) &&
1475 !isBasedOnAlloca(ptr);
1476
1477 cir::PtrOffsetFlags flags = cir::PtrOffsetFlags::None;
1478 if (!isSectionSubscript)
1479 flags |= cir::PtrOffsetFlags::InBounds;
1480 if (!signedIndices)
1481 flags |= cir::PtrOffsetFlags::NoUnsignedWrap;
1482 return flags;
1483}
1484
1485static mlir::Value emitArraySubscriptPtr(CIRGenFunction &cgf,
1486 mlir::Location beginLoc,
1487 mlir::Location endLoc, mlir::Value ptr,
1488 mlir::Type eltTy, mlir::Value idx,
1489 bool signedIndices, bool shouldDecay) {
1490 CIRGenModule &cgm = cgf.getCIRGenModule();
1491 assert(!cir::MissingFeatures::emitCheckedInBoundsGEP());
1492 return cgm.getBuilder().getArrayElement(
1493 arrayLocBegin: beginLoc, arrayLocEnd: endLoc, arrayPtr: ptr, eltTy, idx, shouldDecay,
1494 flags: getArraySubscriptFlags(cgf, ptr, signedIndices));
1495}
1496
1497static Address emitArraySubscriptPtr(CIRGenFunction &cgf,
1498 mlir::Location beginLoc,
1499 mlir::Location endLoc, Address addr,
1500 QualType eltType, mlir::Value idx,
1501 mlir::Location loc, bool signedIndices,
1502 bool shouldDecay) {
1503
1504 // Determine the element size of the statically-sized base. This is
1505 // the thing that the indices are expressed in terms of.
1506 if (const VariableArrayType *vla =
1507 cgf.getContext().getAsVariableArrayType(T: eltType)) {
1508 eltType = CodeGenUtils::getFixedSizeElementType(Ctx: cgf.getContext(), VLA: vla);
1509 }
1510
1511 // We can use that to compute the best alignment of the element.
1512 const CharUnits eltSize = cgf.getContext().getTypeSizeInChars(T: eltType);
1513 const CharUnits eltAlign =
1514 getArrayElementAlign(arrayAlign: addr.getAlignment(), idx, eltSize);
1515
1516 assert(!cir::MissingFeatures::preservedAccessIndexRegion());
1517 const mlir::Value eltPtr = emitArraySubscriptPtr(
1518 cgf, beginLoc, endLoc, ptr: addr.getPointer(), eltTy: addr.getElementType(), idx,
1519 signedIndices, shouldDecay);
1520 const mlir::Type elementType = cgf.convertTypeForMem(t: eltType);
1521 return Address(eltPtr, elementType, eltAlign);
1522}
1523
1524LValue
1525CIRGenFunction::emitArraySubscriptExpr(const clang::ArraySubscriptExpr *e) {
1526 if (e->getType()->getAs<ObjCObjectType>()) {
1527 cgm.errorNYI(e->getSourceRange(), "emitArraySubscriptExpr: ObjCObjectType");
1528 return LValue::makeAddr(address: Address::invalid(), t: e->getType(), baseInfo: LValueBaseInfo());
1529 }
1530
1531 // The index must always be an integer, which is not an aggregate. Emit it
1532 // in lexical order (this complexity is, sadly, required by C++17).
1533 mlir::Value idxPre = (e->getLHS() == e->getIdx())
1534 ? emitScalarExpr(e: e->getIdx())
1535 : mlir::Value();
1536 const bool signedIndices =
1537 e->getIdx()->getType()->isSignedIntegerOrEnumerationType();
1538
1539 auto emitIdxAfterBase = [&, idxPre](bool promote) -> mlir::Value {
1540 mlir::Value idx = idxPre;
1541 if (e->getLHS() != e->getIdx()) {
1542 assert(e->getRHS() == e->getIdx() && "index was neither LHS nor RHS");
1543 idx = emitScalarExpr(e: e->getIdx());
1544 }
1545
1546 assert(!cir::MissingFeatures::sanitizers());
1547
1548 // Extend or truncate the index type to pointer-sized integer.
1549 if (promote) {
1550 // Choose the type we extend or truncate to based on the signedness of the
1551 // index type.
1552 mlir::Type desiredIdxTy = signedIndices ? ptrDiffTy : uIntPtrTy;
1553
1554 if (idx.getType() != desiredIdxTy) {
1555 cir::CastKind kind = mlir::isa<cir::BoolType>(Val: idx.getType())
1556 ? cir::CastKind::bool_to_int
1557 : cir::CastKind::integral;
1558 idx = builder.createOrFold<cir::CastOp>(location: idx.getLoc(), args&: desiredIdxTy,
1559 args&: kind, args&: idx);
1560 }
1561 }
1562
1563 return idx;
1564 };
1565 // This is captured by value above, using it after this is an error, so clear
1566 // it to make sure no one is depending on it (mirrors classic codegen).
1567 idxPre = mlir::Value();
1568
1569 // If the base is a vector type, then we are forming a vector element
1570 // with this subscript.
1571 if (e->getBase()->getType()->isSubscriptableVectorType() &&
1572 !isa<ExtVectorElementExpr>(Val: e->getBase())) {
1573 const LValue lv = emitLValue(e: e->getBase());
1574 const mlir::Value idx = emitIdxAfterBase(/*promote=*/false);
1575 return LValue::makeVectorElt(vecAddress: lv.getAddress(), index: idx, t: e->getBase()->getType(),
1576 baseInfo: lv.getBaseInfo());
1577 }
1578
1579 // The HLSL runtime handles subscript expressions on global resource arrays
1580 // and objects with HLSL buffer layouts.
1581 if (getLangOpts().HLSL) {
1582 cgm.errorNYI(e->getSourceRange(), "emitArraySubscriptExpr: HLSL");
1583 return {};
1584 }
1585
1586 // Handle the extvector case we ignored above.
1587 if (isa<ExtVectorElementExpr>(Val: e->getBase())) {
1588 const LValue lv = emitLValue(e: e->getBase());
1589 const mlir::Value idx = emitIdxAfterBase(/*promote=*/true);
1590 Address addr = emitExtVectorElementLValue(lv, loc: cgm.getLoc(cLoc: e->getExprLoc()));
1591
1592 QualType elementType = lv.getType()->castAs<VectorType>()->getElementType();
1593 addr = emitArraySubscriptPtr(cgf&: *this, beginLoc: cgm.getLoc(cLoc: e->getBeginLoc()),
1594 endLoc: cgm.getLoc(cLoc: e->getEndLoc()), addr, eltType: e->getType(),
1595 idx, loc: cgm.getLoc(cLoc: e->getExprLoc()),
1596 signedIndices, /*shouldDecay=*/false);
1597
1598 return makeAddrLValue(addr, ty: elementType, baseInfo: lv.getBaseInfo());
1599 }
1600
1601 if (const VariableArrayType *vla =
1602 getContext().getAsVariableArrayType(T: e->getType())) {
1603 // The base must be a pointer, which is not an aggregate. Emit
1604 // it. It needs to be emitted first in case it's what captures
1605 // the VLA bounds.
1606 Address addr = emitPointerWithAlignment(expr: e->getBase());
1607 mlir::Value idx = emitIdxAfterBase(/*promote=*/true);
1608
1609 // The element count here is the total number of non-VLA elements.
1610 mlir::Value numElements = getVLASize(type: vla).numElts;
1611 idx = builder.createIntCast(src: idx, newTy: numElements.getType());
1612
1613 // Effectively, the multiply by the VLA size is part of the GEP.
1614 // GEP indexes are signed, and scaling an index isn't permitted to
1615 // signed-overflow, so we use the same semantics for our explicit
1616 // multiply. We suppress this if overflow is not undefined behavior.
1617 cir::OverflowFlags overflowFlags = getLangOpts().PointerOverflowDefined
1618 ? cir::OverflowFlags::None
1619 : cir::OverflowFlags::NoSignedWrap;
1620 idx = builder.createMul(loc: cgm.getLoc(cLoc: e->getExprLoc()), lhs: idx, rhs: numElements,
1621 flags: overflowFlags);
1622
1623 addr = emitArraySubscriptPtr(cgf&: *this, beginLoc: cgm.getLoc(cLoc: e->getBeginLoc()),
1624 endLoc: cgm.getLoc(cLoc: e->getEndLoc()), addr, eltType: e->getType(),
1625 idx, loc: cgm.getLoc(cLoc: e->getExprLoc()),
1626 signedIndices, /*shouldDecay=*/false);
1627
1628 return makeAddrLValue(addr, ty: vla->getElementType(), baseInfo: LValueBaseInfo());
1629 }
1630
1631 if (const Expr *array = getSimpleArrayDecayOperand(e: e->getBase())) {
1632 LValue arrayLV;
1633 if (const auto *ase = dyn_cast<ArraySubscriptExpr>(Val: array))
1634 arrayLV = emitArraySubscriptExpr(e: ase);
1635 else
1636 arrayLV = emitLValue(e: array);
1637 mlir::Value idx = emitIdxAfterBase(/*promote=*/true);
1638
1639 // Propagate the alignment from the array itself to the result.
1640 const Address addr = emitArraySubscriptPtr(
1641 cgf&: *this, beginLoc: cgm.getLoc(cLoc: array->getBeginLoc()), endLoc: cgm.getLoc(cLoc: array->getEndLoc()),
1642 addr: arrayLV.getAddress(), eltType: e->getType(), idx, loc: cgm.getLoc(cLoc: e->getExprLoc()),
1643 signedIndices, /*shouldDecay=*/true);
1644
1645 const LValue lv = LValue::makeAddr(address: addr, t: e->getType(), baseInfo: LValueBaseInfo());
1646
1647 if (getLangOpts().ObjC && getLangOpts().getGC() != LangOptions::NonGC) {
1648 cgm.errorNYI(e->getSourceRange(), "emitArraySubscriptExpr: ObjC with GC");
1649 }
1650
1651 return lv;
1652 }
1653
1654 // The base must be a pointer; emit it with an estimate of its alignment.
1655 assert(e->getBase()->getType()->isPointerType() &&
1656 "The base must be a pointer");
1657
1658 LValueBaseInfo eltBaseInfo;
1659 const Address ptrAddr = emitPointerWithAlignment(expr: e->getBase(), baseInfo: &eltBaseInfo);
1660 const mlir::Value idx = emitIdxAfterBase(/*promote=*/true);
1661 // Propagate the alignment from the array itself to the result.
1662 const Address addxr = emitArraySubscriptPtr(
1663 cgf&: *this, beginLoc: cgm.getLoc(cLoc: e->getBeginLoc()), endLoc: cgm.getLoc(cLoc: e->getEndLoc()), addr: ptrAddr,
1664 eltType: e->getType(), idx, loc: cgm.getLoc(cLoc: e->getExprLoc()), signedIndices,
1665 /*shouldDecay=*/false);
1666
1667 const LValue lv = LValue::makeAddr(address: addxr, t: e->getType(), baseInfo: eltBaseInfo);
1668
1669 if (getLangOpts().ObjC && getLangOpts().getGC() != LangOptions::NonGC) {
1670 cgm.errorNYI(e->getSourceRange(), "emitArraySubscriptExpr: ObjC with GC");
1671 }
1672
1673 return lv;
1674}
1675
1676LValue CIRGenFunction::emitMatrixSubscriptExpr(const MatrixSubscriptExpr *e) {
1677 assert(!e->isIncomplete() &&
1678 "incomplete matrix subscript expressions should be rejected in Sema");
1679 LValue base = emitLValue(e: e->getBase());
1680 mlir::Value rowIdx = emitScalarExpr(e: e->getRowIdx());
1681 mlir::Value colIdx = emitScalarExpr(e: e->getColumnIdx());
1682
1683 if (isMatrixRowMajor(LangOpts: getLangOpts(), T: e->getBase()->getType()))
1684 cgm.errorNYI(e->getSourceRange(),
1685 "emitMatrixSubscriptExpr: row-major matrix");
1686
1687 return LValue::makeMatrixElt(matAddress: base.getAddress(), rowIdx, colIdx,
1688 t: e->getBase()->getType(), baseInfo: base.getBaseInfo());
1689}
1690
1691LValue CIRGenFunction::emitExtVectorElementExpr(const ExtVectorElementExpr *e) {
1692 // Emit the base vector as an l-value.
1693 LValue base;
1694
1695 // ExtVectorElementExpr's base can either be a vector or pointer to vector.
1696 if (e->isArrow()) {
1697 // If it is a pointer to a vector, emit the address and form an lvalue with
1698 // it.
1699 LValueBaseInfo baseInfo;
1700 Address ptr = emitPointerWithAlignment(expr: e->getBase(), baseInfo: &baseInfo);
1701 const auto *clangPtrTy =
1702 e->getBase()->getType()->castAs<clang::PointerType>();
1703 base = makeAddrLValue(addr: ptr, ty: clangPtrTy->getPointeeType(), baseInfo);
1704 base.getQuals().removeObjCGCAttr();
1705 } else if (e->getBase()->isGLValue()) {
1706 // Otherwise, if the base is an lvalue ( as in the case of foo.x.x),
1707 // emit the base as an lvalue.
1708 assert(e->getBase()->getType()->isVectorType());
1709 base = emitLValue(e: e->getBase());
1710 } else {
1711 // Otherwise, the base is a normal rvalue (as in (V+V).x), emit it as such.
1712 assert(e->getBase()->getType()->isVectorType() &&
1713 "Result must be a vector");
1714 mlir::Value vec = emitScalarExpr(e: e->getBase());
1715
1716 // Store the vector to memory (because LValue wants an address).
1717 QualType baseTy = e->getBase()->getType();
1718 Address vecMem = createMemTemp(t: baseTy, loc: vec.getLoc(), name: "tmp");
1719 if (!getLangOpts().HLSL && baseTy->isExtVectorBoolType()) {
1720 cgm.errorNYI(e->getSourceRange(),
1721 "emitExtVectorElementExpr: ExtVectorBoolType & !HLSL");
1722 return {};
1723 }
1724 builder.createStore(loc: vec.getLoc(), val: vec, dst: vecMem);
1725 base = makeAddrLValue(addr: vecMem, ty: baseTy, source: AlignmentSource::Decl);
1726 }
1727
1728 QualType type =
1729 e->getType().withCVRQualifiers(CVR: base.getQuals().getCVRQualifiers());
1730
1731 // Encode the element access list into a vector of unsigned indices.
1732 SmallVector<uint32_t, 4> indices;
1733 e->getEncodedElementAccess(Elts&: indices);
1734
1735 if (base.isSimple()) {
1736 SmallVector<int64_t> attrElts(indices.begin(), indices.end());
1737 mlir::ArrayAttr elts = builder.getI64ArrayAttr(values: attrElts);
1738 return LValue::makeExtVectorElt(vecAddress: base.getAddress(), elts, type,
1739 baseInfo: base.getBaseInfo());
1740 }
1741
1742 if (base.isMatrixRow()) {
1743 cgm.errorNYI(e->getSourceRange(), "emitExtVectorElementExpr: isMatrixRow");
1744 return {};
1745 }
1746
1747 assert(base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
1748 mlir::ArrayAttr baseElts = base.getExtVectorElts();
1749 SmallVector<int64_t> elts;
1750 for (unsigned idx : indices)
1751 elts.push_back(Elt: getAccessedFieldNo(idx, elts: baseElts));
1752 mlir::ArrayAttr cv = builder.getI64ArrayAttr(values: elts);
1753 return LValue::makeExtVectorElt(vecAddress: base.getAddress(), elts: cv, type,
1754 baseInfo: base.getBaseInfo());
1755}
1756
1757LValue CIRGenFunction::emitMatrixElementExpr(const MatrixElementExpr *e) {
1758 LValue base;
1759 if (e->getBase()->isGLValue()) {
1760 base = emitLValue(e: e->getBase());
1761 } else {
1762 cgm.errorNYI(e->getSourceRange(), "emitMatrixElementExpr base not GLValue");
1763 return {};
1764 }
1765
1766 QualType resultType =
1767 e->getType().withCVRQualifiers(CVR: base.getQuals().getCVRQualifiers());
1768
1769 // Encode the element access list into a vector of unsigned indices.
1770 // getEncodedElementAccess returns row-major linearized indices.
1771 SmallVector<uint32_t, 4> indices;
1772 e->getEncodedElementAccess(Elts&: indices);
1773
1774 // getEncodedElementAccess returns row-major linearized indices
1775 // If the matrix memory layout is column-major, convert indices
1776 // to column-major indices.
1777 bool isRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T: e->getBase()->getType());
1778 if (!isRowMajor) {
1779 const auto *matrixTy =
1780 e->getBase()->getType()->castAs<ConstantMatrixType>();
1781 unsigned numCols = matrixTy->getNumColumns();
1782 for (uint32_t &idx : indices) {
1783 // Decompose row-major index: Row = Idx / NumCols, Col = Idx % NumCols
1784 unsigned row = idx / numCols;
1785 unsigned column = idx % numCols;
1786 // Re-linearize as column-major
1787 idx = matrixTy->getColumnMajorFlattenedIndex(Row: row, Column: column);
1788 }
1789 }
1790
1791 if (base.isSimple()) {
1792 Address matrixAddr = base.getAddress();
1793 if (getLangOpts().HLSL &&
1794 e->getBase()->getType().getAddressSpace() == LangAS::hlsl_constant) {
1795 cgm.errorNYI(
1796 e->getSourceRange(),
1797 "emitMatrixElementExpr simple & HLSL address space hlsl_constant");
1798 return {};
1799 }
1800
1801 SmallVector<int64_t> attrElts(indices.begin(), indices.end());
1802 mlir::ArrayAttr elts = builder.getI64ArrayAttr(values: attrElts);
1803 return LValue::makeExtVectorElt(vecAddress: matrixAddr, elts, type: resultType,
1804 baseInfo: base.getBaseInfo());
1805 }
1806
1807 cgm.errorNYI(e->getSourceRange(), "emitMatrixElementExpr not simple");
1808 return {};
1809}
1810
1811LValue CIRGenFunction::emitStringLiteralLValue(const StringLiteral *e,
1812 llvm::StringRef name) {
1813 cir::GlobalOp globalOp = cgm.getGlobalForStringLiteral(s: e, name);
1814 assert(globalOp.getAlignment() && "expected alignment for string literal");
1815 unsigned align = *(globalOp.getAlignment());
1816 mlir::Value addr =
1817 builder.createGetGlobal(loc: getLoc(srcLoc: e->getSourceRange()), global: globalOp);
1818 mlir::ptr::MemorySpaceAttrInterface destAS =
1819 cgm.getTypes().getPointerAddressSpace(pointeeTy: e->getType());
1820 if (mlir::cast<cir::PointerType>(Val: addr.getType()).getAddrSpace() != destAS)
1821 addr = performAddrSpaceCast(
1822 v: addr, destTy: builder.getPointerTo(ty: globalOp.getSymType(), as: destAS));
1823 return makeAddrLValue(
1824 addr: Address(addr, globalOp.getSymType(), CharUnits::fromQuantity(Quantity: align)),
1825 ty: e->getType(), source: AlignmentSource::Decl);
1826}
1827
1828/// Casts are never lvalues unless that cast is to a reference type. If the cast
1829/// is to a reference, we can have the usual lvalue result, otherwise if a cast
1830/// is needed by the code generator in an lvalue context, then it must mean that
1831/// we need the address of an aggregate in order to access one of its members.
1832/// This can happen for all the reasons that casts are permitted with aggregate
1833/// result, including noop aggregate casts, and cast from scalar to union.
1834LValue CIRGenFunction::emitCastLValue(const CastExpr *e) {
1835 switch (e->getCastKind()) {
1836 case CK_ToVoid:
1837 case CK_BitCast:
1838 case CK_LValueToRValueBitCast:
1839 case CK_ArrayToPointerDecay:
1840 case CK_FunctionToPointerDecay:
1841 case CK_NullToMemberPointer:
1842 case CK_NullToPointer:
1843 case CK_IntegralToPointer:
1844 case CK_PointerToIntegral:
1845 case CK_PointerToBoolean:
1846 case CK_IntegralCast:
1847 case CK_BooleanToSignedIntegral:
1848 case CK_IntegralToBoolean:
1849 case CK_IntegralToFloating:
1850 case CK_FloatingToIntegral:
1851 case CK_FloatingToBoolean:
1852 case CK_FloatingCast:
1853 case CK_FloatingRealToComplex:
1854 case CK_FloatingComplexToReal:
1855 case CK_FloatingComplexToBoolean:
1856 case CK_FloatingComplexCast:
1857 case CK_FloatingComplexToIntegralComplex:
1858 case CK_IntegralRealToComplex:
1859 case CK_IntegralComplexToReal:
1860 case CK_IntegralComplexToBoolean:
1861 case CK_IntegralComplexCast:
1862 case CK_IntegralComplexToFloatingComplex:
1863 case CK_DerivedToBaseMemberPointer:
1864 case CK_BaseToDerivedMemberPointer:
1865 case CK_MemberPointerToBoolean:
1866 case CK_ReinterpretMemberPointer:
1867 case CK_AnyPointerToBlockPointerCast:
1868 case CK_ARCProduceObject:
1869 case CK_ARCConsumeObject:
1870 case CK_ARCReclaimReturnedObject:
1871 case CK_ARCExtendBlockObject:
1872 case CK_CopyAndAutoreleaseBlockObject:
1873 case CK_IntToOCLSampler:
1874 case CK_FloatingToFixedPoint:
1875 case CK_FixedPointToFloating:
1876 case CK_FixedPointCast:
1877 case CK_FixedPointToBoolean:
1878 case CK_FixedPointToIntegral:
1879 case CK_IntegralToFixedPoint:
1880 case CK_MatrixCast:
1881 case CK_HLSLVectorTruncation:
1882 case CK_HLSLMatrixTruncation:
1883 case CK_HLSLArrayRValue:
1884 case CK_HLSLElementwiseCast:
1885 case CK_HLSLAggregateSplatCast:
1886 llvm_unreachable("unexpected cast lvalue");
1887
1888 case CK_Dependent:
1889 llvm_unreachable("dependent cast kind in IR gen!");
1890
1891 case CK_BuiltinFnToFnPtr:
1892 llvm_unreachable("builtin functions are handled elsewhere");
1893
1894 case CK_Dynamic: {
1895 LValue lv = emitLValue(e: e->getSubExpr());
1896 Address v = lv.getAddress();
1897 const auto *dce = cast<CXXDynamicCastExpr>(Val: e);
1898 return makeNaturalAlignAddrLValue(val: emitDynamicCast(thisAddr: v, dce), ty: e->getType());
1899 }
1900
1901 // These are never l-values; just use the aggregate emission code.
1902 case CK_ToUnion:
1903 return emitAggExprToLValue(e);
1904
1905 case CK_ConstructorConversion:
1906 case CK_UserDefinedConversion:
1907 case CK_CPointerToObjCPointerCast:
1908 case CK_BlockPointerToObjCPointerCast:
1909 case CK_LValueToRValue:
1910 return emitLValue(e: e->getSubExpr());
1911
1912 case CK_NonAtomicToAtomic:
1913 case CK_AtomicToNonAtomic:
1914 case CK_ObjCObjectLValueCast:
1915 case CK_VectorSplat: {
1916 return emitLValue(e: e->getSubExpr());
1917 }
1918
1919 case CK_AddressSpaceConversion: {
1920 LValue lv = emitLValue(e: e->getSubExpr());
1921 QualType destTy = getContext().getPointerType(T: e->getType());
1922
1923 mlir::Value v = performAddrSpaceCast(v: lv.getPointer(), destTy: convertType(t: destTy));
1924
1925 return makeAddrLValue(addr: Address(v, convertTypeForMem(t: e->getType()),
1926 lv.getAddress().getAlignment()),
1927 ty: e->getType(), baseInfo: lv.getBaseInfo());
1928 }
1929
1930 case CK_LValueBitCast: {
1931 // This must be a reinterpret_cast (or c-style equivalent).
1932 const auto *ce = cast<ExplicitCastExpr>(Val: e);
1933
1934 cgm.emitExplicitCastExprType(e: ce, cgf: this);
1935 LValue LV = emitLValue(e: e->getSubExpr());
1936 Address V = LV.getAddress().withElementType(
1937 builder, ElemTy: convertTypeForMem(t: ce->getTypeAsWritten()->getPointeeType()));
1938
1939 return makeAddrLValue(addr: V, ty: e->getType(), baseInfo: LV.getBaseInfo());
1940 }
1941
1942 case CK_NoOp: {
1943 // CK_NoOp can model a qualification conversion, which can remove an array
1944 // bound and change the IR type.
1945 LValue lv = emitLValue(e: e->getSubExpr());
1946 // Propagate the volatile qualifier to LValue, if exists in e.
1947 if (e->changesVolatileQualification())
1948 lv.getQuals() = e->getType().getQualifiers();
1949 if (lv.isSimple()) {
1950 Address v = lv.getAddress();
1951 if (v.isValid()) {
1952 mlir::Type ty = convertTypeForMem(t: e->getType());
1953 if (v.getElementType() != ty) {
1954 // We have only inspected/reproduced this with complete to incomplete
1955 // array types, so we do an NYI for other cases, so we can make sure
1956 // we're doing a conversion we want to be making.
1957 auto fromTy = dyn_cast<cir::ArrayType>(Val: v.getElementType());
1958 auto toTy = dyn_cast<cir::ArrayType>(Val&: ty);
1959 if (!fromTy || !toTy ||
1960 fromTy.getElementType() != toTy.getElementType() ||
1961 toTy.getSize() != 0)
1962 cgm.errorNYI(e->getSourceRange(),
1963 "emitCastLValue NoOp not array-shrink case");
1964
1965 lv = makeAddrLValue(addr: v.withElementType(builder, ElemTy: ty), ty: e->getType(),
1966 baseInfo: lv.getBaseInfo());
1967 }
1968 }
1969 }
1970 return lv;
1971 }
1972
1973 case CK_UncheckedDerivedToBase:
1974 case CK_DerivedToBase: {
1975 auto *derivedClassDecl = e->getSubExpr()->getType()->castAsCXXRecordDecl();
1976
1977 LValue lv = emitLValue(e: e->getSubExpr());
1978 Address thisAddr = lv.getAddress();
1979
1980 // Perform the derived-to-base conversion
1981 Address baseAddr =
1982 getAddressOfBaseClass(value: thisAddr, derived: derivedClassDecl, path: e->path(),
1983 /*NullCheckValue=*/nullCheckValue: false, loc: e->getExprLoc());
1984
1985 // TODO: Support accesses to members of base classes in TBAA. For now, we
1986 // conservatively pretend that the complete object is of the base class
1987 // type.
1988 assert(!cir::MissingFeatures::opTBAA());
1989 return makeAddrLValue(addr: baseAddr, ty: e->getType(), baseInfo: lv.getBaseInfo());
1990 }
1991
1992 case CK_BaseToDerived: {
1993 const auto *derivedClassDecl = e->getType()->castAsCXXRecordDecl();
1994 LValue lv = emitLValue(e: e->getSubExpr());
1995
1996 // Perform the base-to-derived conversion
1997 Address derived = getAddressOfDerivedClass(
1998 loc: getLoc(srcLoc: e->getSourceRange()), baseAddr: lv.getAddress(), derived: derivedClassDecl,
1999 path: e->path(), /*NullCheckValue=*/nullCheckValue: false);
2000 // C++11 [expr.static.cast]p2: Behavior is undefined if a downcast is
2001 // performed and the object is not of the derived type.
2002 assert(!cir::MissingFeatures::sanitizers());
2003
2004 assert(!cir::MissingFeatures::opTBAA());
2005 return makeAddrLValue(addr: derived, ty: e->getType(), baseInfo: lv.getBaseInfo());
2006 }
2007
2008 case CK_ZeroToOCLOpaqueType:
2009 llvm_unreachable("NULL to OpenCL opaque type lvalue cast is not valid");
2010 }
2011
2012 llvm_unreachable("Invalid cast kind");
2013}
2014
2015static DeclRefExpr *tryToConvertMemberExprToDeclRefExpr(CIRGenFunction &cgf,
2016 const MemberExpr *me) {
2017 if (auto *vd = dyn_cast<VarDecl>(Val: me->getMemberDecl())) {
2018 // Try to emit static variable member expressions as DREs.
2019 return DeclRefExpr::Create(
2020 Context: cgf.getContext(), QualifierLoc: NestedNameSpecifierLoc(), TemplateKWLoc: SourceLocation(), D: vd,
2021 /*RefersToEnclosingVariableOrCapture=*/false, NameLoc: me->getExprLoc(),
2022 T: me->getType(), VK: me->getValueKind(), FoundD: nullptr, TemplateArgs: nullptr, NOUR: me->isNonOdrUse());
2023 }
2024 return nullptr;
2025}
2026
2027LValue CIRGenFunction::emitMemberExpr(const MemberExpr *e) {
2028 if (DeclRefExpr *dre = tryToConvertMemberExprToDeclRefExpr(cgf&: *this, me: e)) {
2029 emitIgnoredExpr(e: e->getBase());
2030 return emitDeclRefLValue(e: dre);
2031 }
2032
2033 Expr *baseExpr = e->getBase();
2034 // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a scalar.
2035 LValue baseLV;
2036 if (e->isArrow()) {
2037 LValueBaseInfo baseInfo;
2038 assert(!cir::MissingFeatures::opTBAA());
2039 Address addr = emitPointerWithAlignment(expr: baseExpr, baseInfo: &baseInfo);
2040 QualType ptrTy = baseExpr->getType()->getPointeeType();
2041 assert(!cir::MissingFeatures::typeChecks());
2042 baseLV = makeAddrLValue(addr, ty: ptrTy, baseInfo);
2043 } else {
2044 assert(!cir::MissingFeatures::typeChecks());
2045 baseLV = emitLValue(e: baseExpr);
2046 }
2047
2048 const NamedDecl *nd = e->getMemberDecl();
2049 if (auto *field = dyn_cast<FieldDecl>(Val: nd)) {
2050 LValue lv = emitLValueForField(base: baseLV, field);
2051 assert(!cir::MissingFeatures::setObjCGCLValueClass());
2052 if (getLangOpts().OpenMP) {
2053 // If the member was explicitly marked as nontemporal, mark it as
2054 // nontemporal. If the base lvalue is marked as nontemporal, mark access
2055 // to children as nontemporal too.
2056 assert(!cir::MissingFeatures::openMPNontemporalDecl());
2057 if (baseLV.isNontemporal())
2058 lv.setNontemporal(true);
2059 }
2060 return lv;
2061 }
2062
2063 if (const auto *fd = dyn_cast<FunctionDecl>(Val: nd))
2064 return emitFunctionDeclLValue(cgf&: *this, e, gd: fd);
2065
2066 llvm_unreachable("Unhandled member declaration!");
2067}
2068
2069/// Evaluate an expression into a given memory location.
2070void CIRGenFunction::emitAnyExprToMem(const Expr *e, Address location,
2071 Qualifiers quals, bool isInit) {
2072 // FIXME: This function should take an LValue as an argument.
2073 switch (getEvaluationKind(type: e->getType())) {
2074 case cir::TEK_Complex: {
2075 LValue lv = makeAddrLValue(addr: location, ty: e->getType());
2076 emitComplexExprIntoLValue(e, dest: lv, isInit);
2077 return;
2078 }
2079
2080 case cir::TEK_Aggregate: {
2081 emitAggExpr(e, slot: AggValueSlot::forAddr(addr: location, quals,
2082 isDestructed: AggValueSlot::IsDestructed_t(isInit),
2083 isAliased: AggValueSlot::IsAliased_t(!isInit),
2084 mayOverlap: AggValueSlot::MayOverlap));
2085 return;
2086 }
2087
2088 case cir::TEK_Scalar: {
2089 RValue rv = RValue::get(v: emitScalarExpr(e));
2090 LValue lv = makeAddrLValue(addr: location, ty: e->getType());
2091 emitStoreThroughLValue(src: rv, dst: lv);
2092 return;
2093 }
2094 }
2095
2096 llvm_unreachable("bad evaluation kind");
2097}
2098
2099static Address createReferenceTemporary(CIRGenFunction &cgf,
2100 const MaterializeTemporaryExpr *m,
2101 const Expr *inner) {
2102 // TODO(cir): cgf.getTargetHooks();
2103 switch (m->getStorageDuration()) {
2104 case SD_FullExpression:
2105 case SD_Automatic: {
2106 QualType ty = inner->getType();
2107
2108 assert(!cir::MissingFeatures::mergeAllConstants());
2109
2110 // The temporary memory should be created in the same scope as the extending
2111 // declaration of the temporary materialization expression.
2112 cir::AllocaOp extDeclAlloca;
2113 if (const ValueDecl *extDecl = m->getExtendingDecl()) {
2114 auto extDeclAddrIter = cgf.localDeclMap.find(Val: extDecl);
2115 if (extDeclAddrIter != cgf.localDeclMap.end())
2116 extDeclAlloca = extDeclAddrIter->second.getUnderlyingAllocaOp();
2117 }
2118 mlir::OpBuilder::InsertPoint ip;
2119 if (extDeclAlloca) {
2120 ip = {extDeclAlloca->getBlock(), extDeclAlloca->getIterator()};
2121 } else if (cgf.isInConditionalBranch() &&
2122 m->getStorageDuration() == SD_FullExpression) {
2123 // Place in the function entry block so the alloca dominates both
2124 // regions of any enclosing cir.cleanup.scope. The default path
2125 // would use curLexScope which may be a ternary branch.
2126 ip = cgf.getBuilder().getBestAllocaInsertPoint(
2127 block: cgf.getCurFunctionEntryBlock());
2128 }
2129 return cgf.createMemTemp(t: ty, loc: cgf.getLoc(srcLoc: m->getSourceRange()),
2130 name: cgf.getCounterRefTmpAsString(), /*alloca=*/nullptr,
2131 ip);
2132 }
2133 case SD_Thread:
2134 case SD_Static: {
2135 auto addr =
2136 mlir::cast<cir::GlobalOp>(Val: cgf.cgm.getAddrOfGlobalTemporary(mte: m, init: inner));
2137 auto getGlobal = cgf.cgm.getBuilder().createGetGlobal(global: addr);
2138 assert(addr.getAlignment().has_value() &&
2139 "This should always have an alignment");
2140 return Address(getGlobal,
2141 clang::CharUnits::fromQuantity(Quantity: addr.getAlignment().value()));
2142 }
2143
2144 case SD_Dynamic:
2145 llvm_unreachable("temporary can't have dynamic storage duration");
2146 }
2147 llvm_unreachable("unknown storage duration");
2148}
2149
2150static void pushTemporaryCleanup(CIRGenFunction &cgf,
2151 const MaterializeTemporaryExpr *m,
2152 const Expr *e, Address referenceTemporary) {
2153 // Objective-C++ ARC:
2154 // If we are binding a reference to a temporary that has ownership, we
2155 // need to perform retain/release operations on the temporary.
2156 //
2157 // FIXME(ogcg): This should be looking at e, not m.
2158 if (m->getType().getObjCLifetime()) {
2159 cgf.cgm.errorNYI(e->getSourceRange(), "pushTemporaryCleanup: ObjCLifetime");
2160 return;
2161 }
2162
2163 const QualType::DestructionKind dk = e->getType().isDestructedType();
2164 if (dk == QualType::DK_none)
2165 return;
2166
2167 switch (m->getStorageDuration()) {
2168 case SD_Static:
2169 case SD_Thread: {
2170 CXXDestructorDecl *referenceTemporaryDtor = nullptr;
2171 if (const auto *classDecl =
2172 e->getType()->getBaseElementTypeUnsafe()->getAsCXXRecordDecl();
2173 classDecl && !classDecl->hasTrivialDestructor())
2174 // Get the destructor for the reference temporary.
2175 referenceTemporaryDtor = classDecl->getDestructor();
2176
2177 if (!referenceTemporaryDtor)
2178 return;
2179
2180 CIRGenModule &cgm = cgf.cgm;
2181 auto globalOp =
2182 mlir::cast<cir::GlobalOp>(Val: cgm.getAddrOfGlobalTemporary(mte: m, init: e));
2183
2184 CIRGenBuilderTy &builder = cgm.getBuilder();
2185 mlir::Location loc = cgm.getLoc(cRange: m->getSourceRange());
2186 auto registerOp = cir::RegisterExitDtorOp::create(
2187 builder, location: loc, object: globalOp.getSymNameAttr().getValue());
2188 mlir::OpBuilder::InsertionGuard guard(builder);
2189 builder.setInsertionPointToStart(
2190 builder.createBlock(parent: &registerOp.getBody()));
2191
2192 mlir::Value tempAddr = builder.createGetGlobal(global: globalOp);
2193 if (e->getType()->isArrayType()) {
2194 Address addr{tempAddr, cgf.convertTypeForMem(t: e->getType()),
2195 referenceTemporary.getAlignment()};
2196 cgf.emitDestroy(addr, type: e->getType(), destroyer: CIRGenFunction::destroyCXXObject);
2197 } else {
2198 GlobalDecl gd(referenceTemporaryDtor, Dtor_Complete);
2199 cir::FuncOp dtorFn = cgm.getAddrAndTypeOfCXXStructor(gd).second;
2200 // Make sure the call and the callee agree on calling convention.
2201 builder.createCallOp(loc, callee: dtorFn, operands: mlir::ValueRange{tempAddr},
2202 /*attrs=*/{}, /*argAttrs=*/{}, /*resAttrs=*/{},
2203 callingConv: dtorFn.getCallingConv());
2204 }
2205 cir::YieldOp::create(builder, location: loc);
2206 break;
2207 }
2208
2209 case SD_FullExpression:
2210 cgf.pushDestroy(kind: NormalAndEHCleanup, addr: referenceTemporary, type: e->getType(),
2211 destroyer: CIRGenFunction::destroyCXXObject);
2212 break;
2213
2214 case SD_Automatic:
2215 cgf.pushLifetimeExtendedDestroy(
2216 kind: NormalAndEHCleanup, addr: referenceTemporary, type: e->getType(),
2217 destroyer: CIRGenFunction::destroyCXXObject, useEHCleanupForArray: cgf.getLangOpts().Exceptions,
2218 extendingDecl: m->getExtendingDecl());
2219 break;
2220
2221 case SD_Dynamic:
2222 llvm_unreachable("temporary cannot have dynamic storage duration");
2223 }
2224}
2225
2226LValue CIRGenFunction::emitMaterializeTemporaryExpr(
2227 const MaterializeTemporaryExpr *m) {
2228 const Expr *e = m->getSubExpr();
2229
2230 assert((!m->getExtendingDecl() || !isa<VarDecl>(m->getExtendingDecl()) ||
2231 !cast<VarDecl>(m->getExtendingDecl())->isARCPseudoStrong()) &&
2232 "Reference should never be pseudo-strong!");
2233
2234 // FIXME: ideally this would use emitAnyExprToMem, however, we cannot do so
2235 // as that will cause the lifetime adjustment to be lost for ARC
2236 auto ownership = m->getType().getObjCLifetime();
2237 if (ownership != Qualifiers::OCL_None &&
2238 ownership != Qualifiers::OCL_ExplicitNone) {
2239 cgm.errorNYI(e->getSourceRange(),
2240 "emitMaterializeTemporaryExpr: ObjCLifetime");
2241 return {};
2242 }
2243
2244 SmallVector<const Expr *, 2> commaLHSs;
2245 SmallVector<SubobjectAdjustment, 2> adjustments;
2246 e = e->skipRValueSubobjectAdjustments(CommaLHS&: commaLHSs, Adjustments&: adjustments);
2247
2248 for (const Expr *ignored : commaLHSs)
2249 emitIgnoredExpr(e: ignored);
2250
2251 if (isa<OpaqueValueExpr>(Val: e)) {
2252 cgm.errorNYI(e->getSourceRange(),
2253 "emitMaterializeTemporaryExpr: OpaqueValueExpr");
2254 return {};
2255 }
2256
2257 // Create and initialize the reference temporary.
2258 Address object = createReferenceTemporary(cgf&: *this, m, inner: e);
2259 cir::GlobalOp var = nullptr;
2260 if (auto getGlobalOp = object.getPointer().getDefiningOp<cir::GetGlobalOp>())
2261 var = mlir::dyn_cast_or_null<cir::GlobalOp>(
2262 Val: cgm.getGlobalValue(ref: getGlobalOp.getName()));
2263
2264 if (var) {
2265 if (!var.getInitialValue().has_value()) {
2266 var.setInitialValueAttr(builder.getZeroInitAttr(ty: var.getSymType()));
2267 assert(!cir::MissingFeatures::pointerAuthentication());
2268 emitAnyExprToMem(e, location: object, quals: Qualifiers(), /*isInitializer=*/isInit: true);
2269 }
2270 } else {
2271 assert(!cir::MissingFeatures::emitLifetimeMarkers());
2272 emitAnyExprToMem(e, location: object, quals: Qualifiers(), /*isInitializer=*/isInit: true);
2273 }
2274 pushTemporaryCleanup(cgf&: *this, m, e, referenceTemporary: object);
2275
2276 // Perform derived-to-base casts and/or field accesses, to get from the
2277 // temporary object we created (and, potentially, for which we extended
2278 // the lifetime) to the subobject we're binding the reference to.
2279 for (SubobjectAdjustment &adjustment : llvm::reverse(C&: adjustments)) {
2280 switch (adjustment.Kind) {
2281 case SubobjectAdjustment::DerivedToBaseAdjustment:
2282 object =
2283 getAddressOfBaseClass(value: object, derived: adjustment.DerivedToBase.DerivedClass,
2284 path: adjustment.DerivedToBase.BasePath->path(),
2285 /*nullCheckValue=*/false, loc: e->getExprLoc());
2286 break;
2287 case SubobjectAdjustment::FieldAdjustment: {
2288 LValue lv = makeAddrLValue(addr: object, ty: e->getType(), source: AlignmentSource::Decl);
2289 lv = emitLValueForField(base: lv, field: adjustment.Field);
2290 assert(lv.isSimple() &&
2291 "materialized temporary field is not a simple lvalue");
2292 object = lv.getAddress();
2293 break;
2294 }
2295 case SubobjectAdjustment::MemberPointerAdjustment: {
2296 mlir::Value ptr = emitScalarExpr(e: adjustment.Ptr.RHS);
2297 object = emitCXXMemberDataPointerAddress(
2298 e, base: object, memberPtr: ptr, memberPtrType: adjustment.Ptr.MPT, /*baseInfo=*/nullptr);
2299 break;
2300 }
2301 }
2302 }
2303
2304 return makeAddrLValue(addr: object, ty: m->getType(), source: AlignmentSource::Decl);
2305}
2306
2307LValue
2308CIRGenFunction::getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e) {
2309 assert(OpaqueValueMapping::shouldBindAsLValue(e));
2310
2311 auto it = opaqueLValues.find(Val: e);
2312 if (it != opaqueLValues.end())
2313 return it->second;
2314
2315 assert(e->isUnique() && "LValue for a nonunique OVE hasn't been emitted");
2316 return emitLValue(e: e->getSourceExpr());
2317}
2318
2319RValue
2320CIRGenFunction::getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e) {
2321 assert(!OpaqueValueMapping::shouldBindAsLValue(e));
2322
2323 auto it = opaqueRValues.find(Val: e);
2324 if (it != opaqueRValues.end())
2325 return it->second;
2326
2327 assert(e->isUnique() && "RValue for a nonunique OVE hasn't been emitted");
2328 return emitAnyExpr(e: e->getSourceExpr());
2329}
2330
2331LValue CIRGenFunction::emitCompoundLiteralLValue(const CompoundLiteralExpr *e) {
2332 if (e->isFileScope()) {
2333 cgm.errorNYI(e->getSourceRange(), "emitCompoundLiteralLValue: FileScope");
2334 return {};
2335 }
2336
2337 if (e->getType()->isVariablyModifiedType())
2338 emitVariablyModifiedType(ty: e->getType());
2339
2340 Address declPtr = createMemTemp(t: e->getType(), loc: getLoc(srcLoc: e->getSourceRange()),
2341 name: ".compoundliteral");
2342 const Expr *initExpr = e->getInitializer();
2343 LValue result = makeAddrLValue(addr: declPtr, ty: e->getType(), source: AlignmentSource::Decl);
2344
2345 emitAnyExprToMem(e: initExpr, location: declPtr, quals: e->getType().getQualifiers(),
2346 /*Init*/ isInit: true);
2347
2348 // Block-scope compound literals are destroyed at the end of the enclosing
2349 // scope in C.
2350 if (!getLangOpts().CPlusPlus && e->getType().isDestructedType()) {
2351 cgm.errorNYI(e->getSourceRange(),
2352 "emitCompoundLiteralLValue: non C++ DestructedType");
2353 return {};
2354 }
2355
2356 return result;
2357}
2358
2359LValue CIRGenFunction::emitCallExprLValue(const CallExpr *e) {
2360 RValue rv = emitCallExpr(e);
2361
2362 if (!rv.isScalar())
2363 return makeAddrLValue(addr: rv.getAggregateAddress(), ty: e->getType(),
2364 source: AlignmentSource::Decl);
2365
2366 assert(e->getCallReturnType(getContext())->isReferenceType() &&
2367 "Can't have a scalar return unless the return type is a "
2368 "reference type!");
2369
2370 return makeNaturalAlignPointeeAddrLValue(v: rv.getValue(), t: e->getType());
2371}
2372
2373LValue
2374CIRGenFunction::emitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *e) {
2375 AggValueSlot slot =
2376 createAggTemp(ty: e->getType(), loc: getLoc(srcLoc: e->getSourceRange()), name: "temp.lvalue");
2377 slot.setExternallyDestructed();
2378 emitAggExpr(e: e->getSubExpr(), slot);
2379 emitCXXTemporary(temporary: e->getTemporary(), tempType: e->getType(), ptr: slot.getAddress());
2380 return makeAddrLValue(addr: slot.getAddress(), ty: e->getType(), source: AlignmentSource::Decl);
2381}
2382
2383LValue CIRGenFunction::emitCXXConstructLValue(const CXXConstructExpr *e) {
2384 assert(e->getType()->getAsCXXRecordDecl()->hasTrivialDestructor() &&
2385 "binding l-value to type which needs a temporary");
2386 AggValueSlot slot = createAggTemp(ty: e->getType(), loc: getLoc(srcLoc: e->getSourceRange()));
2387 emitCXXConstructExpr(e, dest: slot);
2388 return makeAddrLValue(addr: slot.getAddress(), ty: e->getType(), source: AlignmentSource::Decl);
2389}
2390
2391LValue CIRGenFunction::emitBinaryOperatorLValue(const BinaryOperator *e) {
2392 // Comma expressions just emit their LHS then their RHS as an l-value.
2393 if (e->getOpcode() == BO_Comma) {
2394 emitIgnoredExpr(e: e->getLHS());
2395 return emitLValue(e: e->getRHS());
2396 }
2397
2398 if (e->getOpcode() == BO_PtrMemD || e->getOpcode() == BO_PtrMemI)
2399 return emitPointerToDataMemberBinaryExpr(e);
2400
2401 assert(e->getOpcode() == BO_Assign && "unexpected binary l-value");
2402
2403 // Note that in all of these cases, __block variables need the RHS
2404 // evaluated first just in case the variable gets moved by the RHS.
2405
2406 switch (CIRGenFunction::getEvaluationKind(type: e->getType())) {
2407 case cir::TEK_Scalar: {
2408 assert(!cir::MissingFeatures::objCLifetime());
2409 if (e->getLHS()->getType().getObjCLifetime() !=
2410 clang::Qualifiers::ObjCLifetime::OCL_None) {
2411 cgm.errorNYI(e->getSourceRange(), "objc lifetimes");
2412 return {};
2413 }
2414
2415 RValue rv = emitAnyExpr(e: e->getRHS());
2416 LValue lv = emitLValue(e: e->getLHS());
2417
2418 SourceLocRAIIObject loc{*this, e->getSourceRange()};
2419 if (lv.isBitField())
2420 emitStoreThroughBitfieldLValue(src: rv, dst: lv);
2421 else
2422 emitStoreThroughLValue(src: rv, dst: lv);
2423
2424 assert(!cir::MissingFeatures::openMPLastprivateConditional());
2425
2426 return lv;
2427 }
2428
2429 case cir::TEK_Complex: {
2430 return emitComplexAssignmentLValue(e);
2431 }
2432
2433 case cir::TEK_Aggregate:
2434 cgm.errorNYI(e->getSourceRange(), "aggregate lvalues");
2435 return {};
2436 }
2437 llvm_unreachable("bad evaluation kind");
2438}
2439
2440/// Emit code to compute the specified expression which
2441/// can have any type. The result is returned as an RValue struct.
2442RValue CIRGenFunction::emitAnyExpr(const Expr *e, AggValueSlot aggSlot,
2443 bool ignoreResult) {
2444 switch (CIRGenFunction::getEvaluationKind(type: e->getType())) {
2445 case cir::TEK_Scalar:
2446 return RValue::get(v: emitScalarExpr(e, ignoreResultAssign: ignoreResult));
2447 case cir::TEK_Complex:
2448 return RValue::getComplex(v: emitComplexExpr(e));
2449 case cir::TEK_Aggregate: {
2450 if (!ignoreResult && aggSlot.isIgnored())
2451 aggSlot = createAggTemp(ty: e->getType(), loc: getLoc(srcLoc: e->getSourceRange()),
2452 name: getCounterAggTmpAsString());
2453 emitAggExpr(e, slot: aggSlot);
2454 return aggSlot.asRValue();
2455 }
2456 }
2457 llvm_unreachable("bad evaluation kind");
2458}
2459
2460// Returns whether \p fn disables the builtin \p name with no_builtin. An empty
2461// list disables all builtins.
2462static bool hasNoBuiltinAttr(cir::FuncOp fn, StringRef name) {
2463 auto noBuiltins = fn->getAttrOfType<mlir::ArrayAttr>(
2464 name: cir::CIRDialect::getNoBuiltinsAttrName());
2465 if (!noBuiltins)
2466 return false;
2467 return noBuiltins.empty() ||
2468 llvm::is_contained(Range&: noBuiltins,
2469 Element: mlir::StringAttr::get(context: fn.getContext(), bytes: name));
2470}
2471
2472CIRGenCallee CIRGenFunction::emitDirectCallee(const GlobalDecl &gd) {
2473 const auto *fd = cast<FunctionDecl>(Val: gd.getDecl());
2474
2475 if (unsigned builtinID = fd->getBuiltinID()) {
2476 StringRef ident = cgm.getMangledName(gd);
2477 std::string fdInlineName = (ident + ".inline").str();
2478
2479 bool isPredefinedLibFunction =
2480 cgm.getASTContext().BuiltinInfo.isPredefinedLibFunction(ID: builtinID);
2481 auto fn = dyn_cast<cir::FuncOp>(Val: curFn);
2482 bool hasAttributeNoBuiltin = fn && hasNoBuiltinAttr(fn, name: fd->getName());
2483
2484 // When directly calling an inline builtin, call it through it's mangled
2485 // name to make it clear it's not the actual builtin.
2486 if ((!fn || fn.getName() != fdInlineName) &&
2487 CodeGenUtils::onlyHasInlineBuiltinDeclaration(FD: fd)) {
2488 cir::FuncOp clone =
2489 mlir::cast_or_null<cir::FuncOp>(Val: cgm.getGlobalValue(ref: fdInlineName));
2490
2491 if (!clone) {
2492 // Create a forward declaration - the body will be generated in
2493 // generateCode when the function definition is processed
2494 cir::FuncOp calleeFunc = cgm.getRawFunctionPointer(gd);
2495 mlir::OpBuilder::InsertionGuard guard(builder);
2496 builder.setInsertionPointToStart(cgm.getModule().getBody());
2497
2498 clone = cir::FuncOp::create(builder, location: calleeFunc.getLoc(), sym_name: fdInlineName,
2499 type: calleeFunc.getFunctionType());
2500 cgm.insertGlobalSymbol(op: clone);
2501 clone.setLinkageAttr(cir::GlobalLinkageKindAttr::get(
2502 context: &cgm.getMLIRContext(), value: cir::GlobalLinkageKind::InternalLinkage));
2503 clone.setSymVisibility("private");
2504 clone.setInlineKind(cir::InlineKind::AlwaysInline);
2505 }
2506 return CIRGenCallee::forDirect(funcPtr: clone, abstractInfo: gd);
2507 }
2508
2509 // Replaceable builtins provide their own implementation of a builtin. If we
2510 // are in an inline builtin implementation, avoid trivial infinite
2511 // recursion. Honor __attribute__((no_builtin("foo"))) or
2512 // __attribute__((no_builtin)) on the current function unless foo is
2513 // not a predefined library function which means we must generate the
2514 // builtin no matter what.
2515 else if (!isPredefinedLibFunction || !hasAttributeNoBuiltin)
2516 return CIRGenCallee::forBuiltin(builtinID, builtinDecl: fd);
2517 }
2518
2519 cir::FuncOp callee = cgm.getRawFunctionPointer(gd);
2520 if (!callee)
2521 return CIRGenCallee();
2522
2523 if ((cgm.getLangOpts().CUDA || cgm.getLangOpts().HIP) &&
2524 !cgm.getLangOpts().CUDAIsDevice && fd->hasAttr<CUDAGlobalAttr>()) {
2525 mlir::Operation *handle = cgm.getCUDARuntime().getKernelHandle(fn: callee, gd);
2526 callee =
2527 mlir::cast<cir::FuncOp>(Val&: *cgm.getCUDARuntime().getKernelStub(handle));
2528 }
2529
2530 return CIRGenCallee::forDirect(funcPtr: callee, abstractInfo: gd);
2531}
2532
2533mlir::Value CIRGenFunction::getUndefConstant(mlir::Location loc,
2534 mlir::Type cirTy) {
2535 return builder.getConstant(loc, attr: cir::UndefAttr::get(type: cirTy));
2536}
2537
2538RValue CIRGenFunction::getUndefRValue(QualType ty) {
2539 if (ty->isVoidType())
2540 return RValue::get(v: nullptr);
2541
2542 mlir::Location loc = builder.getUnknownLoc();
2543
2544 switch (getEvaluationKind(type: ty)) {
2545 case cir::TEK_Complex: {
2546 QualType elemTy = ty->castAs<ComplexType>()->getElementType();
2547 mlir::Type elemCirTy = convertType(t: elemTy);
2548 mlir::Value undefElem = getUndefConstant(loc, cirTy: elemCirTy);
2549 mlir::Value v = builder.createComplexCreate(loc, real: undefElem, imag: undefElem);
2550 return RValue::getComplex(v);
2551 }
2552
2553 // If this is a use of an undefined aggregate type, the aggregate must have
2554 // an identifiable address. Just because the contents of the value are
2555 // undefined doesn't mean that the address can't be taken and compared.
2556 case cir::TEK_Aggregate: {
2557 Address destPtr = createMemTempWithoutCast(t: ty, loc, name: "undef.agg.tmp");
2558 return RValue::getAggregate(addr: destPtr);
2559 }
2560
2561 case cir::TEK_Scalar:
2562 return RValue::get(v: getUndefConstant(loc, cirTy: convertType(t: ty)));
2563 }
2564 llvm_unreachable("bad evaluation kind");
2565}
2566
2567RValue CIRGenFunction::emitCall(clang::QualType calleeTy,
2568 const CIRGenCallee &origCallee,
2569 const clang::CallExpr *e,
2570 ReturnValueSlot returnValue) {
2571 // Get the actual function type. The callee type will always be a pointer to
2572 // function type or a block pointer type.
2573 assert(calleeTy->isFunctionPointerType() &&
2574 "Callee must have function pointer type!");
2575
2576 calleeTy = getContext().getCanonicalType(T: calleeTy);
2577 auto pointeeTy = cast<PointerType>(Val&: calleeTy)->getPointeeType();
2578
2579 CIRGenCallee callee = origCallee;
2580
2581 if (getLangOpts().CPlusPlus)
2582 assert(!cir::MissingFeatures::sanitizers());
2583
2584 const auto *fnType = cast<FunctionType>(Val&: pointeeTy);
2585
2586 assert(!cir::MissingFeatures::sanitizers());
2587
2588 CallArgList args;
2589 EvaluationOrder order = CodeGenUtils::getCallArgEvaluationOrder(E: e);
2590
2591 // C++23 static-member operators (`static operator()` /
2592 // `static operator[]`) produce a CXXOperatorCallExpr whose first argument
2593 // is the object expression even though the operator is static. Emit the
2594 // object for its side effects and drop it before walking the parameter
2595 // arguments.
2596 auto arguments = e->arguments();
2597 if (const auto *oce = dyn_cast<CXXOperatorCallExpr>(Val: e)) {
2598 if (const auto *md =
2599 dyn_cast_if_present<CXXMethodDecl>(Val: oce->getCalleeDecl());
2600 md && md->isStatic()) {
2601 emitIgnoredExpr(e: e->getArg(Arg: 0));
2602 arguments = llvm::drop_begin(RangeOrContainer&: arguments, N: 1);
2603 }
2604 }
2605
2606 emitCallArgs(args, prototype: dyn_cast<FunctionProtoType>(Val: fnType), argRange: arguments,
2607 callee: e->getDirectCallee(), /*paramsToSkip=*/0, order);
2608
2609 const CIRGenFunctionInfo &funcInfo =
2610 cgm.getTypes().arrangeFreeFunctionCall(args, fnType);
2611
2612 // C99 6.5.2.2p6:
2613 // If the expression that denotes the called function has a type that does
2614 // not include a prototype, [the default argument promotions are performed].
2615 // If the number of arguments does not equal the number of parameters, the
2616 // behavior is undefined. If the function is defined with a type that
2617 // includes a prototype, and either the prototype ends with an ellipsis (,
2618 // ...) or the types of the arguments after promotion are not compatible
2619 // with the types of the parameters, the behavior is undefined. If the
2620 // function is defined with a type that does not include a prototype, and
2621 // the types of the arguments after promotion are not compatible with those
2622 // of the parameters after promotion, the behavior is undefined [except in
2623 // some trivial cases].
2624 // That is, in the general case, we should assume that a call through an
2625 // unprototyped function type works like a *non-variadic* call. The way we
2626 // make this work is to cast to the exxact type fo the promoted arguments.
2627 if (isa<FunctionNoProtoType>(Val: fnType)) {
2628 assert(!cir::MissingFeatures::opCallChain());
2629 assert(!cir::MissingFeatures::addressSpace());
2630 cir::FuncType calleeTy = getTypes().getFunctionType(info: funcInfo);
2631 // get non-variadic function type
2632 calleeTy = cir::FuncType::get(inputs: calleeTy.getInputs(),
2633 returnType: calleeTy.getReturnType(), isVarArg: false);
2634 auto calleePtrTy = cir::PointerType::get(pointee: calleeTy);
2635
2636 mlir::Operation *fn = callee.getFunctionPointer();
2637 mlir::Value addr;
2638 if (auto funcOp = mlir::dyn_cast<cir::FuncOp>(Val: fn)) {
2639 addr = cir::GetGlobalOp::create(
2640 builder, location: getLoc(srcLoc: e->getSourceRange()),
2641 addr: cir::PointerType::get(pointee: funcOp.getFunctionType()), name: funcOp.getSymName());
2642 } else {
2643 addr = fn->getResult(idx: 0);
2644 }
2645
2646 fn = builder.createBitcast(src: addr, newTy: calleePtrTy).getDefiningOp();
2647 callee.setFunctionPointer(fn);
2648 }
2649
2650 assert(!cir::MissingFeatures::opCallFnInfoOpts());
2651
2652 // HIP function pointer contains kernel handle when it is used in triple
2653 // chevron. The kernel stub needs to be loaded from kernel handle and used
2654 // as callee.
2655 const clang::Decl *targetDecl =
2656 origCallee.getAbstractInfo().getCalleeDecl().getDecl();
2657 if (getLangOpts().HIP && !getLangOpts().CUDAIsDevice &&
2658 isa<CUDAKernelCallExpr>(Val: e) &&
2659 (!targetDecl || !isa<FunctionDecl>(Val: targetDecl))) {
2660 mlir::Value handleAddr = callee.getFunctionPointer()->getResult(idx: 0);
2661 mlir::Location loc = getLoc(srcLoc: e->getSourceRange());
2662 auto handlePtrTy = mlir::cast<cir::PointerType>(Val: handleAddr.getType());
2663 mlir::Value handleAddrAddr =
2664 builder.createBitcast(src: handleAddr, newTy: cir::PointerType::get(pointee: handlePtrTy));
2665 cir::LoadOp stub = builder.createLoad(
2666 loc, addr: Address(handleAddrAddr, handlePtrTy, getPointerAlign()));
2667 callee.setFunctionPointer(stub.getOperation());
2668 }
2669
2670 cir::CIRCallOpInterface callOp;
2671 RValue callResult = emitCall(funcInfo, callee, returnValue, args, callOp: &callOp,
2672 isMustTail: e == mustTailCall, clangLoc: e->getSourceRange());
2673
2674 assert(!cir::MissingFeatures::generateDebugInfo());
2675
2676 return callResult;
2677}
2678
2679CIRGenCallee CIRGenFunction::emitCallee(const clang::Expr *e) {
2680 e = e->IgnoreParens();
2681
2682 // Look through function-to-pointer decay.
2683 if (const auto *implicitCast = dyn_cast<ImplicitCastExpr>(Val: e)) {
2684 if (implicitCast->getCastKind() == CK_FunctionToPointerDecay ||
2685 implicitCast->getCastKind() == CK_BuiltinFnToFnPtr) {
2686 return emitCallee(e: implicitCast->getSubExpr());
2687 }
2688 // Classic codegen has some handling here for ptr-auth (as a part of the
2689 // large ptr-auth-qualifier PR (#100830)). In the meantime, other cast kinds
2690 // can fall-through and be handled by the indirect call work below,
2691 // including L-to-R value conversions and atomic conversions.
2692 assert(!MissingFeatures::pointerAuthentication());
2693
2694 } else if (const auto *declRef = dyn_cast<DeclRefExpr>(Val: e)) {
2695 // Resolve direct calls.
2696 if (const auto *funcDecl = dyn_cast<FunctionDecl>(Val: declRef->getDecl()))
2697 return emitDirectCallee(gd: funcDecl);
2698 } else if (auto me = dyn_cast<MemberExpr>(Val: e)) {
2699 if (const auto *fd = dyn_cast<FunctionDecl>(Val: me->getMemberDecl())) {
2700 emitIgnoredExpr(e: me->getBase());
2701 return emitDirectCallee(gd: fd);
2702 }
2703 // Else fall through to the indirect reference handling below.
2704 } else if (auto *pde = dyn_cast<CXXPseudoDestructorExpr>(Val: e)) {
2705 return CIRGenCallee::forPseudoDestructor(expr: pde);
2706 }
2707
2708 // Otherwise, we have an indirect reference.
2709 mlir::Value calleePtr;
2710 QualType functionType;
2711 if (const auto *ptrType = e->getType()->getAs<clang::PointerType>()) {
2712 calleePtr = emitScalarExpr(e);
2713 functionType = ptrType->getPointeeType();
2714 } else {
2715 functionType = e->getType();
2716 calleePtr = emitLValue(e).getPointer();
2717 }
2718 assert(functionType->isFunctionType());
2719
2720 GlobalDecl gd;
2721 if (const auto *vd =
2722 dyn_cast_or_null<VarDecl>(Val: e->getReferencedDeclOfCallee()))
2723 gd = GlobalDecl(vd);
2724
2725 CIRGenCalleeInfo calleeInfo(functionType->getAs<FunctionProtoType>(), gd);
2726 CIRGenCallee callee(calleeInfo, calleePtr.getDefiningOp());
2727 return callee;
2728}
2729
2730RValue CIRGenFunction::emitCallExpr(const clang::CallExpr *e,
2731 ReturnValueSlot returnValue) {
2732 assert(!cir::MissingFeatures::objCBlocks());
2733
2734 if (const auto *ce = dyn_cast<CXXMemberCallExpr>(Val: e))
2735 return emitCXXMemberCallExpr(e: ce, returnValue);
2736
2737 if (const auto *cudaKernelCallExpr = dyn_cast<CUDAKernelCallExpr>(Val: e))
2738 return emitCUDAKernelCallExpr(expr: cudaKernelCallExpr, returnValue);
2739
2740 // A CXXOperatorCallExpr is created even for explicit-object methods or
2741 // static member operators (C++23 `static operator()` / `static
2742 // operator[]`), but those should be treated like ordinary static function
2743 // calls. Only route through the member-call path for ordinary instance
2744 // operators.
2745 if (const auto *operatorCall = dyn_cast<CXXOperatorCallExpr>(Val: e)) {
2746 if (const auto *md =
2747 dyn_cast_if_present<CXXMethodDecl>(Val: operatorCall->getCalleeDecl());
2748 md && md->isImplicitObjectMemberFunction())
2749 return emitCXXOperatorMemberCallExpr(e: operatorCall, md, returnValue);
2750 }
2751
2752 CIRGenCallee callee = emitCallee(e: e->getCallee());
2753
2754 // emitCallee has already diagnosed (errorNYI) why there is no callee.
2755 if (callee.isInvalid())
2756 return getUndefRValue(ty: e->getType());
2757
2758 if (callee.isBuiltin())
2759 return emitBuiltinExpr(gd: callee.getBuiltinDecl(), builtinID: callee.getBuiltinID(), e,
2760 returnValue);
2761
2762 if (callee.isPseudoDestructor())
2763 return emitCXXPseudoDestructorExpr(expr: callee.getPseudoDestructorExpr());
2764
2765 return emitCall(calleeTy: e->getCallee()->getType(), origCallee: callee, e, returnValue);
2766}
2767
2768/// Emit code to compute the specified expression, ignoring the result.
2769void CIRGenFunction::emitIgnoredExpr(const Expr *e) {
2770 if (e->isPRValue()) {
2771 emitAnyExpr(e, aggSlot: AggValueSlot::ignored(), /*ignoreResult=*/true);
2772 return;
2773 }
2774
2775 // Just emit it as an l-value and drop the result.
2776 emitLValue(e);
2777}
2778
2779Address CIRGenFunction::emitArrayToPointerDecay(const Expr *e,
2780 LValueBaseInfo *baseInfo) {
2781 assert(!cir::MissingFeatures::opTBAA());
2782 assert(e->getType()->isArrayType() &&
2783 "Array to pointer decay must have array source type!");
2784
2785 // Expressions of array type can't be bitfields or vector elements.
2786 LValue lv = emitLValue(e);
2787 Address addr = lv.getAddress();
2788
2789 if (e->getType()->isVariableArrayType())
2790 return addr;
2791
2792 // If the array type was an incomplete type, we need to make sure
2793 // the decay ends up being the right type.
2794 mlir::Type arrayTy = convertType(t: e->getType());
2795 assert(mlir::isa<cir::ArrayType>(arrayTy) && "expected array");
2796 addr = addr.withElementType(builder, ElemTy: arrayTy);
2797
2798 // The result of this decay conversion points to an array element within the
2799 // base lvalue. However, since TBAA currently does not support representing
2800 // accesses to elements of member arrays, we conservatively represent accesses
2801 // to the pointee object as if it had no any base lvalue specified.
2802 // TODO: Support TBAA for member arrays.
2803 QualType eltType = e->getType()->castAsArrayTypeUnsafe()->getElementType();
2804 assert(!cir::MissingFeatures::opTBAA());
2805
2806 mlir::Value ptr = builder.maybeBuildArrayDecay(
2807 loc: cgm.getLoc(cRange: e->getSourceRange()), arrayPtr: addr.getPointer(),
2808 eltTy: convertTypeForMem(t: eltType));
2809 return Address(ptr, addr.getAlignment());
2810}
2811
2812/// Given the address of a temporary variable, produce an r-value of its type.
2813RValue CIRGenFunction::convertTempToRValue(Address addr, clang::QualType type,
2814 clang::SourceLocation loc) {
2815 LValue lvalue = makeAddrLValue(addr, ty: type, source: AlignmentSource::Decl);
2816 switch (getEvaluationKind(type)) {
2817 case cir::TEK_Complex:
2818 return RValue::getComplex(v: emitLoadOfComplex(src: lvalue, loc));
2819 case cir::TEK_Aggregate:
2820 return lvalue.asAggregateRValue();
2821 case cir::TEK_Scalar:
2822 return RValue::get(v: emitLoadOfScalar(lvalue, loc));
2823 }
2824 llvm_unreachable("bad evaluation kind");
2825}
2826
2827/// Emit an `if` on a boolean condition, filling `then` and `else` into
2828/// appropriated regions.
2829mlir::LogicalResult CIRGenFunction::emitIfOnBoolExpr(const Expr *cond,
2830 const Stmt *thenS,
2831 const Stmt *elseS) {
2832 mlir::Location thenLoc = getLoc(srcLoc: thenS->getSourceRange());
2833 std::optional<mlir::Location> elseLoc;
2834 if (elseS)
2835 elseLoc = getLoc(srcLoc: elseS->getSourceRange());
2836
2837 mlir::LogicalResult resThen = mlir::success(), resElse = mlir::success();
2838 emitIfOnBoolExpr(
2839 cond, /*thenBuilder=*/
2840 [&](mlir::OpBuilder &, mlir::Location) {
2841 LexicalScope lexScope{*this, thenLoc, builder.getInsertionBlock()};
2842 resThen = emitStmt(s: thenS, /*useCurrentScope=*/true);
2843 },
2844 thenLoc,
2845 /*elseBuilder=*/
2846 [&](mlir::OpBuilder &, mlir::Location) {
2847 assert(elseLoc && "Invalid location for elseS.");
2848 LexicalScope lexScope{*this, *elseLoc, builder.getInsertionBlock()};
2849 resElse = emitStmt(s: elseS, /*useCurrentScope=*/true);
2850 },
2851 elseLoc);
2852
2853 return mlir::LogicalResult::success(IsSuccess: resThen.succeeded() &&
2854 resElse.succeeded());
2855}
2856
2857/// Emit an `if` on a boolean condition, filling `then` and `else` into
2858/// appropriated regions.
2859cir::IfOp CIRGenFunction::emitIfOnBoolExpr(
2860 const clang::Expr *cond, BuilderCallbackRef thenBuilder,
2861 mlir::Location thenLoc, BuilderCallbackRef elseBuilder,
2862 std::optional<mlir::Location> elseLoc) {
2863 // Attempt to be as accurate as possible with IfOp location, generate
2864 // one fused location that has either 2 or 4 total locations, depending
2865 // on else's availability.
2866 SmallVector<mlir::Location, 2> ifLocs{thenLoc};
2867 if (elseLoc)
2868 ifLocs.push_back(Elt: *elseLoc);
2869 mlir::Location loc = mlir::FusedLoc::get(context: &getMLIRContext(), locs: ifLocs);
2870
2871 // Emit the code with the fully general case.
2872 mlir::Value condV = emitOpOnBoolExpr(loc, cond);
2873 return emitIfOnBoolValue(condV, loc, thenBuilder, thenLoc, elseBuilder,
2874 elseLoc);
2875}
2876
2877cir::IfOp CIRGenFunction::emitIfOnBoolValue(
2878 mlir::Value condV, mlir::Location loc, BuilderCallbackRef thenBuilder,
2879 mlir::Location thenLoc, BuilderCallbackRef elseBuilder,
2880 std::optional<mlir::Location> elseLoc) {
2881 cir::IfOp ifOp = cir::IfOp::create(builder, location: loc, cond: condV, withElseRegion: elseLoc.has_value(),
2882 /*thenBuilder=*/thenBuilder,
2883 /*elseBuilder=*/elseBuilder);
2884 terminateStructuredRegionBody(r&: ifOp.getThenRegion(), loc: thenLoc);
2885 assert((elseLoc.has_value() || ifOp.getElseRegion().empty()) &&
2886 "else region created with no else location");
2887 if (elseLoc.has_value())
2888 terminateStructuredRegionBody(r&: ifOp.getElseRegion(), loc: *elseLoc);
2889 return ifOp;
2890}
2891
2892/// TODO(cir): see EmitBranchOnBoolExpr for extra ideas).
2893mlir::Value CIRGenFunction::emitOpOnBoolExpr(mlir::Location loc,
2894 const Expr *cond) {
2895 assert(!cir::MissingFeatures::pgoUse());
2896 assert(!cir::MissingFeatures::generateDebugInfo());
2897 cond = cond->IgnoreParens();
2898
2899 // In LLVM the condition is reversed here for efficient codegen.
2900 // This should be done in CIR prior to LLVM lowering, if we do now
2901 // we can make CIR based diagnostics misleading.
2902 // cir.ternary(!x, t, f) -> cir.ternary(x, f, t)
2903 assert(!cir::MissingFeatures::shouldReverseUnaryCondOnBoolExpr());
2904
2905 if (const ConditionalOperator *condOp = dyn_cast<ConditionalOperator>(Val: cond)) {
2906 Expr *trueExpr = condOp->getTrueExpr();
2907 Expr *falseExpr = condOp->getFalseExpr();
2908 mlir::Value condV = emitOpOnBoolExpr(loc, cond: condOp->getCond());
2909
2910 mlir::Value ternaryOpRes =
2911 cir::TernaryOp::create(
2912 builder, location: loc, cond: condV, /*thenBuilder=*/
2913 trueBuilder: [this, trueExpr](mlir::OpBuilder &b, mlir::Location loc) {
2914 mlir::Value lhs = emitScalarExpr(e: trueExpr);
2915 cir::YieldOp::create(builder&: b, location: loc, args: lhs);
2916 },
2917 /*elseBuilder=*/
2918 falseBuilder: [this, falseExpr](mlir::OpBuilder &b, mlir::Location loc) {
2919 mlir::Value rhs = emitScalarExpr(e: falseExpr);
2920 cir::YieldOp::create(builder&: b, location: loc, args: rhs);
2921 })
2922 .getResult();
2923
2924 return emitScalarConversion(src: ternaryOpRes, srcType: condOp->getType(),
2925 dstType: getContext().BoolTy, loc: condOp->getExprLoc());
2926 }
2927
2928 if (isa<CXXThrowExpr>(Val: cond)) {
2929 cgm.errorNYI(feature: "NYI");
2930 return createDummyValue(loc, qt: cond->getType());
2931 }
2932
2933 // If the branch has a condition wrapped by __builtin_unpredictable,
2934 // create metadata that specifies that the branch is unpredictable.
2935 // Don't bother if not optimizing because that metadata would not be used.
2936 assert(!cir::MissingFeatures::insertBuiltinUnpredictable());
2937
2938 // Emit the code with the fully general case.
2939 return evaluateExprAsBool(e: cond);
2940}
2941
2942mlir::Value CIRGenFunction::emitAlloca(StringRef name, mlir::Type ty,
2943 mlir::Location loc, CharUnits alignment,
2944 bool insertIntoFnEntryBlock,
2945 mlir::Value arraySize) {
2946 mlir::Block *entryBlock = insertIntoFnEntryBlock
2947 ? getCurFunctionEntryBlock()
2948 : curLexScope->getEntryBlock();
2949
2950 // If this is an alloca in the entry basic block of a cir.try and there's
2951 // a surrounding cir.scope, make sure the alloca ends up in the surrounding
2952 // scope instead. This is necessary in order to guarantee all SSA values are
2953 // reachable during cleanups.
2954 if (auto tryOp =
2955 llvm::dyn_cast_if_present<cir::TryOp>(Val: entryBlock->getParentOp())) {
2956 if (auto scopeOp = llvm::dyn_cast<cir::ScopeOp>(Val: tryOp->getParentOp()))
2957 entryBlock = &scopeOp.getScopeRegion().front();
2958 }
2959
2960 return emitAlloca(name, ty, loc, alignment,
2961 ip: builder.getBestAllocaInsertPoint(block: entryBlock), arraySize);
2962}
2963
2964mlir::Value CIRGenFunction::emitAlloca(StringRef name, mlir::Type ty,
2965 mlir::Location loc, CharUnits alignment,
2966 mlir::OpBuilder::InsertPoint ip,
2967 mlir::Value arraySize) {
2968 // CIR uses its own alloca address space rather than follow the target data
2969 // layout like original CodeGen. The data layout awareness should be done in
2970 // the lowering pass instead.
2971 cir::PointerType localVarPtrTy =
2972 builder.getPointerTo(ty, as: getCIRAllocaAddressSpace());
2973 mlir::IntegerAttr alignIntAttr = cgm.getSize(size: alignment);
2974
2975 mlir::Value addr;
2976 {
2977 mlir::OpBuilder::InsertionGuard guard(builder);
2978 builder.restoreInsertionPoint(ip);
2979 addr = builder.createAlloca(loc, /*addr type*/ addrType: localVarPtrTy, name,
2980 alignment: alignIntAttr, dynAllocSize: arraySize);
2981 assert(!cir::MissingFeatures::astVarDeclInterface());
2982 }
2983 return addr;
2984}
2985
2986// Note: this function also emit constructor calls to support a MSVC extensions
2987// allowing explicit constructor function call.
2988RValue CIRGenFunction::emitCXXMemberCallExpr(const CXXMemberCallExpr *ce,
2989 ReturnValueSlot returnValue) {
2990 const Expr *callee = ce->getCallee()->IgnoreParens();
2991
2992 if (isa<BinaryOperator>(Val: callee))
2993 return emitCXXMemberPointerCallExpr(ce, returnValue);
2994
2995 const auto *me = cast<MemberExpr>(Val: callee);
2996 const auto *md = cast<CXXMethodDecl>(Val: me->getMemberDecl());
2997
2998 // Sema represents calls to static member functions as ordinary CallExpr
2999 // nodes rather than CXXMemberCallExpr.
3000 assert(!md->isStatic() &&
3001 "static member call is unexpected in emitCXXMemberCallExpr");
3002
3003 bool hasQualifier = me->hasQualifier();
3004 NestedNameSpecifier qualifier = me->getQualifier();
3005 bool isArrow = me->isArrow();
3006 const Expr *base = me->getBase();
3007
3008 return emitCXXMemberOrOperatorMemberCallExpr(
3009 ce, md, returnValue, hasQualifier, qualifier, isArrow, base);
3010}
3011
3012RValue CIRGenFunction::emitReferenceBindingToExpr(const Expr *e) {
3013 // Emit the expression as an lvalue.
3014 LValue lv = emitLValue(e);
3015 assert(lv.isSimple());
3016 mlir::Value value = lv.getPointer();
3017
3018 assert(!cir::MissingFeatures::sanitizers());
3019
3020 return RValue::get(v: value);
3021}
3022
3023Address CIRGenFunction::emitLoadOfReference(LValue refLVal, mlir::Location loc,
3024 LValueBaseInfo *pointeeBaseInfo) {
3025 if (refLVal.isVolatile())
3026 cgm.errorNYI(loc, feature: "load of volatile reference");
3027
3028 cir::LoadOp load =
3029 cir::LoadOp::create(builder, location: loc, result: refLVal.getAddress().getElementType(),
3030 addr: refLVal.getAddress().getPointer());
3031
3032 assert(!cir::MissingFeatures::opTBAA());
3033
3034 QualType pointeeType = refLVal.getType()->getPointeeType();
3035 CharUnits align = cgm.getNaturalTypeAlignment(t: pointeeType, baseInfo: pointeeBaseInfo);
3036 return Address(load, convertTypeForMem(t: pointeeType), align);
3037}
3038
3039LValue CIRGenFunction::emitLoadOfReferenceLValue(Address refAddr,
3040 mlir::Location loc,
3041 QualType refTy,
3042 AlignmentSource source) {
3043 LValue refLVal = makeAddrLValue(addr: refAddr, ty: refTy, baseInfo: LValueBaseInfo(source));
3044 LValueBaseInfo pointeeBaseInfo;
3045 assert(!cir::MissingFeatures::opTBAA());
3046 Address pointeeAddr = emitLoadOfReference(refLVal, loc, pointeeBaseInfo: &pointeeBaseInfo);
3047 return makeAddrLValue(addr: pointeeAddr, ty: refLVal.getType()->getPointeeType(),
3048 baseInfo: pointeeBaseInfo);
3049}
3050
3051void CIRGenFunction::emitTrap(mlir::Location loc, bool createNewBlock) {
3052 cir::TrapOp::create(builder, location: loc);
3053 if (createNewBlock)
3054 builder.createBlock(parent: builder.getBlock()->getParent());
3055}
3056
3057void CIRGenFunction::emitUnreachable(clang::SourceLocation loc,
3058 bool createNewBlock) {
3059 assert(!cir::MissingFeatures::sanitizers());
3060 cir::UnreachableOp::create(builder, location: getLoc(srcLoc: loc));
3061 if (createNewBlock)
3062 builder.createBlock(parent: builder.getBlock()->getParent());
3063}
3064
3065mlir::Value CIRGenFunction::createDummyValue(mlir::Location loc,
3066 clang::QualType qt) {
3067 mlir::Type t = convertType(t: qt);
3068 CharUnits alignment = getContext().getTypeAlignInChars(T: qt);
3069 return builder.createDummyValue(loc, type: t, alignment);
3070}
3071
3072//===----------------------------------------------------------------------===//
3073// CIR builder helpers
3074//===----------------------------------------------------------------------===//
3075
3076Address CIRGenFunction::createMemTempWithoutCast(QualType ty,
3077 mlir::Location loc,
3078 const Twine &name) {
3079 return createTempAllocaWithoutCast(
3080 ty: convertTypeForMem(t: ty), align: getContext().getTypeAlignInChars(T: ty), loc, name);
3081}
3082
3083Address CIRGenFunction::createMemTemp(QualType ty, mlir::Location loc,
3084 const Twine &name, Address *alloca,
3085 mlir::OpBuilder::InsertPoint ip) {
3086 // FIXME: Should we prefer the preferred type alignment here?
3087 return createMemTemp(t: ty, align: getContext().getTypeAlignInChars(T: ty), loc, name,
3088 alloca, ip);
3089}
3090
3091Address CIRGenFunction::createMemTemp(QualType ty, CharUnits align,
3092 mlir::Location loc, const Twine &name,
3093 Address *alloca,
3094 mlir::OpBuilder::InsertPoint ip) {
3095 Address result =
3096 createTempAlloca(ty: convertTypeForMem(t: ty), /*destAddrSpace=*/{}, align, loc,
3097 name, /*arraySize=*/nullptr, alloca, ip);
3098 return result;
3099}
3100
3101/// This creates a alloca and inserts it into the entry block of the
3102/// current region.
3103Address CIRGenFunction::createTempAllocaWithoutCast(
3104 mlir::Type ty, CharUnits align, mlir::Location loc, const Twine &name,
3105 mlir::Value arraySize, mlir::OpBuilder::InsertPoint ip) {
3106 cir::AllocaOp alloca = ip.isSet()
3107 ? createTempAlloca(ty, loc, name, ip, arraySize)
3108 : createTempAlloca(ty, loc, name, arraySize);
3109 alloca.setAlignmentAttr(cgm.getSize(size: align));
3110 return Address(alloca, ty, align);
3111}
3112
3113Address CIRGenFunction::createTempAlloca(mlir::Type ty, CharUnits align,
3114 mlir::Location loc, const Twine &name,
3115 mlir::Value arraySize,
3116 Address *allocaAddr,
3117 mlir::OpBuilder::InsertPoint ip) {
3118 return createTempAlloca(ty, /*destAddrSpace=*/{}, align, loc, name, arraySize,
3119 alloca: allocaAddr, ip);
3120}
3121
3122Address CIRGenFunction::maybeCastStackAddressSpace(
3123 Address alloca, mlir::ptr::MemorySpaceAttrInterface destAddrSpace,
3124 mlir::Value arraySize) {
3125 if (!destAddrSpace)
3126 destAddrSpace = cir::toCIRAddressSpaceAttr(
3127 ctx&: getMLIRContext(), langAS: cgm.getLangTempAllocaAddressSpace());
3128
3129 // Resolve the default address space through getTargetAddressSpace, as
3130 // classic CodeGen does and as CIRGenTypes::getPointerAddressSpace does for
3131 // default pointer types. This is only non-zero for targets where the default
3132 // address space is not 0 (e.g. generic for SYCL device code).
3133 if (!cir::normalizeDefaultAddressSpace(addrSpace: destAddrSpace))
3134 if (unsigned targetAS = getContext().getTargetAddressSpace(AS: LangAS::Default))
3135 destAddrSpace =
3136 cir::TargetAddressSpaceAttr::get(context: &getMLIRContext(), value: targetAS);
3137
3138 mlir::ptr::MemorySpaceAttrInterface srcAddrSpace = getCIRAllocaAddressSpace();
3139 // Alloca always returns a pointer in alloca address space, which may
3140 // be different from the type defined by the language. For example,
3141 // in C++ the auto variables are in the default address space. Therefore
3142 // cast alloca to the default address space when necessary.
3143 if (srcAddrSpace == destAddrSpace)
3144 return alloca;
3145
3146 mlir::OpBuilder::InsertionGuard guard(builder);
3147 if (cir::AllocaOp allocaOp = alloca.getUnderlyingAllocaOp()) {
3148 builder.setInsertionPointAfter(allocaOp);
3149 } else if (!arraySize) {
3150 mlir::Block *entryBlock = getCurFunctionEntryBlock();
3151 builder.restoreInsertionPoint(ip: builder.getBestAllocaInsertPoint(block: entryBlock));
3152 }
3153
3154 mlir::Type destPtrTy =
3155 builder.getPointerTo(ty: alloca.getElementType(), as: destAddrSpace);
3156 mlir::Value casted = performAddrSpaceCast(v: alloca.getPointer(), destTy: destPtrTy);
3157 return Address(casted, alloca.getElementType(), alloca.getAlignment(),
3158 /*isKnownNonNull=*/true);
3159}
3160
3161/// This creates a alloca and inserts it into the entry block. The alloca is
3162/// casted to the requested language address space if necessary.
3163Address CIRGenFunction::createTempAlloca(
3164 mlir::Type ty, mlir::ptr::MemorySpaceAttrInterface destAddrSpace,
3165 CharUnits align, mlir::Location loc, const Twine &name,
3166 mlir::Value arraySize, Address *allocaAddr,
3167 mlir::OpBuilder::InsertPoint ip) {
3168 Address alloca =
3169 createTempAllocaWithoutCast(ty, align, loc, name, arraySize, ip);
3170 if (allocaAddr)
3171 *allocaAddr = alloca;
3172 return maybeCastStackAddressSpace(alloca, destAddrSpace, arraySize);
3173}
3174
3175/// This creates an alloca and inserts it into the entry block if \p ArraySize
3176/// is nullptr, otherwise inserts it at the current insertion point of the
3177/// builder.
3178cir::AllocaOp CIRGenFunction::createTempAlloca(mlir::Type ty,
3179 mlir::Location loc,
3180 const Twine &name,
3181 mlir::Value arraySize,
3182 bool insertIntoFnEntryBlock) {
3183 return mlir::cast<cir::AllocaOp>(Val: emitAlloca(name: name.str(), ty, loc, alignment: CharUnits(),
3184 insertIntoFnEntryBlock, arraySize)
3185 .getDefiningOp());
3186}
3187
3188/// This creates an alloca and inserts it into the provided insertion point
3189cir::AllocaOp CIRGenFunction::createTempAlloca(mlir::Type ty,
3190 mlir::Location loc,
3191 const Twine &name,
3192 mlir::OpBuilder::InsertPoint ip,
3193 mlir::Value arraySize) {
3194 assert(ip.isSet() && "Insertion point is not set");
3195 return mlir::cast<cir::AllocaOp>(
3196 Val: emitAlloca(name: name.str(), ty, loc, alignment: CharUnits(), ip, arraySize)
3197 .getDefiningOp());
3198}
3199
3200/// CreateDefaultAlignTempAlloca - This creates an alloca with the
3201/// default alignment of the corresponding LLVM type, which is *not*
3202/// guaranteed to be related in any way to the expected alignment of
3203/// an AST type that might have been lowered to Ty.
3204Address CIRGenFunction::createDefaultAlignTempAlloca(mlir::Type ty,
3205 mlir::Location loc,
3206 const Twine &name) {
3207 CharUnits align =
3208 CharUnits::fromQuantity(Quantity: cgm.getDataLayout().getABITypeAlign(ty));
3209 return createTempAlloca(ty, align, loc, name);
3210}
3211
3212/// Try to emit a reference to the given value without producing it as
3213/// an l-value. This is just an optimization, but it avoids us needing
3214/// to emit global copies of variables if they're named without triggering
3215/// a formal use in a context where we can't emit a direct reference to them,
3216/// for instance if a block or lambda or a member of a local class uses a
3217/// const int variable or constexpr variable from an enclosing function.
3218CIRGenFunction::ConstantEmission
3219CIRGenFunction::tryEmitAsConstant(const DeclRefExpr *refExpr) {
3220 const ValueDecl *value = refExpr->getDecl();
3221
3222 // The value needs to be an enum constant or a constant variable.
3223 CodeGenUtils::ConstantEmissionKind cek;
3224 if (isa<ParmVarDecl>(Val: value))
3225 cek = CodeGenUtils::CEK_None;
3226 else if (const auto *var = dyn_cast<VarDecl>(Val: value))
3227 cek = CodeGenUtils::checkVarTypeForConstantEmission(Type: var->getType());
3228 else if (isa<EnumConstantDecl>(Val: value))
3229 cek = CodeGenUtils::CEK_AsValueOnly;
3230 else
3231 cek = CodeGenUtils::CEK_None;
3232 if (cek == CodeGenUtils::CEK_None)
3233 return ConstantEmission();
3234
3235 Expr::EvalResult result;
3236 bool resultIsReference;
3237 QualType resultType;
3238
3239 // It's best to evaluate all the way as an r-value if that's permitted.
3240 if (cek != CodeGenUtils::CEK_AsReferenceOnly &&
3241 refExpr->EvaluateAsRValue(Result&: result, Ctx: getContext())) {
3242 resultIsReference = false;
3243 resultType = refExpr->getType().getUnqualifiedType();
3244 } else if (cek != CodeGenUtils::CEK_AsValueOnly &&
3245 refExpr->EvaluateAsLValue(Result&: result, Ctx: getContext())) {
3246 resultIsReference = true;
3247 resultType = value->getType();
3248 } else {
3249 return ConstantEmission();
3250 }
3251
3252 // In any case, if the initializer has side-effects, abandon ship.
3253 if (result.HasSideEffects)
3254 return ConstantEmission();
3255
3256 // In CUDA/HIP device compilation, a lambda may capture a reference variable
3257 // referencing a global host variable by copy. In this case the lambda should
3258 // make a copy of the value of the global host variable. The DRE of the
3259 // captured reference variable cannot be emitted as load from the host
3260 // global variable as compile time constant, since the host variable is not
3261 // accessible on device. The DRE of the captured reference variable has to be
3262 // loaded from captures.
3263 if (cgm.getLangOpts().CUDAIsDevice && result.Val.isLValue() &&
3264 refExpr->refersToEnclosingVariableOrCapture()) {
3265 auto *md = dyn_cast_or_null<CXXMethodDecl>(Val: curCodeDecl);
3266 if (isLambdaMethod(DC: md) && md->getOverloadedOperator() == OO_Call) {
3267 const APValue::LValueBase &base = result.Val.getLValueBase();
3268 if (const ValueDecl *d = base.dyn_cast<const ValueDecl *>()) {
3269 if (const auto *vd = dyn_cast<const VarDecl>(Val: d)) {
3270 if (!vd->hasAttr<CUDADeviceAttr>())
3271 return ConstantEmission();
3272 }
3273 }
3274 }
3275 }
3276
3277 // Emit as a constant.
3278 // FIXME(cir): have emitAbstract build a TypedAttr instead (this requires
3279 // somewhat heavy refactoring...)
3280 mlir::Attribute c = ConstantEmitter(*this).emitAbstract(
3281 loc: refExpr->getLocation(), value: result.Val, destType: resultType);
3282 mlir::TypedAttr cstToEmit = mlir::dyn_cast_if_present<mlir::TypedAttr>(Val&: c);
3283 if (!cstToEmit)
3284 return ConstantEmission();
3285
3286 // Make sure we emit a debug reference to the global variable.
3287 assert(!cir::MissingFeatures::generateDebugInfo());
3288
3289 // If we emitted a reference constant, we need to dereference that.
3290 if (resultIsReference)
3291 return ConstantEmission::forReference(c: cstToEmit);
3292
3293 return ConstantEmission::forValue(c: cstToEmit);
3294}
3295
3296CIRGenFunction::ConstantEmission
3297CIRGenFunction::tryEmitAsConstant(const MemberExpr *me) {
3298 if (DeclRefExpr *dre = tryToConvertMemberExprToDeclRefExpr(cgf&: *this, me))
3299 return tryEmitAsConstant(refExpr: dre);
3300 return ConstantEmission();
3301}
3302
3303mlir::Value CIRGenFunction::emitScalarConstant(
3304 const CIRGenFunction::ConstantEmission &constant, Expr *e) {
3305 assert(constant && "not a constant");
3306 if (constant.isReference())
3307 return emitLoadOfLValue(lv: constant.getReferenceLValue(cgf&: *this, refExpr: e),
3308 loc: e->getExprLoc())
3309 .getValue();
3310 return builder.getConstant(loc: getLoc(srcLoc: e->getSourceRange()), attr: constant.getValue());
3311}
3312
3313LValue CIRGenFunction::emitPredefinedLValue(const PredefinedExpr *e) {
3314 const StringLiteral *sl = e->getFunctionName();
3315 assert(sl != nullptr && "No StringLiteral name in PredefinedExpr");
3316 auto fn = cast<cir::FuncOp>(Val: curFn);
3317 StringRef fnName = fn.getName();
3318 fnName.consume_front(Prefix: "\01");
3319 std::array<StringRef, 2> nameItems = {
3320 PredefinedExpr::getIdentKindName(IK: e->getIdentKind()), fnName};
3321 std::string gvName = llvm::join(R&: nameItems, Separator: ".");
3322 if (isa_and_nonnull<BlockDecl>(Val: curCodeDecl))
3323 cgm.errorNYI(e->getSourceRange(), "predefined lvalue in block");
3324
3325 return emitStringLiteralLValue(e: sl, name: gvName);
3326}
3327
3328LValue CIRGenFunction::emitOpaqueValueLValue(const OpaqueValueExpr *e) {
3329 assert(OpaqueValueMappingData::shouldBindAsLValue(e));
3330 return getOrCreateOpaqueLValueMapping(e);
3331}
3332
3333namespace {
3334// Handle the case where the condition is a constant evaluatable simple integer,
3335// which means we don't have to separately handle the true/false blocks.
3336std::optional<LValue> handleConditionalOperatorLValueSimpleCase(
3337 CIRGenFunction &cgf, const AbstractConditionalOperator *e) {
3338 const Expr *condExpr = e->getCond();
3339 llvm::APSInt condExprVal;
3340 if (!cgf.constantFoldsToSimpleInteger(cond: condExpr, resultInt&: condExprVal))
3341 return std::nullopt;
3342
3343 const Expr *live = e->getTrueExpr(), *dead = e->getFalseExpr();
3344 if (!condExprVal.getBoolValue())
3345 std::swap(a&: live, b&: dead);
3346
3347 if (cgf.containsLabel(s: dead))
3348 return std::nullopt;
3349
3350 // If the true case is live, we need to track its region.
3351 assert(!cir::MissingFeatures::incrementProfileCounter());
3352 assert(!cir::MissingFeatures::pgoUse());
3353 // If a throw expression we emit it and return an undefined lvalue
3354 // because it can't be used.
3355 if (auto *throwExpr = dyn_cast<CXXThrowExpr>(Val: live->IgnoreParens())) {
3356 cgf.emitCXXThrowExpr(e: throwExpr);
3357 // Return an undefined lvalue - the throw terminates execution
3358 // so this value will never actually be used
3359 mlir::Type elemTy = cgf.convertType(t: dead->getType());
3360 mlir::Value undefPtr =
3361 cgf.getBuilder().getNullPtr(ty: cgf.getBuilder().getPointerTo(ty: elemTy),
3362 loc: cgf.getLoc(srcLoc: throwExpr->getSourceRange()));
3363 return cgf.makeAddrLValue(addr: Address(undefPtr, elemTy, CharUnits::One()),
3364 ty: dead->getType());
3365 }
3366 return cgf.emitLValue(e: live);
3367}
3368
3369/// Emit the operand of a glvalue conditional operator. This is either a glvalue
3370/// or a (possibly-parenthesized) throw-expression. If this is a throw, no
3371/// LValue is returned and the current block has been terminated.
3372static std::optional<LValue> emitLValueOrThrowExpression(CIRGenFunction &cgf,
3373 const Expr *operand) {
3374 if (auto *throwExpr = dyn_cast<CXXThrowExpr>(Val: operand->IgnoreParens())) {
3375 cgf.emitCXXThrowExpr(e: throwExpr);
3376 return std::nullopt;
3377 }
3378
3379 return cgf.emitLValue(e: operand);
3380}
3381} // namespace
3382
3383// Create and generate the 3 blocks for a conditional operator.
3384// Leaves the 'current block' in the continuation basic block.
3385template <typename FuncTy>
3386CIRGenFunction::ConditionalInfo
3387CIRGenFunction::emitConditionalBlocks(const AbstractConditionalOperator *e,
3388 const FuncTy &branchGenFunc) {
3389 ConditionalInfo info;
3390 mlir::Location loc = getLoc(srcLoc: e->getSourceRange());
3391 CIRGenBuilderTy &builder = getBuilder();
3392
3393 mlir::Value condV = emitOpOnBoolExpr(loc, cond: e->getCond());
3394
3395 ConditionalEvaluation eval(*this, loc);
3396
3397 auto emitBranch = [&](mlir::OpBuilder &b, mlir::Location loc,
3398 const Expr *expr, std::optional<LValue> &resultLV) {
3399 CIRGenFunction::LexicalScope lexScope{*this, loc, b.getInsertionBlock()};
3400 curLexScope->setAsTernary();
3401
3402 mlir::Value resultPtr;
3403 {
3404 // Emit any cleanups that were needed on this branch so we can spill
3405 // and reload the return value.
3406 CIRGenFunction::RunCleanupsScope branchCleanups(*this);
3407 assert(!cir::MissingFeatures::incrementProfileCounter());
3408 eval.beginEvaluation();
3409 resultLV = branchGenFunc(*this, expr);
3410 resultPtr = resultLV ? resultLV->getPointer() : mlir::Value();
3411 eval.endEvaluation();
3412 branchCleanups.forceCleanup(valuesToReload: {&resultPtr});
3413 }
3414
3415 // A branch that produced no result is a throw-expression; its region is
3416 // already terminated by cir.unreachable and needs no yield.
3417 if (resultPtr)
3418 cir::YieldOp::create(builder&: b, location: loc, args: resultPtr);
3419 };
3420
3421 cir::TernaryOp ternary = cir::TernaryOp::create(
3422 builder, loc, condV,
3423 /*trueBuilder=*/
3424 [&](mlir::OpBuilder &b, mlir::Location loc) {
3425 emitBranch(b, loc, e->getTrueExpr(), info.lhs);
3426 },
3427 /*falseBuilder=*/
3428 [&](mlir::OpBuilder &b, mlir::Location loc) {
3429 emitBranch(b, loc, e->getFalseExpr(), info.rhs);
3430 });
3431
3432 // Arms end with a yield or cir.unreachable (throw); this is a backstop
3433 // for error (NYI) paths that leave a region open.
3434 terminateStructuredRegionBody(r&: ternary.getTrueRegion(), loc);
3435 terminateStructuredRegionBody(r&: ternary.getFalseRegion(), loc);
3436
3437 info.result = ternary.getResult();
3438 return info;
3439}
3440
3441LValue CIRGenFunction::emitConditionalOperatorLValue(
3442 const AbstractConditionalOperator *expr) {
3443 if (!expr->isGLValue()) {
3444 // ?: here should be an aggregate.
3445 assert(hasAggregateEvaluationKind(expr->getType()) &&
3446 "Unexpected conditional operator!");
3447 return emitAggExprToLValue(e: expr);
3448 }
3449
3450 OpaqueValueMapping binding(*this, expr);
3451 if (std::optional<LValue> res =
3452 handleConditionalOperatorLValueSimpleCase(cgf&: *this, e: expr))
3453 return *res;
3454
3455 ConditionalInfo info =
3456 emitConditionalBlocks(e: expr, branchGenFunc: [](CIRGenFunction &cgf, const Expr *e) {
3457 return emitLValueOrThrowExpression(cgf, operand: e);
3458 });
3459
3460 if ((info.lhs && !info.lhs->isSimple()) ||
3461 (info.rhs && !info.rhs->isSimple())) {
3462 cgm.errorNYI(expr->getSourceRange(),
3463 "unsupported conditional operator with non-simple lvalue");
3464 return LValue();
3465 }
3466
3467 if (info.lhs && info.rhs) {
3468 Address lhsAddr = info.lhs->getAddress();
3469 Address rhsAddr = info.rhs->getAddress();
3470 Address result(info.result, lhsAddr.getElementType(),
3471 std::min(a: lhsAddr.getAlignment(), b: rhsAddr.getAlignment()));
3472 AlignmentSource alignSource =
3473 std::max(a: info.lhs->getBaseInfo().getAlignmentSource(),
3474 b: info.rhs->getBaseInfo().getAlignmentSource());
3475 assert(!cir::MissingFeatures::opTBAA());
3476 return makeAddrLValue(addr: result, ty: expr->getType(), baseInfo: LValueBaseInfo(alignSource));
3477 }
3478
3479 assert((info.lhs || info.rhs) &&
3480 "both operands of glvalue conditional are throw-expressions?");
3481
3482 // One arm threw; the surviving arm's pointer is local to the ternary's
3483 // region, so address the result through the ternary's result value.
3484 const LValue &survivingLV = info.lhs ? *info.lhs : *info.rhs;
3485 Address survivingAddr = survivingLV.getAddress();
3486 Address result(info.result, survivingAddr.getElementType(),
3487 survivingAddr.getAlignment());
3488 assert(!cir::MissingFeatures::opTBAA());
3489 return makeAddrLValue(addr: result, ty: expr->getType(), baseInfo: survivingLV.getBaseInfo());
3490}
3491
3492/// An LValue is a candidate for having its loads and stores be made atomic if
3493/// we are operating under /volatile:ms *and* the LValue itself is volatile and
3494/// performing such an operation can be performed without a libcall.
3495bool CIRGenFunction::isLValueSuitableForInlineAtomic(LValue lv) {
3496 if (!cgm.getLangOpts().MSVolatile)
3497 return false;
3498
3499 cgm.errorNYI(feature: "LValueSuitableForInlineAtomic LangOpts MSVolatile");
3500 return false;
3501}
3502
3503LValue CIRGenFunction::emitCXXTypeidLValue(const CXXTypeidExpr *e) {
3504 return makeNaturalAlignAddrLValue(val: emitCXXTypeidExpr(e), ty: e->getType());
3505}
3506