1//===--- CIRGenAtomic.cpp - Emit CIR for atomic operations ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the code for emitting atomic operations.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CIRGenFunction.h"
14#include "clang/CIR/MissingFeatures.h"
15
16using namespace clang;
17using namespace clang::CIRGen;
18using namespace cir;
19
20namespace {
21class AtomicInfo {
22 CIRGenFunction &cgf;
23 QualType atomicTy;
24 QualType valueTy;
25 uint64_t atomicSizeInBits = 0;
26 uint64_t valueSizeInBits = 0;
27 CharUnits atomicAlign;
28 CharUnits valueAlign;
29 TypeEvaluationKind evaluationKind = cir::TEK_Scalar;
30 bool useLibCall = true;
31 LValue lvalue;
32 mlir::Location loc;
33
34public:
35 AtomicInfo(CIRGenFunction &cgf, LValue &lvalue, mlir::Location loc)
36 : cgf(cgf), loc(loc) {
37 assert(!lvalue.isGlobalReg());
38 ASTContext &ctx = cgf.getContext();
39 if (lvalue.isSimple()) {
40 atomicTy = lvalue.getType();
41 if (auto *ty = atomicTy->getAs<AtomicType>())
42 valueTy = ty->getValueType();
43 else
44 valueTy = atomicTy;
45 evaluationKind = cgf.getEvaluationKind(type: valueTy);
46
47 TypeInfo valueTypeInfo = ctx.getTypeInfo(T: valueTy);
48 TypeInfo atomicTypeInfo = ctx.getTypeInfo(T: atomicTy);
49 uint64_t valueAlignInBits = valueTypeInfo.Align;
50 uint64_t atomicAlignInBits = atomicTypeInfo.Align;
51 valueSizeInBits = valueTypeInfo.Width;
52 atomicSizeInBits = atomicTypeInfo.Width;
53 assert(valueSizeInBits <= atomicSizeInBits);
54 assert(valueAlignInBits <= atomicAlignInBits);
55
56 atomicAlign = ctx.toCharUnitsFromBits(BitSize: atomicAlignInBits);
57 valueAlign = ctx.toCharUnitsFromBits(BitSize: valueAlignInBits);
58 if (lvalue.getAlignment().isZero())
59 lvalue.setAlignment(atomicAlign);
60
61 this->lvalue = lvalue;
62 } else {
63 assert(!cir::MissingFeatures::atomicInfo());
64 cgf.cgm.errorNYI(loc, feature: "AtomicInfo: non-simple lvalue");
65 }
66 useLibCall = !ctx.getTargetInfo().hasBuiltinAtomic(
67 AtomicSizeInBits: atomicSizeInBits, AlignmentInBits: ctx.toBits(CharSize: lvalue.getAlignment()));
68 }
69
70 QualType getValueType() const { return valueTy; }
71 QualType getAtomicType() const { return atomicTy; }
72 CharUnits getAtomicAlignment() const { return atomicAlign; }
73 TypeEvaluationKind getEvaluationKind() const { return evaluationKind; }
74 mlir::Value getAtomicPointer() const {
75 if (lvalue.isSimple())
76 return lvalue.getPointer();
77 assert(!cir::MissingFeatures::atomicInfoGetAtomicPointer());
78 return nullptr;
79 }
80 bool shouldUseLibCall() const { return useLibCall; }
81 const LValue &getAtomicLValue() const { return lvalue; }
82 Address getAtomicAddress() const {
83 mlir::Type elemTy;
84 if (lvalue.isSimple()) {
85 elemTy = lvalue.getAddress().getElementType();
86 } else {
87 assert(!cir::MissingFeatures::atomicInfoGetAtomicAddress());
88 cgf.cgm.errorNYI(loc, feature: "AtomicInfo::getAtomicAddress: non-simple lvalue");
89 }
90 return Address(getAtomicPointer(), elemTy, getAtomicAlignment());
91 }
92
93 /// Is the atomic size larger than the underlying value type?
94 ///
95 /// Note that the absence of padding does not mean that atomic
96 /// objects are completely interchangeable with non-atomic
97 /// objects: we might have promoted the alignment of a type
98 /// without making it bigger.
99 bool hasPadding() const { return (valueSizeInBits != atomicSizeInBits); }
100
101 bool emitMemSetZeroIfNecessary() const;
102
103 mlir::Value getScalarRValValueOrNull(RValue rvalue) const;
104
105 /// Cast the given pointer to an integer pointer suitable for atomic
106 /// operations on the source.
107 Address castToAtomicIntPointer(Address addr) const;
108
109 /// If addr is compatible with the iN that will be used for an atomic
110 /// operation, bitcast it. Otherwise, create a temporary that is suitable and
111 /// copy the value across.
112 Address convertToAtomicIntPointer(Address addr, mlir::Location loc) const;
113
114 /// Turn an atomic-layout object into an r-value.
115 RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
116 SourceLocation loc, bool asValue) const;
117
118 /// Converts a rvalue to integer value.
119 mlir::Value convertRValueToInt(RValue rvalue, mlir::Location loc,
120 bool cmpxchg = false) const;
121
122 RValue convertToValueOrAtomic(mlir::Value intVal, AggValueSlot resultSlot,
123 SourceLocation loc, bool asValue,
124 bool cmpxchg = false) const;
125
126 /// Copy an atomic r-value into atomic-layout memory.
127 void emitCopyIntoMemory(RValue rvalue) const;
128
129 /// Project an l-value down to the value field.
130 LValue projectValue() const {
131 assert(lvalue.isSimple());
132 Address addr = getAtomicAddress();
133 if (hasPadding())
134 addr = cgf.getBuilder().createGetMember(loc, base: addr, /*name=*/"value",
135 /*index=*/0,
136 layout: cgf.cgm.getDataLayout());
137
138 assert(!cir::MissingFeatures::opTBAA());
139 return LValue::makeAddr(address: addr, t: getValueType(), baseInfo: lvalue.getBaseInfo());
140 }
141
142 /// Emits atomic load.
143 /// \returns Loaded value.
144 RValue emitAtomicLoad(AggValueSlot resultSlot, SourceLocation loc,
145 bool asValue, cir::MemOrder order, bool isVolatile);
146
147 /// Materialize an atomic r-value in atomic-layout memory.
148 Address materializeRValue(RValue rvalue, mlir::Location loc) const;
149
150 /// Creates temp alloca for intermediate operations on atomic value.
151 Address createTempAlloca() const;
152
153private:
154 bool requiresMemSetZero(mlir::Type ty) const;
155
156 /// Emits atomic load as a CIR operation.
157 mlir::Value emitAtomicLoadOp(cir::MemOrder order, bool isVolatile,
158 bool cmpxchg = false);
159};
160} // namespace
161
162/// Attach the AMDGPU atomic metadata markers that the current atomic options
163/// call for. The "no.X" metadata is emitted when the corresponding option is
164/// off, since it asserts the absence of that memory kind. The clang::atomic
165/// attribute is what turns the options on and off.
166static void setAMDGPUAtomicMetadata(CIRGenFunction &cgf, mlir::Operation *op,
167 const AtomicExpr *expr) {
168 // TODO: AMDGPUTargetCodeGenInfo::setTargetAtomicMetadata also calls
169 // CGF.AddAMDGPUAvailableVisibleMMRA on every atomic instruction; this is
170 // tied to the AMDGPUAvailableVisible statement attribute, which
171 // CIRGenStmt.cpp's emitAttributedStmt currently rejects via errorNYI.
172 assert(!cir::MissingFeatures::atomicAMDGPUAvailableVisibleMMRA());
173
174 const llvm::Triple &triple = cgf.cgm.getTriple();
175 bool isAMDGCN = triple.isAMDGCN();
176 bool isAMDSPIRV = triple.isSPIRV() && triple.getVendor() == llvm::Triple::AMD;
177 if (!isAMDGCN && !isAMDSPIRV)
178 return;
179
180 // Lowering keeps this only on a flat pointer.
181 mlir::UnitAttr unit = cgf.getBuilder().getUnitAttr();
182 if (isAMDGCN && expr->threadPrivateMemoryAtomicsAreUndefined())
183 op->setAttr(name: cir::CIRDialect::getAMDGPUNoPrivateAddrSpaceAttrName(), value: unit);
184
185 // Only a read-modify-write instruction carries these; a plain load, store or
186 // cmpxchg does not.
187 if (!mlir::isa<cir::AtomicFetchOp, cir::AtomicXchgOp>(Val: op))
188 return;
189
190 clang::AtomicOptions atomicOpts = cgf.cgm.getAtomicOpts();
191 if (!atomicOpts.getOption(Kind: clang::AtomicOptionKind::FineGrainedMemory))
192 op->setAttr(name: cir::CIRDialect::getAMDGPUNoFineGrainedMemoryAttrName(), value: unit);
193 if (!atomicOpts.getOption(Kind: clang::AtomicOptionKind::RemoteMemory))
194 op->setAttr(name: cir::CIRDialect::getAMDGPUNoRemoteMemoryAttrName(), value: unit);
195 // Denormal flushing only matters for a float add.
196 auto fetchOp = mlir::dyn_cast<cir::AtomicFetchOp>(Val: op);
197 if (fetchOp &&
198 atomicOpts.getOption(Kind: clang::AtomicOptionKind::IgnoreDenormalMode) &&
199 fetchOp.getBinop() == cir::AtomicFetchKind::Add &&
200 mlir::isa<cir::SingleType>(Val: fetchOp.getVal().getType()))
201 op->setAttr(name: cir::CIRDialect::getAMDGPUIgnoreDenormalModeAttrName(), value: unit);
202}
203
204// This function emits any expression (scalar, complex, or aggregate)
205// into a temporary alloca.
206static Address emitValToTemp(CIRGenFunction &cgf, Expr *e) {
207 Address declPtr = cgf.createMemTemp(
208 t: e->getType(), loc: cgf.getLoc(srcLoc: e->getSourceRange()), name: ".atomictmp");
209 cgf.emitAnyExprToMem(e, location: declPtr, quals: e->getType().getQualifiers(),
210 /*Init*/ isInitializer: true);
211 return declPtr;
212}
213
214/// Does a store of the given IR type modify the full expected width?
215static bool isFullSizeType(CIRGenModule &cgm, mlir::Type ty,
216 uint64_t expectedSize) {
217 return cgm.getDataLayout().getTypeStoreSize(ty) * 8 == expectedSize;
218}
219
220/// Does the atomic type require memsetting to zero before initialization?
221///
222/// The IR type is provided as a way of making certain queries faster.
223bool AtomicInfo::requiresMemSetZero(mlir::Type ty) const {
224 // If the atomic type has size padding, we definitely need a memset.
225 if (hasPadding())
226 return true;
227
228 // Otherwise, do some simple heuristics to try to avoid it:
229 switch (getEvaluationKind()) {
230 // For scalars and complexes, check whether the store size of the
231 // type uses the full size.
232 case cir::TEK_Scalar:
233 return !isFullSizeType(cgm&: cgf.cgm, ty, expectedSize: atomicSizeInBits);
234 case cir::TEK_Complex:
235 return !isFullSizeType(cgm&: cgf.cgm,
236 ty: mlir::cast<cir::ComplexType>(Val&: ty).getElementType(),
237 expectedSize: atomicSizeInBits / 2);
238 // Padding in structs has an undefined bit pattern. User beware.
239 case cir::TEK_Aggregate:
240 return false;
241 }
242 llvm_unreachable("bad evaluation kind");
243}
244
245Address AtomicInfo::convertToAtomicIntPointer(Address addr,
246 mlir::Location loc) const {
247 mlir::Type ty = addr.getElementType();
248 uint64_t sourceSizeInBits = cgf.cgm.getDataLayout().getTypeSizeInBits(ty);
249 if (sourceSizeInBits != atomicSizeInBits) {
250 CIRGenBuilderTy &builder = cgf.getBuilder();
251
252 Address tmp = createTempAlloca();
253 mlir::Value zero = builder.getConstInt(loc, t: cgf.cgm.uInt8Ty, c: 0);
254 unsigned size =
255 cgf.getContext().toCharUnitsFromBits(BitSize: atomicSizeInBits).getQuantity();
256 mlir::Value memSetSize = builder.getConstInt(loc, t: cgf.cgm.uInt64Ty, c: size);
257 addr = addr.withElementType(builder, ElemTy: cgf.cgm.voidTy);
258 tmp = tmp.withElementType(builder, ElemTy: cgf.cgm.voidTy);
259 builder.createMemSet(loc, dst: tmp, val: zero, len: memSetSize);
260
261 builder.createMemCpy(
262 loc, dst: tmp, src: addr,
263 len: builder.getConstInt(loc, t: cgf.cgm.uInt64Ty,
264 c: std::min(a: atomicSizeInBits, b: sourceSizeInBits) / 8));
265 addr = tmp;
266 }
267
268 return castToAtomicIntPointer(addr);
269}
270
271RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
272 AggValueSlot resultSlot,
273 SourceLocation loc,
274 bool asValue) const {
275 if (lvalue.isSimple()) {
276 if (evaluationKind == TEK_Aggregate)
277 return resultSlot.asRValue();
278
279 // Drill into the padding structure if we have one.
280 if (hasPadding()) {
281 cgf.cgm.errorNYI(loc,
282 "AtomicInfo::convertAtomicTempToRValue: hasPadding");
283 return RValue::get(v: nullptr);
284 }
285
286 // Otherwise, just convert the temporary to an r-value using the
287 // normal conversion routine.
288 return cgf.convertTempToRValue(addr, type: getValueType(), loc);
289 }
290
291 cgf.cgm.errorNYI(
292 loc, "AtomicInfo::convertAtomicTempToRValue: lvalue is not simple");
293 return RValue::get(v: nullptr);
294}
295
296RValue AtomicInfo::emitAtomicLoad(AggValueSlot resultSlot, SourceLocation loc,
297 bool asValue, cir::MemOrder order,
298 bool isVolatile) {
299 // Check whether we should use a library call.
300 if (shouldUseLibCall()) {
301 assert(!cir::MissingFeatures::atomicUseLibCall());
302 cgf.cgm.errorNYI(loc, "emitAtomicLoad: emit atomic lib call");
303 return RValue::get(v: nullptr);
304 }
305
306 // Okay, we're doing this natively.
307 mlir::Value loadOp = emitAtomicLoadOp(order, isVolatile);
308
309 // If we're ignoring an aggregate return, don't do anything.
310 if (getEvaluationKind() == TEK_Aggregate && resultSlot.isIgnored())
311 return RValue::getAggregate(addr: Address::invalid(), isVolatile: false);
312
313 // Okay, turn that back into the original value or atomic (for non-simple
314 // lvalues) type.
315 return convertToValueOrAtomic(intVal: loadOp, resultSlot, loc, asValue);
316}
317
318Address AtomicInfo::createTempAlloca() const {
319 // Remove addrspace info from the atomic pointer element when making the
320 // alloca pointer element.
321 QualType tmpTy = (lvalue.isBitField() && valueSizeInBits > atomicSizeInBits)
322 ? valueTy
323 : atomicTy.getUnqualifiedType();
324 Address tempAlloca =
325 cgf.createMemTemp(t: tmpTy, align: getAtomicAlignment(), loc, name: "atomic-temp");
326
327 // Cast to pointer to value type for bitfields.
328 if (lvalue.isBitField()) {
329 cgf.cgm.errorNYI(loc, feature: "AtomicInfo::createTempAlloca: bitfield lvalue");
330 }
331
332 return tempAlloca;
333}
334
335mlir::Value AtomicInfo::getScalarRValValueOrNull(RValue rvalue) const {
336 if (rvalue.isScalar() && (!hasPadding() || !lvalue.isSimple()))
337 return rvalue.getValue();
338 return nullptr;
339}
340
341Address AtomicInfo::castToAtomicIntPointer(Address addr) const {
342 auto intTy = mlir::dyn_cast<cir::IntType>(Val: addr.getElementType());
343 // Don't bother with int casts if the integer size is the same.
344 if (intTy && intTy.getWidth() == atomicSizeInBits)
345 return addr;
346 auto ty = cgf.getBuilder().getUIntNTy(n: atomicSizeInBits);
347 return addr.withElementType(builder&: cgf.getBuilder(), ElemTy: ty);
348}
349
350bool AtomicInfo::emitMemSetZeroIfNecessary() const {
351 assert(lvalue.isSimple());
352 Address addr = lvalue.getAddress();
353 if (!requiresMemSetZero(ty: addr.getElementType()))
354 return false;
355
356 addr = addr.withElementType(builder&: cgf.getBuilder(), ElemTy: cgf.cgm.voidTy);
357 mlir::Value zero = cgf.getBuilder().getConstInt(loc, t: cgf.cgm.uInt8Ty, c: 0);
358 mlir::Value memSetSize = cgf.getBuilder().getConstInt(
359 loc, t: cgf.cgm.uInt64Ty,
360 c: cgf.getContext().toCharUnitsFromBits(BitSize: atomicSizeInBits).getQuantity());
361
362 cgf.getBuilder().createMemSet(loc, dst: addr, val: zero, len: memSetSize);
363 return true;
364}
365
366/// Return true if \param valueTy is a type that should be casted to integer
367/// around the atomic memory operation. If \param cmpxchg is true, then the
368/// cast of a floating point type is made as that instruction can not have
369/// floating point operands. TODO: Allow compare-and-exchange and FP - see
370/// comment in CIRGenAtomicExpandPass.cpp.
371static bool shouldCastToInt(mlir::Type valueTy, bool cmpxchg) {
372 if (cir::isAnyFloatingPointType(type: valueTy))
373 return isa<cir::FP80Type>(Val: valueTy) || cmpxchg;
374 return !isa<cir::IntType>(Val: valueTy) && !isa<cir::PointerType>(Val: valueTy);
375}
376
377mlir::Value AtomicInfo::emitAtomicLoadOp(cir::MemOrder order, bool isVolatile,
378 bool cmpxchg) {
379 Address addr = getAtomicAddress();
380 if (shouldCastToInt(valueTy: addr.getElementType(), cmpxchg))
381 addr = castToAtomicIntPointer(addr);
382
383 cir::LoadOp op =
384 cgf.getBuilder().createLoad(loc, addr, /*isVolatile=*/isVolatile);
385 op.setMemOrder(order);
386
387 assert(!cir::MissingFeatures::opTBAA());
388 return op;
389}
390
391mlir::Value AtomicInfo::convertRValueToInt(RValue rvalue, mlir::Location loc,
392 bool cmpxchg) const {
393 // If we've got a scalar value of the right size, try to avoid going
394 // through memory. Floats get casted if needed by AtomicExpandPass.
395 if (mlir::Value value = getScalarRValValueOrNull(rvalue)) {
396 if (!shouldCastToInt(valueTy: value.getType(), cmpxchg))
397 return cgf.emitToMemory(value, ty: valueTy);
398
399 cgf.cgm.errorNYI(
400 loc, feature: "AtomicInfo::convertRValueToInt: cast scalar rvalue to int");
401 return nullptr;
402 }
403
404 // Otherwise, we need to go through memory.
405 // Put the r-value in memory.
406 Address addr = materializeRValue(rvalue, loc);
407
408 // Cast the temporary to the atomic int type and pull a value out.
409 addr = castToAtomicIntPointer(addr);
410
411 return cgf.getBuilder().createLoad(loc, addr);
412}
413
414RValue AtomicInfo::convertToValueOrAtomic(mlir::Value intVal,
415 AggValueSlot resultSlot,
416 SourceLocation loc, bool asValue,
417 bool cmpxchg) const {
418 // Try not to in some easy cases.
419 assert((mlir::isa<cir::IntType, cir::PointerType, cir::FPTypeInterface>(
420 intVal.getType())) &&
421 "Expected integer, pointer or floating point value when converting "
422 "result.");
423 bool isWholeValue =
424 !lvalue.isBitField() || lvalue.getBitFieldInfo().size == valueSizeInBits;
425 if (getEvaluationKind() == TEK_Scalar &&
426 ((isWholeValue && !hasPadding()) || !asValue)) {
427 mlir::Type valTy = asValue ? cgf.convertTypeForMem(t: valueTy)
428 : getAtomicAddress().getElementType();
429 if (!shouldCastToInt(valueTy: valTy, cmpxchg)) {
430 assert((!mlir::isa<cir::IntType>(valTy) || intVal.getType() == valTy) &&
431 "Different integer types.");
432 return RValue::get(v: cgf.emitFromMemory(value: intVal, ty: valueTy));
433 }
434
435 cgf.cgm.errorNYI(feature: "convertToValueOrAtomic: convert through bitcast");
436 return RValue::get(v: nullptr);
437 }
438
439 // Create a temporary. This needs to be big enough to hold the
440 // atomic integer.
441 Address temp = Address::invalid();
442 bool tempIsVolatile = false;
443 if (asValue && getEvaluationKind() == TEK_Aggregate) {
444 assert(!resultSlot.isIgnored());
445 temp = resultSlot.getAddress();
446 tempIsVolatile = resultSlot.isVolatile();
447 } else {
448 temp = createTempAlloca();
449 }
450
451 // Slam the integer into the temporary.
452 Address castTemp = castToAtomicIntPointer(addr: temp);
453 cgf.getBuilder().createStore(loc: cgf.getLoc(srcLoc: loc), val: intVal, dst: castTemp,
454 isVolatile: tempIsVolatile);
455 return convertAtomicTempToRValue(addr: temp, resultSlot, loc, asValue);
456}
457
458/// Copy an r-value into memory as part of storing to an atomic type.
459/// This needs to create a bit-pattern suitable for atomic operations.
460void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
461 assert(lvalue.isSimple());
462
463 // If we have an r-value, the rvalue should be of the atomic type,
464 // which means that the caller is responsible for having zeroed
465 // any padding. Just do an aggregate copy of that type.
466 if (rvalue.isAggregate()) {
467 cgf.cgm.errorNYI(feature: "copying aggregate into atomic lvalue");
468 return;
469 }
470
471 // Okay, otherwise we're copying stuff.
472
473 // Zero out the buffer if necessary.
474 emitMemSetZeroIfNecessary();
475
476 // Drill past the padding if present.
477 LValue tempLValue = projectValue();
478
479 // Okay, store the rvalue in.
480 if (rvalue.isScalar()) {
481 cgf.emitStoreOfScalar(value: rvalue.getValue(), lvalue: tempLValue, /*isInit=*/true);
482 } else {
483 cgf.emitStoreOfComplex(loc, v: rvalue.getComplexValue(), dest: tempLValue,
484 /*isInit=*/true);
485 }
486}
487
488/// Materialize an r-value into memory for the purposes of storing it
489/// to an atomic type.
490Address AtomicInfo::materializeRValue(RValue rvalue, mlir::Location loc) const {
491 // Aggregate r-values are already in memory, and EmitAtomicStore
492 // requires them to be values of the atomic type.
493 if (rvalue.isAggregate())
494 return rvalue.getAggregateAddress();
495
496 // Otherwise, make a temporary and materialize into it.
497 LValue tempLV = cgf.makeAddrLValue(addr: createTempAlloca(), ty: getAtomicType());
498 AtomicInfo atomics(cgf, tempLV, loc);
499
500 atomics.emitCopyIntoMemory(rvalue);
501 return tempLV.getAddress();
502}
503
504static void emitDefaultCaseLabel(CIRGenBuilderTy &builder, mlir::Location loc) {
505 mlir::ArrayAttr valuesAttr = builder.getArrayAttr(value: {});
506 mlir::OpBuilder::InsertPoint insertPoint;
507 cir::CaseOp::create(builder, location: loc, value: valuesAttr, kind: cir::CaseOpKind::Default,
508 insertPoint);
509 builder.restoreInsertionPoint(ip: insertPoint);
510}
511
512// Create a "case" operation with the given list of orders as its values. Also
513// create the region that will hold the body of the switch-case label.
514static void emitMemOrderCaseLabel(CIRGenBuilderTy &builder, mlir::Location loc,
515 mlir::Type orderType,
516 llvm::ArrayRef<cir::MemOrder> orders) {
517 llvm::SmallVector<mlir::Attribute, 2> orderAttrs;
518 for (cir::MemOrder order : orders)
519 orderAttrs.push_back(Elt: cir::IntAttr::get(type: orderType, value: static_cast<int>(order)));
520 mlir::ArrayAttr ordersAttr = builder.getArrayAttr(value: orderAttrs);
521
522 mlir::OpBuilder::InsertPoint insertPoint;
523 cir::CaseOp::create(builder, location: loc, value: ordersAttr, kind: cir::CaseOpKind::Anyof,
524 insertPoint);
525 builder.restoreInsertionPoint(ip: insertPoint);
526}
527
528static void emitAtomicCmpXchg(CIRGenFunction &cgf, AtomicExpr *e, bool isWeak,
529 Address dest, Address ptr, Address val1,
530 Address val2, Address expectedResult,
531 uint64_t size, cir::MemOrder successOrder,
532 cir::MemOrder failureOrder,
533 cir::SyncScopeKind scope) {
534 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
535
536 CIRGenBuilderTy &builder = cgf.getBuilder();
537 mlir::Value expected = builder.createLoad(loc, addr: val1);
538 mlir::Value desired = builder.createLoad(loc, addr: val2);
539
540 auto cmpxchg = cir::AtomicCmpXchgOp::create(
541 builder, location: loc, old: expected.getType(), success: builder.getBoolTy(), ptr: ptr.getPointer(),
542 expected, desired,
543 succ_order: cir::MemOrderAttr::get(context: &cgf.getMLIRContext(), value: successOrder),
544 fail_order: cir::MemOrderAttr::get(context: &cgf.getMLIRContext(), value: failureOrder),
545 sync_scope: cir::SyncScopeKindAttr::get(context: &cgf.getMLIRContext(), value: scope),
546 alignment: builder.getI64IntegerAttr(value: ptr.getAlignment().getAsAlign().value()));
547
548 cmpxchg.setIsVolatile(e->isVolatile());
549 cmpxchg.setWeak(isWeak);
550 setAMDGPUAtomicMetadata(cgf, op: cmpxchg, expr: e);
551
552 mlir::Value failed = builder.createNot(value: cmpxchg.getSuccess());
553 cir::IfOp::create(
554 builder, location: loc, cond: failed, /*withElseRegion=*/false,
555 thenBuilder: [&](mlir::OpBuilder &, mlir::Location) {
556 uint64_t expectedSizeInBytes = cgf.cgm.getDataLayout().getTypeStoreSize(
557 ty: expectedResult.getElementType());
558
559 if (expectedSizeInBytes == size) {
560 Address storeAddr = expectedResult.withElementType(
561 builder, ElemTy: cmpxchg.getOld().getType());
562 builder.createStore(loc, val: cmpxchg.getOld(), dst: storeAddr);
563 assert(!MissingFeatures::generateDebugInfo());
564 } else {
565 Address oldTmp = cgf.createTempAlloca(
566 ty: cmpxchg.getOld().getType(), align: ptr.getAlignment(), loc, name: "old.tmp");
567 builder.createStore(loc, val: cmpxchg.getOld(), dst: oldTmp);
568 assert(!MissingFeatures::generateDebugInfo());
569
570 Address oldTmpVoid = oldTmp.withElementType(builder, ElemTy: cgf.cgm.voidTy);
571 Address expectedVoid =
572 expectedResult.withElementType(builder, ElemTy: cgf.cgm.voidTy);
573 builder.createMemCpy(
574 loc, dst: expectedVoid, src: oldTmpVoid,
575 len: builder.getConstInt(loc, t: cgf.cgm.uInt64Ty, c: expectedSizeInBytes));
576 }
577 builder.createYield(loc);
578 });
579
580 // Update the memory at Dest with Success's value.
581 cgf.emitStoreOfScalar(value: cmpxchg.getSuccess(),
582 lvalue: cgf.makeAddrLValue(addr: dest, ty: e->getType()),
583 /*isInit=*/false);
584}
585
586static void emitAtomicCmpXchgFailureSet(
587 CIRGenFunction &cgf, AtomicExpr *e, bool isWeak, Address dest, Address ptr,
588 Address val1, Address val2, Address expectedResult, Expr *failureOrderExpr,
589 uint64_t size, cir::MemOrder successOrder, cir::SyncScopeKind scope) {
590 Expr::EvalResult failureOrderEval;
591 if (failureOrderExpr->EvaluateAsInt(Result&: failureOrderEval, Ctx: cgf.getContext())) {
592 uint64_t failureOrderInt = failureOrderEval.Val.getInt().getZExtValue();
593
594 cir::MemOrder failureOrder;
595 if (!cir::isValidCIRAtomicOrderingCABI(value: failureOrderInt)) {
596 failureOrder = cir::MemOrder::Relaxed;
597 } else {
598 switch ((cir::MemOrder)failureOrderInt) {
599 case cir::MemOrder::Relaxed:
600 // 31.7.2.18: "The failure argument shall not be memory_order_release
601 // nor memory_order_acq_rel". Fallback to monotonic.
602 case cir::MemOrder::Release:
603 case cir::MemOrder::AcquireRelease:
604 failureOrder = cir::MemOrder::Relaxed;
605 break;
606 case cir::MemOrder::Consume:
607 case cir::MemOrder::Acquire:
608 failureOrder = cir::MemOrder::Acquire;
609 break;
610 case cir::MemOrder::SequentiallyConsistent:
611 failureOrder = cir::MemOrder::SequentiallyConsistent;
612 break;
613 }
614 }
615
616 // Prior to c++17, "the failure argument shall be no stronger than the
617 // success argument". This condition has been lifted and the only
618 // precondition is 31.7.2.18. Effectively treat this as a DR and skip
619 // language version checks.
620 emitAtomicCmpXchg(cgf, e, isWeak, dest, ptr, val1, val2, expectedResult,
621 size, successOrder, failureOrder, scope);
622 return;
623 }
624
625 // The failure memory order is not a compile time constant. The CIR atomic ops
626 // require a constant value, so that memory order is known at compile time. In
627 // this case, we can switch based on the memory order and call each variant
628 // individually.
629 mlir::Value failureOrderVal = cgf.emitScalarExpr(e: failureOrderExpr);
630 mlir::Location atomicLoc = cgf.getLoc(srcLoc: e->getSourceRange());
631 cir::SwitchOp::create(
632 builder&: cgf.getBuilder(), location: atomicLoc, condition: failureOrderVal,
633 switchBuilder: [&](mlir::OpBuilder &b, mlir::Location loc, mlir::OperationState &os) {
634 mlir::Block *switchBlock = cgf.getBuilder().getBlock();
635
636 // case cir::MemOrder::Relaxed:
637 // // 31.7.2.18: "The failure argument shall not be
638 // memory_order_release
639 // // nor memory_order_acq_rel". Fallback to monotonic.
640 // case cir::MemOrder::Release:
641 // case cir::MemOrder::AcquireRelease:
642 // Note: Since there are 3 options, this makes sense to just emit as a
643 // 'default', which prevents user code from 'falling off' of this,
644 // which seems reasonable. Also, 'relaxed' being the default behavior
645 // is also probably the least harmful.
646 emitDefaultCaseLabel(builder&: cgf.getBuilder(), loc: atomicLoc);
647 emitAtomicCmpXchg(cgf, e, isWeak, dest, ptr, val1, val2, expectedResult,
648 size, successOrder, failureOrder: cir::MemOrder::Relaxed, scope);
649 cgf.getBuilder().createBreak(loc: atomicLoc);
650 cgf.getBuilder().setInsertionPointToEnd(switchBlock);
651
652 // case cir::MemOrder::Consume:
653 // case cir::MemOrder::Acquire:
654 emitMemOrderCaseLabel(builder&: cgf.getBuilder(), loc, orderType: failureOrderVal.getType(),
655 orders: {cir::MemOrder::Consume, cir::MemOrder::Acquire});
656 emitAtomicCmpXchg(cgf, e, isWeak, dest, ptr, val1, val2, expectedResult,
657 size, successOrder, failureOrder: cir::MemOrder::Acquire, scope);
658 cgf.getBuilder().createBreak(loc: atomicLoc);
659 cgf.getBuilder().setInsertionPointToEnd(switchBlock);
660
661 // case cir::MemOrder::SequentiallyConsistent:
662 emitMemOrderCaseLabel(builder&: cgf.getBuilder(), loc, orderType: failureOrderVal.getType(),
663 orders: {cir::MemOrder::SequentiallyConsistent});
664 emitAtomicCmpXchg(cgf, e, isWeak, dest, ptr, val1, val2, expectedResult,
665 size, successOrder,
666 failureOrder: cir::MemOrder::SequentiallyConsistent, scope);
667 cgf.getBuilder().createBreak(loc: atomicLoc);
668 cgf.getBuilder().setInsertionPointToEnd(switchBlock);
669
670 cgf.getBuilder().createYield(loc: atomicLoc);
671 });
672}
673
674// A version of the emitAtomicCmpXchgFailureSet function that ALSO checks
675// whether it is 'weak' or not (by adding an 'if' around it, and calling
676// emitAtomicCmpXchgFailureSet 2x).
677static void emitAtomicCmpXchgFailureSetCheckWeak(
678 CIRGenFunction &cgf, AtomicExpr *e, Expr *isWeakExpr, Address dest,
679 Address ptr, Address val1, Address val2, Address expectedResult,
680 Expr *failureOrderExpr, uint64_t size, cir::MemOrder successOrder,
681 cir::SyncScopeKind scope) {
682 mlir::Value isWeakVal = cgf.emitScalarExpr(e: isWeakExpr);
683 // The AST seems to be inserting a 'bool' cast (even in C mode) here, so we'll
684 // just emit it like a scalar.
685 assert(isWeakVal.getType() == cgf.getBuilder().getBoolTy());
686 mlir::Location atomicLoc = cgf.getLoc(srcLoc: e->getSourceRange());
687
688 // Unlike classic compiler, we use an 'if' here instead of a switch, simply to
689 // make this more readable/logical, plus we don't allow switch over a bool in
690 // CIR.
691 cir::IfOp::create(
692 builder&: cgf.getBuilder(), location: atomicLoc, cond: isWeakVal, /*elseRegion=*/withElseRegion: true,
693 thenBuilder: [&](mlir::OpBuilder &b, mlir::Location loc) {
694 emitAtomicCmpXchgFailureSet(cgf, e, /*isWeak=*/true, dest, ptr, val1,
695 val2, expectedResult, failureOrderExpr,
696 size, successOrder, scope);
697 cgf.getBuilder().createYield(loc: atomicLoc);
698 },
699 elseBuilder: [&](mlir::OpBuilder &b, mlir::Location loc) {
700 emitAtomicCmpXchgFailureSet(cgf, e, /*isWeak=*/false, dest, ptr, val1,
701 val2, expectedResult, failureOrderExpr,
702 size, successOrder, scope);
703 cgf.getBuilder().createYield(loc: atomicLoc);
704 });
705}
706
707static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
708 Address ptr, Address val1, Address val2,
709 Address expectedResult, Expr *isWeakExpr,
710 Expr *failureOrderExpr, int64_t size,
711 cir::MemOrder order, cir::SyncScopeKind scope) {
712 assert(!cir::MissingFeatures::atomicSyncScopeID());
713 llvm::StringRef opName;
714
715 CIRGenBuilderTy &builder = cgf.getBuilder();
716 mlir::Location loc = cgf.getLoc(srcLoc: expr->getSourceRange());
717 auto orderAttr = cir::MemOrderAttr::get(context: builder.getContext(), value: order);
718 auto scopeAttr = cir::SyncScopeKindAttr::get(context: builder.getContext(), value: scope);
719 cir::AtomicFetchKindAttr fetchAttr;
720 bool fetchFirst = true;
721
722 auto handleFetchOp = [&](cir::AtomicFetchKind kind) {
723 opName = cir::AtomicFetchOp::getOperationName();
724 fetchAttr = cir::AtomicFetchKindAttr::get(context: builder.getContext(), value: kind);
725 };
726
727 switch (expr->getOp()) {
728 case AtomicExpr::AO__c11_atomic_init:
729 llvm_unreachable("already handled!");
730
731 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
732 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
733 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
734 emitAtomicCmpXchgFailureSet(cgf, e: expr, /*isWeak=*/false, dest, ptr, val1,
735 val2, expectedResult, failureOrderExpr, size,
736 successOrder: order, scope);
737 return;
738
739 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
740 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
741 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
742 emitAtomicCmpXchgFailureSet(cgf, e: expr, /*isWeak=*/true, dest, ptr, val1,
743 val2, expectedResult, failureOrderExpr, size,
744 successOrder: order, scope);
745 return;
746
747 case AtomicExpr::AO__atomic_compare_exchange:
748 case AtomicExpr::AO__atomic_compare_exchange_n:
749 case AtomicExpr::AO__scoped_atomic_compare_exchange:
750 case AtomicExpr::AO__scoped_atomic_compare_exchange_n: {
751 bool isWeak = false;
752 if (isWeakExpr->EvaluateAsBooleanCondition(Result&: isWeak, Ctx: cgf.getContext())) {
753 emitAtomicCmpXchgFailureSet(cgf, e: expr, isWeak, dest, ptr, val1, val2,
754 expectedResult, failureOrderExpr, size, successOrder: order,
755 scope);
756 } else {
757 emitAtomicCmpXchgFailureSetCheckWeak(
758 cgf, e: expr, isWeakExpr, dest, ptr, val1, val2, expectedResult,
759 failureOrderExpr, size, successOrder: order, scope);
760 }
761 return;
762 }
763
764 case AtomicExpr::AO__c11_atomic_load:
765 case AtomicExpr::AO__atomic_load_n:
766 case AtomicExpr::AO__atomic_load:
767 case AtomicExpr::AO__scoped_atomic_load_n:
768 case AtomicExpr::AO__scoped_atomic_load:
769 case AtomicExpr::AO__hip_atomic_load:
770 case AtomicExpr::AO__opencl_atomic_load: {
771 cir::LoadOp load =
772 builder.createLoad(loc, addr: ptr, /*isVolatile=*/expr->isVolatile());
773
774 load->setAttr(name: "mem_order", value: orderAttr);
775 load->setAttr(name: "sync_scope", value: scopeAttr);
776
777 builder.createStore(loc, val: load->getResult(idx: 0), dst: dest);
778 return;
779 }
780
781 case AtomicExpr::AO__c11_atomic_store:
782 case AtomicExpr::AO__atomic_store_n:
783 case AtomicExpr::AO__atomic_store:
784 case AtomicExpr::AO__scoped_atomic_store:
785 case AtomicExpr::AO__scoped_atomic_store_n:
786 case AtomicExpr::AO__hip_atomic_store:
787 case AtomicExpr::AO__opencl_atomic_store: {
788 cir::LoadOp loadVal1 = builder.createLoad(loc, addr: val1);
789
790 assert(!cir::MissingFeatures::atomicSyncScopeID());
791
792 builder.createStore(loc, val: loadVal1, dst: ptr, isVolatile: expr->isVolatile(),
793 /*isNontemporal=*/false,
794 /*align=*/mlir::IntegerAttr{}, scope: scopeAttr, order: orderAttr);
795 return;
796 }
797
798 case AtomicExpr::AO__c11_atomic_exchange:
799 case AtomicExpr::AO__atomic_exchange_n:
800 case AtomicExpr::AO__atomic_exchange:
801 case AtomicExpr::AO__scoped_atomic_exchange_n:
802 case AtomicExpr::AO__scoped_atomic_exchange:
803 case AtomicExpr::AO__hip_atomic_exchange:
804 case AtomicExpr::AO__opencl_atomic_exchange:
805 opName = cir::AtomicXchgOp::getOperationName();
806 break;
807
808 case AtomicExpr::AO__atomic_add_fetch:
809 case AtomicExpr::AO__scoped_atomic_add_fetch:
810 fetchFirst = false;
811 [[fallthrough]];
812 case AtomicExpr::AO__c11_atomic_fetch_add:
813 case AtomicExpr::AO__atomic_fetch_add:
814 case AtomicExpr::AO__scoped_atomic_fetch_add:
815 case AtomicExpr::AO__hip_atomic_fetch_add:
816 case AtomicExpr::AO__opencl_atomic_fetch_add:
817 handleFetchOp(cir::AtomicFetchKind::Add);
818 break;
819
820 case AtomicExpr::AO__atomic_sub_fetch:
821 case AtomicExpr::AO__scoped_atomic_sub_fetch:
822 fetchFirst = false;
823 [[fallthrough]];
824 case AtomicExpr::AO__c11_atomic_fetch_sub:
825 case AtomicExpr::AO__atomic_fetch_sub:
826 case AtomicExpr::AO__scoped_atomic_fetch_sub:
827 case AtomicExpr::AO__hip_atomic_fetch_sub:
828 case AtomicExpr::AO__opencl_atomic_fetch_sub:
829 handleFetchOp(cir::AtomicFetchKind::Sub);
830 break;
831
832 case AtomicExpr::AO__atomic_min_fetch:
833 case AtomicExpr::AO__scoped_atomic_min_fetch:
834 fetchFirst = false;
835 [[fallthrough]];
836 case AtomicExpr::AO__c11_atomic_fetch_min:
837 case AtomicExpr::AO__atomic_fetch_min:
838 case AtomicExpr::AO__scoped_atomic_fetch_min:
839 case AtomicExpr::AO__hip_atomic_fetch_min:
840 case AtomicExpr::AO__opencl_atomic_fetch_min:
841 handleFetchOp(cir::AtomicFetchKind::Min);
842 break;
843
844 case AtomicExpr::AO__atomic_fetch_fminimum:
845 case AtomicExpr::AO__scoped_atomic_fetch_fminimum:
846 assert(expr->getValueType()->isFloatingType() &&
847 "fminimum operations only support floating-point types");
848 handleFetchOp(cir::AtomicFetchKind::Minimum);
849 break;
850
851 case AtomicExpr::AO__atomic_fetch_fminimum_num:
852 case AtomicExpr::AO__scoped_atomic_fetch_fminimum_num:
853 assert(expr->getValueType()->isFloatingType() &&
854 "fminimum_num operations only support floating-point types");
855 handleFetchOp(cir::AtomicFetchKind::MinimumNum);
856 break;
857
858 case AtomicExpr::AO__atomic_max_fetch:
859 case AtomicExpr::AO__scoped_atomic_max_fetch:
860 fetchFirst = false;
861 [[fallthrough]];
862 case AtomicExpr::AO__c11_atomic_fetch_max:
863 case AtomicExpr::AO__atomic_fetch_max:
864 case AtomicExpr::AO__scoped_atomic_fetch_max:
865 case AtomicExpr::AO__hip_atomic_fetch_max:
866 case AtomicExpr::AO__opencl_atomic_fetch_max:
867 handleFetchOp(cir::AtomicFetchKind::Max);
868 break;
869
870 case AtomicExpr::AO__atomic_fetch_fmaximum:
871 case AtomicExpr::AO__scoped_atomic_fetch_fmaximum:
872 assert(expr->getValueType()->isFloatingType() &&
873 "fmaximum operations only support floating-point types");
874 handleFetchOp(cir::AtomicFetchKind::Maximum);
875 break;
876
877 case AtomicExpr::AO__atomic_fetch_fmaximum_num:
878 case AtomicExpr::AO__scoped_atomic_fetch_fmaximum_num:
879 assert(expr->getValueType()->isFloatingType() &&
880 "fmaximum_num operations only support floating-point types");
881 handleFetchOp(cir::AtomicFetchKind::MaximumNum);
882 break;
883
884 case AtomicExpr::AO__atomic_and_fetch:
885 case AtomicExpr::AO__scoped_atomic_and_fetch:
886 fetchFirst = false;
887 [[fallthrough]];
888 case AtomicExpr::AO__c11_atomic_fetch_and:
889 case AtomicExpr::AO__atomic_fetch_and:
890 case AtomicExpr::AO__scoped_atomic_fetch_and:
891 case AtomicExpr::AO__hip_atomic_fetch_and:
892 case AtomicExpr::AO__opencl_atomic_fetch_and:
893 handleFetchOp(cir::AtomicFetchKind::And);
894 break;
895
896 case AtomicExpr::AO__atomic_or_fetch:
897 case AtomicExpr::AO__scoped_atomic_or_fetch:
898 fetchFirst = false;
899 [[fallthrough]];
900 case AtomicExpr::AO__c11_atomic_fetch_or:
901 case AtomicExpr::AO__atomic_fetch_or:
902 case AtomicExpr::AO__scoped_atomic_fetch_or:
903 case AtomicExpr::AO__hip_atomic_fetch_or:
904 case AtomicExpr::AO__opencl_atomic_fetch_or:
905 handleFetchOp(cir::AtomicFetchKind::Or);
906 break;
907
908 case AtomicExpr::AO__atomic_xor_fetch:
909 case AtomicExpr::AO__scoped_atomic_xor_fetch:
910 fetchFirst = false;
911 [[fallthrough]];
912 case AtomicExpr::AO__c11_atomic_fetch_xor:
913 case AtomicExpr::AO__atomic_fetch_xor:
914 case AtomicExpr::AO__scoped_atomic_fetch_xor:
915 case AtomicExpr::AO__hip_atomic_fetch_xor:
916 case AtomicExpr::AO__opencl_atomic_fetch_xor:
917 handleFetchOp(cir::AtomicFetchKind::Xor);
918 break;
919
920 case AtomicExpr::AO__atomic_nand_fetch:
921 case AtomicExpr::AO__scoped_atomic_nand_fetch:
922 fetchFirst = false;
923 [[fallthrough]];
924 case AtomicExpr::AO__c11_atomic_fetch_nand:
925 case AtomicExpr::AO__atomic_fetch_nand:
926 case AtomicExpr::AO__scoped_atomic_fetch_nand:
927 handleFetchOp(cir::AtomicFetchKind::Nand);
928 break;
929
930 case AtomicExpr::AO__atomic_test_and_set: {
931 auto op = cir::AtomicTestAndSetOp::create(
932 builder, location: loc, ptr: ptr.getPointer(), mem_order: order,
933 alignment: builder.getI64IntegerAttr(value: ptr.getAlignment().getQuantity()),
934 is_volatile: expr->isVolatile());
935 builder.createStore(loc, val: op, dst: dest);
936 return;
937 }
938
939 case AtomicExpr::AO__atomic_clear: {
940 cir::AtomicClearOp::create(
941 builder, location: loc, ptr: ptr.getPointer(), mem_order: order,
942 alignment: builder.getI64IntegerAttr(value: ptr.getAlignment().getQuantity()),
943 is_volatile: expr->isVolatile());
944 return;
945 }
946
947 case AtomicExpr::AO__atomic_fetch_uinc:
948 case AtomicExpr::AO__scoped_atomic_fetch_uinc:
949 handleFetchOp(cir::AtomicFetchKind::UIncWrap);
950 break;
951
952 case AtomicExpr::AO__atomic_fetch_udec:
953 case AtomicExpr::AO__scoped_atomic_fetch_udec:
954 handleFetchOp(cir::AtomicFetchKind::UDecWrap);
955 break;
956
957 case AtomicExpr::AO__opencl_atomic_init:
958 cgf.cgm.errorNYI(expr->getSourceRange(), "emitAtomicOp: expr op NYI");
959 return;
960 }
961
962 assert(!opName.empty() && "expected operation name to build");
963 mlir::Value loadVal1 = builder.createLoad(loc, addr: val1);
964
965 SmallVector<mlir::Value> atomicOperands = {ptr.getPointer(), loadVal1};
966 SmallVector<mlir::Type> atomicResTys = {loadVal1.getType()};
967 mlir::Operation *rmwOp = builder.create(loc, opName: builder.getStringAttr(bytes: opName),
968 operands: atomicOperands, types: atomicResTys);
969
970 if (fetchAttr)
971 rmwOp->setAttr(name: "binop", value: fetchAttr);
972 rmwOp->setAttr(name: "mem_order", value: orderAttr);
973 rmwOp->setAttr(name: "sync_scope", value: scopeAttr);
974 if (expr->isVolatile())
975 rmwOp->setAttr(name: "is_volatile", value: builder.getUnitAttr());
976 if (fetchFirst && opName == cir::AtomicFetchOp::getOperationName())
977 rmwOp->setAttr(name: "fetch_first", value: builder.getUnitAttr());
978
979 setAMDGPUAtomicMetadata(cgf, op: rmwOp, expr);
980
981 mlir::Value result = rmwOp->getResult(idx: 0);
982
983 builder.createStore(loc, val: result, dst: dest);
984}
985
986// Map clang sync scope to CIR sync scope.
987static cir::SyncScopeKind convertSyncScopeToCIR(CIRGenFunction &cgf,
988 SourceRange range,
989 clang::SyncScope scope) {
990 switch (scope) {
991 case clang::SyncScope::SingleScope:
992 return cir::SyncScopeKind::SingleThread;
993 case clang::SyncScope::SystemScope:
994 return cir::SyncScopeKind::System;
995 case clang::SyncScope::DeviceScope:
996 return cir::SyncScopeKind::Device;
997 case clang::SyncScope::WorkgroupScope:
998 return cir::SyncScopeKind::Workgroup;
999 case clang::SyncScope::WavefrontScope:
1000 return cir::SyncScopeKind::Wavefront;
1001 case clang::SyncScope::ClusterScope:
1002 return cir::SyncScopeKind::Cluster;
1003
1004 case clang::SyncScope::HIPSingleThread:
1005 return cir::SyncScopeKind::HIPSingleThread;
1006 case clang::SyncScope::HIPSystem:
1007 return cir::SyncScopeKind::HIPSystem;
1008 case clang::SyncScope::HIPAgent:
1009 return cir::SyncScopeKind::HIPAgent;
1010 case clang::SyncScope::HIPWorkgroup:
1011 return cir::SyncScopeKind::HIPWorkgroup;
1012 case clang::SyncScope::HIPWavefront:
1013 return cir::SyncScopeKind::HIPWavefront;
1014 case clang::SyncScope::HIPCluster:
1015 return cir::SyncScopeKind::HIPCluster;
1016
1017 case clang::SyncScope::OpenCLWorkGroup:
1018 return cir::SyncScopeKind::OpenCLWorkGroup;
1019 case clang::SyncScope::OpenCLDevice:
1020 return cir::SyncScopeKind::OpenCLDevice;
1021 case clang::SyncScope::OpenCLAllSVMDevices:
1022 return cir::SyncScopeKind::OpenCLAllSVMDevices;
1023 case clang::SyncScope::OpenCLSubGroup:
1024 return cir::SyncScopeKind::OpenCLSubGroup;
1025 }
1026
1027 llvm_unreachable("unhandled sync scope");
1028}
1029
1030static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
1031 Address ptr, Address val1, Address val2,
1032 Address expectedResult, Expr *isWeakExpr,
1033 Expr *failureOrderExpr, int64_t size,
1034 cir::MemOrder order,
1035 const std::optional<Expr::EvalResult> &scopeConst,
1036 mlir::Value scopeValue) {
1037 std::unique_ptr<AtomicScopeModel> scopeModel = expr->getScopeModel();
1038
1039 if (!scopeModel) {
1040 emitAtomicOp(cgf, expr, dest, ptr, val1, val2, expectedResult, isWeakExpr,
1041 failureOrderExpr, size, order, scope: cir::SyncScopeKind::System);
1042 return;
1043 }
1044
1045 if (scopeConst.has_value()) {
1046 cir::SyncScopeKind mappedScope = convertSyncScopeToCIR(
1047 cgf, range: expr->getScope()->getSourceRange(),
1048 scope: scopeModel->map(S: scopeConst->Val.getInt().getZExtValue()));
1049 emitAtomicOp(cgf, expr, dest, ptr, val1, val2, expectedResult, isWeakExpr,
1050 failureOrderExpr, size, order, scope: mappedScope);
1051 return;
1052 }
1053
1054 // The sync scope is not a compile-time constant. Emit a switch statement to
1055 // handle each possible value of the sync scope.
1056 CIRGenBuilderTy &builder = cgf.getBuilder();
1057 mlir::Location loc = cgf.getLoc(srcLoc: expr->getSourceRange());
1058 llvm::ArrayRef<unsigned> allScopes = scopeModel->getRuntimeValues();
1059 unsigned fallback = scopeModel->getFallBackValue();
1060
1061 cir::SwitchOp::create(
1062 builder, location: loc, condition: scopeValue,
1063 switchBuilder: [&](mlir::OpBuilder &, mlir::Location loc, mlir::OperationState &) {
1064 mlir::Block *switchBlock = builder.getBlock();
1065
1066 // Default case -- use fallback scope
1067 cir::SyncScopeKind fallbackScope = convertSyncScopeToCIR(
1068 cgf, range: expr->getScope()->getSourceRange(), scope: scopeModel->map(S: fallback));
1069 emitDefaultCaseLabel(builder, loc);
1070 emitAtomicOp(cgf, expr, dest, ptr, val1, val2, expectedResult,
1071 isWeakExpr, failureOrderExpr, size, order, scope: fallbackScope);
1072 builder.createBreak(loc);
1073 builder.setInsertionPointToEnd(switchBlock);
1074
1075 // Emit a switch case for each non-fallback runtime scope value
1076 for (unsigned scope : allScopes) {
1077 if (scope == fallback)
1078 continue;
1079
1080 cir::SyncScopeKind cirScope = convertSyncScopeToCIR(
1081 cgf, range: expr->getScope()->getSourceRange(), scope: scopeModel->map(S: scope));
1082
1083 mlir::ArrayAttr casesAttr = builder.getArrayAttr(
1084 value: {cir::IntAttr::get(type: scopeValue.getType(), value: scope)});
1085 mlir::OpBuilder::InsertPoint insertPoint;
1086 cir::CaseOp::create(builder, location: loc, value: casesAttr, kind: cir::CaseOpKind::Equal,
1087 insertPoint);
1088
1089 builder.restoreInsertionPoint(ip: insertPoint);
1090 emitAtomicOp(cgf, expr, dest, ptr, val1, val2, expectedResult,
1091 isWeakExpr, failureOrderExpr, size, order, scope: cirScope);
1092 builder.createBreak(loc);
1093 builder.setInsertionPointToEnd(switchBlock);
1094 }
1095
1096 builder.createYield(loc);
1097 });
1098}
1099
1100static std::optional<cir::MemOrder>
1101getEffectiveAtomicMemOrder(cir::MemOrder oriOrder, bool isStore, bool isLoad,
1102 bool isFence) {
1103 // Some memory orders are not supported by partial atomic operation:
1104 // {memory_order_releaxed} is not valid for fence operations.
1105 // {memory_order_consume, memory_order_acquire} are not valid for write-only
1106 // operations.
1107 // {memory_order_release} is not valid for read-only operations.
1108 // {memory_order_acq_rel} is only valid for read-write operations.
1109 if (isStore) {
1110 if (oriOrder == cir::MemOrder::Consume ||
1111 oriOrder == cir::MemOrder::Acquire ||
1112 oriOrder == cir::MemOrder::AcquireRelease)
1113 return std::nullopt;
1114 } else if (isLoad) {
1115 if (oriOrder == cir::MemOrder::Release ||
1116 oriOrder == cir::MemOrder::AcquireRelease)
1117 return std::nullopt;
1118 } else if (isFence) {
1119 if (oriOrder == cir::MemOrder::Relaxed)
1120 return std::nullopt;
1121 }
1122 // memory_order_consume is not implemented, it is always treated like
1123 // memory_order_acquire
1124 if (oriOrder == cir::MemOrder::Consume)
1125 return cir::MemOrder::Acquire;
1126 return oriOrder;
1127}
1128
1129static void emitAtomicExprWithDynamicMemOrder(
1130 CIRGenFunction &cgf, mlir::Value order, bool isStore, bool isLoad,
1131 bool isFence, llvm::function_ref<void(cir::MemOrder)> emitAtomicOpFn) {
1132 if (!order)
1133 return;
1134 // The memory order is not known at compile-time. The atomic operations
1135 // can't handle runtime memory orders; the memory order must be hard coded.
1136 // Generate a "switch" statement that converts a runtime value into a
1137 // compile-time value.
1138 CIRGenBuilderTy &builder = cgf.getBuilder();
1139 cir::SwitchOp::create(
1140 builder, location: order.getLoc(), condition: order,
1141 switchBuilder: [&](mlir::OpBuilder &, mlir::Location loc, mlir::OperationState &) {
1142 mlir::Block *switchBlock = builder.getBlock();
1143
1144 auto emitMemOrderCase = [&](llvm::ArrayRef<cir::MemOrder> caseOrders) {
1145 // Checking there are same effective memory order for each case.
1146 for (int i = 1, e = caseOrders.size(); i < e; i++)
1147 assert((getEffectiveAtomicMemOrder(caseOrders[i - 1], isStore,
1148 isLoad, isFence) ==
1149 getEffectiveAtomicMemOrder(caseOrders[i], isStore, isLoad,
1150 isFence)) &&
1151 "Effective memory order must be same!");
1152 // Emit case label and atomic opeartion if neccessary.
1153 if (caseOrders.empty()) {
1154 emitDefaultCaseLabel(builder, loc);
1155 // There is no good way to report an unsupported memory order at
1156 // runtime, hence the fallback to memory_order_relaxed.
1157 if (!isFence)
1158 emitAtomicOpFn(cir::MemOrder::Relaxed);
1159 } else if (std::optional<cir::MemOrder> actualOrder =
1160 getEffectiveAtomicMemOrder(oriOrder: caseOrders[0], isStore,
1161 isLoad, isFence)) {
1162 // Included in default case.
1163 if (!isFence && actualOrder == cir::MemOrder::Relaxed)
1164 return;
1165 // Creating case operation for effective memory order. If there are
1166 // multiple cases in `caseOrders`, the actual order of each case
1167 // must be same, this needs to be guaranteed by the caller.
1168 emitMemOrderCaseLabel(builder, loc, orderType: order.getType(), orders: caseOrders);
1169 emitAtomicOpFn(actualOrder.value());
1170 } else {
1171 // Do nothing if (!caseOrders.empty() && !actualOrder)
1172 return;
1173 }
1174 builder.createBreak(loc);
1175 builder.setInsertionPointToEnd(switchBlock);
1176 };
1177
1178 emitMemOrderCase(/*default:*/ {});
1179 emitMemOrderCase({cir::MemOrder::Relaxed});
1180 emitMemOrderCase({cir::MemOrder::Consume, cir::MemOrder::Acquire});
1181 emitMemOrderCase({cir::MemOrder::Release});
1182 emitMemOrderCase({cir::MemOrder::AcquireRelease});
1183 emitMemOrderCase({cir::MemOrder::SequentiallyConsistent});
1184
1185 builder.createYield(loc);
1186 });
1187}
1188
1189void CIRGenFunction::emitAtomicExprWithMemOrder(
1190 const Expr *memOrder, bool isStore, bool isLoad, bool isFence,
1191 llvm::function_ref<void(cir::MemOrder)> emitAtomicOpFn) {
1192 // Emit the memory order operand, and try to evaluate it as a constant.
1193 Expr::EvalResult eval;
1194 if (memOrder->EvaluateAsInt(Result&: eval, Ctx: getContext())) {
1195 uint64_t constOrder = eval.Val.getInt().getZExtValue();
1196 // We should not ever get to a case where the ordering isn't a valid CABI
1197 // value, but it's hard to enforce that in general.
1198 if (!cir::isValidCIRAtomicOrderingCABI(value: constOrder))
1199 return;
1200 cir::MemOrder oriOrder = static_cast<cir::MemOrder>(constOrder);
1201 if (std::optional<cir::MemOrder> actualOrder =
1202 getEffectiveAtomicMemOrder(oriOrder, isStore, isLoad, isFence))
1203 emitAtomicOpFn(actualOrder.value());
1204 return;
1205 }
1206
1207 // Otherwise, handle variable memory ordering. Emit `SwitchOp` to convert
1208 // dynamic value to static value.
1209 mlir::Value dynOrder = emitScalarExpr(e: memOrder);
1210 emitAtomicExprWithDynamicMemOrder(cgf&: *this, order: dynOrder, isStore, isLoad, isFence,
1211 emitAtomicOpFn);
1212}
1213
1214static RValue emitAtomicLibCall(CIRGenFunction &cgf, llvm::StringRef funcName,
1215 QualType resultType, CallArgList &args) {
1216 const CIRGenFunctionInfo &fnInfo =
1217 cgf.cgm.getTypes().arrangeBuiltinFunctionCall(resultType, args);
1218 cir::FuncType fnTy = cgf.cgm.getTypes().getFunctionType(info: fnInfo);
1219
1220 mlir::NamedAttrList fnAttrs;
1221 assert(!cir::MissingFeatures::opFuncExtraAttrs());
1222
1223 cir::FuncOp fn = cgf.cgm.createRuntimeFunction(ty: fnTy, name: funcName, extraAttrs: fnAttrs);
1224 auto callee = CIRGenCallee::forDirect(funcPtr: fn);
1225 return cgf.emitCall(funcInfo: fnInfo, callee, returnValue: ReturnValueSlot(), args,
1226 /*isMustTail=*/false);
1227}
1228
1229static RValue emitLibCallForAtomicExpr(CIRGenFunction &cgf, AtomicExpr *e,
1230 Address atomicPtr, Address dest,
1231 Address val1, Address val2,
1232 uint64_t atomicTySize,
1233 QualType resultTy) {
1234 mlir::Location loc = cgf.getLoc(srcLoc: e->getSourceRange());
1235
1236 CallArgList args;
1237 // For non-optimized library calls, the size is the first parameter.
1238 args.add(
1239 rvalue: RValue::get(v: cgf.getBuilder().getConstInt(loc, t: cgf.sizeTy, c: atomicTySize)),
1240 type: cgf.getContext().getSizeType());
1241
1242 // The atomic address is the second parameter.
1243 // The OpenCL atomic library functions only accept pointer arguments to
1244 // generic address space.
1245 auto castToGenericAddrSpace = [&](mlir::Value v, QualType pt) {
1246 if (!e->isOpenCL())
1247 return cgf.getBuilder().createPtrBitcast(src: v, newPointeeTy: cgf.voidTy);
1248
1249 assert(!cir::MissingFeatures::openCL());
1250 cgf.cgm.errorNYI(loc, feature: "emitLibCallForAtomicExpr: openCL");
1251 return cgf.getBuilder().createPtrBitcast(src: v, newPointeeTy: cgf.voidTy);
1252 };
1253 args.add(rvalue: RValue::get(v: castToGenericAddrSpace(atomicPtr.emitRawPointer(),
1254 e->getPtr()->getType())),
1255 type: cgf.getContext().VoidPtrTy);
1256
1257 mlir::Value order = cgf.emitScalarExpr(e: e->getOrder());
1258
1259 // The next 1-3 parameters are op-dependent.
1260 llvm::StringRef calleeName;
1261 QualType retTy;
1262 bool hasRetTy = false;
1263 switch (e->getOp()) {
1264 case AtomicExpr::AO__c11_atomic_init:
1265 case AtomicExpr::AO__opencl_atomic_init:
1266 llvm_unreachable("Already handled!");
1267
1268 // There is only one libcall for compare an exchange, because there is no
1269 // optimisation benefit possible from a libcall version of a weak compare
1270 // and exchange.
1271 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1272 // void *desired, int success, int failure)
1273 case AtomicExpr::AO__atomic_compare_exchange:
1274 case AtomicExpr::AO__atomic_compare_exchange_n:
1275 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1276 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1277 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1278 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1279 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1280 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1281 case AtomicExpr::AO__scoped_atomic_compare_exchange:
1282 case AtomicExpr::AO__scoped_atomic_compare_exchange_n: {
1283 calleeName = "__atomic_compare_exchange";
1284 retTy = cgf.getContext().BoolTy;
1285 hasRetTy = true;
1286 mlir::Value orderFail = cgf.emitScalarExpr(e: e->getOrderFail());
1287 args.add(rvalue: RValue::get(v: castToGenericAddrSpace(val1.emitRawPointer(),
1288 e->getVal1()->getType())),
1289 type: cgf.getContext().VoidPtrTy);
1290 args.add(rvalue: RValue::get(v: castToGenericAddrSpace(val2.emitRawPointer(),
1291 e->getVal2()->getType())),
1292 type: cgf.getContext().VoidPtrTy);
1293 args.add(rvalue: RValue::get(v: order), type: cgf.getContext().IntTy);
1294 order = orderFail;
1295 break;
1296 }
1297
1298 // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1299 // int order)
1300 case AtomicExpr::AO__atomic_exchange:
1301 case AtomicExpr::AO__atomic_exchange_n:
1302 case AtomicExpr::AO__c11_atomic_exchange:
1303 case AtomicExpr::AO__hip_atomic_exchange:
1304 case AtomicExpr::AO__opencl_atomic_exchange:
1305 case AtomicExpr::AO__scoped_atomic_exchange:
1306 case AtomicExpr::AO__scoped_atomic_exchange_n:
1307 calleeName = "__atomic_exchange";
1308 args.add(rvalue: RValue::get(v: castToGenericAddrSpace(val1.emitRawPointer(),
1309 e->getVal1()->getType())),
1310 type: cgf.getContext().VoidPtrTy);
1311 break;
1312
1313 // void __atomic_store(size_t size, void *mem, void *val, int order)
1314 case AtomicExpr::AO__atomic_store:
1315 case AtomicExpr::AO__atomic_store_n:
1316 case AtomicExpr::AO__c11_atomic_store:
1317 case AtomicExpr::AO__hip_atomic_store:
1318 case AtomicExpr::AO__opencl_atomic_store:
1319 case AtomicExpr::AO__scoped_atomic_store:
1320 case AtomicExpr::AO__scoped_atomic_store_n:
1321 calleeName = "__atomic_store";
1322 retTy = cgf.getContext().VoidTy;
1323 hasRetTy = true;
1324 args.add(rvalue: RValue::get(v: castToGenericAddrSpace(val1.emitRawPointer(),
1325 e->getVal1()->getType())),
1326 type: cgf.getContext().VoidPtrTy);
1327 break;
1328
1329 // void __atomic_load(size_t size, void *mem, void *return, int order)
1330 case AtomicExpr::AO__atomic_load:
1331 case AtomicExpr::AO__atomic_load_n:
1332 case AtomicExpr::AO__c11_atomic_load:
1333 case AtomicExpr::AO__hip_atomic_load:
1334 case AtomicExpr::AO__opencl_atomic_load:
1335 case AtomicExpr::AO__scoped_atomic_load:
1336 case AtomicExpr::AO__scoped_atomic_load_n:
1337 calleeName = "__atomic_load";
1338 break;
1339
1340 case AtomicExpr::AO__atomic_add_fetch:
1341 case AtomicExpr::AO__scoped_atomic_add_fetch:
1342 case AtomicExpr::AO__atomic_fetch_add:
1343 case AtomicExpr::AO__c11_atomic_fetch_add:
1344 case AtomicExpr::AO__hip_atomic_fetch_add:
1345 case AtomicExpr::AO__opencl_atomic_fetch_add:
1346 case AtomicExpr::AO__scoped_atomic_fetch_add:
1347 case AtomicExpr::AO__atomic_and_fetch:
1348 case AtomicExpr::AO__scoped_atomic_and_fetch:
1349 case AtomicExpr::AO__atomic_fetch_and:
1350 case AtomicExpr::AO__c11_atomic_fetch_and:
1351 case AtomicExpr::AO__hip_atomic_fetch_and:
1352 case AtomicExpr::AO__opencl_atomic_fetch_and:
1353 case AtomicExpr::AO__scoped_atomic_fetch_and:
1354 case AtomicExpr::AO__atomic_or_fetch:
1355 case AtomicExpr::AO__scoped_atomic_or_fetch:
1356 case AtomicExpr::AO__atomic_fetch_or:
1357 case AtomicExpr::AO__c11_atomic_fetch_or:
1358 case AtomicExpr::AO__hip_atomic_fetch_or:
1359 case AtomicExpr::AO__opencl_atomic_fetch_or:
1360 case AtomicExpr::AO__scoped_atomic_fetch_or:
1361 case AtomicExpr::AO__atomic_sub_fetch:
1362 case AtomicExpr::AO__scoped_atomic_sub_fetch:
1363 case AtomicExpr::AO__atomic_fetch_sub:
1364 case AtomicExpr::AO__c11_atomic_fetch_sub:
1365 case AtomicExpr::AO__hip_atomic_fetch_sub:
1366 case AtomicExpr::AO__opencl_atomic_fetch_sub:
1367 case AtomicExpr::AO__scoped_atomic_fetch_sub:
1368 case AtomicExpr::AO__atomic_xor_fetch:
1369 case AtomicExpr::AO__scoped_atomic_xor_fetch:
1370 case AtomicExpr::AO__atomic_fetch_xor:
1371 case AtomicExpr::AO__c11_atomic_fetch_xor:
1372 case AtomicExpr::AO__hip_atomic_fetch_xor:
1373 case AtomicExpr::AO__opencl_atomic_fetch_xor:
1374 case AtomicExpr::AO__scoped_atomic_fetch_xor:
1375 case AtomicExpr::AO__atomic_nand_fetch:
1376 case AtomicExpr::AO__atomic_fetch_nand:
1377 case AtomicExpr::AO__c11_atomic_fetch_nand:
1378 case AtomicExpr::AO__scoped_atomic_fetch_nand:
1379 case AtomicExpr::AO__scoped_atomic_nand_fetch:
1380 case AtomicExpr::AO__atomic_min_fetch:
1381 case AtomicExpr::AO__atomic_fetch_min:
1382 case AtomicExpr::AO__c11_atomic_fetch_min:
1383 case AtomicExpr::AO__hip_atomic_fetch_min:
1384 case AtomicExpr::AO__opencl_atomic_fetch_min:
1385 case AtomicExpr::AO__scoped_atomic_fetch_min:
1386 case AtomicExpr::AO__scoped_atomic_min_fetch:
1387 case AtomicExpr::AO__atomic_max_fetch:
1388 case AtomicExpr::AO__atomic_fetch_max:
1389 case AtomicExpr::AO__c11_atomic_fetch_max:
1390 case AtomicExpr::AO__hip_atomic_fetch_max:
1391 case AtomicExpr::AO__opencl_atomic_fetch_max:
1392 case AtomicExpr::AO__scoped_atomic_fetch_max:
1393 case AtomicExpr::AO__scoped_atomic_max_fetch:
1394 case AtomicExpr::AO__scoped_atomic_fetch_uinc:
1395 case AtomicExpr::AO__scoped_atomic_fetch_udec:
1396 case AtomicExpr::AO__atomic_fetch_fmaximum:
1397 case AtomicExpr::AO__atomic_fetch_fmaximum_num:
1398 case AtomicExpr::AO__atomic_fetch_fminimum:
1399 case AtomicExpr::AO__atomic_fetch_fminimum_num:
1400 case AtomicExpr::AO__scoped_atomic_fetch_fmaximum:
1401 case AtomicExpr::AO__scoped_atomic_fetch_fmaximum_num:
1402 case AtomicExpr::AO__scoped_atomic_fetch_fminimum:
1403 case AtomicExpr::AO__scoped_atomic_fetch_fminimum_num:
1404 case AtomicExpr::AO__atomic_test_and_set:
1405 case AtomicExpr::AO__atomic_clear:
1406 case AtomicExpr::AO__atomic_fetch_uinc:
1407 case AtomicExpr::AO__atomic_fetch_udec:
1408 llvm_unreachable("Integral atomic operations always become atomicrmw!");
1409 }
1410
1411 if (e->isOpenCL()) {
1412 assert(!cir::MissingFeatures::openCL());
1413 cgf.cgm.errorNYI(loc, feature: "emitLibCallForAtomicExpr: openCL");
1414 return RValue::get(v: nullptr);
1415 }
1416
1417 // By default, assume we return a value of the atomic type.
1418 if (!hasRetTy) {
1419 // Value is returned through parameter before the order.
1420 retTy = cgf.getContext().VoidTy;
1421 args.add(rvalue: RValue::get(v: castToGenericAddrSpace(dest.emitRawPointer(), retTy)),
1422 type: cgf.getContext().VoidPtrTy);
1423 }
1424
1425 // Order is always the last parameter.
1426 args.add(rvalue: RValue::get(v: order), type: cgf.getContext().IntTy);
1427 if (e->isOpenCL()) {
1428 assert(!cir::MissingFeatures::openCL());
1429 cgf.cgm.errorNYI(loc, feature: "emitLibCallForAtomicExpr: openCL");
1430 return RValue::get(v: nullptr);
1431 }
1432
1433 RValue res = emitAtomicLibCall(cgf, funcName: calleeName, resultType: retTy, args);
1434
1435 // The value is returned directly from the libcall.
1436 if (e->isCmpXChg())
1437 return res;
1438
1439 if (resultTy->isVoidType())
1440 return RValue::get(v: nullptr);
1441
1442 return cgf.convertTempToRValue(
1443 addr: dest.withElementType(builder&: cgf.getBuilder(), ElemTy: cgf.convertTypeForMem(t: resultTy)),
1444 type: resultTy, loc: e->getExprLoc());
1445}
1446
1447RValue CIRGenFunction::emitAtomicExpr(AtomicExpr *e) {
1448 QualType atomicTy = e->getPtr()->getType()->getPointeeType();
1449 QualType memTy = atomicTy;
1450 if (const auto *ty = atomicTy->getAs<AtomicType>())
1451 memTy = ty->getValueType();
1452
1453 Expr *isWeakExpr = nullptr;
1454 Expr *orderFailExpr = nullptr;
1455
1456 Address val1 = Address::invalid();
1457 Address val2 = Address::invalid();
1458 Address dest = Address::invalid();
1459 Address ptr = emitPointerWithAlignment(expr: e->getPtr());
1460
1461 assert(!cir::MissingFeatures::openCL());
1462 if (e->getOp() == AtomicExpr::AO__c11_atomic_init) {
1463 LValue lvalue = makeAddrLValue(addr: ptr, ty: atomicTy);
1464 emitAtomicInit(init: e->getVal1(), dest: lvalue);
1465 return RValue::get(v: nullptr);
1466 }
1467
1468 TypeInfoChars typeInfo = getContext().getTypeInfoInChars(T: atomicTy);
1469 uint64_t size = typeInfo.Width.getQuantity();
1470
1471 // Emit the sync scope operand, and try to evaluate it as a constant.
1472 mlir::Value scope =
1473 e->getScopeModel() ? emitScalarExpr(e: e->getScope()) : nullptr;
1474 std::optional<Expr::EvalResult> scopeConst;
1475 if (Expr::EvalResult eval;
1476 e->getScopeModel() && e->getScope()->EvaluateAsInt(Result&: eval, Ctx: getContext()))
1477 scopeConst.emplace(args: std::move(eval));
1478
1479 switch (e->getOp()) {
1480 default:
1481 cgm.errorNYI(e->getSourceRange(), "atomic op NYI");
1482 return RValue::get(v: nullptr);
1483
1484 case AtomicExpr::AO__c11_atomic_init:
1485 llvm_unreachable("already handled above with emitAtomicInit");
1486
1487 case AtomicExpr::AO__atomic_load_n:
1488 case AtomicExpr::AO__scoped_atomic_load_n:
1489 case AtomicExpr::AO__c11_atomic_load:
1490 case AtomicExpr::AO__opencl_atomic_load:
1491 case AtomicExpr::AO__hip_atomic_load:
1492 case AtomicExpr::AO__atomic_test_and_set:
1493 case AtomicExpr::AO__atomic_clear:
1494 break;
1495
1496 case AtomicExpr::AO__atomic_load:
1497 case AtomicExpr::AO__scoped_atomic_load:
1498 dest = emitPointerWithAlignment(expr: e->getVal1());
1499 break;
1500
1501 case AtomicExpr::AO__atomic_store:
1502 case AtomicExpr::AO__scoped_atomic_store:
1503 val1 = emitPointerWithAlignment(expr: e->getVal1());
1504 break;
1505
1506 case AtomicExpr::AO__atomic_exchange:
1507 case AtomicExpr::AO__scoped_atomic_exchange:
1508 val1 = emitPointerWithAlignment(expr: e->getVal1());
1509 dest = emitPointerWithAlignment(expr: e->getVal2());
1510 break;
1511
1512 case AtomicExpr::AO__atomic_compare_exchange:
1513 case AtomicExpr::AO__atomic_compare_exchange_n:
1514 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1515 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1516 case AtomicExpr::AO__scoped_atomic_compare_exchange:
1517 case AtomicExpr::AO__scoped_atomic_compare_exchange_n:
1518 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1519 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1520 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1521 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1522 val1 = emitPointerWithAlignment(expr: e->getVal1());
1523 if (e->getOp() == AtomicExpr::AO__atomic_compare_exchange ||
1524 e->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)
1525 val2 = emitPointerWithAlignment(expr: e->getVal2());
1526 else
1527 val2 = emitValToTemp(cgf&: *this, e: e->getVal2());
1528 orderFailExpr = e->getOrderFail();
1529 if (e->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
1530 e->getOp() == AtomicExpr::AO__atomic_compare_exchange ||
1531 e->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange_n ||
1532 e->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)
1533 isWeakExpr = e->getWeak();
1534 break;
1535
1536 case AtomicExpr::AO__c11_atomic_fetch_add:
1537 case AtomicExpr::AO__c11_atomic_fetch_sub:
1538 if (memTy->isPointerType()) {
1539 // For pointer arithmetic, we're required to do a bit of math:
1540 // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
1541 // ... but only for the C11 builtins. The GNU builtins expect the
1542 // user to multiply by sizeof(T).
1543 QualType val1Ty = e->getVal1()->getType();
1544 mlir::Location loc = getLoc(srcLoc: e->getSourceRange());
1545 mlir::Value val1Scalar = emitScalarExpr(e: e->getVal1());
1546 CharUnits pointeeIncAmt =
1547 getContext().getTypeSizeInChars(T: memTy->getPointeeType());
1548 mlir::Value scale = builder.getConstInt(loc, t: val1Scalar.getType(),
1549 c: pointeeIncAmt.getQuantity());
1550 val1Scalar = builder.createMul(loc, lhs: val1Scalar, rhs: scale);
1551 val1 = createMemTemp(t: val1Ty, loc, name: ".atomictmp");
1552 emitStoreOfScalar(value: val1Scalar, lvalue: makeAddrLValue(addr: val1, ty: val1Ty),
1553 /*isInit=*/true);
1554 }
1555 [[fallthrough]];
1556 case AtomicExpr::AO__atomic_fetch_add:
1557 case AtomicExpr::AO__atomic_fetch_sub:
1558 case AtomicExpr::AO__atomic_add_fetch:
1559 case AtomicExpr::AO__atomic_sub_fetch:
1560 if (memTy->isPointerType()) {
1561 // Fetch-and-update atomic operation on pointers should treat the pointer
1562 // value as uintptr_t values
1563 if (!val1.isValid())
1564 val1 = emitValToTemp(cgf&: *this, e: e->getVal1());
1565 ptr = ptr.withElementType(builder, ElemTy: val1.getElementType());
1566 break;
1567 }
1568 [[fallthrough]];
1569 case AtomicExpr::AO__atomic_fetch_max:
1570 case AtomicExpr::AO__atomic_fetch_min:
1571 case AtomicExpr::AO__atomic_max_fetch:
1572 case AtomicExpr::AO__atomic_min_fetch:
1573 case AtomicExpr::AO__c11_atomic_fetch_max:
1574 case AtomicExpr::AO__c11_atomic_fetch_min:
1575 case AtomicExpr::AO__scoped_atomic_fetch_add:
1576 case AtomicExpr::AO__scoped_atomic_fetch_max:
1577 case AtomicExpr::AO__scoped_atomic_fetch_min:
1578 case AtomicExpr::AO__scoped_atomic_fetch_sub:
1579 case AtomicExpr::AO__scoped_atomic_fetch_fminimum:
1580 case AtomicExpr::AO__scoped_atomic_fetch_fmaximum:
1581 case AtomicExpr::AO__scoped_atomic_fetch_fminimum_num:
1582 case AtomicExpr::AO__scoped_atomic_fetch_fmaximum_num:
1583 case AtomicExpr::AO__scoped_atomic_add_fetch:
1584 case AtomicExpr::AO__scoped_atomic_max_fetch:
1585 case AtomicExpr::AO__scoped_atomic_min_fetch:
1586 case AtomicExpr::AO__scoped_atomic_sub_fetch:
1587 [[fallthrough]];
1588
1589 case AtomicExpr::AO__atomic_fetch_and:
1590 case AtomicExpr::AO__atomic_fetch_nand:
1591 case AtomicExpr::AO__atomic_fetch_or:
1592 case AtomicExpr::AO__atomic_fetch_xor:
1593 case AtomicExpr::AO__atomic_and_fetch:
1594 case AtomicExpr::AO__atomic_nand_fetch:
1595 case AtomicExpr::AO__atomic_or_fetch:
1596 case AtomicExpr::AO__atomic_xor_fetch:
1597 case AtomicExpr::AO__atomic_exchange_n:
1598 case AtomicExpr::AO__atomic_store_n:
1599 case AtomicExpr::AO__c11_atomic_fetch_and:
1600 case AtomicExpr::AO__c11_atomic_fetch_nand:
1601 case AtomicExpr::AO__c11_atomic_fetch_or:
1602 case AtomicExpr::AO__c11_atomic_fetch_xor:
1603 case AtomicExpr::AO__c11_atomic_exchange:
1604 case AtomicExpr::AO__c11_atomic_store:
1605 case AtomicExpr::AO__scoped_atomic_fetch_and:
1606 case AtomicExpr::AO__scoped_atomic_fetch_nand:
1607 case AtomicExpr::AO__scoped_atomic_fetch_or:
1608 case AtomicExpr::AO__scoped_atomic_fetch_xor:
1609 case AtomicExpr::AO__scoped_atomic_and_fetch:
1610 case AtomicExpr::AO__scoped_atomic_nand_fetch:
1611 case AtomicExpr::AO__scoped_atomic_or_fetch:
1612 case AtomicExpr::AO__scoped_atomic_xor_fetch:
1613 case AtomicExpr::AO__scoped_atomic_store_n:
1614 case AtomicExpr::AO__scoped_atomic_exchange_n:
1615 case AtomicExpr::AO__atomic_fetch_uinc:
1616 case AtomicExpr::AO__atomic_fetch_udec:
1617 case AtomicExpr::AO__scoped_atomic_fetch_uinc:
1618 case AtomicExpr::AO__scoped_atomic_fetch_udec:
1619 case AtomicExpr::AO__atomic_fetch_fminimum:
1620 case AtomicExpr::AO__atomic_fetch_fmaximum:
1621 case AtomicExpr::AO__atomic_fetch_fminimum_num:
1622 case AtomicExpr::AO__atomic_fetch_fmaximum_num:
1623 case AtomicExpr::AO__hip_atomic_exchange:
1624 case AtomicExpr::AO__hip_atomic_store:
1625 case AtomicExpr::AO__hip_atomic_fetch_add:
1626 case AtomicExpr::AO__hip_atomic_fetch_sub:
1627 case AtomicExpr::AO__hip_atomic_fetch_min:
1628 case AtomicExpr::AO__hip_atomic_fetch_max:
1629 case AtomicExpr::AO__hip_atomic_fetch_and:
1630 case AtomicExpr::AO__hip_atomic_fetch_or:
1631 case AtomicExpr::AO__hip_atomic_fetch_xor:
1632 case AtomicExpr::AO__opencl_atomic_exchange:
1633 case AtomicExpr::AO__opencl_atomic_store:
1634 case AtomicExpr::AO__opencl_atomic_fetch_add:
1635 case AtomicExpr::AO__opencl_atomic_fetch_sub:
1636 case AtomicExpr::AO__opencl_atomic_fetch_min:
1637 case AtomicExpr::AO__opencl_atomic_fetch_max:
1638 case AtomicExpr::AO__opencl_atomic_fetch_and:
1639 case AtomicExpr::AO__opencl_atomic_fetch_or:
1640 case AtomicExpr::AO__opencl_atomic_fetch_xor:
1641 val1 = emitValToTemp(cgf&: *this, e: e->getVal1());
1642 break;
1643 }
1644
1645 QualType resultTy = e->getType().getUnqualifiedType();
1646
1647 bool shouldCastToIntPtrTy =
1648 shouldCastToInt(valueTy: convertTypeForMem(t: memTy), cmpxchg: e->isCmpXChg());
1649
1650 // The inlined atomics only function on iN types, where N is a power of 2. We
1651 // need to make sure (via temporaries if necessary) that all incoming values
1652 // are compatible.
1653 mlir::Location loc = getLoc(srcLoc: e->getSourceRange());
1654 LValue atomicValue = makeAddrLValue(addr: ptr, ty: atomicTy);
1655 AtomicInfo atomics(*this, atomicValue, loc);
1656
1657 // Save val1's address before it is (possibly) converted to a temporary
1658 // sized to the full atomic width below.
1659 Address originalVal1 = val1;
1660
1661 if (shouldCastToIntPtrTy) {
1662 ptr = atomics.castToAtomicIntPointer(addr: ptr);
1663 if (val1.isValid())
1664 val1 = atomics.convertToAtomicIntPointer(addr: val1, loc);
1665 if (val2.isValid())
1666 val2 = atomics.convertToAtomicIntPointer(addr: val2, loc);
1667 }
1668 if (dest.isValid()) {
1669 if (shouldCastToIntPtrTy)
1670 dest = atomics.castToAtomicIntPointer(addr: dest);
1671 } else if (e->isCmpXChg()) {
1672 dest = createMemTemp(t: resultTy, loc, name: "cmpxchg.bool");
1673 } else if (e->getOp() == AtomicExpr::AO__atomic_test_and_set) {
1674 dest = createMemTemp(t: resultTy, loc, name: "test_and_set.bool");
1675 } else if (!resultTy->isVoidType()) {
1676 dest = atomics.createTempAlloca();
1677 if (shouldCastToIntPtrTy)
1678 dest = atomics.castToAtomicIntPointer(addr: dest);
1679 }
1680
1681 bool powerOf2Size = (size & (size - 1)) == 0;
1682 bool useLibCall = !powerOf2Size || (size > 16);
1683
1684 // For atomics larger than 16 bytes, emit a libcall from the frontend. This
1685 // avoids the overhead of dealing with excessively-large value types in IR.
1686 // Non-power-of-2 values also lower to libcall here, as they are not currently
1687 // permitted in IR instructions (although that constraint could be relaxed in
1688 // the future). For other cases where a libcall is required on a given
1689 // platform, we let the backend handle it (this includes handling for all of
1690 // the size-optimized libcall variants, which are only valid up to 16 bytes.)
1691 //
1692 // See: https://llvm.org/docs/Atomics.html#libcalls-atomic
1693 if (useLibCall)
1694 return emitLibCallForAtomicExpr(cgf&: *this, e, atomicPtr: ptr, dest, val1, val2, atomicTySize: size,
1695 resultTy);
1696
1697 bool isStore = e->getOp() == AtomicExpr::AO__c11_atomic_store ||
1698 e->getOp() == AtomicExpr::AO__opencl_atomic_store ||
1699 e->getOp() == AtomicExpr::AO__hip_atomic_store ||
1700 e->getOp() == AtomicExpr::AO__atomic_store ||
1701 e->getOp() == AtomicExpr::AO__atomic_store_n ||
1702 e->getOp() == AtomicExpr::AO__scoped_atomic_store ||
1703 e->getOp() == AtomicExpr::AO__scoped_atomic_store_n ||
1704 e->getOp() == AtomicExpr::AO__atomic_clear;
1705 bool isLoad = e->getOp() == AtomicExpr::AO__c11_atomic_load ||
1706 e->getOp() == AtomicExpr::AO__opencl_atomic_load ||
1707 e->getOp() == AtomicExpr::AO__hip_atomic_load ||
1708 e->getOp() == AtomicExpr::AO__atomic_load ||
1709 e->getOp() == AtomicExpr::AO__atomic_load_n ||
1710 e->getOp() == AtomicExpr::AO__scoped_atomic_load ||
1711 e->getOp() == AtomicExpr::AO__scoped_atomic_load_n;
1712
1713 auto emitAtomicOpCallBackFn = [&](cir::MemOrder memOrder) {
1714 emitAtomicOp(cgf&: *this, expr: e, dest, ptr, val1, val2, expectedResult: originalVal1, isWeakExpr,
1715 failureOrderExpr: orderFailExpr, size, order: memOrder, scopeConst, scopeValue: scope);
1716 };
1717 emitAtomicExprWithMemOrder(memOrder: e->getOrder(), isStore, isLoad, /*isFence*/ false,
1718 emitAtomicOpFn: emitAtomicOpCallBackFn);
1719
1720 if (resultTy->isVoidType())
1721 return RValue::get(v: nullptr);
1722
1723 return convertTempToRValue(
1724 addr: dest.withElementType(builder, ElemTy: convertTypeForMem(t: resultTy)), type: resultTy,
1725 loc: e->getExprLoc());
1726}
1727
1728RValue CIRGenFunction::emitAtomicLoad(LValue lvalue, SourceLocation loc,
1729 AggValueSlot slot) {
1730 if (lvalue.getType()->isAtomicType())
1731 return emitAtomicLoad(lvalue, loc, order: cir::MemOrder::SequentiallyConsistent,
1732 /*isVolatile=*/lvalue.isVolatileQualified(), slot);
1733 return emitAtomicLoad(lvalue, loc, order: cir::MemOrder::Acquire,
1734 /*isVolatile=*/true, slot);
1735}
1736
1737RValue CIRGenFunction::emitAtomicLoad(LValue lvalue, SourceLocation loc,
1738 cir::MemOrder order, bool isVolatile,
1739 AggValueSlot slot) {
1740 AtomicInfo info(*this, lvalue, getLoc(srcLoc: loc));
1741 return info.emitAtomicLoad(resultSlot: slot, loc, /*asValue=*/true, order, isVolatile);
1742}
1743
1744void CIRGenFunction::emitAtomicStore(RValue rvalue, LValue dest, bool isInit) {
1745 bool isVolatile = dest.isVolatileQualified();
1746 auto order = cir::MemOrder::SequentiallyConsistent;
1747 if (!dest.getType()->isAtomicType()) {
1748 assert(!cir::MissingFeatures::atomicMicrosoftVolatile());
1749 }
1750 return emitAtomicStore(rvalue, dest, order, isVolatile, isInit);
1751}
1752
1753/// Emit a store to an l-value of atomic type.
1754///
1755/// Note that the r-value is expected to be an r-value of the atomic type; this
1756/// means that for aggregate r-values, it should include storage for any padding
1757/// that was necessary.
1758void CIRGenFunction::emitAtomicStore(RValue rvalue, LValue dest,
1759 cir::MemOrder order, bool isVolatile,
1760 bool isInit) {
1761 // If this is an aggregate r-value, it should agree in type except
1762 // maybe for address-space qualification.
1763 mlir::Location loc = dest.getPointer().getLoc();
1764 assert(!rvalue.isAggregate() ||
1765 rvalue.getAggregateAddress().getElementType() ==
1766 dest.getAddress().getElementType());
1767
1768 AtomicInfo atomics(*this, dest, loc);
1769 LValue lvalue = atomics.getAtomicLValue();
1770
1771 if (lvalue.isSimple()) {
1772 // If this is an initialization, just put the value there normally.
1773 if (isInit) {
1774 atomics.emitCopyIntoMemory(rvalue);
1775 return;
1776 }
1777
1778 // Check whether we should use a library call.
1779 if (atomics.shouldUseLibCall()) {
1780 assert(!cir::MissingFeatures::atomicUseLibCall());
1781 cgm.errorNYI(loc, feature: "emitAtomicStore: atomic store with library call");
1782 return;
1783 }
1784
1785 // Okay, we're doing this natively.
1786 mlir::Value valueToStore = atomics.convertRValueToInt(rvalue, loc);
1787
1788 // Do the atomic store.
1789 Address addr = atomics.getAtomicAddress();
1790 if (mlir::Value value = atomics.getScalarRValValueOrNull(rvalue)) {
1791 if (shouldCastToInt(valueTy: value.getType(), /*CmpXchg=*/cmpxchg: false)) {
1792 addr = atomics.castToAtomicIntPointer(addr);
1793 valueToStore =
1794 builder.createIntCast(src: valueToStore, newTy: addr.getElementType());
1795 }
1796 }
1797 cir::StoreOp store = builder.createStore(loc, val: valueToStore, dst: addr);
1798
1799 // Initializations don't need to be atomic.
1800 if (!isInit) {
1801 assert(!cir::MissingFeatures::atomicOpenMP());
1802 store.setMemOrder(order);
1803 }
1804
1805 // Other decoration.
1806 if (isVolatile)
1807 store.setIsVolatile(true);
1808
1809 assert(!cir::MissingFeatures::opLoadStoreTbaa());
1810 return;
1811 }
1812
1813 cgm.errorNYI(loc, feature: "emitAtomicStore: non-simple atomic lvalue");
1814 assert(!cir::MissingFeatures::opLoadStoreAtomic());
1815}
1816
1817void CIRGenFunction::emitAtomicInit(Expr *init, LValue dest) {
1818 AtomicInfo atomics(*this, dest, getLoc(srcLoc: init->getSourceRange()));
1819
1820 switch (atomics.getEvaluationKind()) {
1821 case cir::TEK_Scalar: {
1822 mlir::Value value = emitScalarExpr(e: init);
1823 atomics.emitCopyIntoMemory(rvalue: RValue::get(v: value));
1824 return;
1825 }
1826
1827 case cir::TEK_Complex: {
1828 mlir::Value value = emitComplexExpr(e: init);
1829 atomics.emitCopyIntoMemory(rvalue: RValue::get(v: value));
1830 return;
1831 }
1832
1833 case cir::TEK_Aggregate: {
1834 // Fix up the destination if the initializer isn't an expression
1835 // of atomic type.
1836 bool zeroed = false;
1837 if (!init->getType()->isAtomicType()) {
1838 zeroed = atomics.emitMemSetZeroIfNecessary();
1839 dest = atomics.projectValue();
1840 }
1841
1842 // Evaluate the expression directly into the destination.
1843 assert(!cir::MissingFeatures::aggValueSlotGC());
1844 AggValueSlot slot = AggValueSlot::forLValue(
1845 LV: dest, isDestructed: AggValueSlot::IsNotDestructed, isAliased: AggValueSlot::IsNotAliased,
1846 mayOverlap: AggValueSlot::DoesNotOverlap,
1847 isZeroed: zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed);
1848
1849 emitAggExpr(e: init, slot);
1850 return;
1851 }
1852 }
1853
1854 llvm_unreachable("bad evaluation kind");
1855}
1856