1//===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the code for emitting atomic operations.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCall.h"
14#include "CGRecordLayout.h"
15#include "CodeGenFunction.h"
16#include "CodeGenModule.h"
17#include "TargetInfo.h"
18#include "clang/AST/ASTContext.h"
19#include "clang/CodeGen/CGFunctionInfo.h"
20#include "clang/Frontend/FrontendDiagnostic.h"
21#include "llvm/ADT/DenseMap.h"
22#include "llvm/IR/DataLayout.h"
23#include "llvm/IR/Intrinsics.h"
24
25using namespace clang;
26using namespace CodeGen;
27
28namespace {
29 class AtomicInfo {
30 CodeGenFunction &CGF;
31 QualType AtomicTy;
32 QualType ValueTy;
33 uint64_t AtomicSizeInBits;
34 uint64_t ValueSizeInBits;
35 CharUnits AtomicAlign;
36 CharUnits ValueAlign;
37 TypeEvaluationKind EvaluationKind;
38 bool UseLibcall;
39 LValue LVal;
40 CGBitFieldInfo BFI;
41 public:
42 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
43 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
44 EvaluationKind(TEK_Scalar), UseLibcall(true) {
45 assert(!lvalue.isGlobalReg());
46 ASTContext &C = CGF.getContext();
47 if (lvalue.isSimple()) {
48 AtomicTy = lvalue.getType();
49 if (auto *ATy = AtomicTy->getAs<AtomicType>())
50 ValueTy = ATy->getValueType();
51 else
52 ValueTy = AtomicTy;
53 EvaluationKind = CGF.getEvaluationKind(T: ValueTy);
54
55 uint64_t ValueAlignInBits;
56 uint64_t AtomicAlignInBits;
57 TypeInfo ValueTI = C.getTypeInfo(T: ValueTy);
58 ValueSizeInBits = ValueTI.Width;
59 ValueAlignInBits = ValueTI.Align;
60
61 TypeInfo AtomicTI = C.getTypeInfo(T: AtomicTy);
62 AtomicSizeInBits = AtomicTI.Width;
63 AtomicAlignInBits = AtomicTI.Align;
64
65 assert(ValueSizeInBits <= AtomicSizeInBits);
66 assert(ValueAlignInBits <= AtomicAlignInBits);
67
68 AtomicAlign = C.toCharUnitsFromBits(BitSize: AtomicAlignInBits);
69 ValueAlign = C.toCharUnitsFromBits(BitSize: ValueAlignInBits);
70 if (lvalue.getAlignment().isZero())
71 lvalue.setAlignment(AtomicAlign);
72
73 LVal = lvalue;
74 } else if (lvalue.isBitField()) {
75 ValueTy = lvalue.getType();
76 ValueSizeInBits = C.getTypeSize(T: ValueTy);
77 auto &OrigBFI = lvalue.getBitFieldInfo();
78 auto Offset = OrigBFI.Offset % C.toBits(CharSize: lvalue.getAlignment());
79 AtomicSizeInBits = C.toBits(
80 CharSize: C.toCharUnitsFromBits(BitSize: Offset + OrigBFI.Size + C.getCharWidth() - 1)
81 .alignTo(Align: lvalue.getAlignment()));
82 llvm::Value *BitFieldPtr = lvalue.getRawBitFieldPointer(CGF);
83 auto OffsetInChars =
84 (C.toCharUnitsFromBits(BitSize: OrigBFI.Offset) / lvalue.getAlignment()) *
85 lvalue.getAlignment();
86 llvm::Value *StoragePtr = CGF.Builder.CreateConstGEP1_64(
87 Ty: CGF.Int8Ty, Ptr: BitFieldPtr, Idx0: OffsetInChars.getQuantity());
88 StoragePtr = CGF.Builder.CreateAddrSpaceCast(
89 V: StoragePtr, DestTy: CGF.UnqualPtrTy, Name: "atomic_bitfield_base");
90 BFI = OrigBFI;
91 BFI.Offset = Offset;
92 BFI.StorageSize = AtomicSizeInBits;
93 BFI.StorageOffset += OffsetInChars;
94 llvm::Type *StorageTy = CGF.Builder.getIntNTy(N: AtomicSizeInBits);
95 LVal = LValue::MakeBitfield(
96 Addr: Address(StoragePtr, StorageTy, lvalue.getAlignment()), Info: BFI,
97 type: lvalue.getType(), BaseInfo: lvalue.getBaseInfo(), TBAAInfo: lvalue.getTBAAInfo());
98 AtomicTy = C.getIntTypeForBitwidth(DestWidth: AtomicSizeInBits, Signed: OrigBFI.IsSigned);
99 if (AtomicTy.isNull()) {
100 llvm::APInt Size(
101 /*numBits=*/32,
102 C.toCharUnitsFromBits(BitSize: AtomicSizeInBits).getQuantity());
103 AtomicTy = C.getConstantArrayType(EltTy: C.CharTy, ArySize: Size, SizeExpr: nullptr,
104 ASM: ArraySizeModifier::Normal,
105 /*IndexTypeQuals=*/0);
106 }
107 AtomicAlign = ValueAlign = lvalue.getAlignment();
108 } else if (lvalue.isVectorElt()) {
109 ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();
110 ValueSizeInBits = C.getTypeSize(T: ValueTy);
111 AtomicTy = lvalue.getType();
112 AtomicSizeInBits = C.getTypeSize(T: AtomicTy);
113 AtomicAlign = ValueAlign = lvalue.getAlignment();
114 LVal = lvalue;
115 } else {
116 assert(lvalue.isExtVectorElt());
117 ValueTy = lvalue.getType();
118 ValueSizeInBits = C.getTypeSize(T: ValueTy);
119 AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
120 VectorType: lvalue.getType(), NumElts: cast<llvm::FixedVectorType>(
121 Val: lvalue.getExtVectorAddress().getElementType())
122 ->getNumElements());
123 AtomicSizeInBits = C.getTypeSize(T: AtomicTy);
124 AtomicAlign = ValueAlign = lvalue.getAlignment();
125 LVal = lvalue;
126 }
127 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
128 AtomicSizeInBits, AlignmentInBits: C.toBits(CharSize: lvalue.getAlignment()));
129 }
130
131 QualType getAtomicType() const { return AtomicTy; }
132 QualType getValueType() const { return ValueTy; }
133 CharUnits getAtomicAlignment() const { return AtomicAlign; }
134 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
135 uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
136 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
137 bool shouldUseLibcall() const { return UseLibcall; }
138 const LValue &getAtomicLValue() const { return LVal; }
139 llvm::Value *getAtomicPointer() const {
140 if (LVal.isSimple())
141 return LVal.emitRawPointer(CGF);
142 else if (LVal.isBitField())
143 return LVal.getRawBitFieldPointer(CGF);
144 else if (LVal.isVectorElt())
145 return LVal.getRawVectorPointer(CGF);
146 assert(LVal.isExtVectorElt());
147 return LVal.getRawExtVectorPointer(CGF);
148 }
149 Address getAtomicAddress() const {
150 llvm::Type *ElTy;
151 if (LVal.isSimple())
152 ElTy = LVal.getAddress().getElementType();
153 else if (LVal.isBitField())
154 ElTy = LVal.getBitFieldAddress().getElementType();
155 else if (LVal.isVectorElt())
156 ElTy = LVal.getVectorAddress().getElementType();
157 else
158 ElTy = LVal.getExtVectorAddress().getElementType();
159 return Address(getAtomicPointer(), ElTy, getAtomicAlignment());
160 }
161
162 Address getAtomicAddressAsAtomicIntPointer() const {
163 return castToAtomicIntPointer(Addr: getAtomicAddress());
164 }
165
166 /// Is the atomic size larger than the underlying value type?
167 ///
168 /// Note that the absence of padding does not mean that atomic
169 /// objects are completely interchangeable with non-atomic
170 /// objects: we might have promoted the alignment of a type
171 /// without making it bigger.
172 bool hasPadding() const {
173 return (ValueSizeInBits != AtomicSizeInBits);
174 }
175
176 bool emitMemSetZeroIfNecessary() const;
177
178 llvm::Value *getAtomicSizeValue() const {
179 CharUnits size = CGF.getContext().toCharUnitsFromBits(BitSize: AtomicSizeInBits);
180 return CGF.CGM.getSize(numChars: size);
181 }
182
183 /// Cast the given pointer to an integer pointer suitable for atomic
184 /// operations if the source.
185 Address castToAtomicIntPointer(Address Addr) const;
186
187 /// If Addr is compatible with the iN that will be used for an atomic
188 /// operation, bitcast it. Otherwise, create a temporary that is suitable
189 /// and copy the value across.
190 Address convertToAtomicIntPointer(Address Addr) const;
191
192 /// Turn an atomic-layout object into an r-value.
193 RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
194 SourceLocation loc, bool AsValue) const;
195
196 llvm::Value *getScalarRValValueOrNull(RValue RVal) const;
197
198 /// Converts an rvalue to integer value if needed.
199 llvm::Value *convertRValueToInt(RValue RVal, bool CmpXchg = false) const;
200
201 RValue ConvertToValueOrAtomic(llvm::Value *IntVal, AggValueSlot ResultSlot,
202 SourceLocation Loc, bool AsValue,
203 bool CmpXchg = false) const;
204
205 /// Copy an atomic r-value into atomic-layout memory.
206 void emitCopyIntoMemory(RValue rvalue) const;
207
208 /// Project an l-value down to the value field.
209 LValue projectValue() const {
210 assert(LVal.isSimple());
211 Address addr = getAtomicAddress();
212 if (hasPadding())
213 addr = CGF.Builder.CreateStructGEP(Addr: addr, Index: 0);
214
215 return LValue::MakeAddr(Addr: addr, type: getValueType(), Context&: CGF.getContext(),
216 BaseInfo: LVal.getBaseInfo(), TBAAInfo: LVal.getTBAAInfo());
217 }
218
219 /// Emits atomic load.
220 /// \returns Loaded value.
221 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
222 bool AsValue, llvm::AtomicOrdering AO,
223 bool IsVolatile);
224
225 /// Emits atomic compare-and-exchange sequence.
226 /// \param Expected Expected value.
227 /// \param Desired Desired value.
228 /// \param Success Atomic ordering for success operation.
229 /// \param Failure Atomic ordering for failed operation.
230 /// \param IsWeak true if atomic operation is weak, false otherwise.
231 /// \returns Pair of values: previous value from storage (value type) and
232 /// boolean flag (i1 type) with true if success and false otherwise.
233 std::pair<RValue, llvm::Value *>
234 EmitAtomicCompareExchange(RValue Expected, RValue Desired,
235 llvm::AtomicOrdering Success =
236 llvm::AtomicOrdering::SequentiallyConsistent,
237 llvm::AtomicOrdering Failure =
238 llvm::AtomicOrdering::SequentiallyConsistent,
239 bool IsWeak = false);
240
241 /// Emits atomic update.
242 /// \param AO Atomic ordering.
243 /// \param UpdateOp Update operation for the current lvalue.
244 void EmitAtomicUpdate(llvm::AtomicOrdering AO,
245 const llvm::function_ref<RValue(RValue)> &UpdateOp,
246 bool IsVolatile);
247 /// Emits atomic update.
248 /// \param AO Atomic ordering.
249 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
250 bool IsVolatile);
251
252 /// Materialize an atomic r-value in atomic-layout memory.
253 Address materializeRValue(RValue rvalue) const;
254
255 /// Creates temp alloca for intermediate operations on atomic value.
256 Address CreateTempAlloca() const;
257 private:
258 bool requiresMemSetZero(llvm::Type *type) const;
259
260
261 /// Emits atomic load as a libcall.
262 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
263 llvm::AtomicOrdering AO, bool IsVolatile);
264 /// Emits atomic load as LLVM instruction.
265 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile,
266 bool CmpXchg = false);
267 /// Emits atomic compare-and-exchange op as a libcall.
268 llvm::Value *EmitAtomicCompareExchangeLibcall(
269 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
270 llvm::AtomicOrdering Success =
271 llvm::AtomicOrdering::SequentiallyConsistent,
272 llvm::AtomicOrdering Failure =
273 llvm::AtomicOrdering::SequentiallyConsistent);
274 /// Emits atomic compare-and-exchange op as LLVM instruction.
275 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
276 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
277 llvm::AtomicOrdering Success =
278 llvm::AtomicOrdering::SequentiallyConsistent,
279 llvm::AtomicOrdering Failure =
280 llvm::AtomicOrdering::SequentiallyConsistent,
281 bool IsWeak = false);
282 /// Emit atomic update as libcalls.
283 void
284 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
285 const llvm::function_ref<RValue(RValue)> &UpdateOp,
286 bool IsVolatile);
287 /// Emit atomic update as LLVM instructions.
288 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
289 const llvm::function_ref<RValue(RValue)> &UpdateOp,
290 bool IsVolatile);
291 /// Emit atomic update as libcalls.
292 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
293 bool IsVolatile);
294 /// Emit atomic update as LLVM instructions.
295 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
296 bool IsVolatile);
297 };
298}
299
300Address AtomicInfo::CreateTempAlloca() const {
301 Address TempAlloca = CGF.CreateMemTemp(
302 T: (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
303 : AtomicTy,
304 Align: getAtomicAlignment(),
305 Name: "atomic-temp");
306 // Cast to pointer to value type for bitfields.
307 if (LVal.isBitField())
308 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
309 Addr: TempAlloca, Ty: getAtomicAddress().getType(),
310 ElementTy: getAtomicAddress().getElementType());
311 return TempAlloca;
312}
313
314static RValue emitAtomicLibcall(CodeGenFunction &CGF,
315 StringRef fnName,
316 QualType resultType,
317 CallArgList &args) {
318 const CGFunctionInfo &fnInfo =
319 CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
320 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(Info: fnInfo);
321 llvm::AttrBuilder fnAttrB(CGF.getLLVMContext());
322 fnAttrB.addAttribute(Val: llvm::Attribute::NoUnwind);
323 fnAttrB.addAttribute(Val: llvm::Attribute::WillReturn);
324 llvm::AttributeList fnAttrs = llvm::AttributeList::get(
325 C&: CGF.getLLVMContext(), Index: llvm::AttributeList::FunctionIndex, B: fnAttrB);
326
327 llvm::FunctionCallee fn =
328 CGF.CGM.CreateRuntimeFunction(Ty: fnTy, Name: fnName, ExtraAttrs: fnAttrs);
329 auto callee = CGCallee::forDirect(functionPtr: fn);
330 return CGF.EmitCall(CallInfo: fnInfo, Callee: callee, ReturnValue: ReturnValueSlot(), Args: args);
331}
332
333/// Does a store of the given IR type modify the full expected width?
334static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
335 uint64_t expectedSize) {
336 return (CGM.getDataLayout().getTypeStoreSize(Ty: type) * 8 == expectedSize);
337}
338
339/// Does the atomic type require memsetting to zero before initialization?
340///
341/// The IR type is provided as a way of making certain queries faster.
342bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
343 // If the atomic type has size padding, we definitely need a memset.
344 if (hasPadding()) return true;
345
346 // Otherwise, do some simple heuristics to try to avoid it:
347 switch (getEvaluationKind()) {
348 // For scalars and complexes, check whether the store size of the
349 // type uses the full size.
350 case TEK_Scalar:
351 return !isFullSizeType(CGM&: CGF.CGM, type, expectedSize: AtomicSizeInBits);
352 case TEK_Complex:
353 return !isFullSizeType(CGM&: CGF.CGM, type: type->getStructElementType(N: 0),
354 expectedSize: AtomicSizeInBits / 2);
355
356 // Padding in structs has an undefined bit pattern. User beware.
357 case TEK_Aggregate:
358 return false;
359 }
360 llvm_unreachable("bad evaluation kind");
361}
362
363bool AtomicInfo::emitMemSetZeroIfNecessary() const {
364 assert(LVal.isSimple());
365 Address addr = LVal.getAddress();
366 if (!requiresMemSetZero(type: addr.getElementType()))
367 return false;
368
369 CGF.Builder.CreateMemSet(
370 Ptr: addr.emitRawPointer(CGF), Val: llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 0),
371 Size: CGF.getContext().toCharUnitsFromBits(BitSize: AtomicSizeInBits).getQuantity(),
372 Align: LVal.getAlignment().getAsAlign());
373 return true;
374}
375
376static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
377 Address Dest, Address Ptr,
378 Address Val1, Address Val2,
379 uint64_t Size,
380 llvm::AtomicOrdering SuccessOrder,
381 llvm::AtomicOrdering FailureOrder,
382 llvm::SyncScope::ID Scope) {
383 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
384 llvm::Value *Expected = CGF.Builder.CreateLoad(Addr: Val1);
385 llvm::Value *Desired = CGF.Builder.CreateLoad(Addr: Val2);
386
387 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
388 Addr: Ptr, Cmp: Expected, New: Desired, SuccessOrdering: SuccessOrder, FailureOrdering: FailureOrder, SSID: Scope);
389 Pair->setVolatile(E->isVolatile());
390 Pair->setWeak(IsWeak);
391 CGF.getTargetHooks().setTargetAtomicMetadata(CGF, AtomicInst&: *Pair, Expr: E);
392
393 // Cmp holds the result of the compare-exchange operation: true on success,
394 // false on failure.
395 llvm::Value *Old = CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 0);
396 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 1);
397
398 // This basic block is used to hold the store instruction if the operation
399 // failed.
400 llvm::BasicBlock *StoreExpectedBB =
401 CGF.createBasicBlock(name: "cmpxchg.store_expected", parent: CGF.CurFn);
402
403 // This basic block is the exit point of the operation, we should end up
404 // here regardless of whether or not the operation succeeded.
405 llvm::BasicBlock *ContinueBB =
406 CGF.createBasicBlock(name: "cmpxchg.continue", parent: CGF.CurFn);
407
408 // Update Expected if Expected isn't equal to Old, otherwise branch to the
409 // exit point.
410 CGF.Builder.CreateCondBr(Cond: Cmp, True: ContinueBB, False: StoreExpectedBB);
411
412 CGF.Builder.SetInsertPoint(StoreExpectedBB);
413 // Update the memory at Expected with Old's value.
414 auto *I = CGF.Builder.CreateStore(Val: Old, Addr: Val1);
415 CGF.addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Old);
416
417 // Finally, branch to the exit point.
418 CGF.Builder.CreateBr(Dest: ContinueBB);
419
420 CGF.Builder.SetInsertPoint(ContinueBB);
421 // Update the memory at Dest with Cmp's value.
422 CGF.EmitStoreOfScalar(value: Cmp, lvalue: CGF.MakeAddrLValue(Addr: Dest, T: E->getType()));
423}
424
425/// Given an ordering required on success, emit all possible cmpxchg
426/// instructions to cope with the provided (but possibly only dynamically known)
427/// FailureOrder.
428static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
429 bool IsWeak, Address Dest, Address Ptr,
430 Address Val1, Address Val2,
431 llvm::Value *FailureOrderVal,
432 uint64_t Size,
433 llvm::AtomicOrdering SuccessOrder,
434 llvm::SyncScope::ID Scope) {
435 llvm::AtomicOrdering FailureOrder;
436 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(Val: FailureOrderVal)) {
437 auto FOS = FO->getSExtValue();
438 if (!llvm::isValidAtomicOrderingCABI(I: FOS))
439 FailureOrder = llvm::AtomicOrdering::Monotonic;
440 else
441 switch ((llvm::AtomicOrderingCABI)FOS) {
442 case llvm::AtomicOrderingCABI::relaxed:
443 // 31.7.2.18: "The failure argument shall not be memory_order_release
444 // nor memory_order_acq_rel". Fallback to monotonic.
445 case llvm::AtomicOrderingCABI::release:
446 case llvm::AtomicOrderingCABI::acq_rel:
447 FailureOrder = llvm::AtomicOrdering::Monotonic;
448 break;
449 case llvm::AtomicOrderingCABI::consume:
450 case llvm::AtomicOrderingCABI::acquire:
451 FailureOrder = llvm::AtomicOrdering::Acquire;
452 break;
453 case llvm::AtomicOrderingCABI::seq_cst:
454 FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
455 break;
456 }
457 // Prior to c++17, "the failure argument shall be no stronger than the
458 // success argument". This condition has been lifted and the only
459 // precondition is 31.7.2.18. Effectively treat this as a DR and skip
460 // language version checks.
461 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
462 FailureOrder, Scope);
463 return;
464 }
465
466 // Create all the relevant BB's
467 auto *MonotonicBB = CGF.createBasicBlock(name: "monotonic_fail", parent: CGF.CurFn);
468 auto *AcquireBB = CGF.createBasicBlock(name: "acquire_fail", parent: CGF.CurFn);
469 auto *SeqCstBB = CGF.createBasicBlock(name: "seqcst_fail", parent: CGF.CurFn);
470 auto *ContBB = CGF.createBasicBlock(name: "atomic.continue", parent: CGF.CurFn);
471
472 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
473 // doesn't matter unless someone is crazy enough to use something that
474 // doesn't fold to a constant for the ordering.
475 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(V: FailureOrderVal, Dest: MonotonicBB);
476 // Implemented as acquire, since it's the closest in LLVM.
477 SI->addCase(OnVal: CGF.Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::consume),
478 Dest: AcquireBB);
479 SI->addCase(OnVal: CGF.Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::acquire),
480 Dest: AcquireBB);
481 SI->addCase(OnVal: CGF.Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::seq_cst),
482 Dest: SeqCstBB);
483
484 // Emit all the different atomics
485 CGF.Builder.SetInsertPoint(MonotonicBB);
486 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
487 Size, SuccessOrder, FailureOrder: llvm::AtomicOrdering::Monotonic, Scope);
488 CGF.Builder.CreateBr(Dest: ContBB);
489
490 CGF.Builder.SetInsertPoint(AcquireBB);
491 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
492 FailureOrder: llvm::AtomicOrdering::Acquire, Scope);
493 CGF.Builder.CreateBr(Dest: ContBB);
494
495 CGF.Builder.SetInsertPoint(SeqCstBB);
496 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
497 FailureOrder: llvm::AtomicOrdering::SequentiallyConsistent, Scope);
498 CGF.Builder.CreateBr(Dest: ContBB);
499
500 CGF.Builder.SetInsertPoint(ContBB);
501}
502
503/// Duplicate the atomic min/max operation in conventional IR for the builtin
504/// variants that return the new rather than the original value.
505static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder,
506 AtomicExpr::AtomicOp Op,
507 bool IsSigned,
508 llvm::Value *OldVal,
509 llvm::Value *RHS) {
510 llvm::CmpInst::Predicate Pred;
511 switch (Op) {
512 default:
513 llvm_unreachable("Unexpected min/max operation");
514 case AtomicExpr::AO__atomic_max_fetch:
515 case AtomicExpr::AO__scoped_atomic_max_fetch:
516 Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT;
517 break;
518 case AtomicExpr::AO__atomic_min_fetch:
519 case AtomicExpr::AO__scoped_atomic_min_fetch:
520 Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT;
521 break;
522 }
523 llvm::Value *Cmp = Builder.CreateICmp(P: Pred, LHS: OldVal, RHS, Name: "tst");
524 return Builder.CreateSelect(C: Cmp, True: OldVal, False: RHS, Name: "newval");
525}
526
527static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
528 Address Ptr, Address Val1, Address Val2,
529 llvm::Value *IsWeak, llvm::Value *FailureOrder,
530 uint64_t Size, llvm::AtomicOrdering Order,
531 llvm::SyncScope::ID Scope) {
532 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
533 bool PostOpMinMax = false;
534 unsigned PostOp = 0;
535
536 switch (E->getOp()) {
537 case AtomicExpr::AO__c11_atomic_init:
538 case AtomicExpr::AO__opencl_atomic_init:
539 llvm_unreachable("Already handled!");
540
541 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
542 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
543 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
544 emitAtomicCmpXchgFailureSet(CGF, E, IsWeak: false, Dest, Ptr, Val1, Val2,
545 FailureOrderVal: FailureOrder, Size, SuccessOrder: Order, Scope);
546 return;
547 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
548 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
549 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
550 emitAtomicCmpXchgFailureSet(CGF, E, IsWeak: true, Dest, Ptr, Val1, Val2,
551 FailureOrderVal: FailureOrder, Size, SuccessOrder: Order, Scope);
552 return;
553 case AtomicExpr::AO__atomic_compare_exchange:
554 case AtomicExpr::AO__atomic_compare_exchange_n:
555 case AtomicExpr::AO__scoped_atomic_compare_exchange:
556 case AtomicExpr::AO__scoped_atomic_compare_exchange_n: {
557 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(Val: IsWeak)) {
558 emitAtomicCmpXchgFailureSet(CGF, E, IsWeak: IsWeakC->getZExtValue(), Dest, Ptr,
559 Val1, Val2, FailureOrderVal: FailureOrder, Size, SuccessOrder: Order, Scope);
560 } else {
561 // Create all the relevant BB's
562 llvm::BasicBlock *StrongBB =
563 CGF.createBasicBlock(name: "cmpxchg.strong", parent: CGF.CurFn);
564 llvm::BasicBlock *WeakBB = CGF.createBasicBlock(name: "cmxchg.weak", parent: CGF.CurFn);
565 llvm::BasicBlock *ContBB =
566 CGF.createBasicBlock(name: "cmpxchg.continue", parent: CGF.CurFn);
567
568 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(V: IsWeak, Dest: WeakBB);
569 SI->addCase(OnVal: CGF.Builder.getInt1(V: false), Dest: StrongBB);
570
571 CGF.Builder.SetInsertPoint(StrongBB);
572 emitAtomicCmpXchgFailureSet(CGF, E, IsWeak: false, Dest, Ptr, Val1, Val2,
573 FailureOrderVal: FailureOrder, Size, SuccessOrder: Order, Scope);
574 CGF.Builder.CreateBr(Dest: ContBB);
575
576 CGF.Builder.SetInsertPoint(WeakBB);
577 emitAtomicCmpXchgFailureSet(CGF, E, IsWeak: true, Dest, Ptr, Val1, Val2,
578 FailureOrderVal: FailureOrder, Size, SuccessOrder: Order, Scope);
579 CGF.Builder.CreateBr(Dest: ContBB);
580
581 CGF.Builder.SetInsertPoint(ContBB);
582 }
583 return;
584 }
585 case AtomicExpr::AO__c11_atomic_load:
586 case AtomicExpr::AO__opencl_atomic_load:
587 case AtomicExpr::AO__hip_atomic_load:
588 case AtomicExpr::AO__atomic_load_n:
589 case AtomicExpr::AO__atomic_load:
590 case AtomicExpr::AO__scoped_atomic_load_n:
591 case AtomicExpr::AO__scoped_atomic_load: {
592 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr: Ptr);
593 Load->setAtomic(Ordering: Order, SSID: Scope);
594 Load->setVolatile(E->isVolatile());
595 CGF.maybeAttachRangeForLoad(Load, Ty: E->getValueType(), Loc: E->getExprLoc());
596 auto *I = CGF.Builder.CreateStore(Val: Load, Addr: Dest);
597 CGF.addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Load);
598 return;
599 }
600
601 case AtomicExpr::AO__c11_atomic_store:
602 case AtomicExpr::AO__opencl_atomic_store:
603 case AtomicExpr::AO__hip_atomic_store:
604 case AtomicExpr::AO__atomic_store:
605 case AtomicExpr::AO__atomic_store_n:
606 case AtomicExpr::AO__scoped_atomic_store:
607 case AtomicExpr::AO__scoped_atomic_store_n: {
608 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Addr: Val1);
609 llvm::StoreInst *Store = CGF.Builder.CreateStore(Val: LoadVal1, Addr: Ptr);
610 Store->setAtomic(Ordering: Order, SSID: Scope);
611 Store->setVolatile(E->isVolatile());
612 CGF.addInstToCurrentSourceAtom(KeyInstruction: Store, Backup: LoadVal1);
613 return;
614 }
615
616 case AtomicExpr::AO__c11_atomic_exchange:
617 case AtomicExpr::AO__hip_atomic_exchange:
618 case AtomicExpr::AO__opencl_atomic_exchange:
619 case AtomicExpr::AO__atomic_exchange_n:
620 case AtomicExpr::AO__atomic_exchange:
621 case AtomicExpr::AO__scoped_atomic_exchange_n:
622 case AtomicExpr::AO__scoped_atomic_exchange:
623 Op = llvm::AtomicRMWInst::Xchg;
624 break;
625
626 case AtomicExpr::AO__atomic_add_fetch:
627 case AtomicExpr::AO__scoped_atomic_add_fetch:
628 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd
629 : llvm::Instruction::Add;
630 [[fallthrough]];
631 case AtomicExpr::AO__c11_atomic_fetch_add:
632 case AtomicExpr::AO__hip_atomic_fetch_add:
633 case AtomicExpr::AO__opencl_atomic_fetch_add:
634 case AtomicExpr::AO__atomic_fetch_add:
635 case AtomicExpr::AO__scoped_atomic_fetch_add:
636 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd
637 : llvm::AtomicRMWInst::Add;
638 break;
639
640 case AtomicExpr::AO__atomic_sub_fetch:
641 case AtomicExpr::AO__scoped_atomic_sub_fetch:
642 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub
643 : llvm::Instruction::Sub;
644 [[fallthrough]];
645 case AtomicExpr::AO__c11_atomic_fetch_sub:
646 case AtomicExpr::AO__hip_atomic_fetch_sub:
647 case AtomicExpr::AO__opencl_atomic_fetch_sub:
648 case AtomicExpr::AO__atomic_fetch_sub:
649 case AtomicExpr::AO__scoped_atomic_fetch_sub:
650 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub
651 : llvm::AtomicRMWInst::Sub;
652 break;
653
654 case AtomicExpr::AO__atomic_min_fetch:
655 case AtomicExpr::AO__scoped_atomic_min_fetch:
656 PostOpMinMax = true;
657 [[fallthrough]];
658 case AtomicExpr::AO__c11_atomic_fetch_min:
659 case AtomicExpr::AO__hip_atomic_fetch_min:
660 case AtomicExpr::AO__opencl_atomic_fetch_min:
661 case AtomicExpr::AO__atomic_fetch_min:
662 case AtomicExpr::AO__scoped_atomic_fetch_min:
663 Op = E->getValueType()->isFloatingType()
664 ? llvm::AtomicRMWInst::FMin
665 : (E->getValueType()->isSignedIntegerType()
666 ? llvm::AtomicRMWInst::Min
667 : llvm::AtomicRMWInst::UMin);
668 break;
669
670 case AtomicExpr::AO__atomic_max_fetch:
671 case AtomicExpr::AO__scoped_atomic_max_fetch:
672 PostOpMinMax = true;
673 [[fallthrough]];
674 case AtomicExpr::AO__c11_atomic_fetch_max:
675 case AtomicExpr::AO__hip_atomic_fetch_max:
676 case AtomicExpr::AO__opencl_atomic_fetch_max:
677 case AtomicExpr::AO__atomic_fetch_max:
678 case AtomicExpr::AO__scoped_atomic_fetch_max:
679 Op = E->getValueType()->isFloatingType()
680 ? llvm::AtomicRMWInst::FMax
681 : (E->getValueType()->isSignedIntegerType()
682 ? llvm::AtomicRMWInst::Max
683 : llvm::AtomicRMWInst::UMax);
684 break;
685
686 case AtomicExpr::AO__atomic_and_fetch:
687 case AtomicExpr::AO__scoped_atomic_and_fetch:
688 PostOp = llvm::Instruction::And;
689 [[fallthrough]];
690 case AtomicExpr::AO__c11_atomic_fetch_and:
691 case AtomicExpr::AO__hip_atomic_fetch_and:
692 case AtomicExpr::AO__opencl_atomic_fetch_and:
693 case AtomicExpr::AO__atomic_fetch_and:
694 case AtomicExpr::AO__scoped_atomic_fetch_and:
695 Op = llvm::AtomicRMWInst::And;
696 break;
697
698 case AtomicExpr::AO__atomic_or_fetch:
699 case AtomicExpr::AO__scoped_atomic_or_fetch:
700 PostOp = llvm::Instruction::Or;
701 [[fallthrough]];
702 case AtomicExpr::AO__c11_atomic_fetch_or:
703 case AtomicExpr::AO__hip_atomic_fetch_or:
704 case AtomicExpr::AO__opencl_atomic_fetch_or:
705 case AtomicExpr::AO__atomic_fetch_or:
706 case AtomicExpr::AO__scoped_atomic_fetch_or:
707 Op = llvm::AtomicRMWInst::Or;
708 break;
709
710 case AtomicExpr::AO__atomic_xor_fetch:
711 case AtomicExpr::AO__scoped_atomic_xor_fetch:
712 PostOp = llvm::Instruction::Xor;
713 [[fallthrough]];
714 case AtomicExpr::AO__c11_atomic_fetch_xor:
715 case AtomicExpr::AO__hip_atomic_fetch_xor:
716 case AtomicExpr::AO__opencl_atomic_fetch_xor:
717 case AtomicExpr::AO__atomic_fetch_xor:
718 case AtomicExpr::AO__scoped_atomic_fetch_xor:
719 Op = llvm::AtomicRMWInst::Xor;
720 break;
721
722 case AtomicExpr::AO__atomic_nand_fetch:
723 case AtomicExpr::AO__scoped_atomic_nand_fetch:
724 PostOp = llvm::Instruction::And; // the NOT is special cased below
725 [[fallthrough]];
726 case AtomicExpr::AO__c11_atomic_fetch_nand:
727 case AtomicExpr::AO__atomic_fetch_nand:
728 case AtomicExpr::AO__scoped_atomic_fetch_nand:
729 Op = llvm::AtomicRMWInst::Nand;
730 break;
731
732 case AtomicExpr::AO__atomic_test_and_set: {
733 llvm::AtomicRMWInst *RMWI =
734 CGF.emitAtomicRMWInst(Op: llvm::AtomicRMWInst::Xchg, Addr: Ptr,
735 Val: CGF.Builder.getInt8(C: 1), Order, SSID: Scope, AE: E);
736 RMWI->setVolatile(E->isVolatile());
737 llvm::Value *Result = CGF.Builder.CreateIsNotNull(Arg: RMWI, Name: "tobool");
738 auto *I = CGF.Builder.CreateStore(Val: Result, Addr: Dest);
739 CGF.addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Result);
740 return;
741 }
742
743 case AtomicExpr::AO__atomic_clear: {
744 llvm::StoreInst *Store =
745 CGF.Builder.CreateStore(Val: CGF.Builder.getInt8(C: 0), Addr: Ptr);
746 Store->setAtomic(Ordering: Order, SSID: Scope);
747 Store->setVolatile(E->isVolatile());
748 CGF.addInstToCurrentSourceAtom(KeyInstruction: Store, Backup: nullptr);
749 return;
750 }
751 }
752
753 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Addr: Val1);
754 llvm::AtomicRMWInst *RMWI =
755 CGF.emitAtomicRMWInst(Op, Addr: Ptr, Val: LoadVal1, Order, SSID: Scope, AE: E);
756 RMWI->setVolatile(E->isVolatile());
757
758 // For __atomic_*_fetch operations, perform the operation again to
759 // determine the value which was written.
760 llvm::Value *Result = RMWI;
761 if (PostOpMinMax)
762 Result = EmitPostAtomicMinMax(Builder&: CGF.Builder, Op: E->getOp(),
763 IsSigned: E->getValueType()->isSignedIntegerType(),
764 OldVal: RMWI, RHS: LoadVal1);
765 else if (PostOp)
766 Result = CGF.Builder.CreateBinOp(Opc: (llvm::Instruction::BinaryOps)PostOp, LHS: RMWI,
767 RHS: LoadVal1);
768 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch ||
769 E->getOp() == AtomicExpr::AO__scoped_atomic_nand_fetch)
770 Result = CGF.Builder.CreateNot(V: Result);
771 auto *I = CGF.Builder.CreateStore(Val: Result, Addr: Dest);
772 CGF.addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Result);
773}
774
775// This function emits any expression (scalar, complex, or aggregate)
776// into a temporary alloca.
777static Address
778EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
779 Address DeclPtr = CGF.CreateMemTemp(T: E->getType(), Name: ".atomictmp");
780 CGF.EmitAnyExprToMem(E, Location: DeclPtr, Quals: E->getType().getQualifiers(),
781 /*Init*/ IsInitializer: true);
782 return DeclPtr;
783}
784
785static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
786 Address Ptr, Address Val1, Address Val2,
787 llvm::Value *IsWeak, llvm::Value *FailureOrder,
788 uint64_t Size, llvm::AtomicOrdering Order,
789 llvm::Value *Scope) {
790 auto ScopeModel = Expr->getScopeModel();
791
792 // LLVM atomic instructions always have sync scope. If clang atomic
793 // expression has no scope operand, use default LLVM sync scope.
794 if (!ScopeModel) {
795 llvm::SyncScope::ID SS;
796 if (CGF.getLangOpts().OpenCL)
797 // OpenCL approach is: "The functions that do not have memory_scope
798 // argument have the same semantics as the corresponding functions with
799 // the memory_scope argument set to memory_scope_device." See ref.:
800 // https://registry.khronos.org/OpenCL/specs/3.0-unified/html/OpenCL_C.html#atomic-functions
801 SS = CGF.getTargetHooks().getLLVMSyncScopeID(LangOpts: CGF.getLangOpts(),
802 Scope: SyncScope::OpenCLDevice,
803 Ordering: Order, Ctx&: CGF.getLLVMContext());
804 else
805 SS = llvm::SyncScope::System;
806 EmitAtomicOp(CGF, E: Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
807 Order, Scope: SS);
808 return;
809 }
810
811 // Handle constant scope.
812 if (auto SC = dyn_cast<llvm::ConstantInt>(Val: Scope)) {
813 auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
814 LangOpts: CGF.CGM.getLangOpts(), Scope: ScopeModel->map(S: SC->getZExtValue()),
815 Ordering: Order, Ctx&: CGF.CGM.getLLVMContext());
816 EmitAtomicOp(CGF, E: Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
817 Order, Scope: SCID);
818 return;
819 }
820
821 // Handle non-constant scope.
822 auto &Builder = CGF.Builder;
823 auto Scopes = ScopeModel->getRuntimeValues();
824 llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
825 for (auto S : Scopes)
826 BB[S] = CGF.createBasicBlock(name: getAsString(S: ScopeModel->map(S)), parent: CGF.CurFn);
827
828 llvm::BasicBlock *ContBB =
829 CGF.createBasicBlock(name: "atomic.scope.continue", parent: CGF.CurFn);
830
831 auto *SC = Builder.CreateIntCast(V: Scope, DestTy: Builder.getInt32Ty(), isSigned: false);
832 // If unsupported sync scope is encountered at run time, assume a fallback
833 // sync scope value.
834 auto FallBack = ScopeModel->getFallBackValue();
835 llvm::SwitchInst *SI = Builder.CreateSwitch(V: SC, Dest: BB[FallBack]);
836 for (auto S : Scopes) {
837 auto *B = BB[S];
838 if (S != FallBack)
839 SI->addCase(OnVal: Builder.getInt32(C: S), Dest: B);
840
841 Builder.SetInsertPoint(B);
842 EmitAtomicOp(CGF, E: Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
843 Order,
844 Scope: CGF.getTargetHooks().getLLVMSyncScopeID(LangOpts: CGF.CGM.getLangOpts(),
845 Scope: ScopeModel->map(S),
846 Ordering: Order,
847 Ctx&: CGF.getLLVMContext()));
848 Builder.CreateBr(Dest: ContBB);
849 }
850
851 Builder.SetInsertPoint(ContBB);
852}
853
854RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
855 ApplyAtomGroup Grp(getDebugInfo());
856
857 QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
858 QualType MemTy = AtomicTy;
859 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
860 MemTy = AT->getValueType();
861 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
862
863 Address Val1 = Address::invalid();
864 Address Val2 = Address::invalid();
865 Address Dest = Address::invalid();
866 Address Ptr = EmitPointerWithAlignment(Addr: E->getPtr());
867
868 if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
869 E->getOp() == AtomicExpr::AO__opencl_atomic_init) {
870 LValue lvalue = MakeAddrLValue(Addr: Ptr, T: AtomicTy);
871 EmitAtomicInit(E: E->getVal1(), lvalue);
872 return RValue::get(V: nullptr);
873 }
874
875 auto TInfo = getContext().getTypeInfoInChars(T: AtomicTy);
876 uint64_t Size = TInfo.Width.getQuantity();
877 unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
878
879 CharUnits MaxInlineWidth =
880 getContext().toCharUnitsFromBits(BitSize: MaxInlineWidthInBits);
881 DiagnosticsEngine &Diags = CGM.getDiags();
882 bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0;
883 bool Oversized = getContext().toBits(CharSize: TInfo.Width) > MaxInlineWidthInBits;
884 if (Misaligned) {
885 Diags.Report(Loc: E->getBeginLoc(), DiagID: diag::warn_atomic_op_misaligned)
886 << (int)TInfo.Width.getQuantity()
887 << (int)Ptr.getAlignment().getQuantity();
888 }
889 if (Oversized) {
890 Diags.Report(Loc: E->getBeginLoc(), DiagID: diag::warn_atomic_op_oversized)
891 << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity();
892 }
893
894 llvm::Value *Order = EmitScalarExpr(E: E->getOrder());
895 llvm::Value *Scope =
896 E->getScopeModel() ? EmitScalarExpr(E: E->getScope()) : nullptr;
897 bool ShouldCastToIntPtrTy = true;
898
899 switch (E->getOp()) {
900 case AtomicExpr::AO__c11_atomic_init:
901 case AtomicExpr::AO__opencl_atomic_init:
902 llvm_unreachable("Already handled above with EmitAtomicInit!");
903
904 case AtomicExpr::AO__atomic_load_n:
905 case AtomicExpr::AO__scoped_atomic_load_n:
906 case AtomicExpr::AO__c11_atomic_load:
907 case AtomicExpr::AO__opencl_atomic_load:
908 case AtomicExpr::AO__hip_atomic_load:
909 case AtomicExpr::AO__atomic_test_and_set:
910 case AtomicExpr::AO__atomic_clear:
911 break;
912
913 case AtomicExpr::AO__atomic_load:
914 case AtomicExpr::AO__scoped_atomic_load:
915 Dest = EmitPointerWithAlignment(Addr: E->getVal1());
916 break;
917
918 case AtomicExpr::AO__atomic_store:
919 case AtomicExpr::AO__scoped_atomic_store:
920 Val1 = EmitPointerWithAlignment(Addr: E->getVal1());
921 break;
922
923 case AtomicExpr::AO__atomic_exchange:
924 case AtomicExpr::AO__scoped_atomic_exchange:
925 Val1 = EmitPointerWithAlignment(Addr: E->getVal1());
926 Dest = EmitPointerWithAlignment(Addr: E->getVal2());
927 break;
928
929 case AtomicExpr::AO__atomic_compare_exchange:
930 case AtomicExpr::AO__atomic_compare_exchange_n:
931 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
932 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
933 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
934 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
935 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
936 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
937 case AtomicExpr::AO__scoped_atomic_compare_exchange:
938 case AtomicExpr::AO__scoped_atomic_compare_exchange_n:
939 Val1 = EmitPointerWithAlignment(Addr: E->getVal1());
940 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange ||
941 E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)
942 Val2 = EmitPointerWithAlignment(Addr: E->getVal2());
943 else
944 Val2 = EmitValToTemp(CGF&: *this, E: E->getVal2());
945 OrderFail = EmitScalarExpr(E: E->getOrderFail());
946 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
947 E->getOp() == AtomicExpr::AO__atomic_compare_exchange ||
948 E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange_n ||
949 E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)
950 IsWeak = EmitScalarExpr(E: E->getWeak());
951 break;
952
953 case AtomicExpr::AO__c11_atomic_fetch_add:
954 case AtomicExpr::AO__c11_atomic_fetch_sub:
955 case AtomicExpr::AO__hip_atomic_fetch_add:
956 case AtomicExpr::AO__hip_atomic_fetch_sub:
957 case AtomicExpr::AO__opencl_atomic_fetch_add:
958 case AtomicExpr::AO__opencl_atomic_fetch_sub:
959 if (MemTy->isPointerType()) {
960 // For pointer arithmetic, we're required to do a bit of math:
961 // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
962 // ... but only for the C11 builtins. The GNU builtins expect the
963 // user to multiply by sizeof(T).
964 QualType Val1Ty = E->getVal1()->getType();
965 llvm::Value *Val1Scalar = EmitScalarExpr(E: E->getVal1());
966 CharUnits PointeeIncAmt =
967 getContext().getTypeSizeInChars(T: MemTy->getPointeeType());
968 Val1Scalar = Builder.CreateMul(LHS: Val1Scalar, RHS: CGM.getSize(numChars: PointeeIncAmt));
969 auto Temp = CreateMemTemp(T: Val1Ty, Name: ".atomictmp");
970 Val1 = Temp;
971 EmitStoreOfScalar(value: Val1Scalar, lvalue: MakeAddrLValue(Addr: Temp, T: Val1Ty));
972 break;
973 }
974 [[fallthrough]];
975 case AtomicExpr::AO__atomic_fetch_add:
976 case AtomicExpr::AO__atomic_fetch_max:
977 case AtomicExpr::AO__atomic_fetch_min:
978 case AtomicExpr::AO__atomic_fetch_sub:
979 case AtomicExpr::AO__atomic_add_fetch:
980 case AtomicExpr::AO__atomic_max_fetch:
981 case AtomicExpr::AO__atomic_min_fetch:
982 case AtomicExpr::AO__atomic_sub_fetch:
983 case AtomicExpr::AO__c11_atomic_fetch_max:
984 case AtomicExpr::AO__c11_atomic_fetch_min:
985 case AtomicExpr::AO__opencl_atomic_fetch_max:
986 case AtomicExpr::AO__opencl_atomic_fetch_min:
987 case AtomicExpr::AO__hip_atomic_fetch_max:
988 case AtomicExpr::AO__hip_atomic_fetch_min:
989 case AtomicExpr::AO__scoped_atomic_fetch_add:
990 case AtomicExpr::AO__scoped_atomic_fetch_max:
991 case AtomicExpr::AO__scoped_atomic_fetch_min:
992 case AtomicExpr::AO__scoped_atomic_fetch_sub:
993 case AtomicExpr::AO__scoped_atomic_add_fetch:
994 case AtomicExpr::AO__scoped_atomic_max_fetch:
995 case AtomicExpr::AO__scoped_atomic_min_fetch:
996 case AtomicExpr::AO__scoped_atomic_sub_fetch:
997 ShouldCastToIntPtrTy = !MemTy->isFloatingType();
998 [[fallthrough]];
999
1000 case AtomicExpr::AO__atomic_fetch_and:
1001 case AtomicExpr::AO__atomic_fetch_nand:
1002 case AtomicExpr::AO__atomic_fetch_or:
1003 case AtomicExpr::AO__atomic_fetch_xor:
1004 case AtomicExpr::AO__atomic_and_fetch:
1005 case AtomicExpr::AO__atomic_nand_fetch:
1006 case AtomicExpr::AO__atomic_or_fetch:
1007 case AtomicExpr::AO__atomic_xor_fetch:
1008 case AtomicExpr::AO__atomic_store_n:
1009 case AtomicExpr::AO__atomic_exchange_n:
1010 case AtomicExpr::AO__c11_atomic_fetch_and:
1011 case AtomicExpr::AO__c11_atomic_fetch_nand:
1012 case AtomicExpr::AO__c11_atomic_fetch_or:
1013 case AtomicExpr::AO__c11_atomic_fetch_xor:
1014 case AtomicExpr::AO__c11_atomic_store:
1015 case AtomicExpr::AO__c11_atomic_exchange:
1016 case AtomicExpr::AO__hip_atomic_fetch_and:
1017 case AtomicExpr::AO__hip_atomic_fetch_or:
1018 case AtomicExpr::AO__hip_atomic_fetch_xor:
1019 case AtomicExpr::AO__hip_atomic_store:
1020 case AtomicExpr::AO__hip_atomic_exchange:
1021 case AtomicExpr::AO__opencl_atomic_fetch_and:
1022 case AtomicExpr::AO__opencl_atomic_fetch_or:
1023 case AtomicExpr::AO__opencl_atomic_fetch_xor:
1024 case AtomicExpr::AO__opencl_atomic_store:
1025 case AtomicExpr::AO__opencl_atomic_exchange:
1026 case AtomicExpr::AO__scoped_atomic_fetch_and:
1027 case AtomicExpr::AO__scoped_atomic_fetch_nand:
1028 case AtomicExpr::AO__scoped_atomic_fetch_or:
1029 case AtomicExpr::AO__scoped_atomic_fetch_xor:
1030 case AtomicExpr::AO__scoped_atomic_and_fetch:
1031 case AtomicExpr::AO__scoped_atomic_nand_fetch:
1032 case AtomicExpr::AO__scoped_atomic_or_fetch:
1033 case AtomicExpr::AO__scoped_atomic_xor_fetch:
1034 case AtomicExpr::AO__scoped_atomic_store_n:
1035 case AtomicExpr::AO__scoped_atomic_exchange_n:
1036 Val1 = EmitValToTemp(CGF&: *this, E: E->getVal1());
1037 break;
1038 }
1039
1040 QualType RValTy = E->getType().getUnqualifiedType();
1041
1042 // The inlined atomics only function on iN types, where N is a power of 2. We
1043 // need to make sure (via temporaries if necessary) that all incoming values
1044 // are compatible.
1045 LValue AtomicVal = MakeAddrLValue(Addr: Ptr, T: AtomicTy);
1046 AtomicInfo Atomics(*this, AtomicVal);
1047
1048 if (ShouldCastToIntPtrTy) {
1049 Ptr = Atomics.castToAtomicIntPointer(Addr: Ptr);
1050 if (Val1.isValid())
1051 Val1 = Atomics.convertToAtomicIntPointer(Addr: Val1);
1052 if (Val2.isValid())
1053 Val2 = Atomics.convertToAtomicIntPointer(Addr: Val2);
1054 }
1055 if (Dest.isValid()) {
1056 if (ShouldCastToIntPtrTy)
1057 Dest = Atomics.castToAtomicIntPointer(Addr: Dest);
1058 } else if (E->isCmpXChg())
1059 Dest = CreateMemTemp(T: RValTy, Name: "cmpxchg.bool");
1060 else if (!RValTy->isVoidType()) {
1061 Dest = Atomics.CreateTempAlloca();
1062 if (ShouldCastToIntPtrTy)
1063 Dest = Atomics.castToAtomicIntPointer(Addr: Dest);
1064 }
1065
1066 bool PowerOf2Size = (Size & (Size - 1)) == 0;
1067 bool UseLibcall = !PowerOf2Size || (Size > 16);
1068
1069 // For atomics larger than 16 bytes, emit a libcall from the frontend. This
1070 // avoids the overhead of dealing with excessively-large value types in IR.
1071 // Non-power-of-2 values also lower to libcall here, as they are not currently
1072 // permitted in IR instructions (although that constraint could be relaxed in
1073 // the future). For other cases where a libcall is required on a given
1074 // platform, we let the backend handle it (this includes handling for all of
1075 // the size-optimized libcall variants, which are only valid up to 16 bytes.)
1076 //
1077 // See: https://llvm.org/docs/Atomics.html#libcalls-atomic
1078 if (UseLibcall) {
1079 CallArgList Args;
1080 // For non-optimized library calls, the size is the first parameter.
1081 Args.add(rvalue: RValue::get(V: llvm::ConstantInt::get(Ty: SizeTy, V: Size)),
1082 type: getContext().getSizeType());
1083
1084 // The atomic address is the second parameter.
1085 // The OpenCL atomic library functions only accept pointer arguments to
1086 // generic address space.
1087 auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
1088 if (!E->isOpenCL())
1089 return V;
1090 auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace();
1091 if (AS == LangAS::opencl_generic)
1092 return V;
1093 auto DestAS = getContext().getTargetAddressSpace(AS: LangAS::opencl_generic);
1094 auto *DestType = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: DestAS);
1095
1096 return getTargetHooks().performAddrSpaceCast(CGF&: *this, V, SrcAddr: AS, DestTy: DestType,
1097 IsNonNull: false);
1098 };
1099
1100 Args.add(rvalue: RValue::get(V: CastToGenericAddrSpace(Ptr.emitRawPointer(CGF&: *this),
1101 E->getPtr()->getType())),
1102 type: getContext().VoidPtrTy);
1103
1104 // The next 1-3 parameters are op-dependent.
1105 std::string LibCallName;
1106 QualType RetTy;
1107 bool HaveRetTy = false;
1108 switch (E->getOp()) {
1109 case AtomicExpr::AO__c11_atomic_init:
1110 case AtomicExpr::AO__opencl_atomic_init:
1111 llvm_unreachable("Already handled!");
1112
1113 // There is only one libcall for compare an exchange, because there is no
1114 // optimisation benefit possible from a libcall version of a weak compare
1115 // and exchange.
1116 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1117 // void *desired, int success, int failure)
1118 case AtomicExpr::AO__atomic_compare_exchange:
1119 case AtomicExpr::AO__atomic_compare_exchange_n:
1120 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1121 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1122 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1123 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1124 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1125 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1126 case AtomicExpr::AO__scoped_atomic_compare_exchange:
1127 case AtomicExpr::AO__scoped_atomic_compare_exchange_n:
1128 LibCallName = "__atomic_compare_exchange";
1129 RetTy = getContext().BoolTy;
1130 HaveRetTy = true;
1131 Args.add(rvalue: RValue::get(V: CastToGenericAddrSpace(Val1.emitRawPointer(CGF&: *this),
1132 E->getVal1()->getType())),
1133 type: getContext().VoidPtrTy);
1134 Args.add(rvalue: RValue::get(V: CastToGenericAddrSpace(Val2.emitRawPointer(CGF&: *this),
1135 E->getVal2()->getType())),
1136 type: getContext().VoidPtrTy);
1137 Args.add(rvalue: RValue::get(V: Order), type: getContext().IntTy);
1138 Order = OrderFail;
1139 break;
1140 // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1141 // int order)
1142 case AtomicExpr::AO__atomic_exchange:
1143 case AtomicExpr::AO__atomic_exchange_n:
1144 case AtomicExpr::AO__c11_atomic_exchange:
1145 case AtomicExpr::AO__hip_atomic_exchange:
1146 case AtomicExpr::AO__opencl_atomic_exchange:
1147 case AtomicExpr::AO__scoped_atomic_exchange:
1148 case AtomicExpr::AO__scoped_atomic_exchange_n:
1149 LibCallName = "__atomic_exchange";
1150 Args.add(rvalue: RValue::get(V: CastToGenericAddrSpace(Val1.emitRawPointer(CGF&: *this),
1151 E->getVal1()->getType())),
1152 type: getContext().VoidPtrTy);
1153 break;
1154 // void __atomic_store(size_t size, void *mem, void *val, int order)
1155 case AtomicExpr::AO__atomic_store:
1156 case AtomicExpr::AO__atomic_store_n:
1157 case AtomicExpr::AO__c11_atomic_store:
1158 case AtomicExpr::AO__hip_atomic_store:
1159 case AtomicExpr::AO__opencl_atomic_store:
1160 case AtomicExpr::AO__scoped_atomic_store:
1161 case AtomicExpr::AO__scoped_atomic_store_n:
1162 LibCallName = "__atomic_store";
1163 RetTy = getContext().VoidTy;
1164 HaveRetTy = true;
1165 Args.add(rvalue: RValue::get(V: CastToGenericAddrSpace(Val1.emitRawPointer(CGF&: *this),
1166 E->getVal1()->getType())),
1167 type: getContext().VoidPtrTy);
1168 break;
1169 // void __atomic_load(size_t size, void *mem, void *return, int order)
1170 case AtomicExpr::AO__atomic_load:
1171 case AtomicExpr::AO__atomic_load_n:
1172 case AtomicExpr::AO__c11_atomic_load:
1173 case AtomicExpr::AO__hip_atomic_load:
1174 case AtomicExpr::AO__opencl_atomic_load:
1175 case AtomicExpr::AO__scoped_atomic_load:
1176 case AtomicExpr::AO__scoped_atomic_load_n:
1177 LibCallName = "__atomic_load";
1178 break;
1179 case AtomicExpr::AO__atomic_add_fetch:
1180 case AtomicExpr::AO__scoped_atomic_add_fetch:
1181 case AtomicExpr::AO__atomic_fetch_add:
1182 case AtomicExpr::AO__c11_atomic_fetch_add:
1183 case AtomicExpr::AO__hip_atomic_fetch_add:
1184 case AtomicExpr::AO__opencl_atomic_fetch_add:
1185 case AtomicExpr::AO__scoped_atomic_fetch_add:
1186 case AtomicExpr::AO__atomic_and_fetch:
1187 case AtomicExpr::AO__scoped_atomic_and_fetch:
1188 case AtomicExpr::AO__atomic_fetch_and:
1189 case AtomicExpr::AO__c11_atomic_fetch_and:
1190 case AtomicExpr::AO__hip_atomic_fetch_and:
1191 case AtomicExpr::AO__opencl_atomic_fetch_and:
1192 case AtomicExpr::AO__scoped_atomic_fetch_and:
1193 case AtomicExpr::AO__atomic_or_fetch:
1194 case AtomicExpr::AO__scoped_atomic_or_fetch:
1195 case AtomicExpr::AO__atomic_fetch_or:
1196 case AtomicExpr::AO__c11_atomic_fetch_or:
1197 case AtomicExpr::AO__hip_atomic_fetch_or:
1198 case AtomicExpr::AO__opencl_atomic_fetch_or:
1199 case AtomicExpr::AO__scoped_atomic_fetch_or:
1200 case AtomicExpr::AO__atomic_sub_fetch:
1201 case AtomicExpr::AO__scoped_atomic_sub_fetch:
1202 case AtomicExpr::AO__atomic_fetch_sub:
1203 case AtomicExpr::AO__c11_atomic_fetch_sub:
1204 case AtomicExpr::AO__hip_atomic_fetch_sub:
1205 case AtomicExpr::AO__opencl_atomic_fetch_sub:
1206 case AtomicExpr::AO__scoped_atomic_fetch_sub:
1207 case AtomicExpr::AO__atomic_xor_fetch:
1208 case AtomicExpr::AO__scoped_atomic_xor_fetch:
1209 case AtomicExpr::AO__atomic_fetch_xor:
1210 case AtomicExpr::AO__c11_atomic_fetch_xor:
1211 case AtomicExpr::AO__hip_atomic_fetch_xor:
1212 case AtomicExpr::AO__opencl_atomic_fetch_xor:
1213 case AtomicExpr::AO__scoped_atomic_fetch_xor:
1214 case AtomicExpr::AO__atomic_nand_fetch:
1215 case AtomicExpr::AO__atomic_fetch_nand:
1216 case AtomicExpr::AO__c11_atomic_fetch_nand:
1217 case AtomicExpr::AO__scoped_atomic_fetch_nand:
1218 case AtomicExpr::AO__scoped_atomic_nand_fetch:
1219 case AtomicExpr::AO__atomic_min_fetch:
1220 case AtomicExpr::AO__atomic_fetch_min:
1221 case AtomicExpr::AO__c11_atomic_fetch_min:
1222 case AtomicExpr::AO__hip_atomic_fetch_min:
1223 case AtomicExpr::AO__opencl_atomic_fetch_min:
1224 case AtomicExpr::AO__scoped_atomic_fetch_min:
1225 case AtomicExpr::AO__scoped_atomic_min_fetch:
1226 case AtomicExpr::AO__atomic_max_fetch:
1227 case AtomicExpr::AO__atomic_fetch_max:
1228 case AtomicExpr::AO__c11_atomic_fetch_max:
1229 case AtomicExpr::AO__hip_atomic_fetch_max:
1230 case AtomicExpr::AO__opencl_atomic_fetch_max:
1231 case AtomicExpr::AO__scoped_atomic_fetch_max:
1232 case AtomicExpr::AO__scoped_atomic_max_fetch:
1233 case AtomicExpr::AO__atomic_test_and_set:
1234 case AtomicExpr::AO__atomic_clear:
1235 llvm_unreachable("Integral atomic operations always become atomicrmw!");
1236 }
1237
1238 if (E->isOpenCL()) {
1239 LibCallName =
1240 std::string("__opencl") + StringRef(LibCallName).drop_front(N: 1).str();
1241 }
1242 // By default, assume we return a value of the atomic type.
1243 if (!HaveRetTy) {
1244 // Value is returned through parameter before the order.
1245 RetTy = getContext().VoidTy;
1246 Args.add(rvalue: RValue::get(
1247 V: CastToGenericAddrSpace(Dest.emitRawPointer(CGF&: *this), RetTy)),
1248 type: getContext().VoidPtrTy);
1249 }
1250 // Order is always the last parameter.
1251 Args.add(rvalue: RValue::get(V: Order),
1252 type: getContext().IntTy);
1253 if (E->isOpenCL())
1254 Args.add(rvalue: RValue::get(V: Scope), type: getContext().IntTy);
1255
1256 RValue Res = emitAtomicLibcall(CGF&: *this, fnName: LibCallName, resultType: RetTy, args&: Args);
1257 // The value is returned directly from the libcall.
1258 if (E->isCmpXChg())
1259 return Res;
1260
1261 if (RValTy->isVoidType())
1262 return RValue::get(V: nullptr);
1263
1264 return convertTempToRValue(addr: Dest.withElementType(ElemTy: ConvertTypeForMem(T: RValTy)),
1265 type: RValTy, Loc: E->getExprLoc());
1266 }
1267
1268 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1269 E->getOp() == AtomicExpr::AO__opencl_atomic_store ||
1270 E->getOp() == AtomicExpr::AO__hip_atomic_store ||
1271 E->getOp() == AtomicExpr::AO__atomic_store ||
1272 E->getOp() == AtomicExpr::AO__atomic_store_n ||
1273 E->getOp() == AtomicExpr::AO__scoped_atomic_store ||
1274 E->getOp() == AtomicExpr::AO__scoped_atomic_store_n ||
1275 E->getOp() == AtomicExpr::AO__atomic_clear;
1276 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1277 E->getOp() == AtomicExpr::AO__opencl_atomic_load ||
1278 E->getOp() == AtomicExpr::AO__hip_atomic_load ||
1279 E->getOp() == AtomicExpr::AO__atomic_load ||
1280 E->getOp() == AtomicExpr::AO__atomic_load_n ||
1281 E->getOp() == AtomicExpr::AO__scoped_atomic_load ||
1282 E->getOp() == AtomicExpr::AO__scoped_atomic_load_n;
1283
1284 if (isa<llvm::ConstantInt>(Val: Order)) {
1285 auto ord = cast<llvm::ConstantInt>(Val: Order)->getZExtValue();
1286 // We should not ever get to a case where the ordering isn't a valid C ABI
1287 // value, but it's hard to enforce that in general.
1288 if (llvm::isValidAtomicOrderingCABI(I: ord))
1289 switch ((llvm::AtomicOrderingCABI)ord) {
1290 case llvm::AtomicOrderingCABI::relaxed:
1291 EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1292 Order: llvm::AtomicOrdering::Monotonic, Scope);
1293 break;
1294 case llvm::AtomicOrderingCABI::consume:
1295 case llvm::AtomicOrderingCABI::acquire:
1296 if (IsStore)
1297 break; // Avoid crashing on code with undefined behavior
1298 EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1299 Order: llvm::AtomicOrdering::Acquire, Scope);
1300 break;
1301 case llvm::AtomicOrderingCABI::release:
1302 if (IsLoad)
1303 break; // Avoid crashing on code with undefined behavior
1304 EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1305 Order: llvm::AtomicOrdering::Release, Scope);
1306 break;
1307 case llvm::AtomicOrderingCABI::acq_rel:
1308 if (IsLoad || IsStore)
1309 break; // Avoid crashing on code with undefined behavior
1310 EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1311 Order: llvm::AtomicOrdering::AcquireRelease, Scope);
1312 break;
1313 case llvm::AtomicOrderingCABI::seq_cst:
1314 EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1315 Order: llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1316 break;
1317 }
1318 if (RValTy->isVoidType())
1319 return RValue::get(V: nullptr);
1320
1321 return convertTempToRValue(addr: Dest.withElementType(ElemTy: ConvertTypeForMem(T: RValTy)),
1322 type: RValTy, Loc: E->getExprLoc());
1323 }
1324
1325 // Long case, when Order isn't obviously constant.
1326
1327 // Create all the relevant BB's
1328 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1329 *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1330 *SeqCstBB = nullptr;
1331 MonotonicBB = createBasicBlock(name: "monotonic", parent: CurFn);
1332 if (!IsStore)
1333 AcquireBB = createBasicBlock(name: "acquire", parent: CurFn);
1334 if (!IsLoad)
1335 ReleaseBB = createBasicBlock(name: "release", parent: CurFn);
1336 if (!IsLoad && !IsStore)
1337 AcqRelBB = createBasicBlock(name: "acqrel", parent: CurFn);
1338 SeqCstBB = createBasicBlock(name: "seqcst", parent: CurFn);
1339 llvm::BasicBlock *ContBB = createBasicBlock(name: "atomic.continue", parent: CurFn);
1340
1341 // Create the switch for the split
1342 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1343 // doesn't matter unless someone is crazy enough to use something that
1344 // doesn't fold to a constant for the ordering.
1345 Order = Builder.CreateIntCast(V: Order, DestTy: Builder.getInt32Ty(), isSigned: false);
1346 llvm::SwitchInst *SI = Builder.CreateSwitch(V: Order, Dest: MonotonicBB);
1347
1348 // Emit all the different atomics
1349 Builder.SetInsertPoint(MonotonicBB);
1350 EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1351 Order: llvm::AtomicOrdering::Monotonic, Scope);
1352 Builder.CreateBr(Dest: ContBB);
1353 if (!IsStore) {
1354 Builder.SetInsertPoint(AcquireBB);
1355 EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1356 Order: llvm::AtomicOrdering::Acquire, Scope);
1357 Builder.CreateBr(Dest: ContBB);
1358 SI->addCase(OnVal: Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::consume),
1359 Dest: AcquireBB);
1360 SI->addCase(OnVal: Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::acquire),
1361 Dest: AcquireBB);
1362 }
1363 if (!IsLoad) {
1364 Builder.SetInsertPoint(ReleaseBB);
1365 EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1366 Order: llvm::AtomicOrdering::Release, Scope);
1367 Builder.CreateBr(Dest: ContBB);
1368 SI->addCase(OnVal: Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::release),
1369 Dest: ReleaseBB);
1370 }
1371 if (!IsLoad && !IsStore) {
1372 Builder.SetInsertPoint(AcqRelBB);
1373 EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1374 Order: llvm::AtomicOrdering::AcquireRelease, Scope);
1375 Builder.CreateBr(Dest: ContBB);
1376 SI->addCase(OnVal: Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::acq_rel),
1377 Dest: AcqRelBB);
1378 }
1379 Builder.SetInsertPoint(SeqCstBB);
1380 EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1381 Order: llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1382 Builder.CreateBr(Dest: ContBB);
1383 SI->addCase(OnVal: Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::seq_cst),
1384 Dest: SeqCstBB);
1385
1386 // Cleanup and return
1387 Builder.SetInsertPoint(ContBB);
1388 if (RValTy->isVoidType())
1389 return RValue::get(V: nullptr);
1390
1391 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1392 return convertTempToRValue(addr: Dest.withElementType(ElemTy: ConvertTypeForMem(T: RValTy)),
1393 type: RValTy, Loc: E->getExprLoc());
1394}
1395
1396Address AtomicInfo::castToAtomicIntPointer(Address addr) const {
1397 llvm::IntegerType *ty =
1398 llvm::IntegerType::get(C&: CGF.getLLVMContext(), NumBits: AtomicSizeInBits);
1399 return addr.withElementType(ElemTy: ty);
1400}
1401
1402Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1403 llvm::Type *Ty = Addr.getElementType();
1404 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1405 if (SourceSizeInBits != AtomicSizeInBits) {
1406 Address Tmp = CreateTempAlloca();
1407 CGF.Builder.CreateMemCpy(Dest: Tmp, Src: Addr,
1408 Size: std::min(a: AtomicSizeInBits, b: SourceSizeInBits) / 8);
1409 Addr = Tmp;
1410 }
1411
1412 return castToAtomicIntPointer(addr: Addr);
1413}
1414
1415RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1416 AggValueSlot resultSlot,
1417 SourceLocation loc,
1418 bool asValue) const {
1419 if (LVal.isSimple()) {
1420 if (EvaluationKind == TEK_Aggregate)
1421 return resultSlot.asRValue();
1422
1423 // Drill into the padding structure if we have one.
1424 if (hasPadding())
1425 addr = CGF.Builder.CreateStructGEP(Addr: addr, Index: 0);
1426
1427 // Otherwise, just convert the temporary to an r-value using the
1428 // normal conversion routine.
1429 return CGF.convertTempToRValue(addr, type: getValueType(), Loc: loc);
1430 }
1431 if (!asValue)
1432 // Get RValue from temp memory as atomic for non-simple lvalues
1433 return RValue::get(V: CGF.Builder.CreateLoad(Addr: addr));
1434 if (LVal.isBitField())
1435 return CGF.EmitLoadOfBitfieldLValue(
1436 LV: LValue::MakeBitfield(Addr: addr, Info: LVal.getBitFieldInfo(), type: LVal.getType(),
1437 BaseInfo: LVal.getBaseInfo(), TBAAInfo: TBAAAccessInfo()), Loc: loc);
1438 if (LVal.isVectorElt())
1439 return CGF.EmitLoadOfLValue(
1440 V: LValue::MakeVectorElt(vecAddress: addr, Idx: LVal.getVectorIdx(), type: LVal.getType(),
1441 BaseInfo: LVal.getBaseInfo(), TBAAInfo: TBAAAccessInfo()), Loc: loc);
1442 assert(LVal.isExtVectorElt());
1443 return CGF.EmitLoadOfExtVectorElementLValue(V: LValue::MakeExtVectorElt(
1444 Addr: addr, Elts: LVal.getExtVectorElts(), type: LVal.getType(),
1445 BaseInfo: LVal.getBaseInfo(), TBAAInfo: TBAAAccessInfo()));
1446}
1447
1448/// Return true if \param ValTy is a type that should be casted to integer
1449/// around the atomic memory operation. If \param CmpXchg is true, then the
1450/// cast of a floating point type is made as that instruction can not have
1451/// floating point operands. TODO: Allow compare-and-exchange and FP - see
1452/// comment in AtomicExpandPass.cpp.
1453static bool shouldCastToInt(llvm::Type *ValTy, bool CmpXchg) {
1454 if (ValTy->isFloatingPointTy())
1455 return ValTy->isX86_FP80Ty() || CmpXchg;
1456 return !ValTy->isIntegerTy() && !ValTy->isPointerTy();
1457}
1458
1459RValue AtomicInfo::ConvertToValueOrAtomic(llvm::Value *Val,
1460 AggValueSlot ResultSlot,
1461 SourceLocation Loc, bool AsValue,
1462 bool CmpXchg) const {
1463 // Try not to in some easy cases.
1464 assert((Val->getType()->isIntegerTy() || Val->getType()->isPointerTy() ||
1465 Val->getType()->isIEEELikeFPTy()) &&
1466 "Expected integer, pointer or floating point value when converting "
1467 "result.");
1468 if (getEvaluationKind() == TEK_Scalar &&
1469 (((!LVal.isBitField() ||
1470 LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1471 !hasPadding()) ||
1472 !AsValue)) {
1473 auto *ValTy = AsValue
1474 ? CGF.ConvertTypeForMem(T: ValueTy)
1475 : getAtomicAddress().getElementType();
1476 if (!shouldCastToInt(ValTy, CmpXchg)) {
1477 assert((!ValTy->isIntegerTy() || Val->getType() == ValTy) &&
1478 "Different integer types.");
1479 return RValue::get(V: CGF.EmitFromMemory(Value: Val, Ty: ValueTy));
1480 }
1481 if (llvm::CastInst::isBitCastable(SrcTy: Val->getType(), DestTy: ValTy))
1482 return RValue::get(V: CGF.Builder.CreateBitCast(V: Val, DestTy: ValTy));
1483 }
1484
1485 // Create a temporary. This needs to be big enough to hold the
1486 // atomic integer.
1487 Address Temp = Address::invalid();
1488 bool TempIsVolatile = false;
1489 if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1490 assert(!ResultSlot.isIgnored());
1491 Temp = ResultSlot.getAddress();
1492 TempIsVolatile = ResultSlot.isVolatile();
1493 } else {
1494 Temp = CreateTempAlloca();
1495 }
1496
1497 // Slam the integer into the temporary.
1498 Address CastTemp = castToAtomicIntPointer(addr: Temp);
1499 CGF.Builder.CreateStore(Val, Addr: CastTemp)->setVolatile(TempIsVolatile);
1500
1501 return convertAtomicTempToRValue(addr: Temp, resultSlot: ResultSlot, loc: Loc, asValue: AsValue);
1502}
1503
1504void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1505 llvm::AtomicOrdering AO, bool) {
1506 // void __atomic_load(size_t size, void *mem, void *return, int order);
1507 CallArgList Args;
1508 Args.add(rvalue: RValue::get(V: getAtomicSizeValue()), type: CGF.getContext().getSizeType());
1509 Args.add(rvalue: RValue::get(V: getAtomicPointer()), type: CGF.getContext().VoidPtrTy);
1510 Args.add(rvalue: RValue::get(V: AddForLoaded), type: CGF.getContext().VoidPtrTy);
1511 Args.add(
1512 rvalue: RValue::get(V: llvm::ConstantInt::get(Ty: CGF.IntTy, V: (int)llvm::toCABI(AO))),
1513 type: CGF.getContext().IntTy);
1514 emitAtomicLibcall(CGF, fnName: "__atomic_load", resultType: CGF.getContext().VoidTy, args&: Args);
1515}
1516
1517llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1518 bool IsVolatile, bool CmpXchg) {
1519 // Okay, we're doing this natively.
1520 Address Addr = getAtomicAddress();
1521 if (shouldCastToInt(ValTy: Addr.getElementType(), CmpXchg))
1522 Addr = castToAtomicIntPointer(addr: Addr);
1523 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, Name: "atomic-load");
1524 Load->setAtomic(Ordering: AO);
1525
1526 // Other decoration.
1527 if (IsVolatile)
1528 Load->setVolatile(true);
1529 CGF.CGM.DecorateInstructionWithTBAA(Inst: Load, TBAAInfo: LVal.getTBAAInfo());
1530 return Load;
1531}
1532
1533/// An LValue is a candidate for having its loads and stores be made atomic if
1534/// we are operating under /volatile:ms *and* the LValue itself is volatile and
1535/// performing such an operation can be performed without a libcall.
1536bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1537 if (!CGM.getLangOpts().MSVolatile) return false;
1538 AtomicInfo AI(*this, LV);
1539 bool IsVolatile = LV.isVolatile() || hasVolatileMember(T: LV.getType());
1540 // An atomic is inline if we don't need to use a libcall.
1541 bool AtomicIsInline = !AI.shouldUseLibcall();
1542 // MSVC doesn't seem to do this for types wider than a pointer.
1543 if (getContext().getTypeSize(T: LV.getType()) >
1544 getContext().getTypeSize(T: getContext().getIntPtrType()))
1545 return false;
1546 return IsVolatile && AtomicIsInline;
1547}
1548
1549RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1550 AggValueSlot Slot) {
1551 llvm::AtomicOrdering AO;
1552 bool IsVolatile = LV.isVolatileQualified();
1553 if (LV.getType()->isAtomicType()) {
1554 AO = llvm::AtomicOrdering::SequentiallyConsistent;
1555 } else {
1556 AO = llvm::AtomicOrdering::Acquire;
1557 IsVolatile = true;
1558 }
1559 return EmitAtomicLoad(lvalue: LV, loc: SL, AO, IsVolatile, slot: Slot);
1560}
1561
1562RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1563 bool AsValue, llvm::AtomicOrdering AO,
1564 bool IsVolatile) {
1565 // Check whether we should use a library call.
1566 if (shouldUseLibcall()) {
1567 Address TempAddr = Address::invalid();
1568 if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1569 assert(getEvaluationKind() == TEK_Aggregate);
1570 TempAddr = ResultSlot.getAddress();
1571 } else
1572 TempAddr = CreateTempAlloca();
1573
1574 EmitAtomicLoadLibcall(AddForLoaded: TempAddr.emitRawPointer(CGF), AO, IsVolatile);
1575
1576 // Okay, turn that back into the original value or whole atomic (for
1577 // non-simple lvalues) type.
1578 return convertAtomicTempToRValue(addr: TempAddr, resultSlot: ResultSlot, loc: Loc, asValue: AsValue);
1579 }
1580
1581 // Okay, we're doing this natively.
1582 auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1583
1584 // If we're ignoring an aggregate return, don't do anything.
1585 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1586 return RValue::getAggregate(addr: Address::invalid(), isVolatile: false);
1587
1588 // Okay, turn that back into the original value or atomic (for non-simple
1589 // lvalues) type.
1590 return ConvertToValueOrAtomic(Val: Load, ResultSlot, Loc, AsValue);
1591}
1592
1593/// Emit a load from an l-value of atomic type. Note that the r-value
1594/// we produce is an r-value of the atomic *value* type.
1595RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1596 llvm::AtomicOrdering AO, bool IsVolatile,
1597 AggValueSlot resultSlot) {
1598 AtomicInfo Atomics(*this, src);
1599 return Atomics.EmitAtomicLoad(ResultSlot: resultSlot, Loc: loc, /*AsValue=*/true, AO,
1600 IsVolatile);
1601}
1602
1603/// Copy an r-value into memory as part of storing to an atomic type.
1604/// This needs to create a bit-pattern suitable for atomic operations.
1605void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1606 assert(LVal.isSimple());
1607 // If we have an r-value, the rvalue should be of the atomic type,
1608 // which means that the caller is responsible for having zeroed
1609 // any padding. Just do an aggregate copy of that type.
1610 if (rvalue.isAggregate()) {
1611 LValue Dest = CGF.MakeAddrLValue(Addr: getAtomicAddress(), T: getAtomicType());
1612 LValue Src = CGF.MakeAddrLValue(Addr: rvalue.getAggregateAddress(),
1613 T: getAtomicType());
1614 bool IsVolatile = rvalue.isVolatileQualified() ||
1615 LVal.isVolatileQualified();
1616 CGF.EmitAggregateCopy(Dest, Src, EltTy: getAtomicType(),
1617 MayOverlap: AggValueSlot::DoesNotOverlap, isVolatile: IsVolatile);
1618 return;
1619 }
1620
1621 // Okay, otherwise we're copying stuff.
1622
1623 // Zero out the buffer if necessary.
1624 emitMemSetZeroIfNecessary();
1625
1626 // Drill past the padding if present.
1627 LValue TempLVal = projectValue();
1628
1629 // Okay, store the rvalue in.
1630 if (rvalue.isScalar()) {
1631 CGF.EmitStoreOfScalar(value: rvalue.getScalarVal(), lvalue: TempLVal, /*init*/ isInit: true);
1632 } else {
1633 CGF.EmitStoreOfComplex(V: rvalue.getComplexVal(), dest: TempLVal, /*init*/ isInit: true);
1634 }
1635}
1636
1637
1638/// Materialize an r-value into memory for the purposes of storing it
1639/// to an atomic type.
1640Address AtomicInfo::materializeRValue(RValue rvalue) const {
1641 // Aggregate r-values are already in memory, and EmitAtomicStore
1642 // requires them to be values of the atomic type.
1643 if (rvalue.isAggregate())
1644 return rvalue.getAggregateAddress();
1645
1646 // Otherwise, make a temporary and materialize into it.
1647 LValue TempLV = CGF.MakeAddrLValue(Addr: CreateTempAlloca(), T: getAtomicType());
1648 AtomicInfo Atomics(CGF, TempLV);
1649 Atomics.emitCopyIntoMemory(rvalue);
1650 return TempLV.getAddress();
1651}
1652
1653llvm::Value *AtomicInfo::getScalarRValValueOrNull(RValue RVal) const {
1654 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple()))
1655 return RVal.getScalarVal();
1656 return nullptr;
1657}
1658
1659llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal, bool CmpXchg) const {
1660 // If we've got a scalar value of the right size, try to avoid going
1661 // through memory. Floats get casted if needed by AtomicExpandPass.
1662 if (llvm::Value *Value = getScalarRValValueOrNull(RVal)) {
1663 if (!shouldCastToInt(ValTy: Value->getType(), CmpXchg))
1664 return CGF.EmitToMemory(Value, Ty: ValueTy);
1665 else {
1666 llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1667 C&: CGF.getLLVMContext(),
1668 NumBits: LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1669 if (llvm::BitCastInst::isBitCastable(SrcTy: Value->getType(), DestTy: InputIntTy))
1670 return CGF.Builder.CreateBitCast(V: Value, DestTy: InputIntTy);
1671 }
1672 }
1673 // Otherwise, we need to go through memory.
1674 // Put the r-value in memory.
1675 Address Addr = materializeRValue(rvalue: RVal);
1676
1677 // Cast the temporary to the atomic int type and pull a value out.
1678 Addr = castToAtomicIntPointer(addr: Addr);
1679 return CGF.Builder.CreateLoad(Addr);
1680}
1681
1682std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1683 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1684 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1685 // Do the atomic store.
1686 Address Addr = getAtomicAddressAsAtomicIntPointer();
1687 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, Cmp: ExpectedVal, New: DesiredVal,
1688 SuccessOrdering: Success, FailureOrdering: Failure);
1689 // Other decoration.
1690 Inst->setVolatile(LVal.isVolatileQualified());
1691 Inst->setWeak(IsWeak);
1692
1693 // Okay, turn that back into the original value type.
1694 auto *PreviousVal = CGF.Builder.CreateExtractValue(Agg: Inst, /*Idxs=*/0);
1695 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Agg: Inst, /*Idxs=*/1);
1696 return std::make_pair(x&: PreviousVal, y&: SuccessFailureVal);
1697}
1698
1699llvm::Value *
1700AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1701 llvm::Value *DesiredAddr,
1702 llvm::AtomicOrdering Success,
1703 llvm::AtomicOrdering Failure) {
1704 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1705 // void *desired, int success, int failure);
1706 CallArgList Args;
1707 Args.add(rvalue: RValue::get(V: getAtomicSizeValue()), type: CGF.getContext().getSizeType());
1708 Args.add(rvalue: RValue::get(V: getAtomicPointer()), type: CGF.getContext().VoidPtrTy);
1709 Args.add(rvalue: RValue::get(V: ExpectedAddr), type: CGF.getContext().VoidPtrTy);
1710 Args.add(rvalue: RValue::get(V: DesiredAddr), type: CGF.getContext().VoidPtrTy);
1711 Args.add(rvalue: RValue::get(
1712 V: llvm::ConstantInt::get(Ty: CGF.IntTy, V: (int)llvm::toCABI(AO: Success))),
1713 type: CGF.getContext().IntTy);
1714 Args.add(rvalue: RValue::get(
1715 V: llvm::ConstantInt::get(Ty: CGF.IntTy, V: (int)llvm::toCABI(AO: Failure))),
1716 type: CGF.getContext().IntTy);
1717 auto SuccessFailureRVal = emitAtomicLibcall(CGF, fnName: "__atomic_compare_exchange",
1718 resultType: CGF.getContext().BoolTy, args&: Args);
1719
1720 return SuccessFailureRVal.getScalarVal();
1721}
1722
1723std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1724 RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1725 llvm::AtomicOrdering Failure, bool IsWeak) {
1726 // Check whether we should use a library call.
1727 if (shouldUseLibcall()) {
1728 // Produce a source address.
1729 Address ExpectedAddr = materializeRValue(rvalue: Expected);
1730 llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
1731 llvm::Value *DesiredPtr = materializeRValue(rvalue: Desired).emitRawPointer(CGF);
1732 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr: ExpectedPtr, DesiredAddr: DesiredPtr,
1733 Success, Failure);
1734 return std::make_pair(
1735 x: convertAtomicTempToRValue(addr: ExpectedAddr, resultSlot: AggValueSlot::ignored(),
1736 loc: SourceLocation(), /*AsValue=*/asValue: false),
1737 y&: Res);
1738 }
1739
1740 // If we've got a scalar value of the right size, try to avoid going
1741 // through memory.
1742 auto *ExpectedVal = convertRValueToInt(RVal: Expected, /*CmpXchg=*/true);
1743 auto *DesiredVal = convertRValueToInt(RVal: Desired, /*CmpXchg=*/true);
1744 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1745 Failure, IsWeak);
1746 return std::make_pair(
1747 x: ConvertToValueOrAtomic(Val: Res.first, ResultSlot: AggValueSlot::ignored(),
1748 Loc: SourceLocation(), /*AsValue=*/false,
1749 /*CmpXchg=*/true),
1750 y&: Res.second);
1751}
1752
1753static void
1754EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1755 const llvm::function_ref<RValue(RValue)> &UpdateOp,
1756 Address DesiredAddr) {
1757 RValue UpRVal;
1758 LValue AtomicLVal = Atomics.getAtomicLValue();
1759 LValue DesiredLVal;
1760 if (AtomicLVal.isSimple()) {
1761 UpRVal = OldRVal;
1762 DesiredLVal = CGF.MakeAddrLValue(Addr: DesiredAddr, T: AtomicLVal.getType());
1763 } else {
1764 // Build new lvalue for temp address.
1765 Address Ptr = Atomics.materializeRValue(rvalue: OldRVal);
1766 LValue UpdateLVal;
1767 if (AtomicLVal.isBitField()) {
1768 UpdateLVal =
1769 LValue::MakeBitfield(Addr: Ptr, Info: AtomicLVal.getBitFieldInfo(),
1770 type: AtomicLVal.getType(),
1771 BaseInfo: AtomicLVal.getBaseInfo(),
1772 TBAAInfo: AtomicLVal.getTBAAInfo());
1773 DesiredLVal =
1774 LValue::MakeBitfield(Addr: DesiredAddr, Info: AtomicLVal.getBitFieldInfo(),
1775 type: AtomicLVal.getType(), BaseInfo: AtomicLVal.getBaseInfo(),
1776 TBAAInfo: AtomicLVal.getTBAAInfo());
1777 } else if (AtomicLVal.isVectorElt()) {
1778 UpdateLVal = LValue::MakeVectorElt(vecAddress: Ptr, Idx: AtomicLVal.getVectorIdx(),
1779 type: AtomicLVal.getType(),
1780 BaseInfo: AtomicLVal.getBaseInfo(),
1781 TBAAInfo: AtomicLVal.getTBAAInfo());
1782 DesiredLVal = LValue::MakeVectorElt(
1783 vecAddress: DesiredAddr, Idx: AtomicLVal.getVectorIdx(), type: AtomicLVal.getType(),
1784 BaseInfo: AtomicLVal.getBaseInfo(), TBAAInfo: AtomicLVal.getTBAAInfo());
1785 } else {
1786 assert(AtomicLVal.isExtVectorElt());
1787 UpdateLVal = LValue::MakeExtVectorElt(Addr: Ptr, Elts: AtomicLVal.getExtVectorElts(),
1788 type: AtomicLVal.getType(),
1789 BaseInfo: AtomicLVal.getBaseInfo(),
1790 TBAAInfo: AtomicLVal.getTBAAInfo());
1791 DesiredLVal = LValue::MakeExtVectorElt(
1792 Addr: DesiredAddr, Elts: AtomicLVal.getExtVectorElts(), type: AtomicLVal.getType(),
1793 BaseInfo: AtomicLVal.getBaseInfo(), TBAAInfo: AtomicLVal.getTBAAInfo());
1794 }
1795 UpRVal = CGF.EmitLoadOfLValue(V: UpdateLVal, Loc: SourceLocation());
1796 }
1797 // Store new value in the corresponding memory area.
1798 RValue NewRVal = UpdateOp(UpRVal);
1799 if (NewRVal.isScalar()) {
1800 CGF.EmitStoreThroughLValue(Src: NewRVal, Dst: DesiredLVal);
1801 } else {
1802 assert(NewRVal.isComplex());
1803 CGF.EmitStoreOfComplex(V: NewRVal.getComplexVal(), dest: DesiredLVal,
1804 /*isInit=*/false);
1805 }
1806}
1807
1808void AtomicInfo::EmitAtomicUpdateLibcall(
1809 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1810 bool IsVolatile) {
1811 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
1812
1813 Address ExpectedAddr = CreateTempAlloca();
1814
1815 EmitAtomicLoadLibcall(AddForLoaded: ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile);
1816 auto *ContBB = CGF.createBasicBlock(name: "atomic_cont");
1817 auto *ExitBB = CGF.createBasicBlock(name: "atomic_exit");
1818 CGF.EmitBlock(BB: ContBB);
1819 Address DesiredAddr = CreateTempAlloca();
1820 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1821 requiresMemSetZero(type: getAtomicAddress().getElementType())) {
1822 auto *OldVal = CGF.Builder.CreateLoad(Addr: ExpectedAddr);
1823 CGF.Builder.CreateStore(Val: OldVal, Addr: DesiredAddr);
1824 }
1825 auto OldRVal = convertAtomicTempToRValue(addr: ExpectedAddr,
1826 resultSlot: AggValueSlot::ignored(),
1827 loc: SourceLocation(), /*AsValue=*/asValue: false);
1828 EmitAtomicUpdateValue(CGF, Atomics&: *this, OldRVal, UpdateOp, DesiredAddr);
1829 llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
1830 llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF);
1831 auto *Res =
1832 EmitAtomicCompareExchangeLibcall(ExpectedAddr: ExpectedPtr, DesiredAddr: DesiredPtr, Success: AO, Failure);
1833 CGF.Builder.CreateCondBr(Cond: Res, True: ExitBB, False: ContBB);
1834 CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
1835}
1836
1837void AtomicInfo::EmitAtomicUpdateOp(
1838 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1839 bool IsVolatile) {
1840 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
1841
1842 // Do the atomic load.
1843 auto *OldVal = EmitAtomicLoadOp(AO: Failure, IsVolatile, /*CmpXchg=*/true);
1844 // For non-simple lvalues perform compare-and-swap procedure.
1845 auto *ContBB = CGF.createBasicBlock(name: "atomic_cont");
1846 auto *ExitBB = CGF.createBasicBlock(name: "atomic_exit");
1847 auto *CurBB = CGF.Builder.GetInsertBlock();
1848 CGF.EmitBlock(BB: ContBB);
1849 llvm::PHINode *PHI = CGF.Builder.CreatePHI(Ty: OldVal->getType(),
1850 /*NumReservedValues=*/2);
1851 PHI->addIncoming(V: OldVal, BB: CurBB);
1852 Address NewAtomicAddr = CreateTempAlloca();
1853 Address NewAtomicIntAddr =
1854 shouldCastToInt(ValTy: NewAtomicAddr.getElementType(), /*CmpXchg=*/true)
1855 ? castToAtomicIntPointer(addr: NewAtomicAddr)
1856 : NewAtomicAddr;
1857
1858 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1859 requiresMemSetZero(type: getAtomicAddress().getElementType())) {
1860 CGF.Builder.CreateStore(Val: PHI, Addr: NewAtomicIntAddr);
1861 }
1862 auto OldRVal = ConvertToValueOrAtomic(Val: PHI, ResultSlot: AggValueSlot::ignored(),
1863 Loc: SourceLocation(), /*AsValue=*/false,
1864 /*CmpXchg=*/true);
1865 EmitAtomicUpdateValue(CGF, Atomics&: *this, OldRVal, UpdateOp, DesiredAddr: NewAtomicAddr);
1866 auto *DesiredVal = CGF.Builder.CreateLoad(Addr: NewAtomicIntAddr);
1867 // Try to write new value using cmpxchg operation.
1868 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal: PHI, DesiredVal, Success: AO, Failure);
1869 PHI->addIncoming(V: Res.first, BB: CGF.Builder.GetInsertBlock());
1870 CGF.Builder.CreateCondBr(Cond: Res.second, True: ExitBB, False: ContBB);
1871 CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
1872}
1873
1874static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1875 RValue UpdateRVal, Address DesiredAddr) {
1876 LValue AtomicLVal = Atomics.getAtomicLValue();
1877 LValue DesiredLVal;
1878 // Build new lvalue for temp address.
1879 if (AtomicLVal.isBitField()) {
1880 DesiredLVal =
1881 LValue::MakeBitfield(Addr: DesiredAddr, Info: AtomicLVal.getBitFieldInfo(),
1882 type: AtomicLVal.getType(), BaseInfo: AtomicLVal.getBaseInfo(),
1883 TBAAInfo: AtomicLVal.getTBAAInfo());
1884 } else if (AtomicLVal.isVectorElt()) {
1885 DesiredLVal =
1886 LValue::MakeVectorElt(vecAddress: DesiredAddr, Idx: AtomicLVal.getVectorIdx(),
1887 type: AtomicLVal.getType(), BaseInfo: AtomicLVal.getBaseInfo(),
1888 TBAAInfo: AtomicLVal.getTBAAInfo());
1889 } else {
1890 assert(AtomicLVal.isExtVectorElt());
1891 DesiredLVal = LValue::MakeExtVectorElt(
1892 Addr: DesiredAddr, Elts: AtomicLVal.getExtVectorElts(), type: AtomicLVal.getType(),
1893 BaseInfo: AtomicLVal.getBaseInfo(), TBAAInfo: AtomicLVal.getTBAAInfo());
1894 }
1895 // Store new value in the corresponding memory area.
1896 assert(UpdateRVal.isScalar());
1897 CGF.EmitStoreThroughLValue(Src: UpdateRVal, Dst: DesiredLVal);
1898}
1899
1900void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1901 RValue UpdateRVal, bool IsVolatile) {
1902 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
1903
1904 Address ExpectedAddr = CreateTempAlloca();
1905
1906 EmitAtomicLoadLibcall(AddForLoaded: ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile);
1907 auto *ContBB = CGF.createBasicBlock(name: "atomic_cont");
1908 auto *ExitBB = CGF.createBasicBlock(name: "atomic_exit");
1909 CGF.EmitBlock(BB: ContBB);
1910 Address DesiredAddr = CreateTempAlloca();
1911 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1912 requiresMemSetZero(type: getAtomicAddress().getElementType())) {
1913 auto *OldVal = CGF.Builder.CreateLoad(Addr: ExpectedAddr);
1914 CGF.Builder.CreateStore(Val: OldVal, Addr: DesiredAddr);
1915 }
1916 EmitAtomicUpdateValue(CGF, Atomics&: *this, UpdateRVal, DesiredAddr);
1917 llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
1918 llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF);
1919 auto *Res =
1920 EmitAtomicCompareExchangeLibcall(ExpectedAddr: ExpectedPtr, DesiredAddr: DesiredPtr, Success: AO, Failure);
1921 CGF.Builder.CreateCondBr(Cond: Res, True: ExitBB, False: ContBB);
1922 CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
1923}
1924
1925void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1926 bool IsVolatile) {
1927 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
1928
1929 // Do the atomic load.
1930 auto *OldVal = EmitAtomicLoadOp(AO: Failure, IsVolatile, /*CmpXchg=*/true);
1931 // For non-simple lvalues perform compare-and-swap procedure.
1932 auto *ContBB = CGF.createBasicBlock(name: "atomic_cont");
1933 auto *ExitBB = CGF.createBasicBlock(name: "atomic_exit");
1934 auto *CurBB = CGF.Builder.GetInsertBlock();
1935 CGF.EmitBlock(BB: ContBB);
1936 llvm::PHINode *PHI = CGF.Builder.CreatePHI(Ty: OldVal->getType(),
1937 /*NumReservedValues=*/2);
1938 PHI->addIncoming(V: OldVal, BB: CurBB);
1939 Address NewAtomicAddr = CreateTempAlloca();
1940 Address NewAtomicIntAddr = castToAtomicIntPointer(addr: NewAtomicAddr);
1941 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1942 requiresMemSetZero(type: getAtomicAddress().getElementType())) {
1943 CGF.Builder.CreateStore(Val: PHI, Addr: NewAtomicIntAddr);
1944 }
1945 EmitAtomicUpdateValue(CGF, Atomics&: *this, UpdateRVal, DesiredAddr: NewAtomicAddr);
1946 auto *DesiredVal = CGF.Builder.CreateLoad(Addr: NewAtomicIntAddr);
1947 // Try to write new value using cmpxchg operation.
1948 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal: PHI, DesiredVal, Success: AO, Failure);
1949 PHI->addIncoming(V: Res.first, BB: CGF.Builder.GetInsertBlock());
1950 CGF.Builder.CreateCondBr(Cond: Res.second, True: ExitBB, False: ContBB);
1951 CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
1952}
1953
1954void AtomicInfo::EmitAtomicUpdate(
1955 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1956 bool IsVolatile) {
1957 if (shouldUseLibcall()) {
1958 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1959 } else {
1960 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1961 }
1962}
1963
1964void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
1965 bool IsVolatile) {
1966 if (shouldUseLibcall()) {
1967 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1968 } else {
1969 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1970 }
1971}
1972
1973void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
1974 bool isInit) {
1975 bool IsVolatile = lvalue.isVolatileQualified();
1976 llvm::AtomicOrdering AO;
1977 if (lvalue.getType()->isAtomicType()) {
1978 AO = llvm::AtomicOrdering::SequentiallyConsistent;
1979 } else {
1980 AO = llvm::AtomicOrdering::Release;
1981 IsVolatile = true;
1982 }
1983 return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1984}
1985
1986/// Emit a store to an l-value of atomic type.
1987///
1988/// Note that the r-value is expected to be an r-value *of the atomic
1989/// type*; this means that for aggregate r-values, it should include
1990/// storage for any padding that was necessary.
1991void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
1992 llvm::AtomicOrdering AO, bool IsVolatile,
1993 bool isInit) {
1994 // If this is an aggregate r-value, it should agree in type except
1995 // maybe for address-space qualification.
1996 assert(!rvalue.isAggregate() ||
1997 rvalue.getAggregateAddress().getElementType() ==
1998 dest.getAddress().getElementType());
1999
2000 AtomicInfo atomics(*this, dest);
2001 LValue LVal = atomics.getAtomicLValue();
2002
2003 // If this is an initialization, just put the value there normally.
2004 if (LVal.isSimple()) {
2005 if (isInit) {
2006 atomics.emitCopyIntoMemory(rvalue);
2007 return;
2008 }
2009
2010 // Check whether we should use a library call.
2011 if (atomics.shouldUseLibcall()) {
2012 // Produce a source address.
2013 Address srcAddr = atomics.materializeRValue(rvalue);
2014
2015 // void __atomic_store(size_t size, void *mem, void *val, int order)
2016 CallArgList args;
2017 args.add(rvalue: RValue::get(V: atomics.getAtomicSizeValue()),
2018 type: getContext().getSizeType());
2019 args.add(rvalue: RValue::get(V: atomics.getAtomicPointer()), type: getContext().VoidPtrTy);
2020 args.add(rvalue: RValue::get(V: srcAddr.emitRawPointer(CGF&: *this)),
2021 type: getContext().VoidPtrTy);
2022 args.add(
2023 rvalue: RValue::get(V: llvm::ConstantInt::get(Ty: IntTy, V: (int)llvm::toCABI(AO))),
2024 type: getContext().IntTy);
2025 emitAtomicLibcall(CGF&: *this, fnName: "__atomic_store", resultType: getContext().VoidTy, args);
2026 return;
2027 }
2028
2029 // Okay, we're doing this natively.
2030 llvm::Value *ValToStore = atomics.convertRValueToInt(RVal: rvalue);
2031
2032 // Do the atomic store.
2033 Address Addr = atomics.getAtomicAddress();
2034 if (llvm::Value *Value = atomics.getScalarRValValueOrNull(RVal: rvalue))
2035 if (shouldCastToInt(ValTy: Value->getType(), /*CmpXchg=*/false)) {
2036 Addr = atomics.castToAtomicIntPointer(addr: Addr);
2037 ValToStore = Builder.CreateIntCast(V: ValToStore, DestTy: Addr.getElementType(),
2038 /*isSigned=*/false);
2039 }
2040 llvm::StoreInst *store = Builder.CreateStore(Val: ValToStore, Addr);
2041
2042 if (AO == llvm::AtomicOrdering::Acquire)
2043 AO = llvm::AtomicOrdering::Monotonic;
2044 else if (AO == llvm::AtomicOrdering::AcquireRelease)
2045 AO = llvm::AtomicOrdering::Release;
2046 // Initializations don't need to be atomic.
2047 if (!isInit)
2048 store->setAtomic(Ordering: AO);
2049
2050 // Other decoration.
2051 if (IsVolatile)
2052 store->setVolatile(true);
2053 CGM.DecorateInstructionWithTBAA(Inst: store, TBAAInfo: dest.getTBAAInfo());
2054 return;
2055 }
2056
2057 // Emit simple atomic update operation.
2058 atomics.EmitAtomicUpdate(AO, UpdateRVal: rvalue, IsVolatile);
2059}
2060
2061/// Emit a compare-and-exchange op for atomic type.
2062///
2063std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
2064 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
2065 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
2066 AggValueSlot Slot) {
2067 // If this is an aggregate r-value, it should agree in type except
2068 // maybe for address-space qualification.
2069 assert(!Expected.isAggregate() ||
2070 Expected.getAggregateAddress().getElementType() ==
2071 Obj.getAddress().getElementType());
2072 assert(!Desired.isAggregate() ||
2073 Desired.getAggregateAddress().getElementType() ==
2074 Obj.getAddress().getElementType());
2075 AtomicInfo Atomics(*this, Obj);
2076
2077 return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
2078 IsWeak);
2079}
2080
2081llvm::AtomicRMWInst *
2082CodeGenFunction::emitAtomicRMWInst(llvm::AtomicRMWInst::BinOp Op, Address Addr,
2083 llvm::Value *Val, llvm::AtomicOrdering Order,
2084 llvm::SyncScope::ID SSID,
2085 const AtomicExpr *AE) {
2086 llvm::AtomicRMWInst *RMW =
2087 Builder.CreateAtomicRMW(Op, Addr, Val, Ordering: Order, SSID);
2088 getTargetHooks().setTargetAtomicMetadata(CGF&: *this, AtomicInst&: *RMW, Expr: AE);
2089 return RMW;
2090}
2091
2092void CodeGenFunction::EmitAtomicUpdate(
2093 LValue LVal, llvm::AtomicOrdering AO,
2094 const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
2095 AtomicInfo Atomics(*this, LVal);
2096 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2097}
2098
2099void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
2100 AtomicInfo atomics(*this, dest);
2101
2102 switch (atomics.getEvaluationKind()) {
2103 case TEK_Scalar: {
2104 llvm::Value *value = EmitScalarExpr(E: init);
2105 atomics.emitCopyIntoMemory(rvalue: RValue::get(V: value));
2106 return;
2107 }
2108
2109 case TEK_Complex: {
2110 ComplexPairTy value = EmitComplexExpr(E: init);
2111 atomics.emitCopyIntoMemory(rvalue: RValue::getComplex(C: value));
2112 return;
2113 }
2114
2115 case TEK_Aggregate: {
2116 // Fix up the destination if the initializer isn't an expression
2117 // of atomic type.
2118 bool Zeroed = false;
2119 if (!init->getType()->isAtomicType()) {
2120 Zeroed = atomics.emitMemSetZeroIfNecessary();
2121 dest = atomics.projectValue();
2122 }
2123
2124 // Evaluate the expression directly into the destination.
2125 AggValueSlot slot = AggValueSlot::forLValue(
2126 LV: dest, isDestructed: AggValueSlot::IsNotDestructed,
2127 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
2128 mayOverlap: AggValueSlot::DoesNotOverlap,
2129 isZeroed: Zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed);
2130
2131 EmitAggExpr(E: init, AS: slot);
2132 return;
2133 }
2134 }
2135 llvm_unreachable("bad evaluation kind");
2136}
2137