1//===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the code for emitting atomic operations.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCall.h"
14#include "CGRecordLayout.h"
15#include "CodeGenFunction.h"
16#include "CodeGenModule.h"
17#include "TargetInfo.h"
18#include "clang/AST/ASTContext.h"
19#include "clang/Basic/DiagnosticFrontend.h"
20#include "clang/CodeGen/CGFunctionInfo.h"
21#include "llvm/ADT/DenseMap.h"
22#include "llvm/IR/DataLayout.h"
23#include "llvm/IR/Intrinsics.h"
24
25using namespace clang;
26using namespace CodeGen;
27
28namespace {
29 class AtomicInfo {
30 CodeGenFunction &CGF;
31 QualType AtomicTy;
32 QualType ValueTy;
33 uint64_t AtomicSizeInBits;
34 uint64_t ValueSizeInBits;
35 CharUnits AtomicAlign;
36 CharUnits ValueAlign;
37 TypeEvaluationKind EvaluationKind;
38 bool UseLibcall;
39 LValue LVal;
40 CGBitFieldInfo BFI;
41 public:
42 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
43 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
44 EvaluationKind(TEK_Scalar), UseLibcall(true) {
45 assert(!lvalue.isGlobalReg());
46 ASTContext &C = CGF.getContext();
47 if (lvalue.isSimple()) {
48 AtomicTy = lvalue.getType();
49 if (auto *ATy = AtomicTy->getAs<AtomicType>())
50 ValueTy = ATy->getValueType();
51 else
52 ValueTy = AtomicTy;
53 EvaluationKind = CGF.getEvaluationKind(T: ValueTy);
54
55 uint64_t ValueAlignInBits;
56 uint64_t AtomicAlignInBits;
57 TypeInfo ValueTI = C.getTypeInfo(T: ValueTy);
58 ValueSizeInBits = ValueTI.Width;
59 ValueAlignInBits = ValueTI.Align;
60
61 TypeInfo AtomicTI = C.getTypeInfo(T: AtomicTy);
62 AtomicSizeInBits = AtomicTI.Width;
63 AtomicAlignInBits = AtomicTI.Align;
64
65 assert(ValueSizeInBits <= AtomicSizeInBits);
66 assert(ValueAlignInBits <= AtomicAlignInBits);
67
68 AtomicAlign = C.toCharUnitsFromBits(BitSize: AtomicAlignInBits);
69 ValueAlign = C.toCharUnitsFromBits(BitSize: ValueAlignInBits);
70 if (lvalue.getAlignment().isZero())
71 lvalue.setAlignment(AtomicAlign);
72
73 LVal = lvalue;
74 } else if (lvalue.isBitField()) {
75 ValueTy = lvalue.getType();
76 ValueSizeInBits = C.getTypeSize(T: ValueTy);
77 auto &OrigBFI = lvalue.getBitFieldInfo();
78 auto Offset = OrigBFI.Offset % C.toBits(CharSize: lvalue.getAlignment());
79 AtomicSizeInBits = C.toBits(
80 CharSize: C.toCharUnitsFromBits(BitSize: Offset + OrigBFI.Size + C.getCharWidth() - 1)
81 .alignTo(Align: lvalue.getAlignment()));
82 llvm::Value *BitFieldPtr = lvalue.getRawBitFieldPointer(CGF);
83 auto OffsetInChars =
84 (C.toCharUnitsFromBits(BitSize: OrigBFI.Offset) / lvalue.getAlignment()) *
85 lvalue.getAlignment();
86 llvm::Value *StoragePtr = CGF.Builder.CreateConstGEP1_64(
87 Ty: CGF.Int8Ty, Ptr: BitFieldPtr, Idx0: OffsetInChars.getQuantity());
88 StoragePtr = CGF.Builder.CreateAddrSpaceCast(
89 V: StoragePtr, DestTy: CGF.DefaultPtrTy, Name: "atomic_bitfield_base");
90 BFI = OrigBFI;
91 BFI.Offset = Offset;
92 BFI.StorageSize = AtomicSizeInBits;
93 BFI.StorageOffset += OffsetInChars;
94 llvm::Type *StorageTy = CGF.Builder.getIntNTy(N: AtomicSizeInBits);
95 LVal = LValue::MakeBitfield(
96 Addr: Address(StoragePtr, StorageTy, lvalue.getAlignment()), Info: BFI,
97 type: lvalue.getType(), BaseInfo: lvalue.getBaseInfo(), TBAAInfo: lvalue.getTBAAInfo());
98 AtomicTy = C.getIntTypeForBitwidth(DestWidth: AtomicSizeInBits, Signed: OrigBFI.IsSigned);
99 if (AtomicTy.isNull()) {
100 llvm::APInt Size(
101 /*numBits=*/32,
102 C.toCharUnitsFromBits(BitSize: AtomicSizeInBits).getQuantity());
103 AtomicTy = C.getConstantArrayType(EltTy: C.CharTy, ArySize: Size, SizeExpr: nullptr,
104 ASM: ArraySizeModifier::Normal,
105 /*IndexTypeQuals=*/0);
106 }
107 AtomicAlign = ValueAlign = lvalue.getAlignment();
108 } else if (lvalue.isVectorElt()) {
109 ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();
110 ValueSizeInBits = C.getTypeSize(T: ValueTy);
111 AtomicTy = lvalue.getType();
112 AtomicSizeInBits = C.getTypeSize(T: AtomicTy);
113 AtomicAlign = ValueAlign = lvalue.getAlignment();
114 LVal = lvalue;
115 } else {
116 assert(lvalue.isExtVectorElt());
117 ValueTy = lvalue.getType();
118 ValueSizeInBits = C.getTypeSize(T: ValueTy);
119 AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
120 VectorType: lvalue.getType(), NumElts: cast<llvm::FixedVectorType>(
121 Val: lvalue.getExtVectorAddress().getElementType())
122 ->getNumElements());
123 AtomicSizeInBits = C.getTypeSize(T: AtomicTy);
124 AtomicAlign = ValueAlign = lvalue.getAlignment();
125 LVal = lvalue;
126 }
127 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
128 AtomicSizeInBits, AlignmentInBits: C.toBits(CharSize: lvalue.getAlignment()));
129 }
130
131 QualType getAtomicType() const { return AtomicTy; }
132 QualType getValueType() const { return ValueTy; }
133 CharUnits getAtomicAlignment() const { return AtomicAlign; }
134 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
135 uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
136 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
137 bool shouldUseLibcall() const { return UseLibcall; }
138 const LValue &getAtomicLValue() const { return LVal; }
139 llvm::Value *getAtomicPointer() const {
140 if (LVal.isSimple())
141 return LVal.emitRawPointer(CGF);
142 else if (LVal.isBitField())
143 return LVal.getRawBitFieldPointer(CGF);
144 else if (LVal.isVectorElt())
145 return LVal.getRawVectorPointer(CGF);
146 assert(LVal.isExtVectorElt());
147 return LVal.getRawExtVectorPointer(CGF);
148 }
149 Address getAtomicAddress() const {
150 llvm::Type *ElTy;
151 if (LVal.isSimple())
152 ElTy = LVal.getAddress().getElementType();
153 else if (LVal.isBitField())
154 ElTy = LVal.getBitFieldAddress().getElementType();
155 else if (LVal.isVectorElt())
156 ElTy = LVal.getVectorAddress().getElementType();
157 else
158 ElTy = LVal.getExtVectorAddress().getElementType();
159 return Address(getAtomicPointer(), ElTy, getAtomicAlignment());
160 }
161
162 Address getAtomicAddressAsAtomicIntPointer() const {
163 return castToAtomicIntPointer(Addr: getAtomicAddress());
164 }
165
166 /// Is the atomic size larger than the underlying value type?
167 ///
168 /// Note that the absence of padding does not mean that atomic
169 /// objects are completely interchangeable with non-atomic
170 /// objects: we might have promoted the alignment of a type
171 /// without making it bigger.
172 bool hasPadding() const {
173 return (ValueSizeInBits != AtomicSizeInBits);
174 }
175
176 bool emitMemSetZeroIfNecessary() const;
177
178 llvm::Value *getAtomicSizeValue() const {
179 CharUnits size = CGF.getContext().toCharUnitsFromBits(BitSize: AtomicSizeInBits);
180 return CGF.CGM.getSize(numChars: size);
181 }
182
183 /// Cast the given pointer to an integer pointer suitable for atomic
184 /// operations if the source.
185 Address castToAtomicIntPointer(Address Addr) const;
186
187 /// If Addr is compatible with the iN that will be used for an atomic
188 /// operation, bitcast it. Otherwise, create a temporary that is suitable
189 /// and copy the value across.
190 Address convertToAtomicIntPointer(Address Addr) const;
191
192 /// Turn an atomic-layout object into an r-value.
193 RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
194 SourceLocation loc, bool AsValue) const;
195
196 llvm::Value *getScalarRValValueOrNull(RValue RVal) const;
197
198 /// Converts an rvalue to integer value if needed.
199 llvm::Value *convertRValueToInt(RValue RVal, bool CmpXchg = false) const;
200
201 RValue ConvertToValueOrAtomic(llvm::Value *IntVal, AggValueSlot ResultSlot,
202 SourceLocation Loc, bool AsValue,
203 bool CmpXchg = false) const;
204
205 /// Copy an atomic r-value into atomic-layout memory.
206 void emitCopyIntoMemory(RValue rvalue) const;
207
208 /// Project an l-value down to the value field.
209 LValue projectValue() const {
210 assert(LVal.isSimple());
211 Address addr = getAtomicAddress();
212 if (hasPadding())
213 addr = CGF.Builder.CreateStructGEP(Addr: addr, Index: 0);
214
215 return LValue::MakeAddr(Addr: addr, type: getValueType(), Context&: CGF.getContext(),
216 BaseInfo: LVal.getBaseInfo(), TBAAInfo: LVal.getTBAAInfo());
217 }
218
219 /// Emits atomic load.
220 /// \returns Loaded value.
221 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
222 bool AsValue, llvm::AtomicOrdering AO,
223 bool IsVolatile);
224
225 /// Emits atomic compare-and-exchange sequence.
226 /// \param Expected Expected value.
227 /// \param Desired Desired value.
228 /// \param Success Atomic ordering for success operation.
229 /// \param Failure Atomic ordering for failed operation.
230 /// \param IsWeak true if atomic operation is weak, false otherwise.
231 /// \returns Pair of values: previous value from storage (value type) and
232 /// boolean flag (i1 type) with true if success and false otherwise.
233 std::pair<RValue, llvm::Value *>
234 EmitAtomicCompareExchange(RValue Expected, RValue Desired,
235 llvm::AtomicOrdering Success =
236 llvm::AtomicOrdering::SequentiallyConsistent,
237 llvm::AtomicOrdering Failure =
238 llvm::AtomicOrdering::SequentiallyConsistent,
239 bool IsWeak = false);
240
241 /// Emits atomic update.
242 /// \param AO Atomic ordering.
243 /// \param UpdateOp Update operation for the current lvalue.
244 void EmitAtomicUpdate(llvm::AtomicOrdering AO,
245 const llvm::function_ref<RValue(RValue)> &UpdateOp,
246 bool IsVolatile);
247 /// Emits atomic update.
248 /// \param AO Atomic ordering.
249 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
250 bool IsVolatile);
251
252 /// Materialize an atomic r-value in atomic-layout memory.
253 Address materializeRValue(RValue rvalue) const;
254
255 /// Creates temp alloca for intermediate operations on atomic value.
256 Address CreateTempAlloca() const;
257 private:
258 bool requiresMemSetZero(llvm::Type *type) const;
259
260
261 /// Emits atomic load as a libcall.
262 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
263 llvm::AtomicOrdering AO, bool IsVolatile);
264 /// Emits atomic load as LLVM instruction.
265 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile,
266 bool CmpXchg = false);
267 /// Emits atomic compare-and-exchange op as a libcall.
268 llvm::Value *EmitAtomicCompareExchangeLibcall(
269 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
270 llvm::AtomicOrdering Success =
271 llvm::AtomicOrdering::SequentiallyConsistent,
272 llvm::AtomicOrdering Failure =
273 llvm::AtomicOrdering::SequentiallyConsistent);
274 /// Emits atomic compare-and-exchange op as LLVM instruction.
275 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
276 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
277 llvm::AtomicOrdering Success =
278 llvm::AtomicOrdering::SequentiallyConsistent,
279 llvm::AtomicOrdering Failure =
280 llvm::AtomicOrdering::SequentiallyConsistent,
281 bool IsWeak = false);
282 /// Emit atomic update as libcalls.
283 void
284 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
285 const llvm::function_ref<RValue(RValue)> &UpdateOp,
286 bool IsVolatile);
287 /// Emit atomic update as LLVM instructions.
288 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
289 const llvm::function_ref<RValue(RValue)> &UpdateOp,
290 bool IsVolatile);
291 /// Emit atomic update as libcalls.
292 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
293 bool IsVolatile);
294 /// Emit atomic update as LLVM instructions.
295 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
296 bool IsVolatile);
297 };
298}
299
300Address AtomicInfo::CreateTempAlloca() const {
301 // Remove addrspace info from the atomic pointer element when making the
302 // alloca pointer element.
303 QualType TmpTy = (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits)
304 ? ValueTy
305 : AtomicTy.getUnqualifiedType();
306 Address TempAlloca =
307 CGF.CreateMemTempWithoutCast(T: TmpTy, Align: getAtomicAlignment(), Name: "atomic-temp");
308 // Cast to pointer to value type for bitfields.
309 if (LVal.isBitField())
310 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
311 Addr: TempAlloca, Ty: getAtomicAddress().getType(),
312 ElementTy: getAtomicAddress().getElementType());
313 return TempAlloca;
314}
315
316static RValue emitAtomicLibcall(CodeGenFunction &CGF,
317 StringRef fnName,
318 QualType resultType,
319 CallArgList &args) {
320 const CGFunctionInfo &fnInfo =
321 CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
322 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(Info: fnInfo);
323 llvm::AttrBuilder fnAttrB(CGF.getLLVMContext());
324 fnAttrB.addAttribute(Val: llvm::Attribute::NoUnwind);
325 fnAttrB.addAttribute(Val: llvm::Attribute::WillReturn);
326 llvm::AttributeList fnAttrs = llvm::AttributeList::get(
327 C&: CGF.getLLVMContext(), Index: llvm::AttributeList::FunctionIndex, B: fnAttrB);
328
329 llvm::FunctionCallee fn =
330 CGF.CGM.CreateRuntimeFunction(Ty: fnTy, Name: fnName, ExtraAttrs: fnAttrs);
331 auto callee = CGCallee::forDirect(functionPtr: fn);
332 return CGF.EmitCall(CallInfo: fnInfo, Callee: callee, ReturnValue: ReturnValueSlot(), Args: args);
333}
334
335/// Does a store of the given IR type modify the full expected width?
336static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
337 uint64_t expectedSize) {
338 return (CGM.getDataLayout().getTypeStoreSize(Ty: type) * 8 == expectedSize);
339}
340
341/// Does the atomic type require memsetting to zero before initialization?
342///
343/// The IR type is provided as a way of making certain queries faster.
344bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
345 // If the atomic type has size padding, we definitely need a memset.
346 if (hasPadding()) return true;
347
348 // Otherwise, do some simple heuristics to try to avoid it:
349 switch (getEvaluationKind()) {
350 // For scalars and complexes, check whether the store size of the
351 // type uses the full size.
352 case TEK_Scalar:
353 return !isFullSizeType(CGM&: CGF.CGM, type, expectedSize: AtomicSizeInBits);
354 case TEK_Complex:
355 return !isFullSizeType(CGM&: CGF.CGM, type: type->getStructElementType(N: 0),
356 expectedSize: AtomicSizeInBits / 2);
357
358 // Padding in structs has an undefined bit pattern. User beware.
359 case TEK_Aggregate:
360 return false;
361 }
362 llvm_unreachable("bad evaluation kind");
363}
364
365bool AtomicInfo::emitMemSetZeroIfNecessary() const {
366 assert(LVal.isSimple());
367 Address addr = LVal.getAddress();
368 if (!requiresMemSetZero(type: addr.getElementType()))
369 return false;
370
371 CGF.Builder.CreateMemSet(
372 Ptr: addr.emitRawPointer(CGF), Val: llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 0),
373 Size: CGF.getContext().toCharUnitsFromBits(BitSize: AtomicSizeInBits).getQuantity(),
374 Align: LVal.getAlignment().getAsAlign());
375 return true;
376}
377
378static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
379 Address Dest, Address Ptr, Address Val1,
380 Address Val2, Address ExpectedResult,
381 uint64_t Size, llvm::AtomicOrdering SuccessOrder,
382 llvm::AtomicOrdering FailureOrder,
383 llvm::SyncScope::ID Scope) {
384 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
385 llvm::Value *Expected = CGF.Builder.CreateLoad(Addr: Val1);
386 llvm::Value *Desired = CGF.Builder.CreateLoad(Addr: Val2);
387
388 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
389 Addr: Ptr, Cmp: Expected, New: Desired, SuccessOrdering: SuccessOrder, FailureOrdering: FailureOrder, SSID: Scope);
390 Pair->setVolatile(E->isVolatile());
391 Pair->setWeak(IsWeak);
392 CGF.getTargetHooks().setTargetAtomicMetadata(CGF, AtomicInst&: *Pair, Expr: E);
393
394 // Cmp holds the result of the compare-exchange operation: true on success,
395 // false on failure.
396 llvm::Value *Old = CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 0);
397 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 1);
398
399 // This basic block is used to hold the store instruction if the operation
400 // failed.
401 llvm::BasicBlock *StoreExpectedBB =
402 CGF.createBasicBlock(name: "cmpxchg.store_expected", parent: CGF.CurFn);
403
404 // This basic block is the exit point of the operation, we should end up
405 // here regardless of whether or not the operation succeeded.
406 llvm::BasicBlock *ContinueBB =
407 CGF.createBasicBlock(name: "cmpxchg.continue", parent: CGF.CurFn);
408
409 // Update Expected if Expected isn't equal to Old, otherwise branch to the
410 // exit point.
411 CGF.Builder.CreateCondBr(Cond: Cmp, True: ContinueBB, False: StoreExpectedBB);
412
413 CGF.Builder.SetInsertPoint(StoreExpectedBB);
414 // Update the memory at Expected with Old's value.
415 llvm::Type *ExpectedType = ExpectedResult.getElementType();
416 const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
417 uint64_t ExpectedSizeInBytes = DL.getTypeStoreSize(Ty: ExpectedType);
418
419 if (ExpectedSizeInBytes == Size) {
420 // Sizes match: store directly
421 auto *I = CGF.Builder.CreateStore(Val: Old, Addr: ExpectedResult);
422 CGF.addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Old);
423 } else {
424 // store only the first ExpectedSizeInBytes bytes of Old
425 llvm::Type *OldType = Old->getType();
426
427 // Allocate temporary storage for Old value
428 Address OldTmp =
429 CGF.CreateTempAlloca(Ty: OldType, align: Ptr.getAlignment(), Name: "old.tmp");
430
431 // Store Old into this temporary
432 auto *I = CGF.Builder.CreateStore(Val: Old, Addr: OldTmp);
433 CGF.addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Old);
434
435 // Perform memcpy for first ExpectedSizeInBytes bytes
436 CGF.Builder.CreateMemCpy(Dest: ExpectedResult, Src: OldTmp, Size: ExpectedSizeInBytes,
437 /*isVolatile=*/IsVolatile: false);
438 }
439
440 // Finally, branch to the exit point.
441 CGF.Builder.CreateBr(Dest: ContinueBB);
442
443 CGF.Builder.SetInsertPoint(ContinueBB);
444 // Update the memory at Dest with Cmp's value.
445 CGF.EmitStoreOfScalar(value: Cmp, lvalue: CGF.MakeAddrLValue(Addr: Dest, T: E->getType()));
446}
447
448/// Given an ordering required on success, emit all possible cmpxchg
449/// instructions to cope with the provided (but possibly only dynamically known)
450/// FailureOrder.
451static void emitAtomicCmpXchgFailureSet(
452 CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, Address Dest, Address Ptr,
453 Address Val1, Address Val2, Address ExpectedResult,
454 llvm::Value *FailureOrderVal, uint64_t Size,
455 llvm::AtomicOrdering SuccessOrder, llvm::SyncScope::ID Scope) {
456 llvm::AtomicOrdering FailureOrder;
457 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(Val: FailureOrderVal)) {
458 auto FOS = FO->getSExtValue();
459 if (!llvm::isValidAtomicOrderingCABI(I: FOS))
460 FailureOrder = llvm::AtomicOrdering::Monotonic;
461 else
462 switch ((llvm::AtomicOrderingCABI)FOS) {
463 case llvm::AtomicOrderingCABI::relaxed:
464 // 31.7.2.18: "The failure argument shall not be memory_order_release
465 // nor memory_order_acq_rel". Fallback to monotonic.
466 case llvm::AtomicOrderingCABI::release:
467 case llvm::AtomicOrderingCABI::acq_rel:
468 FailureOrder = llvm::AtomicOrdering::Monotonic;
469 break;
470 case llvm::AtomicOrderingCABI::consume:
471 case llvm::AtomicOrderingCABI::acquire:
472 FailureOrder = llvm::AtomicOrdering::Acquire;
473 break;
474 case llvm::AtomicOrderingCABI::seq_cst:
475 FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
476 break;
477 }
478 // Prior to c++17, "the failure argument shall be no stronger than the
479 // success argument". This condition has been lifted and the only
480 // precondition is 31.7.2.18. Effectively treat this as a DR and skip
481 // language version checks.
482 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, ExpectedResult,
483 Size, SuccessOrder, FailureOrder, Scope);
484 return;
485 }
486
487 // Create all the relevant BB's
488 auto *MonotonicBB = CGF.createBasicBlock(name: "monotonic_fail", parent: CGF.CurFn);
489 auto *AcquireBB = CGF.createBasicBlock(name: "acquire_fail", parent: CGF.CurFn);
490 auto *SeqCstBB = CGF.createBasicBlock(name: "seqcst_fail", parent: CGF.CurFn);
491 auto *ContBB = CGF.createBasicBlock(name: "atomic.continue", parent: CGF.CurFn);
492
493 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
494 // doesn't matter unless someone is crazy enough to use something that
495 // doesn't fold to a constant for the ordering.
496 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(V: FailureOrderVal, Dest: MonotonicBB);
497 // Implemented as acquire, since it's the closest in LLVM.
498 SI->addCase(OnVal: CGF.Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::consume),
499 Dest: AcquireBB);
500 SI->addCase(OnVal: CGF.Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::acquire),
501 Dest: AcquireBB);
502 SI->addCase(OnVal: CGF.Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::seq_cst),
503 Dest: SeqCstBB);
504
505 // Emit all the different atomics
506 CGF.Builder.SetInsertPoint(MonotonicBB);
507 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, ExpectedResult, Size,
508 SuccessOrder, FailureOrder: llvm::AtomicOrdering::Monotonic, Scope);
509 CGF.Builder.CreateBr(Dest: ContBB);
510
511 CGF.Builder.SetInsertPoint(AcquireBB);
512 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, ExpectedResult, Size,
513 SuccessOrder, FailureOrder: llvm::AtomicOrdering::Acquire, Scope);
514 CGF.Builder.CreateBr(Dest: ContBB);
515
516 CGF.Builder.SetInsertPoint(SeqCstBB);
517 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, ExpectedResult, Size,
518 SuccessOrder, FailureOrder: llvm::AtomicOrdering::SequentiallyConsistent,
519 Scope);
520 CGF.Builder.CreateBr(Dest: ContBB);
521
522 CGF.Builder.SetInsertPoint(ContBB);
523}
524
525/// Duplicate the atomic min/max operation in conventional IR for the builtin
526/// variants that return the new rather than the original value.
527static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder,
528 AtomicExpr::AtomicOp Op,
529 bool IsSigned,
530 llvm::Value *OldVal,
531 llvm::Value *RHS) {
532 const bool IsFP = OldVal->getType()->isFloatingPointTy();
533
534 if (IsFP) {
535 llvm::Intrinsic::ID IID = (Op == AtomicExpr::AO__atomic_max_fetch ||
536 Op == AtomicExpr::AO__scoped_atomic_max_fetch)
537 ? llvm::Intrinsic::maxnum
538 : llvm::Intrinsic::minnum;
539 return Builder.CreateBinaryIntrinsic(ID: IID, LHS: OldVal, RHS, FMFSource: llvm::FMFSource(),
540 Name: "newval");
541 }
542
543 llvm::CmpInst::Predicate Pred;
544 switch (Op) {
545 default:
546 llvm_unreachable("Unexpected min/max operation");
547 case AtomicExpr::AO__atomic_max_fetch:
548 case AtomicExpr::AO__scoped_atomic_max_fetch:
549 Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT;
550 break;
551 case AtomicExpr::AO__atomic_min_fetch:
552 case AtomicExpr::AO__scoped_atomic_min_fetch:
553 Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT;
554 break;
555 }
556 llvm::Value *Cmp = Builder.CreateICmp(P: Pred, LHS: OldVal, RHS, Name: "tst");
557 return Builder.CreateSelect(C: Cmp, True: OldVal, False: RHS, Name: "newval");
558}
559
560static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
561 Address Ptr, Address Val1, Address Val2,
562 Address ExpectedResult, llvm::Value *IsWeak,
563 llvm::Value *FailureOrder, uint64_t Size,
564 llvm::AtomicOrdering Order,
565 llvm::SyncScope::ID Scope) {
566 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
567 bool PostOpMinMax = false;
568 unsigned PostOp = 0;
569
570 switch (E->getOp()) {
571 case AtomicExpr::AO__c11_atomic_init:
572 case AtomicExpr::AO__opencl_atomic_init:
573 llvm_unreachable("Already handled!");
574
575 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
576 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
577 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
578 emitAtomicCmpXchgFailureSet(CGF, E, IsWeak: false, Dest, Ptr, Val1, Val2,
579 ExpectedResult, FailureOrderVal: FailureOrder, Size, SuccessOrder: Order,
580 Scope);
581 return;
582 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
583 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
584 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
585 emitAtomicCmpXchgFailureSet(CGF, E, IsWeak: true, Dest, Ptr, Val1, Val2,
586 ExpectedResult, FailureOrderVal: FailureOrder, Size, SuccessOrder: Order,
587 Scope);
588 return;
589 case AtomicExpr::AO__atomic_compare_exchange:
590 case AtomicExpr::AO__atomic_compare_exchange_n:
591 case AtomicExpr::AO__scoped_atomic_compare_exchange:
592 case AtomicExpr::AO__scoped_atomic_compare_exchange_n: {
593 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(Val: IsWeak)) {
594 emitAtomicCmpXchgFailureSet(CGF, E, IsWeak: IsWeakC->getZExtValue(), Dest, Ptr,
595 Val1, Val2, ExpectedResult, FailureOrderVal: FailureOrder,
596 Size, SuccessOrder: Order, Scope);
597 } else {
598 // Create all the relevant BB's
599 llvm::BasicBlock *StrongBB =
600 CGF.createBasicBlock(name: "cmpxchg.strong", parent: CGF.CurFn);
601 llvm::BasicBlock *WeakBB = CGF.createBasicBlock(name: "cmxchg.weak", parent: CGF.CurFn);
602 llvm::BasicBlock *ContBB =
603 CGF.createBasicBlock(name: "cmpxchg.continue", parent: CGF.CurFn);
604
605 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(V: IsWeak, Dest: WeakBB);
606 SI->addCase(OnVal: CGF.Builder.getInt1(V: false), Dest: StrongBB);
607
608 CGF.Builder.SetInsertPoint(StrongBB);
609 emitAtomicCmpXchgFailureSet(CGF, E, IsWeak: false, Dest, Ptr, Val1, Val2,
610 ExpectedResult, FailureOrderVal: FailureOrder, Size, SuccessOrder: Order,
611 Scope);
612 CGF.Builder.CreateBr(Dest: ContBB);
613
614 CGF.Builder.SetInsertPoint(WeakBB);
615 emitAtomicCmpXchgFailureSet(CGF, E, IsWeak: true, Dest, Ptr, Val1, Val2,
616 ExpectedResult, FailureOrderVal: FailureOrder, Size, SuccessOrder: Order,
617 Scope);
618 CGF.Builder.CreateBr(Dest: ContBB);
619
620 CGF.Builder.SetInsertPoint(ContBB);
621 }
622 return;
623 }
624 case AtomicExpr::AO__c11_atomic_load:
625 case AtomicExpr::AO__opencl_atomic_load:
626 case AtomicExpr::AO__hip_atomic_load:
627 case AtomicExpr::AO__atomic_load_n:
628 case AtomicExpr::AO__atomic_load:
629 case AtomicExpr::AO__scoped_atomic_load_n:
630 case AtomicExpr::AO__scoped_atomic_load: {
631 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr: Ptr);
632 Load->setAtomic(Ordering: Order, SSID: Scope);
633 Load->setVolatile(E->isVolatile());
634 CGF.maybeAttachRangeForLoad(Load, Ty: E->getValueType(), Loc: E->getExprLoc());
635 auto *I = CGF.Builder.CreateStore(Val: Load, Addr: Dest);
636 CGF.addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Load);
637 return;
638 }
639
640 case AtomicExpr::AO__c11_atomic_store:
641 case AtomicExpr::AO__opencl_atomic_store:
642 case AtomicExpr::AO__hip_atomic_store:
643 case AtomicExpr::AO__atomic_store:
644 case AtomicExpr::AO__atomic_store_n:
645 case AtomicExpr::AO__scoped_atomic_store:
646 case AtomicExpr::AO__scoped_atomic_store_n: {
647 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Addr: Val1);
648 llvm::StoreInst *Store = CGF.Builder.CreateStore(Val: LoadVal1, Addr: Ptr);
649 Store->setAtomic(Ordering: Order, SSID: Scope);
650 Store->setVolatile(E->isVolatile());
651 CGF.addInstToCurrentSourceAtom(KeyInstruction: Store, Backup: LoadVal1);
652 return;
653 }
654
655 case AtomicExpr::AO__c11_atomic_exchange:
656 case AtomicExpr::AO__hip_atomic_exchange:
657 case AtomicExpr::AO__opencl_atomic_exchange:
658 case AtomicExpr::AO__atomic_exchange_n:
659 case AtomicExpr::AO__atomic_exchange:
660 case AtomicExpr::AO__scoped_atomic_exchange_n:
661 case AtomicExpr::AO__scoped_atomic_exchange:
662 Op = llvm::AtomicRMWInst::Xchg;
663 break;
664
665 case AtomicExpr::AO__atomic_add_fetch:
666 case AtomicExpr::AO__scoped_atomic_add_fetch:
667 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd
668 : llvm::Instruction::Add;
669 [[fallthrough]];
670 case AtomicExpr::AO__c11_atomic_fetch_add:
671 case AtomicExpr::AO__hip_atomic_fetch_add:
672 case AtomicExpr::AO__opencl_atomic_fetch_add:
673 case AtomicExpr::AO__atomic_fetch_add:
674 case AtomicExpr::AO__scoped_atomic_fetch_add:
675 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd
676 : llvm::AtomicRMWInst::Add;
677 break;
678
679 case AtomicExpr::AO__atomic_sub_fetch:
680 case AtomicExpr::AO__scoped_atomic_sub_fetch:
681 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub
682 : llvm::Instruction::Sub;
683 [[fallthrough]];
684 case AtomicExpr::AO__c11_atomic_fetch_sub:
685 case AtomicExpr::AO__hip_atomic_fetch_sub:
686 case AtomicExpr::AO__opencl_atomic_fetch_sub:
687 case AtomicExpr::AO__atomic_fetch_sub:
688 case AtomicExpr::AO__scoped_atomic_fetch_sub:
689 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub
690 : llvm::AtomicRMWInst::Sub;
691 break;
692
693 case AtomicExpr::AO__atomic_min_fetch:
694 case AtomicExpr::AO__scoped_atomic_min_fetch:
695 PostOpMinMax = true;
696 [[fallthrough]];
697 case AtomicExpr::AO__c11_atomic_fetch_min:
698 case AtomicExpr::AO__hip_atomic_fetch_min:
699 case AtomicExpr::AO__opencl_atomic_fetch_min:
700 case AtomicExpr::AO__atomic_fetch_min:
701 case AtomicExpr::AO__scoped_atomic_fetch_min:
702 Op = E->getValueType()->isFloatingType()
703 ? llvm::AtomicRMWInst::FMin
704 : (E->getValueType()->isSignedIntegerType()
705 ? llvm::AtomicRMWInst::Min
706 : llvm::AtomicRMWInst::UMin);
707 break;
708
709 case AtomicExpr::AO__atomic_fetch_fminimum:
710 case AtomicExpr::AO__scoped_atomic_fetch_fminimum:
711 assert(E->getValueType()->isFloatingType() &&
712 "fminimum operations only support floating-point types");
713 Op = llvm::AtomicRMWInst::FMinimum;
714 break;
715
716 case AtomicExpr::AO__atomic_fetch_fminimum_num:
717 case AtomicExpr::AO__scoped_atomic_fetch_fminimum_num:
718 assert(E->getValueType()->isFloatingType() &&
719 "fminimum_num operations only support floating-point types");
720 Op = llvm::AtomicRMWInst::FMinimumNum;
721 break;
722
723 case AtomicExpr::AO__atomic_max_fetch:
724 case AtomicExpr::AO__scoped_atomic_max_fetch:
725 PostOpMinMax = true;
726 [[fallthrough]];
727 case AtomicExpr::AO__c11_atomic_fetch_max:
728 case AtomicExpr::AO__hip_atomic_fetch_max:
729 case AtomicExpr::AO__opencl_atomic_fetch_max:
730 case AtomicExpr::AO__atomic_fetch_max:
731 case AtomicExpr::AO__scoped_atomic_fetch_max:
732 Op = E->getValueType()->isFloatingType()
733 ? llvm::AtomicRMWInst::FMax
734 : (E->getValueType()->isSignedIntegerType()
735 ? llvm::AtomicRMWInst::Max
736 : llvm::AtomicRMWInst::UMax);
737 break;
738
739 case AtomicExpr::AO__atomic_fetch_fmaximum:
740 case AtomicExpr::AO__scoped_atomic_fetch_fmaximum:
741 assert(E->getValueType()->isFloatingType() &&
742 "fmaximum operations only support floating-point types");
743 Op = llvm::AtomicRMWInst::FMaximum;
744 break;
745
746 case AtomicExpr::AO__atomic_fetch_fmaximum_num:
747 case AtomicExpr::AO__scoped_atomic_fetch_fmaximum_num:
748 assert(E->getValueType()->isFloatingType() &&
749 "fmaximum_num operations only support floating-point types");
750 Op = llvm::AtomicRMWInst::FMaximumNum;
751 break;
752
753 case AtomicExpr::AO__atomic_and_fetch:
754 case AtomicExpr::AO__scoped_atomic_and_fetch:
755 PostOp = llvm::Instruction::And;
756 [[fallthrough]];
757 case AtomicExpr::AO__c11_atomic_fetch_and:
758 case AtomicExpr::AO__hip_atomic_fetch_and:
759 case AtomicExpr::AO__opencl_atomic_fetch_and:
760 case AtomicExpr::AO__atomic_fetch_and:
761 case AtomicExpr::AO__scoped_atomic_fetch_and:
762 Op = llvm::AtomicRMWInst::And;
763 break;
764
765 case AtomicExpr::AO__atomic_or_fetch:
766 case AtomicExpr::AO__scoped_atomic_or_fetch:
767 PostOp = llvm::Instruction::Or;
768 [[fallthrough]];
769 case AtomicExpr::AO__c11_atomic_fetch_or:
770 case AtomicExpr::AO__hip_atomic_fetch_or:
771 case AtomicExpr::AO__opencl_atomic_fetch_or:
772 case AtomicExpr::AO__atomic_fetch_or:
773 case AtomicExpr::AO__scoped_atomic_fetch_or:
774 Op = llvm::AtomicRMWInst::Or;
775 break;
776
777 case AtomicExpr::AO__atomic_xor_fetch:
778 case AtomicExpr::AO__scoped_atomic_xor_fetch:
779 PostOp = llvm::Instruction::Xor;
780 [[fallthrough]];
781 case AtomicExpr::AO__c11_atomic_fetch_xor:
782 case AtomicExpr::AO__hip_atomic_fetch_xor:
783 case AtomicExpr::AO__opencl_atomic_fetch_xor:
784 case AtomicExpr::AO__atomic_fetch_xor:
785 case AtomicExpr::AO__scoped_atomic_fetch_xor:
786 Op = llvm::AtomicRMWInst::Xor;
787 break;
788
789 case AtomicExpr::AO__atomic_nand_fetch:
790 case AtomicExpr::AO__scoped_atomic_nand_fetch:
791 PostOp = llvm::Instruction::And; // the NOT is special cased below
792 [[fallthrough]];
793 case AtomicExpr::AO__c11_atomic_fetch_nand:
794 case AtomicExpr::AO__atomic_fetch_nand:
795 case AtomicExpr::AO__scoped_atomic_fetch_nand:
796 Op = llvm::AtomicRMWInst::Nand;
797 break;
798
799 case AtomicExpr::AO__atomic_fetch_uinc:
800 case AtomicExpr::AO__scoped_atomic_fetch_uinc:
801 Op = llvm::AtomicRMWInst::UIncWrap;
802 break;
803 case AtomicExpr::AO__atomic_fetch_udec:
804 case AtomicExpr::AO__scoped_atomic_fetch_udec:
805 Op = llvm::AtomicRMWInst::UDecWrap;
806 break;
807
808 case AtomicExpr::AO__atomic_test_and_set: {
809 llvm::AtomicRMWInst *RMWI =
810 CGF.emitAtomicRMWInst(Op: llvm::AtomicRMWInst::Xchg, Addr: Ptr,
811 Val: CGF.Builder.getInt8(C: 1), Order, SSID: Scope, AE: E);
812 RMWI->setVolatile(E->isVolatile());
813 llvm::Value *Result = CGF.EmitToMemory(
814 Value: CGF.Builder.CreateIsNotNull(Arg: RMWI, Name: "tobool"), Ty: E->getType());
815 auto *I = CGF.Builder.CreateStore(Val: Result, Addr: Dest);
816 CGF.addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Result);
817 return;
818 }
819
820 case AtomicExpr::AO__atomic_clear: {
821 llvm::StoreInst *Store =
822 CGF.Builder.CreateStore(Val: CGF.Builder.getInt8(C: 0), Addr: Ptr);
823 Store->setAtomic(Ordering: Order, SSID: Scope);
824 Store->setVolatile(E->isVolatile());
825 CGF.addInstToCurrentSourceAtom(KeyInstruction: Store, Backup: nullptr);
826 return;
827 }
828 }
829
830 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Addr: Val1);
831 llvm::AtomicRMWInst *RMWI =
832 CGF.emitAtomicRMWInst(Op, Addr: Ptr, Val: LoadVal1, Order, SSID: Scope, AE: E);
833 RMWI->setVolatile(E->isVolatile());
834
835 // For __atomic_*_fetch operations, perform the operation again to
836 // determine the value which was written.
837 llvm::Value *Result = RMWI;
838 if (PostOpMinMax)
839 Result = EmitPostAtomicMinMax(Builder&: CGF.Builder, Op: E->getOp(),
840 IsSigned: E->getValueType()->isSignedIntegerType(),
841 OldVal: RMWI, RHS: LoadVal1);
842 else if (PostOp)
843 Result = CGF.Builder.CreateBinOp(Opc: (llvm::Instruction::BinaryOps)PostOp, LHS: RMWI,
844 RHS: LoadVal1);
845 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch ||
846 E->getOp() == AtomicExpr::AO__scoped_atomic_nand_fetch)
847 Result = CGF.Builder.CreateNot(V: Result);
848 auto *I = CGF.Builder.CreateStore(Val: Result, Addr: Dest);
849 CGF.addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Result);
850}
851
852// This function emits any expression (scalar, complex, or aggregate)
853// into a temporary alloca.
854static Address
855EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
856 Address DeclPtr = CGF.CreateMemTempWithoutCast(T: E->getType(), Name: ".atomictmp");
857 CGF.EmitAnyExprToMem(E, Location: DeclPtr, Quals: E->getType().getQualifiers(),
858 /*Init*/ IsInitializer: true);
859 return DeclPtr;
860}
861
862/// Return true if \param ValTy is a type that should be casted to integer
863/// around the atomic memory operation. If \param CmpXchg is true, then the
864/// cast of a floating point type is made as that instruction can not have
865/// floating point operands. TODO: Allow compare-and-exchange and FP - see
866/// comment in AtomicExpandPass.cpp.
867static bool shouldCastToInt(llvm::Type *ValTy, bool CmpXchg) {
868 if (ValTy->isFloatingPointTy())
869 return ValTy->isX86_FP80Ty() || CmpXchg;
870 return !ValTy->isIntegerTy() && !ValTy->isPointerTy();
871}
872
873static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
874 Address Ptr, Address Val1, Address Val2,
875 Address OriginalVal1, llvm::Value *IsWeak,
876 llvm::Value *FailureOrder, uint64_t Size,
877 llvm::AtomicOrdering Order, llvm::Value *Scope) {
878 auto ScopeModel = Expr->getScopeModel();
879
880 // LLVM atomic instructions always have sync scope. If clang atomic
881 // expression has no scope operand, use default LLVM sync scope.
882 if (!ScopeModel) {
883 llvm::SyncScope::ID SS;
884 if (CGF.getLangOpts().OpenCL)
885 // OpenCL approach is: "The functions that do not have memory_scope
886 // argument have the same semantics as the corresponding functions with
887 // the memory_scope argument set to memory_scope_device." See ref.:
888 // https://registry.khronos.org/OpenCL/specs/3.0-unified/html/OpenCL_C.html#atomic-functions
889 SS = CGF.getTargetHooks().getLLVMSyncScopeID(LangOpts: CGF.getLangOpts(),
890 Scope: SyncScope::OpenCLDevice,
891 Ordering: Order, Ctx&: CGF.getLLVMContext());
892 else
893 SS = llvm::SyncScope::System;
894 EmitAtomicOp(CGF, E: Expr, Dest, Ptr, Val1, Val2, ExpectedResult: OriginalVal1, IsWeak,
895 FailureOrder, Size, Order, Scope: SS);
896 return;
897 }
898
899 // Handle constant scope.
900 if (auto SC = dyn_cast<llvm::ConstantInt>(Val: Scope)) {
901 auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
902 LangOpts: CGF.CGM.getLangOpts(), Scope: ScopeModel->map(S: SC->getZExtValue()),
903 Ordering: Order, Ctx&: CGF.CGM.getLLVMContext());
904 EmitAtomicOp(CGF, E: Expr, Dest, Ptr, Val1, Val2, ExpectedResult: OriginalVal1, IsWeak,
905 FailureOrder, Size, Order, Scope: SCID);
906 return;
907 }
908
909 // Handle non-constant scope.
910 auto &Builder = CGF.Builder;
911 auto Scopes = ScopeModel->getRuntimeValues();
912 llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
913 for (auto S : Scopes)
914 BB[S] = CGF.createBasicBlock(name: getAsString(S: ScopeModel->map(S)), parent: CGF.CurFn);
915
916 llvm::BasicBlock *ContBB =
917 CGF.createBasicBlock(name: "atomic.scope.continue", parent: CGF.CurFn);
918
919 auto *SC = Builder.CreateIntCast(V: Scope, DestTy: Builder.getInt32Ty(), isSigned: false);
920 // If unsupported sync scope is encountered at run time, assume a fallback
921 // sync scope value.
922 auto FallBack = ScopeModel->getFallBackValue();
923 llvm::SwitchInst *SI = Builder.CreateSwitch(V: SC, Dest: BB[FallBack]);
924 for (auto S : Scopes) {
925 auto *B = BB[S];
926 if (S != FallBack)
927 SI->addCase(OnVal: Builder.getInt32(C: S), Dest: B);
928
929 Builder.SetInsertPoint(B);
930 EmitAtomicOp(CGF, E: Expr, Dest, Ptr, Val1, Val2, ExpectedResult: OriginalVal1, IsWeak,
931 FailureOrder, Size, Order,
932 Scope: CGF.getTargetHooks().getLLVMSyncScopeID(
933 LangOpts: CGF.CGM.getLangOpts(), Scope: ScopeModel->map(S), Ordering: Order,
934 Ctx&: CGF.getLLVMContext()));
935 Builder.CreateBr(Dest: ContBB);
936 }
937
938 Builder.SetInsertPoint(ContBB);
939}
940
941RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
942 ApplyAtomGroup Grp(getDebugInfo());
943
944 QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
945 QualType MemTy = AtomicTy;
946 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
947 MemTy = AT->getValueType();
948 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
949
950 Address Val1 = Address::invalid();
951 Address Val2 = Address::invalid();
952 Address Dest = Address::invalid();
953 Address Ptr = EmitPointerWithAlignment(Addr: E->getPtr());
954
955 if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
956 E->getOp() == AtomicExpr::AO__opencl_atomic_init) {
957 LValue lvalue = MakeAddrLValue(Addr: Ptr, T: AtomicTy);
958 EmitAtomicInit(E: E->getVal1(), lvalue);
959 return RValue::get(V: nullptr);
960 }
961
962 auto TInfo = getContext().getTypeInfoInChars(T: AtomicTy);
963 uint64_t Size = TInfo.Width.getQuantity();
964 unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
965
966 CharUnits MaxInlineWidth =
967 getContext().toCharUnitsFromBits(BitSize: MaxInlineWidthInBits);
968 DiagnosticsEngine &Diags = CGM.getDiags();
969 bool Misaligned = !Ptr.getAlignment().isMultipleOf(N: TInfo.Width);
970 bool Oversized = getContext().toBits(CharSize: TInfo.Width) > MaxInlineWidthInBits;
971 if (Misaligned) {
972 Diags.Report(Loc: E->getBeginLoc(), DiagID: diag::warn_atomic_op_misaligned)
973 << (int)TInfo.Width.getQuantity()
974 << (int)Ptr.getAlignment().getQuantity();
975 }
976 if (Oversized) {
977 Diags.Report(Loc: E->getBeginLoc(), DiagID: diag::warn_atomic_op_oversized)
978 << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity();
979 }
980
981 llvm::Value *Order = EmitScalarExpr(E: E->getOrder());
982 llvm::Value *Scope =
983 E->getScopeModel() ? EmitScalarExpr(E: E->getScope()) : nullptr;
984
985 switch (E->getOp()) {
986 case AtomicExpr::AO__c11_atomic_init:
987 case AtomicExpr::AO__opencl_atomic_init:
988 llvm_unreachable("Already handled above with EmitAtomicInit!");
989
990 case AtomicExpr::AO__atomic_load_n:
991 case AtomicExpr::AO__scoped_atomic_load_n:
992 case AtomicExpr::AO__c11_atomic_load:
993 case AtomicExpr::AO__opencl_atomic_load:
994 case AtomicExpr::AO__hip_atomic_load:
995 case AtomicExpr::AO__atomic_test_and_set:
996 case AtomicExpr::AO__atomic_clear:
997 break;
998
999 case AtomicExpr::AO__atomic_load:
1000 case AtomicExpr::AO__scoped_atomic_load:
1001 Dest = EmitPointerWithAlignment(Addr: E->getVal1());
1002 break;
1003
1004 case AtomicExpr::AO__atomic_store:
1005 case AtomicExpr::AO__scoped_atomic_store:
1006 Val1 = EmitPointerWithAlignment(Addr: E->getVal1());
1007 break;
1008
1009 case AtomicExpr::AO__atomic_exchange:
1010 case AtomicExpr::AO__scoped_atomic_exchange:
1011 Val1 = EmitPointerWithAlignment(Addr: E->getVal1());
1012 Dest = EmitPointerWithAlignment(Addr: E->getVal2());
1013 break;
1014
1015 case AtomicExpr::AO__atomic_compare_exchange:
1016 case AtomicExpr::AO__atomic_compare_exchange_n:
1017 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1018 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1019 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1020 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1021 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1022 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1023 case AtomicExpr::AO__scoped_atomic_compare_exchange:
1024 case AtomicExpr::AO__scoped_atomic_compare_exchange_n:
1025 Val1 = EmitPointerWithAlignment(Addr: E->getVal1());
1026 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange ||
1027 E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)
1028 Val2 = EmitPointerWithAlignment(Addr: E->getVal2());
1029 else
1030 Val2 = EmitValToTemp(CGF&: *this, E: E->getVal2());
1031 OrderFail = EmitScalarExpr(E: E->getOrderFail());
1032 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
1033 E->getOp() == AtomicExpr::AO__atomic_compare_exchange ||
1034 E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange_n ||
1035 E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)
1036 IsWeak = EmitScalarExpr(E: E->getWeak());
1037 break;
1038
1039 case AtomicExpr::AO__c11_atomic_fetch_add:
1040 case AtomicExpr::AO__c11_atomic_fetch_sub:
1041 case AtomicExpr::AO__hip_atomic_fetch_add:
1042 case AtomicExpr::AO__hip_atomic_fetch_sub:
1043 case AtomicExpr::AO__opencl_atomic_fetch_add:
1044 case AtomicExpr::AO__opencl_atomic_fetch_sub:
1045 if (MemTy->isPointerType()) {
1046 // For pointer arithmetic, we're required to do a bit of math:
1047 // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
1048 // ... but only for the C11 builtins. The GNU builtins expect the
1049 // user to multiply by sizeof(T).
1050 QualType Val1Ty = E->getVal1()->getType();
1051 llvm::Value *Val1Scalar = EmitScalarExpr(E: E->getVal1());
1052 CharUnits PointeeIncAmt =
1053 getContext().getTypeSizeInChars(T: MemTy->getPointeeType());
1054 Val1Scalar = Builder.CreateMul(LHS: Val1Scalar, RHS: CGM.getSize(numChars: PointeeIncAmt));
1055 auto Temp = CreateMemTempWithoutCast(T: Val1Ty, Name: ".atomictmp");
1056 Val1 = Temp;
1057 EmitStoreOfScalar(value: Val1Scalar, lvalue: MakeAddrLValue(Addr: Temp, T: Val1Ty));
1058 break;
1059 }
1060 [[fallthrough]];
1061 case AtomicExpr::AO__atomic_fetch_add:
1062 case AtomicExpr::AO__atomic_fetch_max:
1063 case AtomicExpr::AO__atomic_fetch_min:
1064 case AtomicExpr::AO__atomic_fetch_sub:
1065 case AtomicExpr::AO__atomic_add_fetch:
1066 case AtomicExpr::AO__atomic_max_fetch:
1067 case AtomicExpr::AO__atomic_min_fetch:
1068 case AtomicExpr::AO__atomic_sub_fetch:
1069 case AtomicExpr::AO__c11_atomic_fetch_max:
1070 case AtomicExpr::AO__c11_atomic_fetch_min:
1071 case AtomicExpr::AO__opencl_atomic_fetch_max:
1072 case AtomicExpr::AO__opencl_atomic_fetch_min:
1073 case AtomicExpr::AO__hip_atomic_fetch_max:
1074 case AtomicExpr::AO__hip_atomic_fetch_min:
1075 case AtomicExpr::AO__scoped_atomic_fetch_add:
1076 case AtomicExpr::AO__scoped_atomic_fetch_max:
1077 case AtomicExpr::AO__scoped_atomic_fetch_min:
1078 case AtomicExpr::AO__scoped_atomic_fetch_sub:
1079 case AtomicExpr::AO__scoped_atomic_fetch_fminimum:
1080 case AtomicExpr::AO__scoped_atomic_fetch_fmaximum:
1081 case AtomicExpr::AO__scoped_atomic_fetch_fminimum_num:
1082 case AtomicExpr::AO__scoped_atomic_fetch_fmaximum_num:
1083 case AtomicExpr::AO__scoped_atomic_add_fetch:
1084 case AtomicExpr::AO__scoped_atomic_max_fetch:
1085 case AtomicExpr::AO__scoped_atomic_min_fetch:
1086 case AtomicExpr::AO__scoped_atomic_sub_fetch:
1087 [[fallthrough]];
1088
1089 case AtomicExpr::AO__atomic_fetch_and:
1090 case AtomicExpr::AO__atomic_fetch_nand:
1091 case AtomicExpr::AO__atomic_fetch_or:
1092 case AtomicExpr::AO__atomic_fetch_xor:
1093 case AtomicExpr::AO__atomic_fetch_uinc:
1094 case AtomicExpr::AO__atomic_fetch_udec:
1095 case AtomicExpr::AO__atomic_and_fetch:
1096 case AtomicExpr::AO__atomic_nand_fetch:
1097 case AtomicExpr::AO__atomic_or_fetch:
1098 case AtomicExpr::AO__atomic_xor_fetch:
1099 case AtomicExpr::AO__atomic_store_n:
1100 case AtomicExpr::AO__atomic_exchange_n:
1101 case AtomicExpr::AO__c11_atomic_fetch_and:
1102 case AtomicExpr::AO__c11_atomic_fetch_nand:
1103 case AtomicExpr::AO__c11_atomic_fetch_or:
1104 case AtomicExpr::AO__c11_atomic_fetch_xor:
1105 case AtomicExpr::AO__c11_atomic_store:
1106 case AtomicExpr::AO__c11_atomic_exchange:
1107 case AtomicExpr::AO__hip_atomic_fetch_and:
1108 case AtomicExpr::AO__hip_atomic_fetch_or:
1109 case AtomicExpr::AO__hip_atomic_fetch_xor:
1110 case AtomicExpr::AO__hip_atomic_store:
1111 case AtomicExpr::AO__hip_atomic_exchange:
1112 case AtomicExpr::AO__opencl_atomic_fetch_and:
1113 case AtomicExpr::AO__opencl_atomic_fetch_or:
1114 case AtomicExpr::AO__opencl_atomic_fetch_xor:
1115 case AtomicExpr::AO__opencl_atomic_store:
1116 case AtomicExpr::AO__opencl_atomic_exchange:
1117 case AtomicExpr::AO__scoped_atomic_fetch_and:
1118 case AtomicExpr::AO__scoped_atomic_fetch_nand:
1119 case AtomicExpr::AO__scoped_atomic_fetch_or:
1120 case AtomicExpr::AO__scoped_atomic_fetch_xor:
1121 case AtomicExpr::AO__scoped_atomic_and_fetch:
1122 case AtomicExpr::AO__scoped_atomic_nand_fetch:
1123 case AtomicExpr::AO__scoped_atomic_or_fetch:
1124 case AtomicExpr::AO__scoped_atomic_xor_fetch:
1125 case AtomicExpr::AO__scoped_atomic_store_n:
1126 case AtomicExpr::AO__scoped_atomic_exchange_n:
1127 case AtomicExpr::AO__scoped_atomic_fetch_uinc:
1128 case AtomicExpr::AO__scoped_atomic_fetch_udec:
1129 case AtomicExpr::AO__atomic_fetch_fminimum:
1130 case AtomicExpr::AO__atomic_fetch_fmaximum:
1131 case AtomicExpr::AO__atomic_fetch_fminimum_num:
1132 case AtomicExpr::AO__atomic_fetch_fmaximum_num:
1133 Val1 = EmitValToTemp(CGF&: *this, E: E->getVal1());
1134 break;
1135 }
1136
1137 QualType RValTy = E->getType().getUnqualifiedType();
1138 bool ShouldCastToIntPtrTy =
1139 shouldCastToInt(ValTy: ConvertTypeForMem(T: MemTy), CmpXchg: E->isCmpXChg());
1140
1141 // The inlined atomics only function on iN types, where N is a power of 2. We
1142 // need to make sure (via temporaries if necessary) that all incoming values
1143 // are compatible.
1144 LValue AtomicVal = MakeAddrLValue(Addr: Ptr, T: AtomicTy);
1145 AtomicInfo Atomics(*this, AtomicVal);
1146
1147 Address OriginalVal1 = Val1;
1148 if (ShouldCastToIntPtrTy) {
1149 Ptr = Atomics.castToAtomicIntPointer(Addr: Ptr);
1150 if (Val1.isValid())
1151 Val1 = Atomics.convertToAtomicIntPointer(Addr: Val1);
1152 if (Val2.isValid())
1153 Val2 = Atomics.convertToAtomicIntPointer(Addr: Val2);
1154 }
1155 if (Dest.isValid()) {
1156 if (ShouldCastToIntPtrTy)
1157 Dest = Atomics.castToAtomicIntPointer(Addr: Dest);
1158 } else if (E->isCmpXChg())
1159 Dest = CreateMemTempWithoutCast(T: RValTy, Name: "cmpxchg.bool");
1160 else if (!RValTy->isVoidType()) {
1161 Dest = Atomics.CreateTempAlloca();
1162 if (ShouldCastToIntPtrTy)
1163 Dest = Atomics.castToAtomicIntPointer(Addr: Dest);
1164 }
1165
1166 bool PowerOf2Size = (Size & (Size - 1)) == 0;
1167 bool UseLibcall = !PowerOf2Size || (Size > 16);
1168
1169 // For atomics larger than 16 bytes, emit a libcall from the frontend. This
1170 // avoids the overhead of dealing with excessively-large value types in IR.
1171 // Non-power-of-2 values also lower to libcall here, as they are not currently
1172 // permitted in IR instructions (although that constraint could be relaxed in
1173 // the future). For other cases where a libcall is required on a given
1174 // platform, we let the backend handle it (this includes handling for all of
1175 // the size-optimized libcall variants, which are only valid up to 16 bytes.)
1176 //
1177 // See: https://llvm.org/docs/Atomics.html#libcalls-atomic
1178 if (UseLibcall) {
1179 CallArgList Args;
1180 // For non-optimized library calls, the size is the first parameter.
1181 Args.add(rvalue: RValue::get(V: llvm::ConstantInt::get(Ty: SizeTy, V: Size)),
1182 type: getContext().getSizeType());
1183
1184 // The atomic address is the second parameter.
1185 // The OpenCL atomic library functions only accept pointer arguments to
1186 // generic address space.
1187 auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
1188 if (!E->isOpenCL())
1189 return V;
1190 auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace();
1191 if (AS == LangAS::opencl_generic)
1192 return V;
1193 auto DestAS = getContext().getTargetAddressSpace(AS: LangAS::opencl_generic);
1194 auto *DestType = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: DestAS);
1195
1196 return performAddrSpaceCast(Src: V, DestTy: DestType);
1197 };
1198
1199 Args.add(rvalue: RValue::get(V: CastToGenericAddrSpace(Ptr.emitRawPointer(CGF&: *this),
1200 E->getPtr()->getType())),
1201 type: getContext().VoidPtrTy);
1202
1203 // The next 1-3 parameters are op-dependent.
1204 std::string LibCallName;
1205 QualType RetTy;
1206 bool HaveRetTy = false;
1207 switch (E->getOp()) {
1208 case AtomicExpr::AO__c11_atomic_init:
1209 case AtomicExpr::AO__opencl_atomic_init:
1210 llvm_unreachable("Already handled!");
1211
1212 // There is only one libcall for compare an exchange, because there is no
1213 // optimisation benefit possible from a libcall version of a weak compare
1214 // and exchange.
1215 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1216 // void *desired, int success, int failure)
1217 case AtomicExpr::AO__atomic_compare_exchange:
1218 case AtomicExpr::AO__atomic_compare_exchange_n:
1219 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1220 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1221 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1222 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1223 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1224 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1225 case AtomicExpr::AO__scoped_atomic_compare_exchange:
1226 case AtomicExpr::AO__scoped_atomic_compare_exchange_n:
1227 LibCallName = "__atomic_compare_exchange";
1228 RetTy = getContext().BoolTy;
1229 HaveRetTy = true;
1230 Args.add(rvalue: RValue::get(V: CastToGenericAddrSpace(Val1.emitRawPointer(CGF&: *this),
1231 E->getVal1()->getType())),
1232 type: getContext().VoidPtrTy);
1233 Args.add(rvalue: RValue::get(V: CastToGenericAddrSpace(Val2.emitRawPointer(CGF&: *this),
1234 E->getVal2()->getType())),
1235 type: getContext().VoidPtrTy);
1236 Args.add(rvalue: RValue::get(V: Order), type: getContext().IntTy);
1237 Order = OrderFail;
1238 break;
1239 // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1240 // int order)
1241 case AtomicExpr::AO__atomic_exchange:
1242 case AtomicExpr::AO__atomic_exchange_n:
1243 case AtomicExpr::AO__c11_atomic_exchange:
1244 case AtomicExpr::AO__hip_atomic_exchange:
1245 case AtomicExpr::AO__opencl_atomic_exchange:
1246 case AtomicExpr::AO__scoped_atomic_exchange:
1247 case AtomicExpr::AO__scoped_atomic_exchange_n:
1248 LibCallName = "__atomic_exchange";
1249 Args.add(rvalue: RValue::get(V: CastToGenericAddrSpace(Val1.emitRawPointer(CGF&: *this),
1250 E->getVal1()->getType())),
1251 type: getContext().VoidPtrTy);
1252 break;
1253 // void __atomic_store(size_t size, void *mem, void *val, int order)
1254 case AtomicExpr::AO__atomic_store:
1255 case AtomicExpr::AO__atomic_store_n:
1256 case AtomicExpr::AO__c11_atomic_store:
1257 case AtomicExpr::AO__hip_atomic_store:
1258 case AtomicExpr::AO__opencl_atomic_store:
1259 case AtomicExpr::AO__scoped_atomic_store:
1260 case AtomicExpr::AO__scoped_atomic_store_n:
1261 LibCallName = "__atomic_store";
1262 RetTy = getContext().VoidTy;
1263 HaveRetTy = true;
1264 Args.add(rvalue: RValue::get(V: CastToGenericAddrSpace(Val1.emitRawPointer(CGF&: *this),
1265 E->getVal1()->getType())),
1266 type: getContext().VoidPtrTy);
1267 break;
1268 // void __atomic_load(size_t size, void *mem, void *return, int order)
1269 case AtomicExpr::AO__atomic_load:
1270 case AtomicExpr::AO__atomic_load_n:
1271 case AtomicExpr::AO__c11_atomic_load:
1272 case AtomicExpr::AO__hip_atomic_load:
1273 case AtomicExpr::AO__opencl_atomic_load:
1274 case AtomicExpr::AO__scoped_atomic_load:
1275 case AtomicExpr::AO__scoped_atomic_load_n:
1276 LibCallName = "__atomic_load";
1277 break;
1278 case AtomicExpr::AO__atomic_add_fetch:
1279 case AtomicExpr::AO__scoped_atomic_add_fetch:
1280 case AtomicExpr::AO__atomic_fetch_add:
1281 case AtomicExpr::AO__c11_atomic_fetch_add:
1282 case AtomicExpr::AO__hip_atomic_fetch_add:
1283 case AtomicExpr::AO__opencl_atomic_fetch_add:
1284 case AtomicExpr::AO__scoped_atomic_fetch_add:
1285 case AtomicExpr::AO__atomic_and_fetch:
1286 case AtomicExpr::AO__scoped_atomic_and_fetch:
1287 case AtomicExpr::AO__atomic_fetch_and:
1288 case AtomicExpr::AO__c11_atomic_fetch_and:
1289 case AtomicExpr::AO__hip_atomic_fetch_and:
1290 case AtomicExpr::AO__opencl_atomic_fetch_and:
1291 case AtomicExpr::AO__scoped_atomic_fetch_and:
1292 case AtomicExpr::AO__atomic_or_fetch:
1293 case AtomicExpr::AO__scoped_atomic_or_fetch:
1294 case AtomicExpr::AO__atomic_fetch_or:
1295 case AtomicExpr::AO__c11_atomic_fetch_or:
1296 case AtomicExpr::AO__hip_atomic_fetch_or:
1297 case AtomicExpr::AO__opencl_atomic_fetch_or:
1298 case AtomicExpr::AO__scoped_atomic_fetch_or:
1299 case AtomicExpr::AO__atomic_sub_fetch:
1300 case AtomicExpr::AO__scoped_atomic_sub_fetch:
1301 case AtomicExpr::AO__atomic_fetch_sub:
1302 case AtomicExpr::AO__c11_atomic_fetch_sub:
1303 case AtomicExpr::AO__hip_atomic_fetch_sub:
1304 case AtomicExpr::AO__opencl_atomic_fetch_sub:
1305 case AtomicExpr::AO__scoped_atomic_fetch_sub:
1306 case AtomicExpr::AO__atomic_xor_fetch:
1307 case AtomicExpr::AO__scoped_atomic_xor_fetch:
1308 case AtomicExpr::AO__atomic_fetch_xor:
1309 case AtomicExpr::AO__c11_atomic_fetch_xor:
1310 case AtomicExpr::AO__hip_atomic_fetch_xor:
1311 case AtomicExpr::AO__opencl_atomic_fetch_xor:
1312 case AtomicExpr::AO__scoped_atomic_fetch_xor:
1313 case AtomicExpr::AO__atomic_nand_fetch:
1314 case AtomicExpr::AO__atomic_fetch_nand:
1315 case AtomicExpr::AO__c11_atomic_fetch_nand:
1316 case AtomicExpr::AO__scoped_atomic_fetch_nand:
1317 case AtomicExpr::AO__scoped_atomic_nand_fetch:
1318 case AtomicExpr::AO__atomic_min_fetch:
1319 case AtomicExpr::AO__atomic_fetch_min:
1320 case AtomicExpr::AO__c11_atomic_fetch_min:
1321 case AtomicExpr::AO__hip_atomic_fetch_min:
1322 case AtomicExpr::AO__opencl_atomic_fetch_min:
1323 case AtomicExpr::AO__scoped_atomic_fetch_min:
1324 case AtomicExpr::AO__scoped_atomic_min_fetch:
1325 case AtomicExpr::AO__atomic_max_fetch:
1326 case AtomicExpr::AO__atomic_fetch_max:
1327 case AtomicExpr::AO__c11_atomic_fetch_max:
1328 case AtomicExpr::AO__hip_atomic_fetch_max:
1329 case AtomicExpr::AO__opencl_atomic_fetch_max:
1330 case AtomicExpr::AO__scoped_atomic_fetch_max:
1331 case AtomicExpr::AO__scoped_atomic_max_fetch:
1332 case AtomicExpr::AO__atomic_fetch_fminimum:
1333 case AtomicExpr::AO__atomic_fetch_fmaximum:
1334 case AtomicExpr::AO__atomic_fetch_fminimum_num:
1335 case AtomicExpr::AO__atomic_fetch_fmaximum_num:
1336 case AtomicExpr::AO__scoped_atomic_fetch_fminimum:
1337 case AtomicExpr::AO__scoped_atomic_fetch_fmaximum:
1338 case AtomicExpr::AO__scoped_atomic_fetch_fminimum_num:
1339 case AtomicExpr::AO__scoped_atomic_fetch_fmaximum_num:
1340 case AtomicExpr::AO__scoped_atomic_fetch_uinc:
1341 case AtomicExpr::AO__scoped_atomic_fetch_udec:
1342 case AtomicExpr::AO__atomic_test_and_set:
1343 case AtomicExpr::AO__atomic_clear:
1344 case AtomicExpr::AO__atomic_fetch_uinc:
1345 case AtomicExpr::AO__atomic_fetch_udec:
1346 llvm_unreachable("Integral atomic operations always become atomicrmw!");
1347 }
1348
1349 if (E->isOpenCL()) {
1350 LibCallName =
1351 std::string("__opencl") + StringRef(LibCallName).drop_front(N: 1).str();
1352 }
1353 // By default, assume we return a value of the atomic type.
1354 if (!HaveRetTy) {
1355 // Value is returned through parameter before the order.
1356 RetTy = getContext().VoidTy;
1357 Args.add(rvalue: RValue::get(
1358 V: CastToGenericAddrSpace(Dest.emitRawPointer(CGF&: *this), RetTy)),
1359 type: getContext().VoidPtrTy);
1360 }
1361 // Order is always the last parameter.
1362 Args.add(rvalue: RValue::get(V: Order),
1363 type: getContext().IntTy);
1364 if (E->isOpenCL())
1365 Args.add(rvalue: RValue::get(V: Scope), type: getContext().IntTy);
1366
1367 RValue Res = emitAtomicLibcall(CGF&: *this, fnName: LibCallName, resultType: RetTy, args&: Args);
1368 // The value is returned directly from the libcall.
1369 if (E->isCmpXChg())
1370 return Res;
1371
1372 if (RValTy->isVoidType())
1373 return RValue::get(V: nullptr);
1374
1375 return convertTempToRValue(addr: Dest.withElementType(ElemTy: ConvertTypeForMem(T: RValTy)),
1376 type: RValTy, Loc: E->getExprLoc());
1377 }
1378
1379 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1380 E->getOp() == AtomicExpr::AO__opencl_atomic_store ||
1381 E->getOp() == AtomicExpr::AO__hip_atomic_store ||
1382 E->getOp() == AtomicExpr::AO__atomic_store ||
1383 E->getOp() == AtomicExpr::AO__atomic_store_n ||
1384 E->getOp() == AtomicExpr::AO__scoped_atomic_store ||
1385 E->getOp() == AtomicExpr::AO__scoped_atomic_store_n ||
1386 E->getOp() == AtomicExpr::AO__atomic_clear;
1387 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1388 E->getOp() == AtomicExpr::AO__opencl_atomic_load ||
1389 E->getOp() == AtomicExpr::AO__hip_atomic_load ||
1390 E->getOp() == AtomicExpr::AO__atomic_load ||
1391 E->getOp() == AtomicExpr::AO__atomic_load_n ||
1392 E->getOp() == AtomicExpr::AO__scoped_atomic_load ||
1393 E->getOp() == AtomicExpr::AO__scoped_atomic_load_n;
1394
1395 if (isa<llvm::ConstantInt>(Val: Order)) {
1396 auto ord = cast<llvm::ConstantInt>(Val: Order)->getZExtValue();
1397 // We should not ever get to a case where the ordering isn't a valid C ABI
1398 // value, but it's hard to enforce that in general.
1399 if (llvm::isValidAtomicOrderingCABI(I: ord))
1400 switch ((llvm::AtomicOrderingCABI)ord) {
1401 case llvm::AtomicOrderingCABI::relaxed:
1402 EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,
1403 FailureOrder: OrderFail, Size, Order: llvm::AtomicOrdering::Monotonic, Scope);
1404 break;
1405 case llvm::AtomicOrderingCABI::consume:
1406 case llvm::AtomicOrderingCABI::acquire:
1407 if (IsStore)
1408 break; // Avoid crashing on code with undefined behavior
1409 EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,
1410 FailureOrder: OrderFail, Size, Order: llvm::AtomicOrdering::Acquire, Scope);
1411 break;
1412 case llvm::AtomicOrderingCABI::release:
1413 if (IsLoad)
1414 break; // Avoid crashing on code with undefined behavior
1415 EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,
1416 FailureOrder: OrderFail, Size, Order: llvm::AtomicOrdering::Release, Scope);
1417 break;
1418 case llvm::AtomicOrderingCABI::acq_rel:
1419 if (IsLoad || IsStore)
1420 break; // Avoid crashing on code with undefined behavior
1421 EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,
1422 FailureOrder: OrderFail, Size, Order: llvm::AtomicOrdering::AcquireRelease,
1423 Scope);
1424 break;
1425 case llvm::AtomicOrderingCABI::seq_cst:
1426 EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,
1427 FailureOrder: OrderFail, Size,
1428 Order: llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1429 break;
1430 }
1431 if (RValTy->isVoidType())
1432 return RValue::get(V: nullptr);
1433
1434 return convertTempToRValue(addr: Dest.withElementType(ElemTy: ConvertTypeForMem(T: RValTy)),
1435 type: RValTy, Loc: E->getExprLoc());
1436 }
1437
1438 // Long case, when Order isn't obviously constant.
1439
1440 // Create all the relevant BB's
1441 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1442 *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1443 *SeqCstBB = nullptr;
1444 MonotonicBB = createBasicBlock(name: "monotonic", parent: CurFn);
1445 if (!IsStore)
1446 AcquireBB = createBasicBlock(name: "acquire", parent: CurFn);
1447 if (!IsLoad)
1448 ReleaseBB = createBasicBlock(name: "release", parent: CurFn);
1449 if (!IsLoad && !IsStore)
1450 AcqRelBB = createBasicBlock(name: "acqrel", parent: CurFn);
1451 SeqCstBB = createBasicBlock(name: "seqcst", parent: CurFn);
1452 llvm::BasicBlock *ContBB = createBasicBlock(name: "atomic.continue", parent: CurFn);
1453
1454 // Create the switch for the split
1455 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1456 // doesn't matter unless someone is crazy enough to use something that
1457 // doesn't fold to a constant for the ordering.
1458 Order = Builder.CreateIntCast(V: Order, DestTy: Builder.getInt32Ty(), isSigned: false);
1459 llvm::SwitchInst *SI = Builder.CreateSwitch(V: Order, Dest: MonotonicBB);
1460
1461 // Emit all the different atomics
1462 Builder.SetInsertPoint(MonotonicBB);
1463 EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak, FailureOrder: OrderFail,
1464 Size, Order: llvm::AtomicOrdering::Monotonic, Scope);
1465 Builder.CreateBr(Dest: ContBB);
1466 if (!IsStore) {
1467 Builder.SetInsertPoint(AcquireBB);
1468 EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,
1469 FailureOrder: OrderFail, Size, Order: llvm::AtomicOrdering::Acquire, Scope);
1470 Builder.CreateBr(Dest: ContBB);
1471 SI->addCase(OnVal: Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::consume),
1472 Dest: AcquireBB);
1473 SI->addCase(OnVal: Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::acquire),
1474 Dest: AcquireBB);
1475 }
1476 if (!IsLoad) {
1477 Builder.SetInsertPoint(ReleaseBB);
1478 EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,
1479 FailureOrder: OrderFail, Size, Order: llvm::AtomicOrdering::Release, Scope);
1480 Builder.CreateBr(Dest: ContBB);
1481 SI->addCase(OnVal: Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::release),
1482 Dest: ReleaseBB);
1483 }
1484 if (!IsLoad && !IsStore) {
1485 Builder.SetInsertPoint(AcqRelBB);
1486 EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,
1487 FailureOrder: OrderFail, Size, Order: llvm::AtomicOrdering::AcquireRelease, Scope);
1488 Builder.CreateBr(Dest: ContBB);
1489 SI->addCase(OnVal: Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::acq_rel),
1490 Dest: AcqRelBB);
1491 }
1492 Builder.SetInsertPoint(SeqCstBB);
1493 EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak, FailureOrder: OrderFail,
1494 Size, Order: llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1495 Builder.CreateBr(Dest: ContBB);
1496 SI->addCase(OnVal: Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::seq_cst),
1497 Dest: SeqCstBB);
1498
1499 // Cleanup and return
1500 Builder.SetInsertPoint(ContBB);
1501 if (RValTy->isVoidType())
1502 return RValue::get(V: nullptr);
1503
1504 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1505 return convertTempToRValue(addr: Dest.withElementType(ElemTy: ConvertTypeForMem(T: RValTy)),
1506 type: RValTy, Loc: E->getExprLoc());
1507}
1508
1509Address AtomicInfo::castToAtomicIntPointer(Address addr) const {
1510 llvm::IntegerType *ty =
1511 llvm::IntegerType::get(C&: CGF.getLLVMContext(), NumBits: AtomicSizeInBits);
1512 return addr.withElementType(ElemTy: ty);
1513}
1514
1515Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1516 llvm::Type *Ty = Addr.getElementType();
1517 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1518 if (SourceSizeInBits != AtomicSizeInBits) {
1519 Address Tmp = CreateTempAlloca();
1520 CGF.Builder.CreateMemSet(
1521 Ptr: Tmp.emitRawPointer(CGF), Val: llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 0),
1522 Size: CGF.getContext().toCharUnitsFromBits(BitSize: AtomicSizeInBits).getQuantity(),
1523 Align: Tmp.getAlignment().getAsAlign());
1524
1525 CGF.Builder.CreateMemCpy(Dest: Tmp, Src: Addr,
1526 Size: std::min(a: AtomicSizeInBits, b: SourceSizeInBits) / 8);
1527 Addr = Tmp;
1528 }
1529
1530 return castToAtomicIntPointer(addr: Addr);
1531}
1532
1533RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1534 AggValueSlot resultSlot,
1535 SourceLocation loc,
1536 bool asValue) const {
1537 if (LVal.isSimple()) {
1538 if (EvaluationKind == TEK_Aggregate)
1539 return resultSlot.asRValue();
1540
1541 // Drill into the padding structure if we have one.
1542 if (hasPadding())
1543 addr = CGF.Builder.CreateStructGEP(Addr: addr, Index: 0);
1544
1545 // Otherwise, just convert the temporary to an r-value using the
1546 // normal conversion routine.
1547 return CGF.convertTempToRValue(addr, type: getValueType(), Loc: loc);
1548 }
1549 if (!asValue)
1550 // Get RValue from temp memory as atomic for non-simple lvalues
1551 return RValue::get(V: CGF.Builder.CreateLoad(Addr: addr));
1552 if (LVal.isBitField())
1553 return CGF.EmitLoadOfBitfieldLValue(
1554 LV: LValue::MakeBitfield(Addr: addr, Info: LVal.getBitFieldInfo(), type: LVal.getType(),
1555 BaseInfo: LVal.getBaseInfo(), TBAAInfo: TBAAAccessInfo()), Loc: loc);
1556 if (LVal.isVectorElt())
1557 return CGF.EmitLoadOfLValue(
1558 V: LValue::MakeVectorElt(vecAddress: addr, Idx: LVal.getVectorIdx(), type: LVal.getType(),
1559 BaseInfo: LVal.getBaseInfo(), TBAAInfo: TBAAAccessInfo()), Loc: loc);
1560 assert(LVal.isExtVectorElt());
1561 return CGF.EmitLoadOfExtVectorElementLValue(V: LValue::MakeExtVectorElt(
1562 Addr: addr, Elts: LVal.getExtVectorElts(), type: LVal.getType(),
1563 BaseInfo: LVal.getBaseInfo(), TBAAInfo: TBAAAccessInfo()));
1564}
1565
1566RValue AtomicInfo::ConvertToValueOrAtomic(llvm::Value *Val,
1567 AggValueSlot ResultSlot,
1568 SourceLocation Loc, bool AsValue,
1569 bool CmpXchg) const {
1570 // Try not to in some easy cases.
1571 assert((Val->getType()->isIntegerTy() || Val->getType()->isPointerTy() ||
1572 Val->getType()->isIEEELikeFPTy()) &&
1573 "Expected integer, pointer or floating point value when converting "
1574 "result.");
1575 if (getEvaluationKind() == TEK_Scalar &&
1576 (((!LVal.isBitField() ||
1577 LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1578 !hasPadding()) ||
1579 !AsValue)) {
1580 auto *ValTy = AsValue
1581 ? CGF.ConvertTypeForMem(T: ValueTy)
1582 : getAtomicAddress().getElementType();
1583 if (!shouldCastToInt(ValTy, CmpXchg)) {
1584 assert((!ValTy->isIntegerTy() || Val->getType() == ValTy) &&
1585 "Different integer types.");
1586 return RValue::get(V: CGF.EmitFromMemory(Value: Val, Ty: ValueTy));
1587 }
1588 if (llvm::CastInst::isBitCastable(SrcTy: Val->getType(), DestTy: ValTy))
1589 return RValue::get(V: CGF.Builder.CreateBitCast(V: Val, DestTy: ValTy));
1590 }
1591
1592 // Create a temporary. This needs to be big enough to hold the
1593 // atomic integer.
1594 Address Temp = Address::invalid();
1595 bool TempIsVolatile = false;
1596 if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1597 assert(!ResultSlot.isIgnored());
1598 Temp = ResultSlot.getAddress();
1599 TempIsVolatile = ResultSlot.isVolatile();
1600 } else {
1601 Temp = CreateTempAlloca();
1602 }
1603
1604 // Slam the integer into the temporary.
1605 Address CastTemp = castToAtomicIntPointer(addr: Temp);
1606 CGF.Builder.CreateStore(Val, Addr: CastTemp)->setVolatile(TempIsVolatile);
1607
1608 return convertAtomicTempToRValue(addr: Temp, resultSlot: ResultSlot, loc: Loc, asValue: AsValue);
1609}
1610
1611void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1612 llvm::AtomicOrdering AO, bool) {
1613 // void __atomic_load(size_t size, void *mem, void *return, int order);
1614 CallArgList Args;
1615 Args.add(rvalue: RValue::get(V: getAtomicSizeValue()), type: CGF.getContext().getSizeType());
1616 Args.add(rvalue: RValue::get(V: getAtomicPointer()), type: CGF.getContext().VoidPtrTy);
1617 Args.add(rvalue: RValue::get(V: AddForLoaded), type: CGF.getContext().VoidPtrTy);
1618 Args.add(
1619 rvalue: RValue::get(V: llvm::ConstantInt::get(Ty: CGF.IntTy, V: (int)llvm::toCABI(AO))),
1620 type: CGF.getContext().IntTy);
1621 emitAtomicLibcall(CGF, fnName: "__atomic_load", resultType: CGF.getContext().VoidTy, args&: Args);
1622}
1623
1624llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1625 bool IsVolatile, bool CmpXchg) {
1626 // Okay, we're doing this natively.
1627 Address Addr = getAtomicAddress();
1628 if (shouldCastToInt(ValTy: Addr.getElementType(), CmpXchg))
1629 Addr = castToAtomicIntPointer(addr: Addr);
1630 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, Name: "atomic-load");
1631 Load->setAtomic(Ordering: AO);
1632
1633 // Other decoration.
1634 if (IsVolatile)
1635 Load->setVolatile(true);
1636 CGF.CGM.DecorateInstructionWithTBAA(Inst: Load, TBAAInfo: LVal.getTBAAInfo());
1637 return Load;
1638}
1639
1640/// An LValue is a candidate for having its loads and stores be made atomic if
1641/// we are operating under /volatile:ms *and* the LValue itself is volatile and
1642/// performing such an operation can be performed without a libcall.
1643bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1644 if (!CGM.getLangOpts().MSVolatile) return false;
1645 AtomicInfo AI(*this, LV);
1646 bool IsVolatile = LV.isVolatile() || hasVolatileMember(T: LV.getType());
1647 // An atomic is inline if we don't need to use a libcall.
1648 bool AtomicIsInline = !AI.shouldUseLibcall();
1649 // MSVC doesn't seem to do this for types wider than a pointer.
1650 if (getContext().getTypeSize(T: LV.getType()) >
1651 getContext().getTypeSize(T: getContext().getIntPtrType()))
1652 return false;
1653 return IsVolatile && AtomicIsInline;
1654}
1655
1656RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1657 AggValueSlot Slot) {
1658 llvm::AtomicOrdering AO;
1659 bool IsVolatile = LV.isVolatileQualified();
1660 if (LV.getType()->isAtomicType()) {
1661 AO = llvm::AtomicOrdering::SequentiallyConsistent;
1662 } else {
1663 AO = llvm::AtomicOrdering::Acquire;
1664 IsVolatile = true;
1665 }
1666 return EmitAtomicLoad(lvalue: LV, loc: SL, AO, IsVolatile, slot: Slot);
1667}
1668
1669RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1670 bool AsValue, llvm::AtomicOrdering AO,
1671 bool IsVolatile) {
1672 // Check whether we should use a library call.
1673 if (shouldUseLibcall()) {
1674 Address TempAddr = Address::invalid();
1675 if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1676 assert(getEvaluationKind() == TEK_Aggregate);
1677 TempAddr = ResultSlot.getAddress();
1678 } else
1679 TempAddr = CreateTempAlloca();
1680
1681 EmitAtomicLoadLibcall(AddForLoaded: TempAddr.emitRawPointer(CGF), AO, IsVolatile);
1682
1683 // Okay, turn that back into the original value or whole atomic (for
1684 // non-simple lvalues) type.
1685 return convertAtomicTempToRValue(addr: TempAddr, resultSlot: ResultSlot, loc: Loc, asValue: AsValue);
1686 }
1687
1688 // Okay, we're doing this natively.
1689 auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1690
1691 // If we're ignoring an aggregate return, don't do anything.
1692 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1693 return RValue::getAggregate(addr: Address::invalid(), isVolatile: false);
1694
1695 // Okay, turn that back into the original value or atomic (for non-simple
1696 // lvalues) type.
1697 return ConvertToValueOrAtomic(Val: Load, ResultSlot, Loc, AsValue);
1698}
1699
1700/// Emit a load from an l-value of atomic type. Note that the r-value
1701/// we produce is an r-value of the atomic *value* type.
1702RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1703 llvm::AtomicOrdering AO, bool IsVolatile,
1704 AggValueSlot resultSlot) {
1705 AtomicInfo Atomics(*this, src);
1706 return Atomics.EmitAtomicLoad(ResultSlot: resultSlot, Loc: loc, /*AsValue=*/true, AO,
1707 IsVolatile);
1708}
1709
1710/// Copy an r-value into memory as part of storing to an atomic type.
1711/// This needs to create a bit-pattern suitable for atomic operations.
1712void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1713 assert(LVal.isSimple());
1714 // If we have an r-value, the rvalue should be of the atomic type,
1715 // which means that the caller is responsible for having zeroed
1716 // any padding. Just do an aggregate copy of that type.
1717 if (rvalue.isAggregate()) {
1718 LValue Dest = CGF.MakeAddrLValue(Addr: getAtomicAddress(), T: getAtomicType());
1719 LValue Src = CGF.MakeAddrLValue(Addr: rvalue.getAggregateAddress(),
1720 T: getAtomicType());
1721 bool IsVolatile = rvalue.isVolatileQualified() ||
1722 LVal.isVolatileQualified();
1723 CGF.EmitAggregateCopy(Dest, Src, EltTy: getAtomicType(),
1724 MayOverlap: AggValueSlot::DoesNotOverlap, isVolatile: IsVolatile);
1725 return;
1726 }
1727
1728 // Okay, otherwise we're copying stuff.
1729
1730 // Zero out the buffer if necessary.
1731 emitMemSetZeroIfNecessary();
1732
1733 // Drill past the padding if present.
1734 LValue TempLVal = projectValue();
1735
1736 // Okay, store the rvalue in.
1737 if (rvalue.isScalar()) {
1738 CGF.EmitStoreOfScalar(value: rvalue.getScalarVal(), lvalue: TempLVal, /*init*/ isInit: true);
1739 } else {
1740 CGF.EmitStoreOfComplex(V: rvalue.getComplexVal(), dest: TempLVal, /*init*/ isInit: true);
1741 }
1742}
1743
1744
1745/// Materialize an r-value into memory for the purposes of storing it
1746/// to an atomic type.
1747Address AtomicInfo::materializeRValue(RValue rvalue) const {
1748 // Aggregate r-values are already in memory, and EmitAtomicStore
1749 // requires them to be values of the atomic type.
1750 if (rvalue.isAggregate())
1751 return rvalue.getAggregateAddress();
1752
1753 // Otherwise, make a temporary and materialize into it.
1754 LValue TempLV = CGF.MakeAddrLValue(Addr: CreateTempAlloca(), T: getAtomicType());
1755 AtomicInfo Atomics(CGF, TempLV);
1756 Atomics.emitCopyIntoMemory(rvalue);
1757 return TempLV.getAddress();
1758}
1759
1760llvm::Value *AtomicInfo::getScalarRValValueOrNull(RValue RVal) const {
1761 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple()))
1762 return RVal.getScalarVal();
1763 return nullptr;
1764}
1765
1766llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal, bool CmpXchg) const {
1767 // If we've got a scalar value of the right size, try to avoid going
1768 // through memory. Floats get casted if needed by AtomicExpandPass.
1769 if (llvm::Value *Value = getScalarRValValueOrNull(RVal)) {
1770 if (!shouldCastToInt(ValTy: Value->getType(), CmpXchg))
1771 return CGF.EmitToMemory(Value, Ty: ValueTy);
1772 else {
1773 llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1774 C&: CGF.getLLVMContext(),
1775 NumBits: LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1776 if (llvm::BitCastInst::isBitCastable(SrcTy: Value->getType(), DestTy: InputIntTy))
1777 return CGF.Builder.CreateBitCast(V: Value, DestTy: InputIntTy);
1778 }
1779 }
1780 // Otherwise, we need to go through memory.
1781 // Put the r-value in memory.
1782 Address Addr = materializeRValue(rvalue: RVal);
1783
1784 // Cast the temporary to the atomic int type and pull a value out.
1785 Addr = castToAtomicIntPointer(addr: Addr);
1786 return CGF.Builder.CreateLoad(Addr);
1787}
1788
1789std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1790 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1791 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1792 // Do the atomic store.
1793 Address Addr = getAtomicAddressAsAtomicIntPointer();
1794 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, Cmp: ExpectedVal, New: DesiredVal,
1795 SuccessOrdering: Success, FailureOrdering: Failure);
1796 // Other decoration.
1797 Inst->setVolatile(LVal.isVolatileQualified());
1798 Inst->setWeak(IsWeak);
1799
1800 // Okay, turn that back into the original value type.
1801 auto *PreviousVal = CGF.Builder.CreateExtractValue(Agg: Inst, /*Idxs=*/0);
1802 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Agg: Inst, /*Idxs=*/1);
1803 return std::make_pair(x&: PreviousVal, y&: SuccessFailureVal);
1804}
1805
1806llvm::Value *
1807AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1808 llvm::Value *DesiredAddr,
1809 llvm::AtomicOrdering Success,
1810 llvm::AtomicOrdering Failure) {
1811 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1812 // void *desired, int success, int failure);
1813 CallArgList Args;
1814 Args.add(rvalue: RValue::get(V: getAtomicSizeValue()), type: CGF.getContext().getSizeType());
1815 Args.add(rvalue: RValue::get(V: getAtomicPointer()), type: CGF.getContext().VoidPtrTy);
1816 Args.add(rvalue: RValue::get(V: ExpectedAddr), type: CGF.getContext().VoidPtrTy);
1817 Args.add(rvalue: RValue::get(V: DesiredAddr), type: CGF.getContext().VoidPtrTy);
1818 Args.add(rvalue: RValue::get(
1819 V: llvm::ConstantInt::get(Ty: CGF.IntTy, V: (int)llvm::toCABI(AO: Success))),
1820 type: CGF.getContext().IntTy);
1821 Args.add(rvalue: RValue::get(
1822 V: llvm::ConstantInt::get(Ty: CGF.IntTy, V: (int)llvm::toCABI(AO: Failure))),
1823 type: CGF.getContext().IntTy);
1824 auto SuccessFailureRVal = emitAtomicLibcall(CGF, fnName: "__atomic_compare_exchange",
1825 resultType: CGF.getContext().BoolTy, args&: Args);
1826
1827 return SuccessFailureRVal.getScalarVal();
1828}
1829
1830std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1831 RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1832 llvm::AtomicOrdering Failure, bool IsWeak) {
1833 // Check whether we should use a library call.
1834 if (shouldUseLibcall()) {
1835 // Produce a source address.
1836 Address ExpectedAddr = materializeRValue(rvalue: Expected);
1837 llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
1838 llvm::Value *DesiredPtr = materializeRValue(rvalue: Desired).emitRawPointer(CGF);
1839 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr: ExpectedPtr, DesiredAddr: DesiredPtr,
1840 Success, Failure);
1841 return std::make_pair(
1842 x: convertAtomicTempToRValue(addr: ExpectedAddr, resultSlot: AggValueSlot::ignored(),
1843 loc: SourceLocation(), /*AsValue=*/asValue: false),
1844 y&: Res);
1845 }
1846
1847 // If we've got a scalar value of the right size, try to avoid going
1848 // through memory.
1849 auto *ExpectedVal = convertRValueToInt(RVal: Expected, /*CmpXchg=*/true);
1850 auto *DesiredVal = convertRValueToInt(RVal: Desired, /*CmpXchg=*/true);
1851 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1852 Failure, IsWeak);
1853 return std::make_pair(
1854 x: ConvertToValueOrAtomic(Val: Res.first, ResultSlot: AggValueSlot::ignored(),
1855 Loc: SourceLocation(), /*AsValue=*/false,
1856 /*CmpXchg=*/true),
1857 y&: Res.second);
1858}
1859
1860static void
1861EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1862 const llvm::function_ref<RValue(RValue)> &UpdateOp,
1863 Address DesiredAddr) {
1864 RValue UpRVal;
1865 LValue AtomicLVal = Atomics.getAtomicLValue();
1866 LValue DesiredLVal;
1867 if (AtomicLVal.isSimple()) {
1868 UpRVal = OldRVal;
1869 DesiredLVal = CGF.MakeAddrLValue(Addr: DesiredAddr, T: AtomicLVal.getType());
1870 } else {
1871 // Build new lvalue for temp address.
1872 Address Ptr = Atomics.materializeRValue(rvalue: OldRVal);
1873 LValue UpdateLVal;
1874 if (AtomicLVal.isBitField()) {
1875 UpdateLVal =
1876 LValue::MakeBitfield(Addr: Ptr, Info: AtomicLVal.getBitFieldInfo(),
1877 type: AtomicLVal.getType(),
1878 BaseInfo: AtomicLVal.getBaseInfo(),
1879 TBAAInfo: AtomicLVal.getTBAAInfo());
1880 DesiredLVal =
1881 LValue::MakeBitfield(Addr: DesiredAddr, Info: AtomicLVal.getBitFieldInfo(),
1882 type: AtomicLVal.getType(), BaseInfo: AtomicLVal.getBaseInfo(),
1883 TBAAInfo: AtomicLVal.getTBAAInfo());
1884 } else if (AtomicLVal.isVectorElt()) {
1885 UpdateLVal = LValue::MakeVectorElt(vecAddress: Ptr, Idx: AtomicLVal.getVectorIdx(),
1886 type: AtomicLVal.getType(),
1887 BaseInfo: AtomicLVal.getBaseInfo(),
1888 TBAAInfo: AtomicLVal.getTBAAInfo());
1889 DesiredLVal = LValue::MakeVectorElt(
1890 vecAddress: DesiredAddr, Idx: AtomicLVal.getVectorIdx(), type: AtomicLVal.getType(),
1891 BaseInfo: AtomicLVal.getBaseInfo(), TBAAInfo: AtomicLVal.getTBAAInfo());
1892 } else {
1893 assert(AtomicLVal.isExtVectorElt());
1894 UpdateLVal = LValue::MakeExtVectorElt(Addr: Ptr, Elts: AtomicLVal.getExtVectorElts(),
1895 type: AtomicLVal.getType(),
1896 BaseInfo: AtomicLVal.getBaseInfo(),
1897 TBAAInfo: AtomicLVal.getTBAAInfo());
1898 DesiredLVal = LValue::MakeExtVectorElt(
1899 Addr: DesiredAddr, Elts: AtomicLVal.getExtVectorElts(), type: AtomicLVal.getType(),
1900 BaseInfo: AtomicLVal.getBaseInfo(), TBAAInfo: AtomicLVal.getTBAAInfo());
1901 }
1902 UpRVal = CGF.EmitLoadOfLValue(V: UpdateLVal, Loc: SourceLocation());
1903 }
1904 // Store new value in the corresponding memory area.
1905 RValue NewRVal = UpdateOp(UpRVal);
1906 if (NewRVal.isScalar()) {
1907 CGF.EmitStoreThroughLValue(Src: NewRVal, Dst: DesiredLVal);
1908 } else {
1909 assert(NewRVal.isComplex());
1910 CGF.EmitStoreOfComplex(V: NewRVal.getComplexVal(), dest: DesiredLVal,
1911 /*isInit=*/false);
1912 }
1913}
1914
1915void AtomicInfo::EmitAtomicUpdateLibcall(
1916 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1917 bool IsVolatile) {
1918 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
1919
1920 Address ExpectedAddr = CreateTempAlloca();
1921
1922 EmitAtomicLoadLibcall(AddForLoaded: ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile);
1923 auto *ContBB = CGF.createBasicBlock(name: "atomic_cont");
1924 auto *ExitBB = CGF.createBasicBlock(name: "atomic_exit");
1925 CGF.EmitBlock(BB: ContBB);
1926 Address DesiredAddr = CreateTempAlloca();
1927 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1928 requiresMemSetZero(type: getAtomicAddress().getElementType())) {
1929 auto *OldVal = CGF.Builder.CreateLoad(Addr: ExpectedAddr);
1930 CGF.Builder.CreateStore(Val: OldVal, Addr: DesiredAddr);
1931 }
1932 auto OldRVal = convertAtomicTempToRValue(addr: ExpectedAddr,
1933 resultSlot: AggValueSlot::ignored(),
1934 loc: SourceLocation(), /*AsValue=*/asValue: false);
1935 EmitAtomicUpdateValue(CGF, Atomics&: *this, OldRVal, UpdateOp, DesiredAddr);
1936 llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
1937 llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF);
1938 auto *Res =
1939 EmitAtomicCompareExchangeLibcall(ExpectedAddr: ExpectedPtr, DesiredAddr: DesiredPtr, Success: AO, Failure);
1940 CGF.Builder.CreateCondBr(Cond: Res, True: ExitBB, False: ContBB);
1941 CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
1942}
1943
1944void AtomicInfo::EmitAtomicUpdateOp(
1945 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1946 bool IsVolatile) {
1947 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
1948
1949 // Do the atomic load.
1950 auto *OldVal = EmitAtomicLoadOp(AO: Failure, IsVolatile, /*CmpXchg=*/true);
1951 // For non-simple lvalues perform compare-and-swap procedure.
1952 auto *ContBB = CGF.createBasicBlock(name: "atomic_cont");
1953 auto *ExitBB = CGF.createBasicBlock(name: "atomic_exit");
1954 auto *CurBB = CGF.Builder.GetInsertBlock();
1955 CGF.EmitBlock(BB: ContBB);
1956 llvm::PHINode *PHI = CGF.Builder.CreatePHI(Ty: OldVal->getType(),
1957 /*NumReservedValues=*/2);
1958 PHI->addIncoming(V: OldVal, BB: CurBB);
1959 Address NewAtomicAddr = CreateTempAlloca();
1960 Address NewAtomicIntAddr =
1961 shouldCastToInt(ValTy: NewAtomicAddr.getElementType(), /*CmpXchg=*/true)
1962 ? castToAtomicIntPointer(addr: NewAtomicAddr)
1963 : NewAtomicAddr;
1964
1965 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1966 requiresMemSetZero(type: getAtomicAddress().getElementType())) {
1967 CGF.Builder.CreateStore(Val: PHI, Addr: NewAtomicIntAddr);
1968 }
1969 auto OldRVal = ConvertToValueOrAtomic(Val: PHI, ResultSlot: AggValueSlot::ignored(),
1970 Loc: SourceLocation(), /*AsValue=*/false,
1971 /*CmpXchg=*/true);
1972 EmitAtomicUpdateValue(CGF, Atomics&: *this, OldRVal, UpdateOp, DesiredAddr: NewAtomicAddr);
1973 auto *DesiredVal = CGF.Builder.CreateLoad(Addr: NewAtomicIntAddr);
1974 // Try to write new value using cmpxchg operation.
1975 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal: PHI, DesiredVal, Success: AO, Failure);
1976 PHI->addIncoming(V: Res.first, BB: CGF.Builder.GetInsertBlock());
1977 CGF.Builder.CreateCondBr(Cond: Res.second, True: ExitBB, False: ContBB);
1978 CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
1979}
1980
1981static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1982 RValue UpdateRVal, Address DesiredAddr) {
1983 LValue AtomicLVal = Atomics.getAtomicLValue();
1984 LValue DesiredLVal;
1985 // Build new lvalue for temp address.
1986 if (AtomicLVal.isBitField()) {
1987 DesiredLVal =
1988 LValue::MakeBitfield(Addr: DesiredAddr, Info: AtomicLVal.getBitFieldInfo(),
1989 type: AtomicLVal.getType(), BaseInfo: AtomicLVal.getBaseInfo(),
1990 TBAAInfo: AtomicLVal.getTBAAInfo());
1991 } else if (AtomicLVal.isVectorElt()) {
1992 DesiredLVal =
1993 LValue::MakeVectorElt(vecAddress: DesiredAddr, Idx: AtomicLVal.getVectorIdx(),
1994 type: AtomicLVal.getType(), BaseInfo: AtomicLVal.getBaseInfo(),
1995 TBAAInfo: AtomicLVal.getTBAAInfo());
1996 } else {
1997 assert(AtomicLVal.isExtVectorElt());
1998 DesiredLVal = LValue::MakeExtVectorElt(
1999 Addr: DesiredAddr, Elts: AtomicLVal.getExtVectorElts(), type: AtomicLVal.getType(),
2000 BaseInfo: AtomicLVal.getBaseInfo(), TBAAInfo: AtomicLVal.getTBAAInfo());
2001 }
2002 // Store new value in the corresponding memory area.
2003 assert(UpdateRVal.isScalar());
2004 CGF.EmitStoreThroughLValue(Src: UpdateRVal, Dst: DesiredLVal);
2005}
2006
2007void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
2008 RValue UpdateRVal, bool IsVolatile) {
2009 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
2010
2011 Address ExpectedAddr = CreateTempAlloca();
2012
2013 EmitAtomicLoadLibcall(AddForLoaded: ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile);
2014 auto *ContBB = CGF.createBasicBlock(name: "atomic_cont");
2015 auto *ExitBB = CGF.createBasicBlock(name: "atomic_exit");
2016 CGF.EmitBlock(BB: ContBB);
2017 Address DesiredAddr = CreateTempAlloca();
2018 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
2019 requiresMemSetZero(type: getAtomicAddress().getElementType())) {
2020 auto *OldVal = CGF.Builder.CreateLoad(Addr: ExpectedAddr);
2021 CGF.Builder.CreateStore(Val: OldVal, Addr: DesiredAddr);
2022 }
2023 EmitAtomicUpdateValue(CGF, Atomics&: *this, UpdateRVal, DesiredAddr);
2024 llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
2025 llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF);
2026 auto *Res =
2027 EmitAtomicCompareExchangeLibcall(ExpectedAddr: ExpectedPtr, DesiredAddr: DesiredPtr, Success: AO, Failure);
2028 CGF.Builder.CreateCondBr(Cond: Res, True: ExitBB, False: ContBB);
2029 CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
2030}
2031
2032void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
2033 bool IsVolatile) {
2034 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
2035
2036 // Do the atomic load.
2037 auto *OldVal = EmitAtomicLoadOp(AO: Failure, IsVolatile, /*CmpXchg=*/true);
2038 // For non-simple lvalues perform compare-and-swap procedure.
2039 auto *ContBB = CGF.createBasicBlock(name: "atomic_cont");
2040 auto *ExitBB = CGF.createBasicBlock(name: "atomic_exit");
2041 auto *CurBB = CGF.Builder.GetInsertBlock();
2042 CGF.EmitBlock(BB: ContBB);
2043 llvm::PHINode *PHI = CGF.Builder.CreatePHI(Ty: OldVal->getType(),
2044 /*NumReservedValues=*/2);
2045 PHI->addIncoming(V: OldVal, BB: CurBB);
2046 Address NewAtomicAddr = CreateTempAlloca();
2047 Address NewAtomicIntAddr = castToAtomicIntPointer(addr: NewAtomicAddr);
2048 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
2049 requiresMemSetZero(type: getAtomicAddress().getElementType())) {
2050 CGF.Builder.CreateStore(Val: PHI, Addr: NewAtomicIntAddr);
2051 }
2052 EmitAtomicUpdateValue(CGF, Atomics&: *this, UpdateRVal, DesiredAddr: NewAtomicAddr);
2053 auto *DesiredVal = CGF.Builder.CreateLoad(Addr: NewAtomicIntAddr);
2054 // Try to write new value using cmpxchg operation.
2055 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal: PHI, DesiredVal, Success: AO, Failure);
2056 PHI->addIncoming(V: Res.first, BB: CGF.Builder.GetInsertBlock());
2057 CGF.Builder.CreateCondBr(Cond: Res.second, True: ExitBB, False: ContBB);
2058 CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
2059}
2060
2061void AtomicInfo::EmitAtomicUpdate(
2062 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
2063 bool IsVolatile) {
2064 if (shouldUseLibcall()) {
2065 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
2066 } else {
2067 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
2068 }
2069}
2070
2071void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
2072 bool IsVolatile) {
2073 if (shouldUseLibcall()) {
2074 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
2075 } else {
2076 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
2077 }
2078}
2079
2080void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
2081 bool isInit) {
2082 bool IsVolatile = lvalue.isVolatileQualified();
2083 llvm::AtomicOrdering AO;
2084 if (lvalue.getType()->isAtomicType()) {
2085 AO = llvm::AtomicOrdering::SequentiallyConsistent;
2086 } else {
2087 AO = llvm::AtomicOrdering::Release;
2088 IsVolatile = true;
2089 }
2090 return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
2091}
2092
2093/// Emit a store to an l-value of atomic type.
2094///
2095/// Note that the r-value is expected to be an r-value *of the atomic
2096/// type*; this means that for aggregate r-values, it should include
2097/// storage for any padding that was necessary.
2098void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
2099 llvm::AtomicOrdering AO, bool IsVolatile,
2100 bool isInit) {
2101 // If this is an aggregate r-value, it should agree in type except
2102 // maybe for address-space qualification.
2103 assert(!rvalue.isAggregate() ||
2104 rvalue.getAggregateAddress().getElementType() ==
2105 dest.getAddress().getElementType());
2106
2107 AtomicInfo atomics(*this, dest);
2108 LValue LVal = atomics.getAtomicLValue();
2109
2110 // If this is an initialization, just put the value there normally.
2111 if (LVal.isSimple()) {
2112 if (isInit) {
2113 atomics.emitCopyIntoMemory(rvalue);
2114 return;
2115 }
2116
2117 // Check whether we should use a library call.
2118 if (atomics.shouldUseLibcall()) {
2119 // Produce a source address.
2120 Address srcAddr = atomics.materializeRValue(rvalue);
2121
2122 // void __atomic_store(size_t size, void *mem, void *val, int order)
2123 CallArgList args;
2124 args.add(rvalue: RValue::get(V: atomics.getAtomicSizeValue()),
2125 type: getContext().getSizeType());
2126 args.add(rvalue: RValue::get(V: atomics.getAtomicPointer()), type: getContext().VoidPtrTy);
2127 args.add(rvalue: RValue::get(V: srcAddr.emitRawPointer(CGF&: *this)),
2128 type: getContext().VoidPtrTy);
2129 args.add(
2130 rvalue: RValue::get(V: llvm::ConstantInt::get(Ty: IntTy, V: (int)llvm::toCABI(AO))),
2131 type: getContext().IntTy);
2132 emitAtomicLibcall(CGF&: *this, fnName: "__atomic_store", resultType: getContext().VoidTy, args);
2133 return;
2134 }
2135
2136 // Okay, we're doing this natively.
2137 llvm::Value *ValToStore = atomics.convertRValueToInt(RVal: rvalue);
2138
2139 // Do the atomic store.
2140 Address Addr = atomics.getAtomicAddress();
2141 if (llvm::Value *Value = atomics.getScalarRValValueOrNull(RVal: rvalue))
2142 if (shouldCastToInt(ValTy: Value->getType(), /*CmpXchg=*/false)) {
2143 Addr = atomics.castToAtomicIntPointer(addr: Addr);
2144 ValToStore = Builder.CreateIntCast(V: ValToStore, DestTy: Addr.getElementType(),
2145 /*isSigned=*/false);
2146 }
2147 llvm::StoreInst *store = Builder.CreateStore(Val: ValToStore, Addr);
2148
2149 if (AO == llvm::AtomicOrdering::Acquire)
2150 AO = llvm::AtomicOrdering::Monotonic;
2151 else if (AO == llvm::AtomicOrdering::AcquireRelease)
2152 AO = llvm::AtomicOrdering::Release;
2153 // Initializations don't need to be atomic.
2154 if (!isInit)
2155 store->setAtomic(Ordering: AO);
2156
2157 // Other decoration.
2158 if (IsVolatile)
2159 store->setVolatile(true);
2160 CGM.DecorateInstructionWithTBAA(Inst: store, TBAAInfo: dest.getTBAAInfo());
2161 return;
2162 }
2163
2164 // Emit simple atomic update operation.
2165 atomics.EmitAtomicUpdate(AO, UpdateRVal: rvalue, IsVolatile);
2166}
2167
2168/// Emit a compare-and-exchange op for atomic type.
2169///
2170std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
2171 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
2172 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
2173 AggValueSlot Slot) {
2174 // If this is an aggregate r-value, it should agree in type except
2175 // maybe for address-space qualification.
2176 assert(!Expected.isAggregate() ||
2177 Expected.getAggregateAddress().getElementType() ==
2178 Obj.getAddress().getElementType());
2179 assert(!Desired.isAggregate() ||
2180 Desired.getAggregateAddress().getElementType() ==
2181 Obj.getAddress().getElementType());
2182 AtomicInfo Atomics(*this, Obj);
2183
2184 return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
2185 IsWeak);
2186}
2187
2188llvm::AtomicRMWInst *
2189CodeGenFunction::emitAtomicRMWInst(llvm::AtomicRMWInst::BinOp Op, Address Addr,
2190 llvm::Value *Val, llvm::AtomicOrdering Order,
2191 llvm::SyncScope::ID SSID,
2192 const AtomicExpr *AE) {
2193 llvm::AtomicRMWInst *RMW =
2194 Builder.CreateAtomicRMW(Op, Addr, Val, Ordering: Order, SSID);
2195 getTargetHooks().setTargetAtomicMetadata(CGF&: *this, AtomicInst&: *RMW, Expr: AE);
2196 return RMW;
2197}
2198
2199void CodeGenFunction::EmitAtomicUpdate(
2200 LValue LVal, llvm::AtomicOrdering AO,
2201 const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
2202 AtomicInfo Atomics(*this, LVal);
2203 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2204}
2205
2206void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
2207 AtomicInfo atomics(*this, dest);
2208
2209 switch (atomics.getEvaluationKind()) {
2210 case TEK_Scalar: {
2211 llvm::Value *value = EmitScalarExpr(E: init);
2212 atomics.emitCopyIntoMemory(rvalue: RValue::get(V: value));
2213 return;
2214 }
2215
2216 case TEK_Complex: {
2217 ComplexPairTy value = EmitComplexExpr(E: init);
2218 atomics.emitCopyIntoMemory(rvalue: RValue::getComplex(C: value));
2219 return;
2220 }
2221
2222 case TEK_Aggregate: {
2223 // Fix up the destination if the initializer isn't an expression
2224 // of atomic type.
2225 bool Zeroed = false;
2226 if (!init->getType()->isAtomicType()) {
2227 Zeroed = atomics.emitMemSetZeroIfNecessary();
2228 dest = atomics.projectValue();
2229 }
2230
2231 // Evaluate the expression directly into the destination.
2232 AggValueSlot slot = AggValueSlot::forLValue(
2233 LV: dest, isDestructed: AggValueSlot::IsNotDestructed,
2234 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
2235 mayOverlap: AggValueSlot::DoesNotOverlap,
2236 isZeroed: Zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed);
2237
2238 EmitAggExpr(E: init, AS: slot);
2239 return;
2240 }
2241 }
2242 llvm_unreachable("bad evaluation kind");
2243}
2244