1//===- AtomicExpandPass.cpp - Expand atomic instructions ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass (at IR level) to replace atomic instructions with
10// __atomic_* library calls, or target specific instruction which implement the
11// same semantics in a way which better fits the target backend. This can
12// include the use of (intrinsic-based) load-linked/store-conditional loops,
13// AtomicCmpXchg, or type coercions.
14//
15//===----------------------------------------------------------------------===//
16
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/STLFunctionalExtras.h"
19#include "llvm/ADT/SmallString.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/Analysis/InstSimplifyFolder.h"
22#include "llvm/Analysis/OptimizationRemarkEmitter.h"
23#include "llvm/Analysis/VectorUtils.h"
24#include "llvm/CodeGen/AtomicExpand.h"
25#include "llvm/CodeGen/TargetLowering.h"
26#include "llvm/CodeGen/TargetPassConfig.h"
27#include "llvm/CodeGen/TargetSubtargetInfo.h"
28#include "llvm/CodeGen/ValueTypes.h"
29#include "llvm/IR/Attributes.h"
30#include "llvm/IR/BasicBlock.h"
31#include "llvm/IR/Constant.h"
32#include "llvm/IR/Constants.h"
33#include "llvm/IR/DataLayout.h"
34#include "llvm/IR/DerivedTypes.h"
35#include "llvm/IR/Function.h"
36#include "llvm/IR/IRBuilder.h"
37#include "llvm/IR/Instruction.h"
38#include "llvm/IR/Instructions.h"
39#include "llvm/IR/MDBuilder.h"
40#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
41#include "llvm/IR/Module.h"
42#include "llvm/IR/ProfDataUtils.h"
43#include "llvm/IR/Type.h"
44#include "llvm/IR/User.h"
45#include "llvm/IR/Value.h"
46#include "llvm/InitializePasses.h"
47#include "llvm/Pass.h"
48#include "llvm/Support/Alignment.h"
49#include "llvm/Support/AtomicOrdering.h"
50#include "llvm/Support/Casting.h"
51#include "llvm/Support/Debug.h"
52#include "llvm/Support/ErrorHandling.h"
53#include "llvm/Support/raw_ostream.h"
54#include "llvm/Target/TargetMachine.h"
55#include "llvm/Transforms/Utils/LowerAtomic.h"
56#include <cassert>
57#include <cstdint>
58#include <iterator>
59
60using namespace llvm;
61
62#define DEBUG_TYPE "atomic-expand"
63
64namespace {
65
66class AtomicExpandImpl {
67 const TargetLowering *TLI = nullptr;
68 const LibcallLoweringInfo *LibcallLowering = nullptr;
69 const DataLayout *DL = nullptr;
70 bool SingleThreaded = false;
71
72private:
73 /// Callback type for emitting a cmpxchg instruction during RMW expansion.
74 /// Parameters: (Builder, Addr, Loaded, NewVal, AddrAlign, MemOpOrder,
75 /// SSID, IsVolatile, /* OUT */ Success, /* OUT */ NewLoaded,
76 /// MetadataSrc)
77 using CreateCmpXchgInstFun = function_ref<void(
78 IRBuilderBase &, Value *, Value *, Value *, Align, AtomicOrdering,
79 SyncScope::ID, bool, Value *&, Value *&, Instruction *)>;
80
81 void handleFailure(Instruction &FailedInst, const Twine &Msg,
82 Instruction *DiagnosticInst = nullptr) const {
83 LLVMContext &Ctx = FailedInst.getContext();
84
85 // TODO: Do not use generic error type.
86 Ctx.emitError(I: DiagnosticInst ? DiagnosticInst : &FailedInst, ErrorStr: Msg);
87
88 if (!FailedInst.getType()->isVoidTy())
89 FailedInst.replaceAllUsesWith(V: PoisonValue::get(T: FailedInst.getType()));
90 FailedInst.eraseFromParent();
91 }
92
93 template <typename Inst>
94 void handleUnsupportedAtomicSize(Inst *I, const Twine &AtomicOpName,
95 Instruction *DiagnosticInst = nullptr) const;
96
97 bool bracketInstWithFences(Instruction *I, AtomicOrdering Order);
98 bool tryInsertTrailingSeqCstFence(Instruction *AtomicI);
99 template <typename AtomicInst>
100 bool tryInsertFencesForAtomic(AtomicInst *AtomicI, bool OrderingRequiresFence,
101 AtomicOrdering NewOrdering);
102 IntegerType *getCorrespondingIntegerType(Type *T, const DataLayout &DL);
103 LoadInst *convertAtomicLoadToIntegerType(LoadInst *LI);
104 bool tryExpandAtomicLoad(LoadInst *LI);
105 bool expandAtomicLoadToLL(LoadInst *LI);
106 bool expandAtomicLoadToCmpXchg(LoadInst *LI);
107 StoreInst *convertAtomicStoreToIntegerType(StoreInst *SI);
108 bool tryExpandAtomicStore(StoreInst *SI);
109 void expandAtomicStoreToXChg(StoreInst *SI);
110 bool tryExpandAtomicRMW(AtomicRMWInst *AI);
111 void expandElementwiseAtomicRMW(AtomicRMWInst *AI);
112 void expandAtomicSubToAdd(AtomicRMWInst *AI);
113 AtomicRMWInst *convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI);
114 Value *
115 insertRMWLLSCLoop(IRBuilderBase &Builder, Type *ResultTy, Value *Addr,
116 Align AddrAlign, AtomicOrdering MemOpOrder,
117 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp);
118 void expandAtomicOpToLLSC(
119 Instruction *I, Type *ResultTy, Value *Addr, Align AddrAlign,
120 AtomicOrdering MemOpOrder,
121 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp);
122 void expandPartwordAtomicRMW(
123 AtomicRMWInst *I, TargetLoweringBase::AtomicExpansionKind ExpansionKind);
124 AtomicRMWInst *widenPartwordAtomicRMW(AtomicRMWInst *AI);
125 bool expandPartwordCmpXchg(AtomicCmpXchgInst *I);
126 void expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI);
127 void expandAtomicCmpXchgToMaskedIntrinsic(AtomicCmpXchgInst *CI);
128
129 AtomicCmpXchgInst *convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI);
130 Value *insertRMWCmpXchgLoop(
131 IRBuilderBase &Builder, Type *ResultType, Value *Addr, Align AddrAlign,
132 AtomicOrdering MemOpOrder, SyncScope::ID SSID, bool IsVolatile,
133 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp,
134 CreateCmpXchgInstFun CreateCmpXchg, Instruction *MetadataSrc);
135 bool tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI);
136
137 bool expandAtomicCmpXchg(AtomicCmpXchgInst *CI);
138 bool isIdempotentRMW(AtomicRMWInst *RMWI);
139 bool simplifyIdempotentRMW(AtomicRMWInst *RMWI);
140
141 bool expandAtomicOpToLibcall(Instruction *I, unsigned Size, Align Alignment,
142 Value *PointerOperand, Value *ValueOperand,
143 Value *CASExpected, AtomicOrdering Ordering,
144 AtomicOrdering Ordering2,
145 ArrayRef<RTLIB::Libcall> Libcalls);
146 void expandAtomicLoadToLibcall(LoadInst *LI);
147 void expandAtomicStoreToLibcall(StoreInst *LI);
148 void expandAtomicRMWToLibcall(AtomicRMWInst *I);
149 void expandAtomicCASToLibcall(AtomicCmpXchgInst *I,
150 const Twine &AtomicOpName = "cmpxchg",
151 Instruction *DiagnosticInst = nullptr);
152
153 bool expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
154 CreateCmpXchgInstFun CreateCmpXchg);
155
156 bool lowerToNonAtomic(Instruction *I);
157 bool processAtomicInstr(Instruction *I);
158
159public:
160 bool run(Function &F, const ModuleLibcallLoweringInfo &LibcallResult,
161 const TargetMachine *TM);
162};
163
164class AtomicExpandLegacy : public FunctionPass {
165public:
166 static char ID; // Pass identification, replacement for typeid
167
168 AtomicExpandLegacy() : FunctionPass(ID) {}
169
170 void getAnalysisUsage(AnalysisUsage &AU) const override {
171 AU.addRequired<LibcallLoweringInfoWrapper>();
172 FunctionPass::getAnalysisUsage(AU);
173 }
174
175 bool runOnFunction(Function &F) override;
176};
177
178// IRBuilder to be used for replacement atomic instructions.
179struct ReplacementIRBuilder
180 : IRBuilder<InstSimplifyFolder, IRBuilderCallbackInserter> {
181 MDNode *MMRAMD = nullptr;
182 MDNode *PCSectionsMD = nullptr;
183
184 // Preserves the DebugLoc from I, and preserves still valid metadata.
185 // Enable StrictFP builder mode when appropriate.
186 explicit ReplacementIRBuilder(Instruction *I, const DataLayout &DL)
187 : IRBuilder(
188 I->getIterator(), InstSimplifyFolder(DL),
189 IRBuilderCallbackInserter([this](Instruction *I) { addMD(I); })) {
190 if (I->getFunction()->hasFnAttribute(Kind: Attribute::StrictFP))
191 this->setIsFPConstrained(true);
192
193 MMRAMD = I->getMetadata(KindID: LLVMContext::MD_mmra);
194 PCSectionsMD = I->getMetadata(KindID: LLVMContext::MD_pcsections);
195 }
196
197 void addMD(Instruction *I) {
198 if (canInstructionHaveMMRAs(I: *I))
199 I->setMetadata(KindID: LLVMContext::MD_mmra, Node: MMRAMD);
200 I->setMetadata(KindID: LLVMContext::MD_pcsections, Node: PCSectionsMD);
201 }
202};
203
204} // end anonymous namespace
205
206char AtomicExpandLegacy::ID = 0;
207
208char &llvm::AtomicExpandID = AtomicExpandLegacy::ID;
209
210INITIALIZE_PASS_BEGIN(AtomicExpandLegacy, DEBUG_TYPE,
211 "Expand Atomic instructions", false, false)
212INITIALIZE_PASS_DEPENDENCY(LibcallLoweringInfoWrapper)
213INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
214INITIALIZE_PASS_END(AtomicExpandLegacy, DEBUG_TYPE,
215 "Expand Atomic instructions", false, false)
216
217// Helper functions to retrieve the size of atomic instructions.
218static unsigned getAtomicOpSize(LoadInst *LI) {
219 const DataLayout &DL = LI->getDataLayout();
220 return DL.getTypeStoreSize(Ty: LI->getType());
221}
222
223static unsigned getAtomicOpSize(StoreInst *SI) {
224 const DataLayout &DL = SI->getDataLayout();
225 return DL.getTypeStoreSize(Ty: SI->getValueOperand()->getType());
226}
227
228static unsigned getAtomicOpSize(AtomicRMWInst *RMWI) {
229 const DataLayout &DL = RMWI->getDataLayout();
230 return DL.getTypeStoreSize(Ty: RMWI->getValOperand()->getType());
231}
232
233static unsigned getAtomicOpSize(AtomicCmpXchgInst *CASI) {
234 const DataLayout &DL = CASI->getDataLayout();
235 return DL.getTypeStoreSize(Ty: CASI->getCompareOperand()->getType());
236}
237
238/// Copy metadata that's safe to preserve when widening atomics.
239static void copyMetadataForAtomic(Instruction &Dest,
240 const Instruction &Source) {
241 SmallVector<std::pair<unsigned, MDNode *>, 8> MD;
242 Source.getAllMetadata(MDs&: MD);
243 LLVMContext &Ctx = Dest.getContext();
244 MDBuilder MDB(Ctx);
245
246 for (auto [ID, N] : MD) {
247 switch (ID) {
248 case LLVMContext::MD_dbg:
249 case LLVMContext::MD_tbaa:
250 case LLVMContext::MD_tbaa_struct:
251 case LLVMContext::MD_alias_scope:
252 case LLVMContext::MD_mem_cache_hint:
253 case LLVMContext::MD_noalias:
254 case LLVMContext::MD_noalias_addrspace:
255 case LLVMContext::MD_access_group:
256 case LLVMContext::MD_mmra:
257 case LLVMContext::MD_atomic_ignore_denormal_mode:
258 Dest.setMetadata(KindID: ID, Node: N);
259 break;
260 default:
261 if (ID == Ctx.getMDKindID(Name: "amdgpu.no.remote.memory"))
262 Dest.setMetadata(KindID: ID, Node: N);
263 else if (ID == Ctx.getMDKindID(Name: "amdgpu.no.fine.grained.memory"))
264 Dest.setMetadata(KindID: ID, Node: N);
265 else if (ID == Ctx.getMDKindID(Name: "amdgpu.cfs"))
266 Dest.setMetadata(KindID: ID, Node: N);
267 break;
268 }
269 }
270}
271
272template <typename Inst>
273static bool atomicSizeSupported(const TargetLowering *TLI, Inst *I) {
274 unsigned Size = getAtomicOpSize(I);
275 Align Alignment = I->getAlign();
276 unsigned MaxSize = TLI->getMaxAtomicSizeInBitsSupported() / 8;
277 return Size <= MaxSize && TLI->isAtomicAlignmentSupported(Alignment, SizeInBytes: Size);
278}
279
280template <typename Inst>
281static void writeUnsupportedAtomicSizeReason(const TargetLowering *TLI, Inst *I,
282 raw_ostream &OS) {
283 unsigned Size = getAtomicOpSize(I);
284 Align Alignment = I->getAlign();
285 bool NeedSeparator = false;
286
287 if (!TLI->isAtomicAlignmentSupported(Alignment, SizeInBytes: Size)) {
288 OS << "instruction alignment " << Alignment.value()
289 << " is smaller than the required " << Size
290 << "-byte alignment for this atomic operation";
291 NeedSeparator = true;
292 }
293
294 unsigned MaxSize = TLI->getMaxAtomicSizeInBitsSupported() / 8;
295 if (Size > MaxSize) {
296 if (NeedSeparator)
297 OS << "; ";
298 OS << "target supports atomics up to " << MaxSize
299 << " bytes, but this atomic accesses " << Size << " bytes";
300 }
301}
302
303template <typename Inst>
304void AtomicExpandImpl::handleUnsupportedAtomicSize(
305 Inst *I, const Twine &AtomicOpName, Instruction *DiagnosticInst) const {
306 assert(!atomicSizeSupported(TLI, I) && "expected unsupported atomic size");
307 SmallString<128> FailureReason;
308 raw_svector_ostream OS(FailureReason);
309 writeUnsupportedAtomicSizeReason(TLI, I, OS);
310 handleFailure(FailedInst&: *I, Msg: Twine("unsupported ") + AtomicOpName + ": " + FailureReason,
311 DiagnosticInst);
312}
313
314bool AtomicExpandImpl::tryInsertTrailingSeqCstFence(Instruction *AtomicI) {
315 if (!TLI->shouldInsertTrailingSeqCstFenceForAtomicStore(I: AtomicI))
316 return false;
317
318 IRBuilder Builder(AtomicI);
319 if (auto *TrailingFence = TLI->emitTrailingFence(
320 Builder, Inst: AtomicI, Ord: AtomicOrdering::SequentiallyConsistent)) {
321 TrailingFence->moveAfter(MovePos: AtomicI);
322 return true;
323 }
324 return false;
325}
326
327template <typename AtomicInst>
328bool AtomicExpandImpl::tryInsertFencesForAtomic(AtomicInst *AtomicI,
329 bool OrderingRequiresFence,
330 AtomicOrdering NewOrdering) {
331 bool ShouldInsertFences = TLI->shouldInsertFencesForAtomic(I: AtomicI);
332 if (OrderingRequiresFence && ShouldInsertFences) {
333 AtomicOrdering FenceOrdering = AtomicI->getOrdering();
334 AtomicI->setOrdering(NewOrdering);
335 return bracketInstWithFences(I: AtomicI, Order: FenceOrdering);
336 }
337 if (!ShouldInsertFences)
338 return tryInsertTrailingSeqCstFence(AtomicI);
339 return false;
340}
341
342/// In a single-threaded environment, atomic operations can be lowered to their
343/// non-atomic equivalents: fences are removed, and atomic loads, stores, RMW,
344/// and cmpxchg become plain memory operations.
345bool AtomicExpandImpl::lowerToNonAtomic(Instruction *I) {
346 if (auto *FI = dyn_cast<FenceInst>(Val: I)) {
347 FI->eraseFromParent();
348 return true;
349 }
350
351 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: I))
352 return lowerAtomicCmpXchgInst(CXI);
353
354 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: I))
355 return lowerAtomicRMWInst(RMWI);
356
357 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
358 if (LI->isAtomic()) {
359 LI->setAtomic(Ordering: AtomicOrdering::NotAtomic);
360 LI->setElementwise(false);
361 return true;
362 }
363
364 return false;
365 }
366
367 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
368 if (SI->isAtomic()) {
369 SI->setAtomic(Ordering: AtomicOrdering::NotAtomic);
370 SI->setElementwise(false);
371 return true;
372 }
373
374 return false;
375 }
376
377 return false;
378}
379
380bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
381 if (SingleThreaded)
382 return lowerToNonAtomic(I);
383
384 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
385 if (!LI->isAtomic())
386 return false;
387
388 if (!atomicSizeSupported(TLI, I: LI)) {
389 expandAtomicLoadToLibcall(LI);
390 return true;
391 }
392
393 bool MadeChange = false;
394 if (TLI->shouldCastAtomicLoadInIR(LI) ==
395 TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
396 LI = convertAtomicLoadToIntegerType(LI);
397 MadeChange = true;
398 }
399
400 MadeChange |=
401 tryInsertFencesForAtomic(AtomicI: LI, OrderingRequiresFence: isAcquireOrStronger(AO: LI->getOrdering()),
402 NewOrdering: TLI->atomicOperationOrderAfterFenceSplit(I: LI));
403
404 MadeChange |= tryExpandAtomicLoad(LI);
405 return MadeChange;
406 }
407
408 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
409 if (!SI->isAtomic())
410 return false;
411
412 if (!atomicSizeSupported(TLI, I: SI)) {
413 expandAtomicStoreToLibcall(LI: SI);
414 return true;
415 }
416
417 bool MadeChange = false;
418 if (TLI->shouldCastAtomicStoreInIR(SI) ==
419 TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
420 SI = convertAtomicStoreToIntegerType(SI);
421 MadeChange = true;
422 }
423
424 MadeChange |=
425 tryInsertFencesForAtomic(AtomicI: SI, OrderingRequiresFence: isReleaseOrStronger(AO: SI->getOrdering()),
426 NewOrdering: TLI->atomicOperationOrderAfterFenceSplit(I: SI));
427
428 MadeChange |= tryExpandAtomicStore(SI);
429 return MadeChange;
430 }
431
432 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: I)) {
433 bool IsElementwiseExpand =
434 RMWI->isElementwise() &&
435 TLI->shouldExpandAtomicRMWInIR(RMW: RMWI) ==
436 TargetLoweringBase::AtomicExpansionKind::Expand;
437
438 if (!atomicSizeSupported(TLI, I: RMWI)) {
439 // Elementwise expansion may split an atomicrmw into smaller, supported
440 // atomic sizes.
441 if (!IsElementwiseExpand) {
442 expandAtomicRMWToLibcall(I: RMWI);
443 return true;
444 }
445 }
446
447 bool MadeChange = false;
448 if (!IsElementwiseExpand &&
449 TLI->shouldCastAtomicRMWIInIR(RMWI) ==
450 TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
451 RMWI = convertAtomicXchgToIntegerType(RMWI);
452 MadeChange = true;
453 }
454
455 MadeChange |= tryInsertFencesForAtomic(
456 AtomicI: RMWI, OrderingRequiresFence: isStrongerThanMonotonic(AO: RMWI->getOrdering()),
457 NewOrdering: TLI->atomicOperationOrderAfterFenceSplit(I: RMWI));
458
459 // There are two different ways of expanding RMW instructions:
460 // - into a load if it is idempotent
461 // - into a Cmpxchg/LL-SC loop otherwise
462 // we try them in that order.
463 MadeChange |= (isIdempotentRMW(RMWI) && simplifyIdempotentRMW(RMWI)) ||
464 tryExpandAtomicRMW(AI: RMWI);
465 return MadeChange;
466 }
467
468 if (auto *CASI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
469 if (!atomicSizeSupported(TLI, I: CASI)) {
470 expandAtomicCASToLibcall(I: CASI);
471 return true;
472 }
473
474 // TODO: when we're ready to make the change at the IR level, we can
475 // extend convertCmpXchgToInteger for floating point too.
476 bool MadeChange = false;
477 if (CASI->getCompareOperand()->getType()->isPointerTy()) {
478 // TODO: add a TLI hook to control this so that each target can
479 // convert to lowering the original type one at a time.
480 CASI = convertCmpXchgToIntegerType(CI: CASI);
481 MadeChange = true;
482 }
483
484 auto CmpXchgExpansion = TLI->shouldExpandAtomicCmpXchgInIR(AI: CASI);
485 if (TLI->shouldInsertFencesForAtomic(I: CASI)) {
486 if (CmpXchgExpansion == TargetLoweringBase::AtomicExpansionKind::None &&
487 (isReleaseOrStronger(AO: CASI->getSuccessOrdering()) ||
488 isAcquireOrStronger(AO: CASI->getSuccessOrdering()) ||
489 isAcquireOrStronger(AO: CASI->getFailureOrdering()))) {
490 // If a compare and swap is lowered to LL/SC, we can do smarter fence
491 // insertion, with a stronger one on the success path than on the
492 // failure path. As a result, fence insertion is directly done by
493 // expandAtomicCmpXchg in that case.
494 AtomicOrdering FenceOrdering = CASI->getMergedOrdering();
495 AtomicOrdering CASOrdering =
496 TLI->atomicOperationOrderAfterFenceSplit(I: CASI);
497 CASI->setSuccessOrdering(CASOrdering);
498 CASI->setFailureOrdering(CASOrdering);
499 MadeChange |= bracketInstWithFences(I: CASI, Order: FenceOrdering);
500 }
501 } else if (CmpXchgExpansion !=
502 TargetLoweringBase::AtomicExpansionKind::LLSC) {
503 // CmpXchg LLSC is handled in expandAtomicCmpXchg().
504 MadeChange |= tryInsertTrailingSeqCstFence(AtomicI: CASI);
505 }
506
507 MadeChange |= tryExpandAtomicCmpXchg(CI: CASI);
508 return MadeChange;
509 }
510
511 return false;
512}
513
514bool AtomicExpandImpl::run(Function &F,
515 const ModuleLibcallLoweringInfo &LibcallResult,
516 const TargetMachine *TM) {
517 SingleThreaded = F.getParent()->getThreadModel() == ThreadModel::Single;
518
519 const auto *Subtarget = TM->getSubtargetImpl(F);
520 // In a single-threaded environment atomics are lowered to non-atomic form
521 if (!SingleThreaded && !Subtarget->enableAtomicExpand())
522 return false;
523 TLI = Subtarget->getTargetLowering();
524 LibcallLowering = &getLibcallLowering(ModuleInfo: LibcallResult, Subtarget: *Subtarget);
525 DL = &F.getDataLayout();
526
527 bool MadeChange = false;
528
529 for (Function::iterator BBI = F.begin(), BBE = F.end(); BBI != BBE; ++BBI) {
530 BasicBlock *BB = &*BBI;
531
532 BasicBlock::reverse_iterator Next;
533
534 for (BasicBlock::reverse_iterator I = BB->rbegin(), E = BB->rend(); I != E;
535 I = Next) {
536 Instruction &Inst = *I;
537 Next = std::next(x: I);
538
539 if (processAtomicInstr(I: &Inst)) {
540 MadeChange = true;
541
542 // New blocks may have been inserted.
543 BBE = F.end();
544 }
545 }
546 }
547
548 return MadeChange;
549}
550
551bool AtomicExpandLegacy::runOnFunction(Function &F) {
552
553 auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
554 if (!TPC)
555 return false;
556 auto *TM = &TPC->getTM<TargetMachine>();
557
558 const ModuleLibcallLoweringInfo &LibcallResult =
559 getAnalysis<LibcallLoweringInfoWrapper>().getResult(M: *F.getParent());
560 AtomicExpandImpl AE;
561 return AE.run(F, LibcallResult, TM);
562}
563
564FunctionPass *llvm::createAtomicExpandLegacyPass() {
565 return new AtomicExpandLegacy();
566}
567
568PreservedAnalyses AtomicExpandPass::run(Function &F,
569 FunctionAnalysisManager &FAM) {
570 auto &MAMProxy = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: F);
571
572 const ModuleLibcallLoweringInfo *LibcallResult =
573 MAMProxy.getCachedResult<LibcallLoweringModuleAnalysis>(IR&: *F.getParent());
574
575 if (!LibcallResult) {
576 F.getContext().emitError(ErrorStr: "'" + LibcallLoweringModuleAnalysis::name() +
577 "' analysis required");
578 return PreservedAnalyses::all();
579 }
580
581 AtomicExpandImpl AE;
582
583 bool Changed = AE.run(F, LibcallResult: *LibcallResult, TM);
584 if (!Changed)
585 return PreservedAnalyses::all();
586
587 return PreservedAnalyses::none();
588}
589
590bool AtomicExpandImpl::bracketInstWithFences(Instruction *I,
591 AtomicOrdering Order) {
592 ReplacementIRBuilder Builder(I, *DL);
593
594 auto LeadingFence = TLI->emitLeadingFence(Builder, Inst: I, Ord: Order);
595
596 auto TrailingFence = TLI->emitTrailingFence(Builder, Inst: I, Ord: Order);
597 // We have a guard here because not every atomic operation generates a
598 // trailing fence.
599 if (TrailingFence)
600 TrailingFence->moveAfter(MovePos: I);
601
602 return (LeadingFence || TrailingFence);
603}
604
605/// Get the iX type with the same bitwidth as T.
606IntegerType *
607AtomicExpandImpl::getCorrespondingIntegerType(Type *T, const DataLayout &DL) {
608 EVT VT = TLI->getMemValueType(DL, Ty: T);
609 unsigned BitWidth = VT.getStoreSizeInBits();
610 assert(BitWidth == VT.getSizeInBits() && "must be a power of two");
611 return IntegerType::get(C&: T->getContext(), NumBits: BitWidth);
612}
613
614/// Convert an atomic load of a non-integral type to an integer load of the
615/// equivalent bitwidth. See the function comment on
616/// convertAtomicStoreToIntegerType for background.
617LoadInst *AtomicExpandImpl::convertAtomicLoadToIntegerType(LoadInst *LI) {
618 auto *M = LI->getModule();
619 Type *NewTy = getCorrespondingIntegerType(T: LI->getType(), DL: M->getDataLayout());
620
621 ReplacementIRBuilder Builder(LI, *DL);
622
623 Value *Addr = LI->getPointerOperand();
624
625 auto *NewLI = Builder.CreateLoad(Ty: NewTy, Ptr: Addr, Props: LI->getProperties());
626 LLVM_DEBUG(dbgs() << "Replaced " << *LI << " with " << *NewLI << "\n");
627
628 Value *NewVal = LI->getType()->isPtrOrPtrVectorTy()
629 ? Builder.CreateIntToPtr(V: NewLI, DestTy: LI->getType())
630 : Builder.CreateBitCast(V: NewLI, DestTy: LI->getType());
631 LI->replaceAllUsesWith(V: NewVal);
632 LI->eraseFromParent();
633 return NewLI;
634}
635
636AtomicRMWInst *
637AtomicExpandImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
638 assert(RMWI->getOperation() == AtomicRMWInst::Xchg);
639
640 auto *M = RMWI->getModule();
641 Type *NewTy =
642 getCorrespondingIntegerType(T: RMWI->getType(), DL: M->getDataLayout());
643
644 ReplacementIRBuilder Builder(RMWI, *DL);
645
646 Value *Addr = RMWI->getPointerOperand();
647 Value *Val = RMWI->getValOperand();
648 Value *NewVal = Builder.CreateBitPreservingCastChain(DL: *DL, V: Val, NewTy);
649
650 auto *NewRMWI = Builder.CreateAtomicRMW(Op: AtomicRMWInst::Xchg, Ptr: Addr, Val: NewVal,
651 Align: RMWI->getAlign(), Ordering: RMWI->getOrdering(),
652 SSID: RMWI->getSyncScopeID());
653 NewRMWI->setVolatile(RMWI->isVolatile());
654 copyMetadataForAtomic(Dest&: *NewRMWI, Source: *RMWI);
655 LLVM_DEBUG(dbgs() << "Replaced " << *RMWI << " with " << *NewRMWI << "\n");
656
657 if (!RMWI->use_empty()) {
658 Value *NewRVal =
659 Builder.CreateBitPreservingCastChain(DL: *DL, V: NewRMWI, NewTy: RMWI->getType());
660 RMWI->replaceAllUsesWith(V: NewRVal);
661 }
662
663 RMWI->eraseFromParent();
664 return NewRMWI;
665}
666
667/// Generic Expand handling for elementwise vector atomicrmw instructions.
668/// Halve an `atomicrmw elementwise <N x T>` and feed each half back through the
669/// normal atomic expansion pipeline. The target's shouldExpandAtomicRMWInIR()
670/// is re-queried at the halved width and may return any expansion kind there
671/// (e.g. None to preserve as a native vector atomic, Expand to keep halving,
672/// CmpXChg to emit one wide cmpxchg loop at the halved width). Two cases:
673/// N == 1: collapse directly to one scalar atomicrmw T
674/// N > 1: split into two atomicrmw elementwise <N/2 x T> (low at Ptr, high
675/// at gep inbounds <N/2 x T>, Ptr, 1); reassemble via a
676/// shufflevector.
677/// Note that halving works because the vectors must always be a power of two.
678///
679/// Each new atomicrmw inherits the original ordering/syncscope/volatility and
680/// metadata, and is fed back through processAtomicInstr() so further expansion
681/// fires per-half.
682void AtomicExpandImpl::expandElementwiseAtomicRMW(AtomicRMWInst *AI) {
683 assert(AI->isElementwise());
684 auto *VecTy = cast<FixedVectorType>(Val: AI->getType());
685 Type *LaneTy = VecTy->getElementType();
686 const unsigned NumLanes = VecTy->getNumElements();
687 Value *Ptr = AI->getPointerOperand();
688 Value *Val = AI->getValOperand();
689
690 ReplacementIRBuilder Builder(AI, *DL);
691 Builder.disableGEPCanonicalization();
692 IntegerType *IdxTy =
693 DL->getIndexType(C&: AI->getContext(), AddressSpace: AI->getPointerAddressSpace());
694
695 auto CreateRMWInstruction = [&](Value *HalfPtr, Value *HalfVal, Align A,
696 bool Elementwise) -> AtomicRMWInst * {
697 auto *NewAI =
698 Builder.CreateAtomicRMW(Op: AI->getOperation(), Ptr: HalfPtr, Val: HalfVal, Align: A,
699 Ordering: AI->getOrdering(), SSID: AI->getSyncScopeID());
700 NewAI->setVolatile(AI->isVolatile());
701 NewAI->setElementwise(Elementwise);
702 copyMetadataForAtomic(Dest&: *NewAI, Source: *AI);
703 return NewAI;
704 };
705
706 Constant *IdxZero = ConstantInt::get(Ty: IdxTy, V: 0);
707 Constant *IdxOne = ConstantInt::get(Ty: IdxTy, V: 1);
708
709 // N == 1: collapse to a single scalar atomicrmw T and wrap the result back
710 // into a <1 x T> vector.
711 if (NumLanes == 1) {
712 Value *LaneVal = Builder.CreateExtractElement(Vec: Val, Idx: IdxZero, Name: "lane.val");
713 AtomicRMWInst *LaneRMW = CreateRMWInstruction(Ptr, LaneVal, AI->getAlign(),
714 /*Elementwise=*/false);
715 Value *Result = Builder.CreateInsertElement(Vec: PoisonValue::get(T: VecTy),
716 NewElt: LaneRMW, Idx: IdxZero, Name: "lane.old");
717 AI->replaceAllUsesWith(V: Result);
718 AI->eraseFromParent();
719 processAtomicInstr(I: LaneRMW);
720 return;
721 }
722
723 // N > 1: split into two <N/2 x T> elementwise atomicrmws and recurse via
724 // processAtomicInstr().
725 assert(isPowerOf2_32(NumLanes) &&
726 "elementwise atomicrmw vector length must be a power of two");
727 const unsigned HalfLanes = NumLanes / 2;
728 auto *HalfVecTy = FixedVectorType::get(ElementType: LaneTy, NumElts: HalfLanes);
729 const uint64_t HalfBytes = DL->getTypeStoreSize(Ty: HalfVecTy).getFixedValue();
730
731 Value *LoVal = Builder.CreateShuffleVector(
732 V: Val, Mask: createSequentialMask(Start: 0, NumInts: HalfLanes, NumUndefs: 0), Name: "lo.val");
733 Value *HiVal = Builder.CreateShuffleVector(
734 V: Val, Mask: createSequentialMask(Start: HalfLanes, NumInts: HalfLanes, NumUndefs: 0), Name: "hi.val");
735
736 Value *HiPtr = Builder.CreateInBoundsGEP(Ty: HalfVecTy, Ptr, IdxList: IdxOne, Name: "hi.ptr");
737 Align LoAlign = AI->getAlign();
738 Align HiAlign = commonAlignment(A: LoAlign, Offset: HalfBytes);
739
740 AtomicRMWInst *LoRMW =
741 CreateRMWInstruction(Ptr, LoVal, LoAlign, /*Elementwise=*/true);
742 AtomicRMWInst *HiRMW =
743 CreateRMWInstruction(HiPtr, HiVal, HiAlign, /*Elementwise=*/true);
744
745 Value *Result = concatenateVectors(Builder, Vecs: {LoRMW, HiRMW});
746 Result->setName("old");
747
748 AI->replaceAllUsesWith(V: Result);
749 AI->eraseFromParent();
750 processAtomicInstr(I: LoRMW);
751 processAtomicInstr(I: HiRMW);
752}
753
754bool AtomicExpandImpl::tryExpandAtomicLoad(LoadInst *LI) {
755 switch (TLI->shouldExpandAtomicLoadInIR(LI)) {
756 case TargetLoweringBase::AtomicExpansionKind::None:
757 return false;
758 case TargetLoweringBase::AtomicExpansionKind::LLSC:
759 expandAtomicOpToLLSC(
760 I: LI, ResultTy: LI->getType(), Addr: LI->getPointerOperand(), AddrAlign: LI->getAlign(),
761 MemOpOrder: LI->getOrdering(),
762 PerformOp: [](IRBuilderBase &Builder, Value *Loaded) { return Loaded; });
763 return true;
764 case TargetLoweringBase::AtomicExpansionKind::LLOnly:
765 return expandAtomicLoadToLL(LI);
766 case TargetLoweringBase::AtomicExpansionKind::CmpXChg:
767 return expandAtomicLoadToCmpXchg(LI);
768 case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
769 LI->setAtomic(Ordering: AtomicOrdering::NotAtomic);
770 return true;
771 case TargetLoweringBase::AtomicExpansionKind::CustomExpand:
772 TLI->emitExpandAtomicLoad(LI);
773 return true;
774 default:
775 llvm_unreachable("Unhandled case in tryExpandAtomicLoad");
776 }
777}
778
779bool AtomicExpandImpl::tryExpandAtomicStore(StoreInst *SI) {
780 switch (TLI->shouldExpandAtomicStoreInIR(SI)) {
781 case TargetLoweringBase::AtomicExpansionKind::None:
782 return false;
783 case TargetLoweringBase::AtomicExpansionKind::CustomExpand:
784 TLI->emitExpandAtomicStore(SI);
785 return true;
786 case TargetLoweringBase::AtomicExpansionKind::Expand:
787 expandAtomicStoreToXChg(SI);
788 return true;
789 case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
790 SI->setAtomic(Ordering: AtomicOrdering::NotAtomic);
791 return true;
792 default:
793 llvm_unreachable("Unhandled case in tryExpandAtomicStore");
794 }
795}
796
797bool AtomicExpandImpl::expandAtomicLoadToLL(LoadInst *LI) {
798 ReplacementIRBuilder Builder(LI, *DL);
799
800 // On some architectures, load-linked instructions are atomic for larger
801 // sizes than normal loads. For example, the only 64-bit load guaranteed
802 // to be single-copy atomic by ARM is an ldrexd (A3.5.3).
803 Value *Val = TLI->emitLoadLinked(Builder, ValueTy: LI->getType(),
804 Addr: LI->getPointerOperand(), Ord: LI->getOrdering());
805 TLI->emitAtomicCmpXchgNoStoreLLBalance(Builder);
806
807 LI->replaceAllUsesWith(V: Val);
808 LI->eraseFromParent();
809
810 return true;
811}
812
813bool AtomicExpandImpl::expandAtomicLoadToCmpXchg(LoadInst *LI) {
814 ReplacementIRBuilder Builder(LI, *DL);
815 AtomicOrdering Order = LI->getOrdering();
816 if (Order == AtomicOrdering::Unordered)
817 Order = AtomicOrdering::Monotonic;
818
819 Value *Addr = LI->getPointerOperand();
820 Type *Ty = LI->getType();
821
822 // cmpxchg supports only integer and pointer operands. If the load type is
823 // FP or vector, run the cmpxchg on the same-sized integer and bitcast the
824 // result back; mirrors createCmpXchgInstFun.
825 bool NeedBitcast = Ty->isFloatingPointTy() || Ty->isVectorTy();
826 Type *CmpXchgTy = Ty;
827 if (NeedBitcast)
828 CmpXchgTy = getCorrespondingIntegerType(T: Ty, DL: *DL);
829 Constant *DummyVal = Constant::getNullValue(Ty: CmpXchgTy);
830
831 AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
832 Ptr: Addr, Cmp: DummyVal, New: DummyVal, Align: LI->getAlign(), SuccessOrdering: Order,
833 FailureOrdering: AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: Order),
834 SSID: LI->getSyncScopeID());
835 Pair->setVolatile(LI->isVolatile());
836 Value *Loaded = Builder.CreateExtractValue(Agg: Pair, Idxs: 0, Name: "loaded");
837 if (NeedBitcast)
838 Loaded = Builder.CreateBitPreservingCastChain(DL: *DL, V: Loaded, NewTy: Ty);
839
840 LI->replaceAllUsesWith(V: Loaded);
841 LI->eraseFromParent();
842
843 return true;
844}
845
846/// Convert an atomic store of a non-integral type to an integer store of the
847/// equivalent bitwidth. We used to not support floating point or vector
848/// atomics in the IR at all. The backends learned to deal with the bitcast
849/// idiom because that was the only way of expressing the notion of a atomic
850/// float or vector store. The long term plan is to teach each backend to
851/// instruction select from the original atomic store, but as a migration
852/// mechanism, we convert back to the old format which the backends understand.
853/// Each backend will need individual work to recognize the new format.
854StoreInst *AtomicExpandImpl::convertAtomicStoreToIntegerType(StoreInst *SI) {
855 ReplacementIRBuilder Builder(SI, *DL);
856 auto *M = SI->getModule();
857 Type *NewTy = getCorrespondingIntegerType(T: SI->getValueOperand()->getType(),
858 DL: M->getDataLayout());
859 Value *NewVal = SI->getValueOperand()->getType()->isPtrOrPtrVectorTy()
860 ? Builder.CreatePtrToInt(V: SI->getValueOperand(), DestTy: NewTy)
861 : Builder.CreateBitCast(V: SI->getValueOperand(), DestTy: NewTy);
862
863 Value *Addr = SI->getPointerOperand();
864
865 StoreInst *NewSI = Builder.CreateStore(Val: NewVal, Ptr: Addr, Props: SI->getProperties());
866 copyMetadataForAtomic(Dest&: *NewSI, Source: *SI);
867 LLVM_DEBUG(dbgs() << "Replaced " << *SI << " with " << *NewSI << "\n");
868 SI->eraseFromParent();
869 return NewSI;
870}
871
872void AtomicExpandImpl::expandAtomicStoreToXChg(StoreInst *SI) {
873 // This function is only called on atomic stores that are too large to be
874 // atomic if implemented as a native store. So we replace them by an
875 // atomic swap, that can be implemented for example as a ldrex/strex on ARM
876 // or lock cmpxchg8/16b on X86, as these are atomic for larger sizes.
877 // It is the responsibility of the target to only signal expansion via
878 // shouldExpandAtomicRMW in cases where this is required and possible.
879 ReplacementIRBuilder Builder(SI, *DL);
880 AtomicOrdering Ordering = SI->getOrdering();
881 assert(Ordering != AtomicOrdering::NotAtomic);
882 AtomicOrdering RMWOrdering = Ordering == AtomicOrdering::Unordered
883 ? AtomicOrdering::Monotonic
884 : Ordering;
885 AtomicRMWInst *AI = Builder.CreateAtomicRMW(
886 Op: AtomicRMWInst::Xchg, Ptr: SI->getPointerOperand(), Val: SI->getValueOperand(),
887 Align: SI->getAlign(), Ordering: RMWOrdering, SSID: SI->getSyncScopeID());
888 AI->setVolatile(SI->isVolatile());
889 SI->eraseFromParent();
890
891 if (TLI->shouldCastAtomicRMWIInIR(RMWI: AI) ==
892 TargetLoweringBase::AtomicExpansionKind::CastToInteger)
893 AI = convertAtomicXchgToIntegerType(RMWI: AI);
894
895 // Now we have an appropriate swap instruction, lower it as usual.
896 tryExpandAtomicRMW(AI);
897}
898
899static void createCmpXchgInstFun(IRBuilderBase &Builder, Value *Addr,
900 Value *Loaded, Value *NewVal, Align AddrAlign,
901 AtomicOrdering MemOpOrder, SyncScope::ID SSID,
902 bool IsVolatile, Value *&Success,
903 Value *&NewLoaded, Instruction *MetadataSrc) {
904 Type *OrigTy = NewVal->getType();
905
906 // This code can go away when cmpxchg supports FP and vector types.
907 assert(!OrigTy->isPointerTy());
908 bool NeedBitcast = OrigTy->isFloatingPointTy() || OrigTy->isVectorTy();
909 if (NeedBitcast) {
910 IntegerType *IntTy = Builder.getIntNTy(N: OrigTy->getPrimitiveSizeInBits());
911 NewVal = Builder.CreateBitCast(V: NewVal, DestTy: IntTy);
912 Loaded = Builder.CreateBitCast(V: Loaded, DestTy: IntTy);
913 }
914
915 AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
916 Ptr: Addr, Cmp: Loaded, New: NewVal, Align: AddrAlign, SuccessOrdering: MemOpOrder,
917 FailureOrdering: AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: MemOpOrder), SSID);
918 Pair->setVolatile(IsVolatile);
919 if (MetadataSrc)
920 copyMetadataForAtomic(Dest&: *Pair, Source: *MetadataSrc);
921
922 Success = Builder.CreateExtractValue(Agg: Pair, Idxs: 1, Name: "success");
923 NewLoaded = Builder.CreateExtractValue(Agg: Pair, Idxs: 0, Name: "newloaded");
924
925 if (NeedBitcast)
926 NewLoaded = Builder.CreateBitCast(V: NewLoaded, DestTy: OrigTy);
927}
928
929void AtomicExpandImpl::expandAtomicSubToAdd(AtomicRMWInst *AI) {
930 ReplacementIRBuilder Builder(AI, *DL);
931 AtomicRMWInst::BinOp NewOp;
932 Value *NewVal;
933
934 switch (AI->getOperation()) {
935 case AtomicRMWInst::Sub:
936 NewOp = AtomicRMWInst::Add;
937 NewVal = Builder.CreateNeg(V: AI->getValOperand(), Name: "neg");
938 break;
939 case AtomicRMWInst::FSub:
940 NewOp = AtomicRMWInst::FAdd;
941 NewVal = Builder.CreateFNeg(V: AI->getValOperand(), Name: "fneg");
942 break;
943 default:
944 llvm_unreachable("unsupported atomicrmw expansion");
945 }
946
947 AI->setOperation(NewOp);
948 AI->setOperand(i_nocapture: 1, Val_nocapture: NewVal);
949}
950
951bool AtomicExpandImpl::tryExpandAtomicRMW(AtomicRMWInst *AI) {
952 LLVMContext &Ctx = AI->getModule()->getContext();
953 TargetLowering::AtomicExpansionKind Kind = TLI->shouldExpandAtomicRMWInIR(RMW: AI);
954 switch (Kind) {
955 case TargetLoweringBase::AtomicExpansionKind::None:
956 return false;
957 case TargetLoweringBase::AtomicExpansionKind::LLSC: {
958 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
959 unsigned ValueSize = getAtomicOpSize(RMWI: AI);
960 if (ValueSize < MinCASSize) {
961 expandPartwordAtomicRMW(I: AI,
962 ExpansionKind: TargetLoweringBase::AtomicExpansionKind::LLSC);
963 } else {
964 auto PerformOp = [&](IRBuilderBase &Builder, Value *Loaded) {
965 return buildAtomicRMWValue(Op: AI->getOperation(), Builder, Loaded,
966 Val: AI->getValOperand());
967 };
968 expandAtomicOpToLLSC(I: AI, ResultTy: AI->getType(), Addr: AI->getPointerOperand(),
969 AddrAlign: AI->getAlign(), MemOpOrder: AI->getOrdering(), PerformOp);
970 }
971 return true;
972 }
973 case TargetLoweringBase::AtomicExpansionKind::CmpXChg: {
974 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
975 unsigned ValueSize = getAtomicOpSize(RMWI: AI);
976 if (ValueSize < MinCASSize) {
977 expandPartwordAtomicRMW(I: AI,
978 ExpansionKind: TargetLoweringBase::AtomicExpansionKind::CmpXChg);
979 } else {
980 SmallVector<StringRef> SSNs;
981 Ctx.getSyncScopeNames(SSNs);
982 auto MemScope = SSNs[AI->getSyncScopeID()].empty()
983 ? "system"
984 : SSNs[AI->getSyncScopeID()];
985 OptimizationRemarkEmitter ORE(AI->getFunction());
986 ORE.emit(RemarkBuilder: [&]() {
987 return OptimizationRemark(DEBUG_TYPE, "Passed", AI)
988 << "A compare and swap loop was generated for an atomic "
989 << AI->getOperationName(Op: AI->getOperation()) << " operation at "
990 << MemScope << " memory scope";
991 });
992 expandAtomicRMWToCmpXchg(AI, CreateCmpXchg: createCmpXchgInstFun);
993 }
994 return true;
995 }
996 case TargetLoweringBase::AtomicExpansionKind::MaskedIntrinsic: {
997 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
998 unsigned ValueSize = getAtomicOpSize(RMWI: AI);
999 if (ValueSize < MinCASSize) {
1000 AtomicRMWInst::BinOp Op = AI->getOperation();
1001 // Widen And/Or/Xor and give the target another chance at expanding it.
1002 if (Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||
1003 Op == AtomicRMWInst::And) {
1004 tryExpandAtomicRMW(AI: widenPartwordAtomicRMW(AI));
1005 return true;
1006 }
1007 }
1008 expandAtomicRMWToMaskedIntrinsic(AI);
1009 return true;
1010 }
1011 case TargetLoweringBase::AtomicExpansionKind::BitTestIntrinsic: {
1012 TLI->emitBitTestAtomicRMWIntrinsic(AI);
1013 return true;
1014 }
1015 case TargetLoweringBase::AtomicExpansionKind::CmpArithIntrinsic: {
1016 TLI->emitCmpArithAtomicRMWIntrinsic(AI);
1017 return true;
1018 }
1019 case TargetLoweringBase::AtomicExpansionKind::Expand:
1020 if (AI->isElementwise()) {
1021 expandElementwiseAtomicRMW(AI);
1022 return true;
1023 }
1024 expandAtomicSubToAdd(AI);
1025 return true;
1026 case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
1027 return lowerAtomicRMWInst(RMWI: AI);
1028 case TargetLoweringBase::AtomicExpansionKind::CustomExpand:
1029 TLI->emitExpandAtomicRMW(AI);
1030 return true;
1031 default:
1032 llvm_unreachable("Unhandled case in tryExpandAtomicRMW");
1033 }
1034}
1035
1036namespace {
1037
1038struct PartwordMaskValues {
1039 // These three fields are guaranteed to be set by createMaskInstrs.
1040 Type *WordType = nullptr;
1041 Type *ValueType = nullptr;
1042 Type *IntValueType = nullptr;
1043 Value *AlignedAddr = nullptr;
1044 Align AlignedAddrAlignment;
1045 // The remaining fields can be null.
1046 Value *ShiftAmt = nullptr;
1047 Value *Mask = nullptr;
1048 Value *Inv_Mask = nullptr;
1049};
1050
1051[[maybe_unused]]
1052raw_ostream &operator<<(raw_ostream &O, const PartwordMaskValues &PMV) {
1053 auto PrintObj = [&O](auto *V) {
1054 if (V)
1055 O << *V;
1056 else
1057 O << "nullptr";
1058 O << '\n';
1059 };
1060 O << "PartwordMaskValues {\n";
1061 O << " WordType: ";
1062 PrintObj(PMV.WordType);
1063 O << " ValueType: ";
1064 PrintObj(PMV.ValueType);
1065 O << " AlignedAddr: ";
1066 PrintObj(PMV.AlignedAddr);
1067 O << " AlignedAddrAlignment: " << PMV.AlignedAddrAlignment.value() << '\n';
1068 O << " ShiftAmt: ";
1069 PrintObj(PMV.ShiftAmt);
1070 O << " Mask: ";
1071 PrintObj(PMV.Mask);
1072 O << " Inv_Mask: ";
1073 PrintObj(PMV.Inv_Mask);
1074 O << "}\n";
1075 return O;
1076}
1077
1078} // end anonymous namespace
1079
1080/// This is a helper function which builds instructions to provide
1081/// values necessary for partword atomic operations. It takes an
1082/// incoming address, Addr, and ValueType, and constructs the address,
1083/// shift-amounts and masks needed to work with a larger value of size
1084/// WordSize.
1085///
1086/// AlignedAddr: Addr rounded down to a multiple of WordSize
1087///
1088/// ShiftAmt: Number of bits to right-shift a WordSize value loaded
1089/// from AlignAddr for it to have the same value as if
1090/// ValueType was loaded from Addr.
1091///
1092/// Mask: Value to mask with the value loaded from AlignAddr to
1093/// include only the part that would've been loaded from Addr.
1094///
1095/// Inv_Mask: The inverse of Mask.
1096static PartwordMaskValues createMaskInstrs(IRBuilderBase &Builder,
1097 Instruction *I, Type *ValueType,
1098 Value *Addr, Align AddrAlign,
1099 unsigned MinWordSize) {
1100 PartwordMaskValues PMV;
1101
1102 Module *M = I->getModule();
1103 LLVMContext &Ctx = M->getContext();
1104 const DataLayout &DL = M->getDataLayout();
1105 unsigned ValueSize = DL.getTypeStoreSize(Ty: ValueType);
1106
1107 PMV.ValueType = PMV.IntValueType = ValueType;
1108 if (PMV.ValueType->isFloatingPointTy() || PMV.ValueType->isVectorTy())
1109 PMV.IntValueType =
1110 Type::getIntNTy(C&: Ctx, N: ValueType->getPrimitiveSizeInBits());
1111
1112 PMV.WordType = MinWordSize > ValueSize ? Type::getIntNTy(C&: Ctx, N: MinWordSize * 8)
1113 : ValueType;
1114 if (PMV.ValueType == PMV.WordType) {
1115 PMV.AlignedAddr = Addr;
1116 PMV.AlignedAddrAlignment = AddrAlign;
1117 PMV.ShiftAmt = ConstantInt::get(Ty: PMV.ValueType, V: 0);
1118 PMV.Mask = ConstantInt::get(Ty: PMV.ValueType, V: ~0, /*isSigned*/ IsSigned: true);
1119 return PMV;
1120 }
1121
1122 PMV.AlignedAddrAlignment = Align(MinWordSize);
1123
1124 assert(ValueSize < MinWordSize);
1125
1126 PointerType *PtrTy = cast<PointerType>(Val: Addr->getType());
1127 IntegerType *IntTy = DL.getIndexType(C&: Ctx, AddressSpace: PtrTy->getAddressSpace());
1128 Value *PtrLSB;
1129
1130 if (AddrAlign < MinWordSize) {
1131 PMV.AlignedAddr = Builder.CreateIntrinsic(
1132 ID: Intrinsic::ptrmask, OverloadTypes: {PtrTy, IntTy},
1133 Args: {Addr, ConstantInt::getSigned(Ty: IntTy, V: ~(uint64_t)(MinWordSize - 1))},
1134 FMFSource: nullptr, Name: "AlignedAddr");
1135
1136 Value *AddrInt = Builder.CreatePtrToInt(V: Addr, DestTy: IntTy);
1137 PtrLSB = Builder.CreateAnd(LHS: AddrInt, RHS: MinWordSize - 1, Name: "PtrLSB");
1138 } else {
1139 // If the alignment is high enough, the LSB are known 0.
1140 PMV.AlignedAddr = Addr;
1141 PtrLSB = ConstantInt::getNullValue(Ty: IntTy);
1142 }
1143
1144 if (DL.isLittleEndian()) {
1145 // turn bytes into bits
1146 PMV.ShiftAmt = Builder.CreateShl(LHS: PtrLSB, RHS: 3);
1147 } else {
1148 // turn bytes into bits, and count from the other side.
1149 PMV.ShiftAmt = Builder.CreateShl(
1150 LHS: Builder.CreateXor(LHS: PtrLSB, RHS: MinWordSize - ValueSize), RHS: 3);
1151 }
1152
1153 PMV.ShiftAmt = Builder.CreateTrunc(V: PMV.ShiftAmt, DestTy: PMV.WordType, Name: "ShiftAmt");
1154 PMV.Mask = Builder.CreateShl(
1155 LHS: ConstantInt::get(Ty: PMV.WordType, V: (1 << (ValueSize * 8)) - 1), RHS: PMV.ShiftAmt,
1156 Name: "Mask");
1157
1158 PMV.Inv_Mask = Builder.CreateNot(V: PMV.Mask, Name: "Inv_Mask");
1159
1160 return PMV;
1161}
1162
1163static Value *extractMaskedValue(IRBuilderBase &Builder, Value *WideWord,
1164 const PartwordMaskValues &PMV) {
1165 assert(WideWord->getType() == PMV.WordType && "Widened type mismatch");
1166 if (PMV.WordType == PMV.ValueType)
1167 return WideWord;
1168
1169 Value *Shift = Builder.CreateLShr(LHS: WideWord, RHS: PMV.ShiftAmt, Name: "shifted");
1170 Value *Trunc = Builder.CreateTrunc(V: Shift, DestTy: PMV.IntValueType, Name: "extracted");
1171 return Builder.CreateBitCast(V: Trunc, DestTy: PMV.ValueType);
1172}
1173
1174static Value *insertMaskedValue(IRBuilderBase &Builder, Value *WideWord,
1175 Value *Updated, const PartwordMaskValues &PMV) {
1176 assert(WideWord->getType() == PMV.WordType && "Widened type mismatch");
1177 assert(Updated->getType() == PMV.ValueType && "Value type mismatch");
1178 if (PMV.WordType == PMV.ValueType)
1179 return Updated;
1180
1181 Updated = Builder.CreateBitCast(V: Updated, DestTy: PMV.IntValueType);
1182
1183 Value *ZExt = Builder.CreateZExt(V: Updated, DestTy: PMV.WordType, Name: "extended");
1184 Value *Shift =
1185 Builder.CreateShl(LHS: ZExt, RHS: PMV.ShiftAmt, Name: "shifted", /*HasNUW*/ true);
1186 Value *And = Builder.CreateAnd(LHS: WideWord, RHS: PMV.Inv_Mask, Name: "unmasked");
1187 Value *Or = Builder.CreateOr(LHS: And, RHS: Shift, Name: "inserted");
1188 return Or;
1189}
1190
1191/// Emit IR to implement a masked version of a given atomicrmw
1192/// operation. (That is, only the bits under the Mask should be
1193/// affected by the operation)
1194static Value *performMaskedAtomicOp(AtomicRMWInst::BinOp Op,
1195 IRBuilderBase &Builder, Value *Loaded,
1196 Value *ValOperand_Shifted, Value *Inc,
1197 const PartwordMaskValues &PMV) {
1198 // TODO: update to use
1199 // https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge in order
1200 // to merge bits from two values without requiring PMV.Inv_Mask.
1201
1202 assert(Op != AtomicRMWInst::Or && Op != AtomicRMWInst::Xor &&
1203 Op != AtomicRMWInst::And &&
1204 "Or/Xor/And handled by widenPartwordAtomicRMW");
1205
1206 if (Op == AtomicRMWInst::Xchg) {
1207 // Clear all the bits we are exchanging out. These are the bits under the
1208 // mask. We can clear them with an `and` of the inverse mask.
1209 Value *Loaded_MaskOut = Builder.CreateAnd(LHS: Loaded, RHS: PMV.Inv_Mask);
1210 // Now that the prevous bits are cleared, we can swap in the new value with
1211 // an `or`.
1212 Value *FinalVal = Builder.CreateOr(LHS: Loaded_MaskOut, RHS: ValOperand_Shifted);
1213 return FinalVal;
1214 }
1215
1216 if (Op == AtomicRMWInst::Nand ||
1217 (!PMV.ValueType->isVectorTy() &&
1218 (Op == AtomicRMWInst::Add || Op == AtomicRMWInst::Sub))) {
1219 // For `Nand` and non-vector `Add` and `Sub`, we can perform the operation
1220 // on the entire word because the extra bits in the unmasked region don't
1221 // affect the computation in the masked region. The operation might still
1222 // overwrite the unmasked region (e.g. from integer overflow or underflow),
1223 // so we have to reapply the unmasked region afterwards.
1224 //
1225 // This trick doesn't work for vector `Add` and `Sub` because we use a
1226 // scalar operation on the entire word. Scalarizing vector `Add` and `Sub`
1227 // isn't legal because the vector versions may have element-wise overflows.
1228 // TODO: For these, can we use a wider vector op with additional lanes?
1229
1230 // Atomic operation across the entire word.
1231 Value *NewVal =
1232 buildAtomicRMWValue(Op, Builder, Loaded, Val: ValOperand_Shifted);
1233 // Reapply the bits in the unmasked region.
1234 Value *NewVal_Masked = Builder.CreateAnd(LHS: NewVal, RHS: PMV.Mask);
1235 Value *Loaded_MaskOut = Builder.CreateAnd(LHS: Loaded, RHS: PMV.Inv_Mask);
1236 Value *FinalVal = Builder.CreateOr(LHS: Loaded_MaskOut, RHS: NewVal_Masked);
1237 return FinalVal;
1238 }
1239
1240 // All other ops operate on the sub-word size. Truncate down to the
1241 // original size, and expand out again after doing the operation. Bitcasts
1242 // will be inserted for FP values.
1243 assert(!ValOperand_Shifted);
1244 Value *Loaded_Extract = extractMaskedValue(Builder, WideWord: Loaded, PMV);
1245 Value *NewVal = buildAtomicRMWValue(Op, Builder, Loaded: Loaded_Extract, Val: Inc);
1246 Value *FinalVal = insertMaskedValue(Builder, WideWord: Loaded, Updated: NewVal, PMV);
1247 return FinalVal;
1248}
1249
1250/// Expand a sub-word atomicrmw operation into an appropriate
1251/// word-sized operation.
1252///
1253/// It will create an LL/SC or cmpxchg loop, as appropriate, the same
1254/// way as a typical atomicrmw expansion. The only difference here is
1255/// that the operation inside of the loop may operate upon only a
1256/// part of the value.
1257void AtomicExpandImpl::expandPartwordAtomicRMW(
1258 AtomicRMWInst *AI, TargetLoweringBase::AtomicExpansionKind ExpansionKind) {
1259 // Widen And/Or/Xor and give the target another chance at expanding it.
1260 AtomicRMWInst::BinOp Op = AI->getOperation();
1261 if (Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||
1262 Op == AtomicRMWInst::And) {
1263 tryExpandAtomicRMW(AI: widenPartwordAtomicRMW(AI));
1264 return;
1265 }
1266 AtomicOrdering MemOpOrder = AI->getOrdering();
1267 SyncScope::ID SSID = AI->getSyncScopeID();
1268
1269 ReplacementIRBuilder Builder(AI, *DL);
1270
1271 PartwordMaskValues PMV =
1272 createMaskInstrs(Builder, I: AI, ValueType: AI->getType(), Addr: AI->getPointerOperand(),
1273 AddrAlign: AI->getAlign(), MinWordSize: TLI->getMinCmpXchgSizeInBits() / 8);
1274
1275 Value *ValOperand_Shifted = nullptr;
1276 bool NeedsShiftedOperand =
1277 Op == AtomicRMWInst::Xchg || Op == AtomicRMWInst::Nand ||
1278 (!PMV.ValueType->isVectorTy() &&
1279 (Op == AtomicRMWInst::Add || Op == AtomicRMWInst::Sub));
1280
1281 if (NeedsShiftedOperand) {
1282 Value *ValOp = Builder.CreateBitCast(V: AI->getValOperand(), DestTy: PMV.IntValueType);
1283 ValOperand_Shifted =
1284 Builder.CreateShl(LHS: Builder.CreateZExt(V: ValOp, DestTy: PMV.WordType), RHS: PMV.ShiftAmt,
1285 Name: "ValOperand_Shifted");
1286 }
1287
1288 auto PerformPartwordOp = [&](IRBuilderBase &Builder, Value *Loaded) {
1289 return performMaskedAtomicOp(Op, Builder, Loaded, ValOperand_Shifted,
1290 Inc: AI->getValOperand(), PMV);
1291 };
1292
1293 Value *OldResult;
1294 if (ExpansionKind == TargetLoweringBase::AtomicExpansionKind::CmpXChg) {
1295 OldResult = insertRMWCmpXchgLoop(Builder, ResultType: PMV.WordType, Addr: PMV.AlignedAddr,
1296 AddrAlign: PMV.AlignedAddrAlignment, MemOpOrder, SSID,
1297 IsVolatile: AI->isVolatile(), PerformOp: PerformPartwordOp,
1298 CreateCmpXchg: createCmpXchgInstFun, MetadataSrc: AI);
1299 } else {
1300 assert(ExpansionKind == TargetLoweringBase::AtomicExpansionKind::LLSC);
1301 OldResult = insertRMWLLSCLoop(Builder, ResultTy: PMV.WordType, Addr: PMV.AlignedAddr,
1302 AddrAlign: PMV.AlignedAddrAlignment, MemOpOrder,
1303 PerformOp: PerformPartwordOp);
1304 }
1305
1306 Value *FinalOldResult = extractMaskedValue(Builder, WideWord: OldResult, PMV);
1307 AI->replaceAllUsesWith(V: FinalOldResult);
1308 AI->eraseFromParent();
1309}
1310
1311// Widen the bitwise atomicrmw (or/xor/and) to the minimum supported width.
1312AtomicRMWInst *AtomicExpandImpl::widenPartwordAtomicRMW(AtomicRMWInst *AI) {
1313 ReplacementIRBuilder Builder(AI, *DL);
1314 AtomicRMWInst::BinOp Op = AI->getOperation();
1315
1316 assert((Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||
1317 Op == AtomicRMWInst::And) &&
1318 "Unable to widen operation");
1319
1320 PartwordMaskValues PMV =
1321 createMaskInstrs(Builder, I: AI, ValueType: AI->getType(), Addr: AI->getPointerOperand(),
1322 AddrAlign: AI->getAlign(), MinWordSize: TLI->getMinCmpXchgSizeInBits() / 8);
1323
1324 Value *ValOp = AI->getValOperand();
1325 if (ValOp->getType()->isVectorTy())
1326 // For vectors, bitcast to the integer type before extending. Note that
1327 // or/xor/and on vectors are equivalent to the same operation on an integer
1328 // that spans the vector, so we can use the integer type for the operation.
1329 ValOp = Builder.CreateBitCast(V: ValOp, DestTy: PMV.IntValueType);
1330 Value *ValOperand_Shifted =
1331 Builder.CreateShl(LHS: Builder.CreateZExt(V: ValOp, DestTy: PMV.WordType), RHS: PMV.ShiftAmt,
1332 Name: "ValOperand_Shifted");
1333
1334 Value *NewOperand;
1335
1336 if (Op == AtomicRMWInst::And)
1337 NewOperand =
1338 Builder.CreateOr(LHS: ValOperand_Shifted, RHS: PMV.Inv_Mask, Name: "AndOperand");
1339 else
1340 NewOperand = ValOperand_Shifted;
1341
1342 AtomicRMWInst *NewAI = Builder.CreateAtomicRMW(
1343 Op, Ptr: PMV.AlignedAddr, Val: NewOperand, Align: PMV.AlignedAddrAlignment,
1344 Ordering: AI->getOrdering(), SSID: AI->getSyncScopeID());
1345
1346 NewAI->setVolatile(AI->isVolatile());
1347 copyMetadataForAtomic(Dest&: *NewAI, Source: *AI);
1348
1349 Value *FinalOldResult = extractMaskedValue(Builder, WideWord: NewAI, PMV);
1350 AI->replaceAllUsesWith(V: FinalOldResult);
1351 AI->eraseFromParent();
1352 return NewAI;
1353}
1354
1355bool AtomicExpandImpl::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
1356 // The basic idea here is that we're expanding a cmpxchg of a
1357 // smaller memory size up to a word-sized cmpxchg. To do this, we
1358 // need to add a retry-loop for strong cmpxchg, so that
1359 // modifications to other parts of the word don't cause a spurious
1360 // failure.
1361
1362 // This generates code like the following:
1363 // [[Setup mask values PMV.*]]
1364 // %NewVal_Shifted = shl i32 %NewVal, %PMV.ShiftAmt
1365 // %Cmp_Shifted = shl i32 %Cmp, %PMV.ShiftAmt
1366 // %InitLoaded = load i32* %addr
1367 // %InitLoaded_MaskOut = and i32 %InitLoaded, %PMV.Inv_Mask
1368 // br partword.cmpxchg.loop
1369 // partword.cmpxchg.loop:
1370 // %Loaded_MaskOut = phi i32 [ %InitLoaded_MaskOut, %entry ],
1371 // [ %OldVal_MaskOut, %partword.cmpxchg.failure ]
1372 // %FullWord_NewVal = or i32 %Loaded_MaskOut, %NewVal_Shifted
1373 // %FullWord_Cmp = or i32 %Loaded_MaskOut, %Cmp_Shifted
1374 // %NewCI = cmpxchg i32* %PMV.AlignedAddr, i32 %FullWord_Cmp,
1375 // i32 %FullWord_NewVal success_ordering failure_ordering
1376 // %OldVal = extractvalue { i32, i1 } %NewCI, 0
1377 // %Success = extractvalue { i32, i1 } %NewCI, 1
1378 // br i1 %Success, label %partword.cmpxchg.end,
1379 // label %partword.cmpxchg.failure
1380 // partword.cmpxchg.failure:
1381 // %OldVal_MaskOut = and i32 %OldVal, %PMV.Inv_Mask
1382 // %ShouldContinue = icmp ne i32 %Loaded_MaskOut, %OldVal_MaskOut
1383 // br i1 %ShouldContinue, label %partword.cmpxchg.loop,
1384 // label %partword.cmpxchg.end
1385 // partword.cmpxchg.end:
1386 // %tmp1 = lshr i32 %OldVal, %PMV.ShiftAmt
1387 // %FinalOldVal = trunc i32 %tmp1 to i8
1388 // %tmp2 = insertvalue { i8, i1 } undef, i8 %FinalOldVal, 0
1389 // %Res = insertvalue { i8, i1 } %25, i1 %Success, 1
1390
1391 Value *Addr = CI->getPointerOperand();
1392 Value *Cmp = CI->getCompareOperand();
1393 Value *NewVal = CI->getNewValOperand();
1394
1395 BasicBlock *BB = CI->getParent();
1396 Function *F = BB->getParent();
1397 ReplacementIRBuilder Builder(CI, *DL);
1398 LLVMContext &Ctx = Builder.getContext();
1399
1400 BasicBlock *EndBB =
1401 BB->splitBasicBlock(I: CI->getIterator(), BBName: "partword.cmpxchg.end");
1402 auto FailureBB =
1403 BasicBlock::Create(Context&: Ctx, Name: "partword.cmpxchg.failure", Parent: F, InsertBefore: EndBB);
1404 auto LoopBB = BasicBlock::Create(Context&: Ctx, Name: "partword.cmpxchg.loop", Parent: F, InsertBefore: FailureBB);
1405
1406 // The split call above "helpfully" added a branch at the end of BB
1407 // (to the wrong place).
1408 std::prev(x: BB->end())->eraseFromParent();
1409 Builder.SetInsertPoint(BB);
1410
1411 PartwordMaskValues PMV =
1412 createMaskInstrs(Builder, I: CI, ValueType: CI->getCompareOperand()->getType(), Addr,
1413 AddrAlign: CI->getAlign(), MinWordSize: TLI->getMinCmpXchgSizeInBits() / 8);
1414
1415 // Shift the incoming values over, into the right location in the word.
1416 Value *NewVal_Shifted =
1417 Builder.CreateShl(LHS: Builder.CreateZExt(V: NewVal, DestTy: PMV.WordType), RHS: PMV.ShiftAmt);
1418 Value *Cmp_Shifted =
1419 Builder.CreateShl(LHS: Builder.CreateZExt(V: Cmp, DestTy: PMV.WordType), RHS: PMV.ShiftAmt);
1420
1421 // Load the entire current word, and mask into place the expected and new
1422 // values
1423 LoadInst *InitLoaded = Builder.CreateLoad(Ty: PMV.WordType, Ptr: PMV.AlignedAddr);
1424 Value *InitLoaded_MaskOut = Builder.CreateAnd(LHS: InitLoaded, RHS: PMV.Inv_Mask);
1425 Builder.CreateBr(Dest: LoopBB);
1426
1427 // partword.cmpxchg.loop:
1428 Builder.SetInsertPoint(LoopBB);
1429 PHINode *Loaded_MaskOut = Builder.CreatePHI(Ty: PMV.WordType, NumReservedValues: 2);
1430 Loaded_MaskOut->addIncoming(V: InitLoaded_MaskOut, BB);
1431
1432 // The initial load must be atomic with the same synchronization scope
1433 // to avoid a data race with concurrent stores. If the instruction being
1434 // emulated is volatile, issue a volatile load.
1435 // addIncoming is done first so that any replaceAllUsesWith calls during
1436 // normalization correctly update the PHI incoming value.
1437 InitLoaded->setVolatile(CI->isVolatile());
1438 if (TLI->shouldIssueAtomicLoadForAtomicEmulationLoop()) {
1439 InitLoaded->setAtomic(Ordering: AtomicOrdering::Monotonic, SSID: CI->getSyncScopeID());
1440 // The newly created load might need to be lowered further. Because it is
1441 // created in the same block as the atomicrmw, the AtomicExpand loop will
1442 // not process it again.
1443 processAtomicInstr(I: InitLoaded);
1444 }
1445
1446 // Mask/Or the expected and new values into place in the loaded word.
1447 Value *FullWord_NewVal = Builder.CreateOr(LHS: Loaded_MaskOut, RHS: NewVal_Shifted);
1448 Value *FullWord_Cmp = Builder.CreateOr(LHS: Loaded_MaskOut, RHS: Cmp_Shifted);
1449 AtomicCmpXchgInst *NewCI = Builder.CreateAtomicCmpXchg(
1450 Ptr: PMV.AlignedAddr, Cmp: FullWord_Cmp, New: FullWord_NewVal, Align: PMV.AlignedAddrAlignment,
1451 SuccessOrdering: CI->getSuccessOrdering(), FailureOrdering: CI->getFailureOrdering(), SSID: CI->getSyncScopeID());
1452 NewCI->setVolatile(CI->isVolatile());
1453 // When we're building a strong cmpxchg, we need a loop, so you
1454 // might think we could use a weak cmpxchg inside. But, using strong
1455 // allows the below comparison for ShouldContinue, and we're
1456 // expecting the underlying cmpxchg to be a machine instruction,
1457 // which is strong anyways.
1458 NewCI->setWeak(CI->isWeak());
1459
1460 Value *OldVal = Builder.CreateExtractValue(Agg: NewCI, Idxs: 0);
1461 Value *Success = Builder.CreateExtractValue(Agg: NewCI, Idxs: 1);
1462
1463 if (CI->isWeak())
1464 Builder.CreateBr(Dest: EndBB);
1465 else
1466 Builder.CreateCondBr(Cond: Success, True: EndBB, False: FailureBB);
1467
1468 // partword.cmpxchg.failure:
1469 Builder.SetInsertPoint(FailureBB);
1470 // Upon failure, verify that the masked-out part of the loaded value
1471 // has been modified. If it didn't, abort the cmpxchg, since the
1472 // masked-in part must've.
1473 Value *OldVal_MaskOut = Builder.CreateAnd(LHS: OldVal, RHS: PMV.Inv_Mask);
1474 Value *ShouldContinue = Builder.CreateICmpNE(LHS: Loaded_MaskOut, RHS: OldVal_MaskOut);
1475 Builder.CreateCondBr(Cond: ShouldContinue, True: LoopBB, False: EndBB);
1476
1477 // Add the second value to the phi from above
1478 Loaded_MaskOut->addIncoming(V: OldVal_MaskOut, BB: FailureBB);
1479
1480 // partword.cmpxchg.end:
1481 Builder.SetInsertPoint(CI);
1482
1483 Value *FinalOldVal = extractMaskedValue(Builder, WideWord: OldVal, PMV);
1484 Value *Res = PoisonValue::get(T: CI->getType());
1485 Res = Builder.CreateInsertValue(Agg: Res, Val: FinalOldVal, Idxs: 0);
1486 Res = Builder.CreateInsertValue(Agg: Res, Val: Success, Idxs: 1);
1487
1488 CI->replaceAllUsesWith(V: Res);
1489 CI->eraseFromParent();
1490 return true;
1491}
1492
1493void AtomicExpandImpl::expandAtomicOpToLLSC(
1494 Instruction *I, Type *ResultType, Value *Addr, Align AddrAlign,
1495 AtomicOrdering MemOpOrder,
1496 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp) {
1497 ReplacementIRBuilder Builder(I, *DL);
1498 Value *Loaded = insertRMWLLSCLoop(Builder, ResultTy: ResultType, Addr, AddrAlign,
1499 MemOpOrder, PerformOp);
1500
1501 I->replaceAllUsesWith(V: Loaded);
1502 I->eraseFromParent();
1503}
1504
1505void AtomicExpandImpl::expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI) {
1506 ReplacementIRBuilder Builder(AI, *DL);
1507
1508 PartwordMaskValues PMV =
1509 createMaskInstrs(Builder, I: AI, ValueType: AI->getType(), Addr: AI->getPointerOperand(),
1510 AddrAlign: AI->getAlign(), MinWordSize: TLI->getMinCmpXchgSizeInBits() / 8);
1511
1512 // The value operand must be sign-extended for signed min/max so that the
1513 // target's signed comparison instructions can be used. Otherwise, just
1514 // zero-ext.
1515 Instruction::CastOps CastOp = Instruction::ZExt;
1516 AtomicRMWInst::BinOp RMWOp = AI->getOperation();
1517 if (RMWOp == AtomicRMWInst::Max || RMWOp == AtomicRMWInst::Min)
1518 CastOp = Instruction::SExt;
1519
1520 Value *ValOperand_Shifted = Builder.CreateShl(
1521 LHS: Builder.CreateCast(Op: CastOp, V: AI->getValOperand(), DestTy: PMV.WordType),
1522 RHS: PMV.ShiftAmt, Name: "ValOperand_Shifted");
1523 Value *OldResult = TLI->emitMaskedAtomicRMWIntrinsic(
1524 Builder, AI, AlignedAddr: PMV.AlignedAddr, Incr: ValOperand_Shifted, Mask: PMV.Mask, ShiftAmt: PMV.ShiftAmt,
1525 Ord: AI->getOrdering());
1526 Value *FinalOldResult = extractMaskedValue(Builder, WideWord: OldResult, PMV);
1527 AI->replaceAllUsesWith(V: FinalOldResult);
1528 AI->eraseFromParent();
1529}
1530
1531void AtomicExpandImpl::expandAtomicCmpXchgToMaskedIntrinsic(
1532 AtomicCmpXchgInst *CI) {
1533 ReplacementIRBuilder Builder(CI, *DL);
1534
1535 PartwordMaskValues PMV = createMaskInstrs(
1536 Builder, I: CI, ValueType: CI->getCompareOperand()->getType(), Addr: CI->getPointerOperand(),
1537 AddrAlign: CI->getAlign(), MinWordSize: TLI->getMinCmpXchgSizeInBits() / 8);
1538
1539 Value *CmpVal_Shifted = Builder.CreateShl(
1540 LHS: Builder.CreateZExt(V: CI->getCompareOperand(), DestTy: PMV.WordType), RHS: PMV.ShiftAmt,
1541 Name: "CmpVal_Shifted");
1542 Value *NewVal_Shifted = Builder.CreateShl(
1543 LHS: Builder.CreateZExt(V: CI->getNewValOperand(), DestTy: PMV.WordType), RHS: PMV.ShiftAmt,
1544 Name: "NewVal_Shifted");
1545 Value *OldVal = TLI->emitMaskedAtomicCmpXchgIntrinsic(
1546 Builder, CI, AlignedAddr: PMV.AlignedAddr, CmpVal: CmpVal_Shifted, NewVal: NewVal_Shifted, Mask: PMV.Mask,
1547 Ord: CI->getMergedOrdering());
1548 Value *FinalOldVal = extractMaskedValue(Builder, WideWord: OldVal, PMV);
1549 Value *Res = PoisonValue::get(T: CI->getType());
1550 Res = Builder.CreateInsertValue(Agg: Res, Val: FinalOldVal, Idxs: 0);
1551 Value *Success = Builder.CreateICmpEQ(
1552 LHS: CmpVal_Shifted, RHS: Builder.CreateAnd(LHS: OldVal, RHS: PMV.Mask), Name: "Success");
1553 Res = Builder.CreateInsertValue(Agg: Res, Val: Success, Idxs: 1);
1554
1555 CI->replaceAllUsesWith(V: Res);
1556 CI->eraseFromParent();
1557}
1558
1559Value *AtomicExpandImpl::insertRMWLLSCLoop(
1560 IRBuilderBase &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
1561 AtomicOrdering MemOpOrder,
1562 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp) {
1563 LLVMContext &Ctx = Builder.getContext();
1564 BasicBlock *BB = Builder.GetInsertBlock();
1565 Function *F = BB->getParent();
1566
1567 assert(AddrAlign >= F->getDataLayout().getTypeStoreSize(ResultTy) &&
1568 "Expected at least natural alignment at this point.");
1569
1570 // Given: atomicrmw some_op iN* %addr, iN %incr ordering
1571 //
1572 // The standard expansion we produce is:
1573 // [...]
1574 // atomicrmw.start:
1575 // %loaded = @load.linked(%addr)
1576 // %new = some_op iN %loaded, %incr
1577 // %stored = @store_conditional(%new, %addr)
1578 // %try_again = icmp i32 ne %stored, 0
1579 // br i1 %try_again, label %loop, label %atomicrmw.end
1580 // atomicrmw.end:
1581 // [...]
1582 BasicBlock *ExitBB =
1583 BB->splitBasicBlock(I: Builder.GetInsertPoint(), BBName: "atomicrmw.end");
1584 BasicBlock *LoopBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.start", Parent: F, InsertBefore: ExitBB);
1585
1586 // The split call above "helpfully" added a branch at the end of BB (to the
1587 // wrong place).
1588 std::prev(x: BB->end())->eraseFromParent();
1589 Builder.SetInsertPoint(BB);
1590 Builder.CreateBr(Dest: LoopBB);
1591
1592 // Start the main loop block now that we've taken care of the preliminaries.
1593 Builder.SetInsertPoint(LoopBB);
1594 Value *Loaded = TLI->emitLoadLinked(Builder, ValueTy: ResultTy, Addr, Ord: MemOpOrder);
1595
1596 Value *NewVal = PerformOp(Builder, Loaded);
1597
1598 Value *StoreSuccess =
1599 TLI->emitStoreConditional(Builder, Val: NewVal, Addr, Ord: MemOpOrder);
1600 Value *TryAgain = Builder.CreateICmpNE(
1601 LHS: StoreSuccess, RHS: ConstantInt::get(Ty: IntegerType::get(C&: Ctx, NumBits: 32), V: 0), Name: "tryagain");
1602
1603 Instruction *CondBr = Builder.CreateCondBr(Cond: TryAgain, True: LoopBB, False: ExitBB);
1604
1605 // Atomic RMW expands to a Load-linked / Store-Conditional loop, because it is
1606 // hard to predict precise branch weigths we mark the branch as "unknown"
1607 // (50/50) to prevent misleading optimizations.
1608 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *CondBr, DEBUG_TYPE);
1609
1610 Builder.SetInsertPoint(ExitBB->begin());
1611 return Loaded;
1612}
1613
1614/// Convert an atomic cmpxchg of a non-integral type to an integer cmpxchg of
1615/// the equivalent bitwidth. We used to not support pointer cmpxchg in the
1616/// IR. As a migration step, we convert back to what use to be the standard
1617/// way to represent a pointer cmpxchg so that we can update backends one by
1618/// one.
1619AtomicCmpXchgInst *
1620AtomicExpandImpl::convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI) {
1621 auto *M = CI->getModule();
1622 Type *NewTy = getCorrespondingIntegerType(T: CI->getCompareOperand()->getType(),
1623 DL: M->getDataLayout());
1624
1625 ReplacementIRBuilder Builder(CI, *DL);
1626
1627 Value *Addr = CI->getPointerOperand();
1628
1629 Value *NewCmp = Builder.CreatePtrToInt(V: CI->getCompareOperand(), DestTy: NewTy);
1630 Value *NewNewVal = Builder.CreatePtrToInt(V: CI->getNewValOperand(), DestTy: NewTy);
1631
1632 auto *NewCI = Builder.CreateAtomicCmpXchg(
1633 Ptr: Addr, Cmp: NewCmp, New: NewNewVal, Align: CI->getAlign(), SuccessOrdering: CI->getSuccessOrdering(),
1634 FailureOrdering: CI->getFailureOrdering(), SSID: CI->getSyncScopeID());
1635 NewCI->setVolatile(CI->isVolatile());
1636 NewCI->setWeak(CI->isWeak());
1637 LLVM_DEBUG(dbgs() << "Replaced " << *CI << " with " << *NewCI << "\n");
1638
1639 Value *OldVal = Builder.CreateExtractValue(Agg: NewCI, Idxs: 0);
1640 Value *Succ = Builder.CreateExtractValue(Agg: NewCI, Idxs: 1);
1641
1642 OldVal = Builder.CreateIntToPtr(V: OldVal, DestTy: CI->getCompareOperand()->getType());
1643
1644 Value *Res = PoisonValue::get(T: CI->getType());
1645 Res = Builder.CreateInsertValue(Agg: Res, Val: OldVal, Idxs: 0);
1646 Res = Builder.CreateInsertValue(Agg: Res, Val: Succ, Idxs: 1);
1647
1648 CI->replaceAllUsesWith(V: Res);
1649 CI->eraseFromParent();
1650 return NewCI;
1651}
1652
1653bool AtomicExpandImpl::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
1654 AtomicOrdering SuccessOrder = CI->getSuccessOrdering();
1655 AtomicOrdering FailureOrder = CI->getFailureOrdering();
1656 Value *Addr = CI->getPointerOperand();
1657 BasicBlock *BB = CI->getParent();
1658 Function *F = BB->getParent();
1659 LLVMContext &Ctx = F->getContext();
1660 // If shouldInsertFencesForAtomic() returns true, then the target does not
1661 // want to deal with memory orders, and emitLeading/TrailingFence should take
1662 // care of everything. Otherwise, emitLeading/TrailingFence are no-op and we
1663 // should preserve the ordering.
1664 bool ShouldInsertFencesForAtomic = TLI->shouldInsertFencesForAtomic(I: CI);
1665 AtomicOrdering MemOpOrder = ShouldInsertFencesForAtomic
1666 ? AtomicOrdering::Monotonic
1667 : CI->getMergedOrdering();
1668
1669 // In implementations which use a barrier to achieve release semantics, we can
1670 // delay emitting this barrier until we know a store is actually going to be
1671 // attempted. The cost of this delay is that we need 2 copies of the block
1672 // emitting the load-linked, affecting code size.
1673 //
1674 // Ideally, this logic would be unconditional except for the minsize check
1675 // since in other cases the extra blocks naturally collapse down to the
1676 // minimal loop. Unfortunately, this puts too much stress on later
1677 // optimisations so we avoid emitting the extra logic in those cases too.
1678 bool HasReleasedLoadBB = !CI->isWeak() && ShouldInsertFencesForAtomic &&
1679 SuccessOrder != AtomicOrdering::Monotonic &&
1680 SuccessOrder != AtomicOrdering::Acquire &&
1681 !F->hasMinSize();
1682
1683 // There's no overhead for sinking the release barrier in a weak cmpxchg, so
1684 // do it even on minsize.
1685 // A fence between the LL and SC may clear the reservation on some targets.
1686 // Strong cmpxchg retries, but weak cmpxchg cannot recover.
1687 bool UseUnconditionalReleaseBarrier =
1688 (F->hasMinSize() && !CI->isWeak()) ||
1689 (CI->isWeak() && TLI->fenceClearsLoadLinkedReservation());
1690
1691 // Given: cmpxchg some_op iN* %addr, iN %desired, iN %new success_ord fail_ord
1692 //
1693 // The full expansion we produce is:
1694 // [...]
1695 // %aligned.addr = ...
1696 // cmpxchg.start:
1697 // %unreleasedload = @load.linked(%aligned.addr)
1698 // %unreleasedload.extract = extract value from %unreleasedload
1699 // %should_store = icmp eq %unreleasedload.extract, %desired
1700 // br i1 %should_store, label %cmpxchg.releasingstore,
1701 // label %cmpxchg.nostore
1702 // cmpxchg.releasingstore:
1703 // fence?
1704 // br label cmpxchg.trystore
1705 // cmpxchg.trystore:
1706 // %loaded.trystore = phi [%unreleasedload, %cmpxchg.releasingstore],
1707 // [%releasedload, %cmpxchg.releasedload]
1708 // %updated.new = insert %new into %loaded.trystore
1709 // %stored = @store_conditional(%updated.new, %aligned.addr)
1710 // %success = icmp eq i32 %stored, 0
1711 // br i1 %success, label %cmpxchg.success,
1712 // label %cmpxchg.releasedload/%cmpxchg.failure
1713 // cmpxchg.releasedload:
1714 // %releasedload = @load.linked(%aligned.addr)
1715 // %releasedload.extract = extract value from %releasedload
1716 // %should_store = icmp eq %releasedload.extract, %desired
1717 // br i1 %should_store, label %cmpxchg.trystore,
1718 // label %cmpxchg.failure
1719 // cmpxchg.success:
1720 // fence?
1721 // br label %cmpxchg.end
1722 // cmpxchg.nostore:
1723 // %loaded.nostore = phi [%unreleasedload, %cmpxchg.start],
1724 // [%releasedload,
1725 // %cmpxchg.releasedload/%cmpxchg.trystore]
1726 // @load_linked_fail_balance()?
1727 // br label %cmpxchg.failure
1728 // cmpxchg.failure:
1729 // fence?
1730 // br label %cmpxchg.end
1731 // cmpxchg.end:
1732 // %loaded.exit = phi [%loaded.nostore, %cmpxchg.failure],
1733 // [%loaded.trystore, %cmpxchg.trystore]
1734 // %success = phi i1 [true, %cmpxchg.success], [false, %cmpxchg.failure]
1735 // %loaded = extract value from %loaded.exit
1736 // %restmp = insertvalue { iN, i1 } undef, iN %loaded, 0
1737 // %res = insertvalue { iN, i1 } %restmp, i1 %success, 1
1738 // [...]
1739 BasicBlock *ExitBB = BB->splitBasicBlock(I: CI->getIterator(), BBName: "cmpxchg.end");
1740 auto FailureBB = BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.failure", Parent: F, InsertBefore: ExitBB);
1741 auto NoStoreBB = BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.nostore", Parent: F, InsertBefore: FailureBB);
1742 auto SuccessBB = BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.success", Parent: F, InsertBefore: NoStoreBB);
1743 auto ReleasedLoadBB =
1744 BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.releasedload", Parent: F, InsertBefore: SuccessBB);
1745 auto TryStoreBB =
1746 BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.trystore", Parent: F, InsertBefore: ReleasedLoadBB);
1747 auto ReleasingStoreBB =
1748 BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.fencedstore", Parent: F, InsertBefore: TryStoreBB);
1749 auto StartBB = BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.start", Parent: F, InsertBefore: ReleasingStoreBB);
1750
1751 ReplacementIRBuilder Builder(CI, *DL);
1752
1753 // The split call above "helpfully" added a branch at the end of BB (to the
1754 // wrong place), but we might want a fence too. It's easiest to just remove
1755 // the branch entirely.
1756 std::prev(x: BB->end())->eraseFromParent();
1757 Builder.SetInsertPoint(BB);
1758 if (ShouldInsertFencesForAtomic && UseUnconditionalReleaseBarrier)
1759 TLI->emitLeadingFence(Builder, Inst: CI, Ord: SuccessOrder);
1760
1761 PartwordMaskValues PMV =
1762 createMaskInstrs(Builder, I: CI, ValueType: CI->getCompareOperand()->getType(), Addr,
1763 AddrAlign: CI->getAlign(), MinWordSize: TLI->getMinCmpXchgSizeInBits() / 8);
1764 Builder.CreateBr(Dest: StartBB);
1765
1766 // Start the main loop block now that we've taken care of the preliminaries.
1767 Builder.SetInsertPoint(StartBB);
1768 Value *UnreleasedLoad =
1769 TLI->emitLoadLinked(Builder, ValueTy: PMV.WordType, Addr: PMV.AlignedAddr, Ord: MemOpOrder);
1770 Value *UnreleasedLoadExtract =
1771 extractMaskedValue(Builder, WideWord: UnreleasedLoad, PMV);
1772 Value *ShouldStore = Builder.CreateICmpEQ(
1773 LHS: UnreleasedLoadExtract, RHS: CI->getCompareOperand(), Name: "should_store");
1774
1775 // If the cmpxchg doesn't actually need any ordering when it fails, we can
1776 // jump straight past that fence instruction (if it exists).
1777 Builder.CreateCondBr(Cond: ShouldStore, True: ReleasingStoreBB, False: NoStoreBB,
1778 BranchWeights: MDBuilder(F->getContext()).createLikelyBranchWeights());
1779
1780 Builder.SetInsertPoint(ReleasingStoreBB);
1781 if (ShouldInsertFencesForAtomic && !UseUnconditionalReleaseBarrier)
1782 TLI->emitLeadingFence(Builder, Inst: CI, Ord: SuccessOrder);
1783 Builder.CreateBr(Dest: TryStoreBB);
1784
1785 Builder.SetInsertPoint(TryStoreBB);
1786 PHINode *LoadedTryStore =
1787 Builder.CreatePHI(Ty: PMV.WordType, NumReservedValues: 2, Name: "loaded.trystore");
1788 LoadedTryStore->addIncoming(V: UnreleasedLoad, BB: ReleasingStoreBB);
1789 Value *NewValueInsert =
1790 insertMaskedValue(Builder, WideWord: LoadedTryStore, Updated: CI->getNewValOperand(), PMV);
1791 Value *StoreSuccess = TLI->emitStoreConditional(Builder, Val: NewValueInsert,
1792 Addr: PMV.AlignedAddr, Ord: MemOpOrder);
1793 StoreSuccess = Builder.CreateICmpEQ(
1794 LHS: StoreSuccess, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: 0), Name: "success");
1795 BasicBlock *RetryBB = HasReleasedLoadBB ? ReleasedLoadBB : StartBB;
1796 Builder.CreateCondBr(Cond: StoreSuccess, True: SuccessBB,
1797 False: CI->isWeak() ? FailureBB : RetryBB,
1798 BranchWeights: MDBuilder(F->getContext()).createLikelyBranchWeights());
1799
1800 Builder.SetInsertPoint(ReleasedLoadBB);
1801 Value *SecondLoad;
1802 if (HasReleasedLoadBB) {
1803 SecondLoad =
1804 TLI->emitLoadLinked(Builder, ValueTy: PMV.WordType, Addr: PMV.AlignedAddr, Ord: MemOpOrder);
1805 Value *SecondLoadExtract = extractMaskedValue(Builder, WideWord: SecondLoad, PMV);
1806 ShouldStore = Builder.CreateICmpEQ(LHS: SecondLoadExtract,
1807 RHS: CI->getCompareOperand(), Name: "should_store");
1808
1809 // If the cmpxchg doesn't actually need any ordering when it fails, we can
1810 // jump straight past that fence instruction (if it exists).
1811 Builder.CreateCondBr(
1812 Cond: ShouldStore, True: TryStoreBB, False: NoStoreBB,
1813 BranchWeights: MDBuilder(F->getContext()).createLikelyBranchWeights());
1814 // Update PHI node in TryStoreBB.
1815 LoadedTryStore->addIncoming(V: SecondLoad, BB: ReleasedLoadBB);
1816 } else
1817 Builder.CreateUnreachable();
1818
1819 // Make sure later instructions don't get reordered with a fence if
1820 // necessary.
1821 Builder.SetInsertPoint(SuccessBB);
1822 if (ShouldInsertFencesForAtomic ||
1823 TLI->shouldInsertTrailingSeqCstFenceForAtomicStore(I: CI))
1824 TLI->emitTrailingFence(Builder, Inst: CI, Ord: SuccessOrder);
1825 Builder.CreateBr(Dest: ExitBB);
1826
1827 Builder.SetInsertPoint(NoStoreBB);
1828 PHINode *LoadedNoStore =
1829 Builder.CreatePHI(Ty: UnreleasedLoad->getType(), NumReservedValues: 2, Name: "loaded.nostore");
1830 LoadedNoStore->addIncoming(V: UnreleasedLoad, BB: StartBB);
1831 if (HasReleasedLoadBB)
1832 LoadedNoStore->addIncoming(V: SecondLoad, BB: ReleasedLoadBB);
1833
1834 // In the failing case, where we don't execute the store-conditional, the
1835 // target might want to balance out the load-linked with a dedicated
1836 // instruction (e.g., on ARM, clearing the exclusive monitor).
1837 TLI->emitAtomicCmpXchgNoStoreLLBalance(Builder);
1838 Builder.CreateBr(Dest: FailureBB);
1839
1840 Builder.SetInsertPoint(FailureBB);
1841 PHINode *LoadedFailure =
1842 Builder.CreatePHI(Ty: UnreleasedLoad->getType(), NumReservedValues: 2, Name: "loaded.failure");
1843 LoadedFailure->addIncoming(V: LoadedNoStore, BB: NoStoreBB);
1844 if (CI->isWeak())
1845 LoadedFailure->addIncoming(V: LoadedTryStore, BB: TryStoreBB);
1846 if (ShouldInsertFencesForAtomic)
1847 TLI->emitTrailingFence(Builder, Inst: CI, Ord: FailureOrder);
1848 Builder.CreateBr(Dest: ExitBB);
1849
1850 // Finally, we have control-flow based knowledge of whether the cmpxchg
1851 // succeeded or not. We expose this to later passes by converting any
1852 // subsequent "icmp eq/ne %loaded, %oldval" into a use of an appropriate
1853 // PHI.
1854 Builder.SetInsertPoint(ExitBB->begin());
1855 PHINode *LoadedExit =
1856 Builder.CreatePHI(Ty: UnreleasedLoad->getType(), NumReservedValues: 2, Name: "loaded.exit");
1857 LoadedExit->addIncoming(V: LoadedTryStore, BB: SuccessBB);
1858 LoadedExit->addIncoming(V: LoadedFailure, BB: FailureBB);
1859 PHINode *Success = Builder.CreatePHI(Ty: Type::getInt1Ty(C&: Ctx), NumReservedValues: 2, Name: "success");
1860 Success->addIncoming(V: ConstantInt::getTrue(Context&: Ctx), BB: SuccessBB);
1861 Success->addIncoming(V: ConstantInt::getFalse(Context&: Ctx), BB: FailureBB);
1862
1863 // This is the "exit value" from the cmpxchg expansion. It may be of
1864 // a type wider than the one in the cmpxchg instruction.
1865 Value *LoadedFull = LoadedExit;
1866
1867 Builder.SetInsertPoint(std::next(x: Success->getIterator()));
1868 Value *Loaded = extractMaskedValue(Builder, WideWord: LoadedFull, PMV);
1869
1870 // Look for any users of the cmpxchg that are just comparing the loaded value
1871 // against the desired one, and replace them with the CFG-derived version.
1872 SmallVector<ExtractValueInst *, 2> PrunedInsts;
1873 for (auto *User : CI->users()) {
1874 ExtractValueInst *EV = dyn_cast<ExtractValueInst>(Val: User);
1875 if (!EV)
1876 continue;
1877
1878 assert(EV->getNumIndices() == 1 && EV->getIndices()[0] <= 1 &&
1879 "weird extraction from { iN, i1 }");
1880
1881 if (EV->getIndices()[0] == 0)
1882 EV->replaceAllUsesWith(V: Loaded);
1883 else
1884 EV->replaceAllUsesWith(V: Success);
1885
1886 PrunedInsts.push_back(Elt: EV);
1887 }
1888
1889 // We can remove the instructions now we're no longer iterating through them.
1890 for (auto *EV : PrunedInsts)
1891 EV->eraseFromParent();
1892
1893 if (!CI->use_empty()) {
1894 // Some use of the full struct return that we don't understand has happened,
1895 // so we've got to reconstruct it properly.
1896 Value *Res;
1897 Res = Builder.CreateInsertValue(Agg: PoisonValue::get(T: CI->getType()), Val: Loaded, Idxs: 0);
1898 Res = Builder.CreateInsertValue(Agg: Res, Val: Success, Idxs: 1);
1899
1900 CI->replaceAllUsesWith(V: Res);
1901 }
1902
1903 CI->eraseFromParent();
1904 return true;
1905}
1906
1907bool AtomicExpandImpl::isIdempotentRMW(AtomicRMWInst *RMWI) {
1908 if (RMWI->isVolatile())
1909 return false;
1910 // TODO: Add floating point support.
1911 auto C = dyn_cast<ConstantInt>(Val: RMWI->getValOperand());
1912 if (!C)
1913 return false;
1914
1915 switch (RMWI->getOperation()) {
1916 case AtomicRMWInst::Add:
1917 case AtomicRMWInst::Sub:
1918 case AtomicRMWInst::Or:
1919 case AtomicRMWInst::Xor:
1920 return C->isZero();
1921 case AtomicRMWInst::And:
1922 return C->isMinusOne();
1923 case AtomicRMWInst::Min:
1924 return C->isMaxValue(IsSigned: true);
1925 case AtomicRMWInst::Max:
1926 return C->isMinValue(IsSigned: true);
1927 case AtomicRMWInst::UMin:
1928 return C->isMaxValue(IsSigned: false);
1929 case AtomicRMWInst::UMax:
1930 return C->isMinValue(IsSigned: false);
1931 default:
1932 return false;
1933 }
1934}
1935
1936bool AtomicExpandImpl::simplifyIdempotentRMW(AtomicRMWInst *RMWI) {
1937 if (auto ResultingLoad = TLI->lowerIdempotentRMWIntoFencedLoad(RMWI)) {
1938 tryExpandAtomicLoad(LI: ResultingLoad);
1939 return true;
1940 }
1941 return false;
1942}
1943
1944Value *AtomicExpandImpl::insertRMWCmpXchgLoop(
1945 IRBuilderBase &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
1946 AtomicOrdering MemOpOrder, SyncScope::ID SSID, bool IsVolatile,
1947 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp,
1948 CreateCmpXchgInstFun CreateCmpXchg, Instruction *MetadataSrc) {
1949 LLVMContext &Ctx = Builder.getContext();
1950 BasicBlock *BB = Builder.GetInsertBlock();
1951 Function *F = BB->getParent();
1952
1953 // Given: atomicrmw some_op iN* %addr, iN %incr ordering
1954 //
1955 // The standard expansion we produce is:
1956 // [...]
1957 // %init_loaded = load atomic iN* %addr
1958 // br label %loop
1959 // loop:
1960 // %loaded = phi iN [ %init_loaded, %entry ], [ %new_loaded, %loop ]
1961 // %new = some_op iN %loaded, %incr
1962 // %pair = cmpxchg iN* %addr, iN %loaded, iN %new
1963 // %new_loaded = extractvalue { iN, i1 } %pair, 0
1964 // %success = extractvalue { iN, i1 } %pair, 1
1965 // br i1 %success, label %atomicrmw.end, label %loop
1966 // atomicrmw.end:
1967 // [...]
1968 BasicBlock *ExitBB =
1969 BB->splitBasicBlock(I: Builder.GetInsertPoint(), BBName: "atomicrmw.end");
1970 BasicBlock *LoopBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.start", Parent: F, InsertBefore: ExitBB);
1971
1972 // The split call above "helpfully" added a branch at the end of BB (to the
1973 // wrong place), but we want a load. It's easiest to just remove
1974 // the branch entirely.
1975 std::prev(x: BB->end())->eraseFromParent();
1976 Builder.SetInsertPoint(BB);
1977 LoadInst *InitLoaded = Builder.CreateAlignedLoad(Ty: ResultTy, Ptr: Addr, Align: AddrAlign);
1978 Builder.CreateBr(Dest: LoopBB);
1979
1980 // Start the main loop block now that we've taken care of the preliminaries.
1981 Builder.SetInsertPoint(LoopBB);
1982 PHINode *Loaded = Builder.CreatePHI(Ty: ResultTy, NumReservedValues: 2, Name: "loaded");
1983 Loaded->addIncoming(V: InitLoaded, BB);
1984
1985 // The initial load must be atomic with the same synchronization scope
1986 // to avoid a data race with concurrent stores. If the instruction being
1987 // emulated is volatile, issue a volatile load.
1988 // addIncoming is done first so that any replaceAllUsesWith calls during
1989 // normalization correctly update the PHI incoming value.
1990 InitLoaded->setVolatile(IsVolatile);
1991 if (TLI->shouldIssueAtomicLoadForAtomicEmulationLoop()) {
1992 InitLoaded->setAtomic(Ordering: AtomicOrdering::Monotonic, SSID);
1993 // The newly created load might need to be lowered further. Because it is
1994 // created in the same block as the atomicrmw, the AtomicExpand loop will
1995 // not process it again.
1996 processAtomicInstr(I: InitLoaded);
1997 }
1998
1999 Value *NewVal = PerformOp(Builder, Loaded);
2000
2001 Value *NewLoaded = nullptr;
2002 Value *Success = nullptr;
2003
2004 CreateCmpXchg(Builder, Addr, Loaded, NewVal, AddrAlign,
2005 MemOpOrder == AtomicOrdering::Unordered
2006 ? AtomicOrdering::Monotonic
2007 : MemOpOrder,
2008 SSID, IsVolatile, Success, NewLoaded, MetadataSrc);
2009 assert(Success && NewLoaded);
2010
2011 Loaded->addIncoming(V: NewLoaded, BB: LoopBB);
2012
2013 Instruction *CondBr = Builder.CreateCondBr(Cond: Success, True: ExitBB, False: LoopBB);
2014
2015 // Atomic RMW expands to a cmpxchg loop, Since precise branch weights
2016 // cannot be easily determined here, we mark the branch as "unknown" (50/50)
2017 // to prevent misleading optimizations.
2018 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *CondBr, DEBUG_TYPE);
2019
2020 Builder.SetInsertPoint(ExitBB->begin());
2021 return NewLoaded;
2022}
2023
2024bool AtomicExpandImpl::tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
2025 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
2026 unsigned ValueSize = getAtomicOpSize(CASI: CI);
2027
2028 switch (TLI->shouldExpandAtomicCmpXchgInIR(AI: CI)) {
2029 default:
2030 llvm_unreachable("Unhandled case in tryExpandAtomicCmpXchg");
2031 case TargetLoweringBase::AtomicExpansionKind::None:
2032 if (ValueSize < MinCASSize)
2033 return expandPartwordCmpXchg(CI);
2034 return false;
2035 case TargetLoweringBase::AtomicExpansionKind::LLSC: {
2036 return expandAtomicCmpXchg(CI);
2037 }
2038 case TargetLoweringBase::AtomicExpansionKind::MaskedIntrinsic:
2039 expandAtomicCmpXchgToMaskedIntrinsic(CI);
2040 return true;
2041 case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
2042 return lowerAtomicCmpXchgInst(CXI: CI);
2043 case TargetLoweringBase::AtomicExpansionKind::CustomExpand: {
2044 TLI->emitExpandAtomicCmpXchg(CI);
2045 return true;
2046 }
2047 }
2048}
2049
2050bool AtomicExpandImpl::expandAtomicRMWToCmpXchg(
2051 AtomicRMWInst *AI, CreateCmpXchgInstFun CreateCmpXchg) {
2052 ReplacementIRBuilder Builder(AI, AI->getDataLayout());
2053 Builder.setIsFPConstrained(
2054 AI->getFunction()->hasFnAttribute(Kind: Attribute::StrictFP));
2055
2056 // FIXME: If FP exceptions are observable, we should force them off for the
2057 // loop for the FP atomics.
2058 Value *Loaded = AtomicExpandImpl::insertRMWCmpXchgLoop(
2059 Builder, ResultTy: AI->getType(), Addr: AI->getPointerOperand(), AddrAlign: AI->getAlign(),
2060 MemOpOrder: AI->getOrdering(), SSID: AI->getSyncScopeID(), IsVolatile: AI->isVolatile(),
2061 PerformOp: [&](IRBuilderBase &Builder, Value *Loaded) {
2062 return buildAtomicRMWValue(Op: AI->getOperation(), Builder, Loaded,
2063 Val: AI->getValOperand());
2064 },
2065 CreateCmpXchg, /*MetadataSrc=*/AI);
2066
2067 AI->replaceAllUsesWith(V: Loaded);
2068 AI->eraseFromParent();
2069 return true;
2070}
2071
2072// In order to use one of the sized library calls such as
2073// __atomic_fetch_add_4, the alignment must be sufficient, the size
2074// must be one of the potentially-specialized sizes, and the value
2075// type must actually exist in C on the target (otherwise, the
2076// function wouldn't actually be defined.)
2077static bool canUseSizedAtomicCall(unsigned Size, Align Alignment,
2078 const DataLayout &DL) {
2079 // TODO: "LargestSize" is an approximation for "largest type that
2080 // you can express in C". It seems to be the case that int128 is
2081 // supported on all 64-bit platforms, otherwise only up to 64-bit
2082 // integers are supported. If we get this wrong, then we'll try to
2083 // call a sized libcall that doesn't actually exist. There should
2084 // really be some more reliable way in LLVM of determining integer
2085 // sizes which are valid in the target's C ABI...
2086 unsigned LargestSize = DL.getLargestLegalIntTypeSizeInBits() >= 64 ? 16 : 8;
2087 return Alignment >= Size &&
2088 (Size == 1 || Size == 2 || Size == 4 || Size == 8 || Size == 16) &&
2089 Size <= LargestSize;
2090}
2091
2092void AtomicExpandImpl::expandAtomicLoadToLibcall(LoadInst *I) {
2093 static const RTLIB::Libcall Libcalls[6] = {
2094 RTLIB::ATOMIC_LOAD, RTLIB::ATOMIC_LOAD_1, RTLIB::ATOMIC_LOAD_2,
2095 RTLIB::ATOMIC_LOAD_4, RTLIB::ATOMIC_LOAD_8, RTLIB::ATOMIC_LOAD_16};
2096 unsigned Size = getAtomicOpSize(LI: I);
2097
2098 bool Expanded = expandAtomicOpToLibcall(
2099 I, Size, Alignment: I->getAlign(), PointerOperand: I->getPointerOperand(), ValueOperand: nullptr, CASExpected: nullptr,
2100 Ordering: I->getOrdering(), Ordering2: AtomicOrdering::NotAtomic, Libcalls);
2101 if (!Expanded)
2102 handleUnsupportedAtomicSize(I, AtomicOpName: "atomic load");
2103}
2104
2105void AtomicExpandImpl::expandAtomicStoreToLibcall(StoreInst *I) {
2106 static const RTLIB::Libcall Libcalls[6] = {
2107 RTLIB::ATOMIC_STORE, RTLIB::ATOMIC_STORE_1, RTLIB::ATOMIC_STORE_2,
2108 RTLIB::ATOMIC_STORE_4, RTLIB::ATOMIC_STORE_8, RTLIB::ATOMIC_STORE_16};
2109 unsigned Size = getAtomicOpSize(SI: I);
2110
2111 bool Expanded = expandAtomicOpToLibcall(
2112 I, Size, Alignment: I->getAlign(), PointerOperand: I->getPointerOperand(), ValueOperand: I->getValueOperand(),
2113 CASExpected: nullptr, Ordering: I->getOrdering(), Ordering2: AtomicOrdering::NotAtomic, Libcalls);
2114 if (!Expanded)
2115 handleUnsupportedAtomicSize(I, AtomicOpName: "atomic store");
2116}
2117
2118void AtomicExpandImpl::expandAtomicCASToLibcall(AtomicCmpXchgInst *I,
2119 const Twine &AtomicOpName,
2120 Instruction *DiagnosticInst) {
2121 static const RTLIB::Libcall Libcalls[6] = {
2122 RTLIB::ATOMIC_COMPARE_EXCHANGE, RTLIB::ATOMIC_COMPARE_EXCHANGE_1,
2123 RTLIB::ATOMIC_COMPARE_EXCHANGE_2, RTLIB::ATOMIC_COMPARE_EXCHANGE_4,
2124 RTLIB::ATOMIC_COMPARE_EXCHANGE_8, RTLIB::ATOMIC_COMPARE_EXCHANGE_16};
2125 unsigned Size = getAtomicOpSize(CASI: I);
2126
2127 bool Expanded = expandAtomicOpToLibcall(
2128 I, Size, Alignment: I->getAlign(), PointerOperand: I->getPointerOperand(), ValueOperand: I->getNewValOperand(),
2129 CASExpected: I->getCompareOperand(), Ordering: I->getSuccessOrdering(), Ordering2: I->getFailureOrdering(),
2130 Libcalls);
2131 if (!Expanded)
2132 handleUnsupportedAtomicSize(I, AtomicOpName, DiagnosticInst);
2133}
2134
2135static ArrayRef<RTLIB::Libcall> GetRMWLibcall(AtomicRMWInst::BinOp Op) {
2136 static const RTLIB::Libcall LibcallsXchg[6] = {
2137 RTLIB::ATOMIC_EXCHANGE, RTLIB::ATOMIC_EXCHANGE_1,
2138 RTLIB::ATOMIC_EXCHANGE_2, RTLIB::ATOMIC_EXCHANGE_4,
2139 RTLIB::ATOMIC_EXCHANGE_8, RTLIB::ATOMIC_EXCHANGE_16};
2140 static const RTLIB::Libcall LibcallsAdd[6] = {
2141 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_ADD_1,
2142 RTLIB::ATOMIC_FETCH_ADD_2, RTLIB::ATOMIC_FETCH_ADD_4,
2143 RTLIB::ATOMIC_FETCH_ADD_8, RTLIB::ATOMIC_FETCH_ADD_16};
2144 static const RTLIB::Libcall LibcallsSub[6] = {
2145 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_SUB_1,
2146 RTLIB::ATOMIC_FETCH_SUB_2, RTLIB::ATOMIC_FETCH_SUB_4,
2147 RTLIB::ATOMIC_FETCH_SUB_8, RTLIB::ATOMIC_FETCH_SUB_16};
2148 static const RTLIB::Libcall LibcallsAnd[6] = {
2149 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_AND_1,
2150 RTLIB::ATOMIC_FETCH_AND_2, RTLIB::ATOMIC_FETCH_AND_4,
2151 RTLIB::ATOMIC_FETCH_AND_8, RTLIB::ATOMIC_FETCH_AND_16};
2152 static const RTLIB::Libcall LibcallsOr[6] = {
2153 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_OR_1,
2154 RTLIB::ATOMIC_FETCH_OR_2, RTLIB::ATOMIC_FETCH_OR_4,
2155 RTLIB::ATOMIC_FETCH_OR_8, RTLIB::ATOMIC_FETCH_OR_16};
2156 static const RTLIB::Libcall LibcallsXor[6] = {
2157 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_XOR_1,
2158 RTLIB::ATOMIC_FETCH_XOR_2, RTLIB::ATOMIC_FETCH_XOR_4,
2159 RTLIB::ATOMIC_FETCH_XOR_8, RTLIB::ATOMIC_FETCH_XOR_16};
2160 static const RTLIB::Libcall LibcallsNand[6] = {
2161 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_NAND_1,
2162 RTLIB::ATOMIC_FETCH_NAND_2, RTLIB::ATOMIC_FETCH_NAND_4,
2163 RTLIB::ATOMIC_FETCH_NAND_8, RTLIB::ATOMIC_FETCH_NAND_16};
2164
2165 switch (Op) {
2166 case AtomicRMWInst::BAD_BINOP:
2167 llvm_unreachable("Should not have BAD_BINOP.");
2168 case AtomicRMWInst::Xchg:
2169 return ArrayRef(LibcallsXchg);
2170 case AtomicRMWInst::Add:
2171 return ArrayRef(LibcallsAdd);
2172 case AtomicRMWInst::Sub:
2173 return ArrayRef(LibcallsSub);
2174 case AtomicRMWInst::And:
2175 return ArrayRef(LibcallsAnd);
2176 case AtomicRMWInst::Or:
2177 return ArrayRef(LibcallsOr);
2178 case AtomicRMWInst::Xor:
2179 return ArrayRef(LibcallsXor);
2180 case AtomicRMWInst::Nand:
2181 return ArrayRef(LibcallsNand);
2182 case AtomicRMWInst::Max:
2183 case AtomicRMWInst::Min:
2184 case AtomicRMWInst::UMax:
2185 case AtomicRMWInst::UMin:
2186 case AtomicRMWInst::FMax:
2187 case AtomicRMWInst::FMin:
2188 case AtomicRMWInst::FMaximum:
2189 case AtomicRMWInst::FMinimum:
2190 case AtomicRMWInst::FMaximumNum:
2191 case AtomicRMWInst::FMinimumNum:
2192 case AtomicRMWInst::FAdd:
2193 case AtomicRMWInst::FSub:
2194 case AtomicRMWInst::UIncWrap:
2195 case AtomicRMWInst::UDecWrap:
2196 case AtomicRMWInst::USubCond:
2197 case AtomicRMWInst::USubSat:
2198 // No atomic libcalls are available for these.
2199 return {};
2200 }
2201 llvm_unreachable("Unexpected AtomicRMW operation.");
2202}
2203
2204void AtomicExpandImpl::expandAtomicRMWToLibcall(AtomicRMWInst *I) {
2205 ArrayRef<RTLIB::Libcall> Libcalls = GetRMWLibcall(Op: I->getOperation());
2206
2207 unsigned Size = getAtomicOpSize(RMWI: I);
2208
2209 bool Success = false;
2210 if (!Libcalls.empty())
2211 Success = expandAtomicOpToLibcall(
2212 I, Size, Alignment: I->getAlign(), PointerOperand: I->getPointerOperand(), ValueOperand: I->getValOperand(),
2213 CASExpected: nullptr, Ordering: I->getOrdering(), Ordering2: AtomicOrdering::NotAtomic, Libcalls);
2214
2215 // The expansion failed: either there were no libcalls at all for
2216 // the operation (min/max), or there were only size-specialized
2217 // libcalls (add/sub/etc) and we needed a generic. So, expand to a
2218 // CAS libcall, via a CAS loop, instead.
2219 if (!Success) {
2220 expandAtomicRMWToCmpXchg(
2221 AI: I, CreateCmpXchg: [this, I](IRBuilderBase &Builder, Value *Addr, Value *Loaded,
2222 Value *NewVal, Align Alignment, AtomicOrdering MemOpOrder,
2223 SyncScope::ID SSID, bool IsVolatile, Value *&Success,
2224 Value *&NewLoaded, Instruction *MetadataSrc) {
2225 // Create the CAS instruction normally...
2226 AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
2227 Ptr: Addr, Cmp: Loaded, New: NewVal, Align: Alignment, SuccessOrdering: MemOpOrder,
2228 FailureOrdering: AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: MemOpOrder), SSID);
2229 Pair->setVolatile(IsVolatile);
2230 if (MetadataSrc)
2231 copyMetadataForAtomic(Dest&: *Pair, Source: *MetadataSrc);
2232
2233 Success = Builder.CreateExtractValue(Agg: Pair, Idxs: 1, Name: "success");
2234 NewLoaded = Builder.CreateExtractValue(Agg: Pair, Idxs: 0, Name: "newloaded");
2235
2236 // ...and then expand the CAS into a libcall.
2237 expandAtomicCASToLibcall(
2238 I: Pair,
2239 AtomicOpName: "atomicrmw " + AtomicRMWInst::getOperationName(Op: I->getOperation()),
2240 DiagnosticInst: MetadataSrc);
2241 });
2242 }
2243}
2244
2245// A helper routine for the above expandAtomic*ToLibcall functions.
2246//
2247// 'Libcalls' contains an array of enum values for the particular
2248// ATOMIC libcalls to be emitted. All of the other arguments besides
2249// 'I' are extracted from the Instruction subclass by the
2250// caller. Depending on the particular call, some will be null.
2251bool AtomicExpandImpl::expandAtomicOpToLibcall(
2252 Instruction *I, unsigned Size, Align Alignment, Value *PointerOperand,
2253 Value *ValueOperand, Value *CASExpected, AtomicOrdering Ordering,
2254 AtomicOrdering Ordering2, ArrayRef<RTLIB::Libcall> Libcalls) {
2255 assert(Libcalls.size() == 6);
2256
2257 LLVMContext &Ctx = I->getContext();
2258 Module *M = I->getModule();
2259 const DataLayout &DL = M->getDataLayout();
2260 IRBuilder<> Builder(I);
2261 IRBuilder<> AllocaBuilder(&I->getFunction()->getEntryBlock().front());
2262
2263 bool UseSizedLibcall = canUseSizedAtomicCall(Size, Alignment, DL);
2264 const Align AllocaAlignment =
2265 DL.getPrefTypeAlign(Ty: Type::getIntNTy(C&: Ctx, N: Size * 8));
2266
2267 assert(Ordering != AtomicOrdering::NotAtomic && "expect atomic MO");
2268 assert((!CASExpected || Ordering2 != AtomicOrdering::NotAtomic) &&
2269 "expect atomic MO");
2270 bool HasResult = I->getType() != Type::getVoidTy(C&: Ctx);
2271
2272 RTLIB::Libcall RTLibType;
2273 if (UseSizedLibcall) {
2274 switch (Size) {
2275 case 1:
2276 RTLibType = Libcalls[1];
2277 break;
2278 case 2:
2279 RTLibType = Libcalls[2];
2280 break;
2281 case 4:
2282 RTLibType = Libcalls[3];
2283 break;
2284 case 8:
2285 RTLibType = Libcalls[4];
2286 break;
2287 case 16:
2288 RTLibType = Libcalls[5];
2289 break;
2290 }
2291 } else if (Libcalls[0] != RTLIB::UNKNOWN_LIBCALL) {
2292 RTLibType = Libcalls[0];
2293 } else {
2294 // Can't use sized function, and there's no generic for this
2295 // operation, so give up.
2296 return false;
2297 }
2298
2299 RTLIB::LibcallImpl LibcallImpl = LibcallLowering->getLibcallImpl(Call: RTLibType);
2300 if (LibcallImpl == RTLIB::Unsupported) {
2301 // This target does not implement the requested atomic libcall so give up.
2302 return false;
2303 }
2304
2305 auto [FnType, Attr] = LibcallLowering->getRuntimeLibcallsInfo().getFunctionTy(
2306 Ctx, TT: M->getTargetTriple(), DL, LibcallImpl);
2307 assert(FnType && "missing signature for atomic libcall");
2308
2309 // Build up the function call. There's two kinds. First, the sized
2310 // variants. These calls are going to be one of the following (with
2311 // N=1,2,4,8,16):
2312 // iN __atomic_load_N(iN *ptr, int ordering)
2313 // void __atomic_store_N(iN *ptr, iN val, int ordering)
2314 // iN __atomic_{exchange|fetch_*}_N(iN *ptr, iN val, int ordering)
2315 // bool __atomic_compare_exchange_N(iN *ptr, iN *expected, iN desired,
2316 // int success_order, int failure_order)
2317 //
2318 // Note that these functions can be used for non-integer atomic
2319 // operations, the values just need to be bitcast to integers on the
2320 // way in and out.
2321 //
2322 // And, then, the generic variants. They look like the following:
2323 // void __atomic_load(size_t size, void *ptr, void *ret, int ordering)
2324 // void __atomic_store(size_t size, void *ptr, void *val, int ordering)
2325 // void __atomic_exchange(size_t size, void *ptr, void *val, void *ret,
2326 // int ordering)
2327 // bool __atomic_compare_exchange(size_t size, void *ptr, void *expected,
2328 // void *desired, int success_order,
2329 // int failure_order)
2330 //
2331 // The different signatures are built up depending on the
2332 // 'UseSizedLibcall', 'CASExpected', 'ValueOperand', and 'HasResult'
2333 // variables.
2334
2335 AllocaInst *AllocaCASExpected = nullptr;
2336 AllocaInst *AllocaValue = nullptr;
2337 AllocaInst *AllocaResult = nullptr;
2338
2339 SmallVector<Value *, 6> Args;
2340
2341 // 'size' argument.
2342 if (!UseSizedLibcall)
2343 Args.push_back(Elt: ConstantInt::get(Ty: FnType->getParamType(i: 0), V: Size));
2344
2345 // 'ptr' argument.
2346 // note: This assumes all address spaces share a common libfunc
2347 // implementation and that addresses are convertable. For systems without
2348 // that property, we'd need to extend this mechanism to support AS-specific
2349 // families of atomic intrinsics.
2350 Value *PtrVal = PointerOperand;
2351 PtrVal = Builder.CreateAddrSpaceCast(V: PtrVal, DestTy: PointerType::getUnqual(C&: Ctx));
2352 Args.push_back(Elt: PtrVal);
2353
2354 // 'expected' argument, if present.
2355 if (CASExpected) {
2356 AllocaCASExpected = AllocaBuilder.CreateAlloca(Ty: CASExpected->getType());
2357 AllocaCASExpected->setAlignment(AllocaAlignment);
2358 Builder.CreateLifetimeStart(Ptr: AllocaCASExpected);
2359 Builder.CreateAlignedStore(Val: CASExpected, Ptr: AllocaCASExpected, Align: AllocaAlignment);
2360 Args.push_back(Elt: AllocaCASExpected);
2361 }
2362
2363 // 'val' argument ('desired' for cas), if present.
2364 if (ValueOperand) {
2365 if (UseSizedLibcall) {
2366 Value *IntValue = Builder.CreateBitPreservingCastChain(
2367 DL, V: ValueOperand, NewTy: FnType->getParamType(i: Args.size()));
2368 Args.push_back(Elt: IntValue);
2369 } else {
2370 AllocaValue = AllocaBuilder.CreateAlloca(Ty: ValueOperand->getType());
2371 AllocaValue->setAlignment(AllocaAlignment);
2372 Builder.CreateLifetimeStart(Ptr: AllocaValue);
2373 Builder.CreateAlignedStore(Val: ValueOperand, Ptr: AllocaValue, Align: AllocaAlignment);
2374 Args.push_back(Elt: AllocaValue);
2375 }
2376 }
2377
2378 // 'ret' argument.
2379 if (!CASExpected && HasResult && !UseSizedLibcall) {
2380 AllocaResult = AllocaBuilder.CreateAlloca(Ty: I->getType());
2381 AllocaResult->setAlignment(AllocaAlignment);
2382 Builder.CreateLifetimeStart(Ptr: AllocaResult);
2383 Args.push_back(Elt: AllocaResult);
2384 }
2385
2386 // 'ordering' ('success_order' for cas) argument.
2387 Args.push_back(Elt: ConstantInt::get(Ty: FnType->getParamType(i: Args.size()),
2388 V: (int)toCABI(AO: Ordering)));
2389
2390 // 'failure_order' argument, if present.
2391 if (CASExpected) {
2392 Args.push_back(Elt: ConstantInt::get(Ty: FnType->getParamType(i: Args.size()),
2393 V: (int)toCABI(AO: Ordering2)));
2394 }
2395
2396 assert(Args.size() == FnType->getNumParams() &&
2397 "atomic libcall signature mismatch");
2398
2399 FunctionCallee LibcallFn = M->getOrInsertFunction(
2400 Name: RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: LibcallImpl), T: FnType,
2401 AttributeList: Attr);
2402 CallInst *Call = Builder.CreateCall(Callee: LibcallFn, Args);
2403 Call->setAttributes(Attr);
2404 Value *Result = Call;
2405
2406 // And then, extract the results...
2407 if (ValueOperand && !UseSizedLibcall)
2408 Builder.CreateLifetimeEnd(Ptr: AllocaValue);
2409
2410 if (CASExpected) {
2411 // The final result from the CAS is {load of 'expected' alloca, bool result
2412 // from call}
2413 Type *FinalResultTy = I->getType();
2414 Value *V = PoisonValue::get(T: FinalResultTy);
2415 Value *ExpectedOut = Builder.CreateAlignedLoad(
2416 Ty: CASExpected->getType(), Ptr: AllocaCASExpected, Align: AllocaAlignment);
2417 Builder.CreateLifetimeEnd(Ptr: AllocaCASExpected);
2418 V = Builder.CreateInsertValue(Agg: V, Val: ExpectedOut, Idxs: 0);
2419 V = Builder.CreateInsertValue(Agg: V, Val: Result, Idxs: 1);
2420 I->replaceAllUsesWith(V);
2421 } else if (HasResult) {
2422 Value *V;
2423 if (UseSizedLibcall) {
2424 // Add bitcasts from Result's scalar type to I's <n x ptr> vector type
2425 auto *PtrTy = dyn_cast<PointerType>(Val: I->getType()->getScalarType());
2426 auto *VTy = dyn_cast<VectorType>(Val: I->getType());
2427 if (VTy && PtrTy && !Result->getType()->isVectorTy()) {
2428 unsigned AS = PtrTy->getAddressSpace();
2429 Value *BC = Builder.CreateBitCast(
2430 V: Result, DestTy: VTy->getWithNewType(EltTy: DL.getIntPtrType(C&: Ctx, AddressSpace: AS)));
2431 V = Builder.CreateIntToPtr(V: BC, DestTy: I->getType());
2432 } else
2433 V = Builder.CreateBitOrPointerCast(V: Result, DestTy: I->getType());
2434 } else {
2435 V = Builder.CreateAlignedLoad(Ty: I->getType(), Ptr: AllocaResult,
2436 Align: AllocaAlignment);
2437 Builder.CreateLifetimeEnd(Ptr: AllocaResult);
2438 }
2439 I->replaceAllUsesWith(V);
2440 }
2441 I->eraseFromParent();
2442 return true;
2443}
2444