1//===- AtomicExpandPass.cpp - Expand atomic instructions ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass (at IR level) to replace atomic instructions with
10// __atomic_* library calls, or target specific instruction which implement the
11// same semantics in a way which better fits the target backend. This can
12// include the use of (intrinsic-based) load-linked/store-conditional loops,
13// AtomicCmpXchg, or type coercions.
14//
15//===----------------------------------------------------------------------===//
16
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/STLFunctionalExtras.h"
19#include "llvm/ADT/SmallString.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/Analysis/InstSimplifyFolder.h"
22#include "llvm/Analysis/OptimizationRemarkEmitter.h"
23#include "llvm/CodeGen/AtomicExpand.h"
24#include "llvm/CodeGen/TargetLowering.h"
25#include "llvm/CodeGen/TargetPassConfig.h"
26#include "llvm/CodeGen/TargetSubtargetInfo.h"
27#include "llvm/CodeGen/ValueTypes.h"
28#include "llvm/IR/Attributes.h"
29#include "llvm/IR/BasicBlock.h"
30#include "llvm/IR/Constant.h"
31#include "llvm/IR/Constants.h"
32#include "llvm/IR/DataLayout.h"
33#include "llvm/IR/DerivedTypes.h"
34#include "llvm/IR/Function.h"
35#include "llvm/IR/IRBuilder.h"
36#include "llvm/IR/Instruction.h"
37#include "llvm/IR/Instructions.h"
38#include "llvm/IR/MDBuilder.h"
39#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
40#include "llvm/IR/Module.h"
41#include "llvm/IR/ProfDataUtils.h"
42#include "llvm/IR/Type.h"
43#include "llvm/IR/User.h"
44#include "llvm/IR/Value.h"
45#include "llvm/InitializePasses.h"
46#include "llvm/Pass.h"
47#include "llvm/Support/AtomicOrdering.h"
48#include "llvm/Support/Casting.h"
49#include "llvm/Support/Debug.h"
50#include "llvm/Support/ErrorHandling.h"
51#include "llvm/Support/raw_ostream.h"
52#include "llvm/Target/TargetMachine.h"
53#include "llvm/Transforms/Utils/LowerAtomic.h"
54#include <cassert>
55#include <cstdint>
56#include <iterator>
57
58using namespace llvm;
59
60#define DEBUG_TYPE "atomic-expand"
61
62namespace {
63
64class AtomicExpandImpl {
65 const TargetLowering *TLI = nullptr;
66 const LibcallLoweringInfo *LibcallLowering = nullptr;
67 const DataLayout *DL = nullptr;
68 bool SingleThreaded = false;
69
70private:
71 /// Callback type for emitting a cmpxchg instruction during RMW expansion.
72 /// Parameters: (Builder, Addr, Loaded, NewVal, AddrAlign, MemOpOrder,
73 /// SSID, IsVolatile, /* OUT */ Success, /* OUT */ NewLoaded,
74 /// MetadataSrc)
75 using CreateCmpXchgInstFun = function_ref<void(
76 IRBuilderBase &, Value *, Value *, Value *, Align, AtomicOrdering,
77 SyncScope::ID, bool, Value *&, Value *&, Instruction *)>;
78
79 void handleFailure(Instruction &FailedInst, const Twine &Msg,
80 Instruction *DiagnosticInst = nullptr) const {
81 LLVMContext &Ctx = FailedInst.getContext();
82
83 // TODO: Do not use generic error type.
84 Ctx.emitError(I: DiagnosticInst ? DiagnosticInst : &FailedInst, ErrorStr: Msg);
85
86 if (!FailedInst.getType()->isVoidTy())
87 FailedInst.replaceAllUsesWith(V: PoisonValue::get(T: FailedInst.getType()));
88 FailedInst.eraseFromParent();
89 }
90
91 template <typename Inst>
92 void handleUnsupportedAtomicSize(Inst *I, const Twine &AtomicOpName,
93 Instruction *DiagnosticInst = nullptr) const;
94
95 bool bracketInstWithFences(Instruction *I, AtomicOrdering Order);
96 bool tryInsertTrailingSeqCstFence(Instruction *AtomicI);
97 template <typename AtomicInst>
98 bool tryInsertFencesForAtomic(AtomicInst *AtomicI, bool OrderingRequiresFence,
99 AtomicOrdering NewOrdering);
100 IntegerType *getCorrespondingIntegerType(Type *T, const DataLayout &DL);
101 LoadInst *convertAtomicLoadToIntegerType(LoadInst *LI);
102 bool tryExpandAtomicLoad(LoadInst *LI);
103 bool expandAtomicLoadToLL(LoadInst *LI);
104 bool expandAtomicLoadToCmpXchg(LoadInst *LI);
105 StoreInst *convertAtomicStoreToIntegerType(StoreInst *SI);
106 bool tryExpandAtomicStore(StoreInst *SI);
107 void expandAtomicStoreToXChg(StoreInst *SI);
108 bool tryExpandAtomicRMW(AtomicRMWInst *AI);
109 AtomicRMWInst *convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI);
110 Value *
111 insertRMWLLSCLoop(IRBuilderBase &Builder, Type *ResultTy, Value *Addr,
112 Align AddrAlign, AtomicOrdering MemOpOrder,
113 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp);
114 void expandAtomicOpToLLSC(
115 Instruction *I, Type *ResultTy, Value *Addr, Align AddrAlign,
116 AtomicOrdering MemOpOrder,
117 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp);
118 void expandPartwordAtomicRMW(
119 AtomicRMWInst *I, TargetLoweringBase::AtomicExpansionKind ExpansionKind);
120 AtomicRMWInst *widenPartwordAtomicRMW(AtomicRMWInst *AI);
121 bool expandPartwordCmpXchg(AtomicCmpXchgInst *I);
122 void expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI);
123 void expandAtomicCmpXchgToMaskedIntrinsic(AtomicCmpXchgInst *CI);
124
125 AtomicCmpXchgInst *convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI);
126 Value *insertRMWCmpXchgLoop(
127 IRBuilderBase &Builder, Type *ResultType, Value *Addr, Align AddrAlign,
128 AtomicOrdering MemOpOrder, SyncScope::ID SSID, bool IsVolatile,
129 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp,
130 CreateCmpXchgInstFun CreateCmpXchg, Instruction *MetadataSrc);
131 bool tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI);
132
133 bool expandAtomicCmpXchg(AtomicCmpXchgInst *CI);
134 bool isIdempotentRMW(AtomicRMWInst *RMWI);
135 bool simplifyIdempotentRMW(AtomicRMWInst *RMWI);
136
137 bool expandAtomicOpToLibcall(Instruction *I, unsigned Size, Align Alignment,
138 Value *PointerOperand, Value *ValueOperand,
139 Value *CASExpected, AtomicOrdering Ordering,
140 AtomicOrdering Ordering2,
141 ArrayRef<RTLIB::Libcall> Libcalls);
142 void expandAtomicLoadToLibcall(LoadInst *LI);
143 void expandAtomicStoreToLibcall(StoreInst *LI);
144 void expandAtomicRMWToLibcall(AtomicRMWInst *I);
145 void expandAtomicCASToLibcall(AtomicCmpXchgInst *I,
146 const Twine &AtomicOpName = "cmpxchg",
147 Instruction *DiagnosticInst = nullptr);
148
149 bool expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
150 CreateCmpXchgInstFun CreateCmpXchg);
151
152 bool lowerToNonAtomic(Instruction *I);
153 bool processAtomicInstr(Instruction *I);
154
155public:
156 bool run(Function &F, const ModuleLibcallLoweringInfo &LibcallResult,
157 const TargetMachine *TM);
158};
159
160class AtomicExpandLegacy : public FunctionPass {
161public:
162 static char ID; // Pass identification, replacement for typeid
163
164 AtomicExpandLegacy() : FunctionPass(ID) {}
165
166 void getAnalysisUsage(AnalysisUsage &AU) const override {
167 AU.addRequired<LibcallLoweringInfoWrapper>();
168 FunctionPass::getAnalysisUsage(AU);
169 }
170
171 bool runOnFunction(Function &F) override;
172};
173
174// IRBuilder to be used for replacement atomic instructions.
175struct ReplacementIRBuilder
176 : IRBuilder<InstSimplifyFolder, IRBuilderCallbackInserter> {
177 MDNode *MMRAMD = nullptr;
178 MDNode *PCSectionsMD = nullptr;
179
180 // Preserves the DebugLoc from I, and preserves still valid metadata.
181 // Enable StrictFP builder mode when appropriate.
182 explicit ReplacementIRBuilder(Instruction *I, const DataLayout &DL)
183 : IRBuilder(
184 I->getContext(), InstSimplifyFolder(DL),
185 IRBuilderCallbackInserter([this](Instruction *I) { addMD(I); })) {
186 SetInsertPoint(I);
187 if (BB->getParent()->getAttributes().hasFnAttr(Kind: Attribute::StrictFP))
188 this->setIsFPConstrained(true);
189
190 MMRAMD = I->getMetadata(KindID: LLVMContext::MD_mmra);
191 PCSectionsMD = I->getMetadata(KindID: LLVMContext::MD_pcsections);
192 }
193
194 void addMD(Instruction *I) {
195 if (canInstructionHaveMMRAs(I: *I))
196 I->setMetadata(KindID: LLVMContext::MD_mmra, Node: MMRAMD);
197 I->setMetadata(KindID: LLVMContext::MD_pcsections, Node: PCSectionsMD);
198 }
199};
200
201} // end anonymous namespace
202
203char AtomicExpandLegacy::ID = 0;
204
205char &llvm::AtomicExpandID = AtomicExpandLegacy::ID;
206
207INITIALIZE_PASS_BEGIN(AtomicExpandLegacy, DEBUG_TYPE,
208 "Expand Atomic instructions", false, false)
209INITIALIZE_PASS_DEPENDENCY(LibcallLoweringInfoWrapper)
210INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
211INITIALIZE_PASS_END(AtomicExpandLegacy, DEBUG_TYPE,
212 "Expand Atomic instructions", false, false)
213
214// Helper functions to retrieve the size of atomic instructions.
215static unsigned getAtomicOpSize(LoadInst *LI) {
216 const DataLayout &DL = LI->getDataLayout();
217 return DL.getTypeStoreSize(Ty: LI->getType());
218}
219
220static unsigned getAtomicOpSize(StoreInst *SI) {
221 const DataLayout &DL = SI->getDataLayout();
222 return DL.getTypeStoreSize(Ty: SI->getValueOperand()->getType());
223}
224
225static unsigned getAtomicOpSize(AtomicRMWInst *RMWI) {
226 const DataLayout &DL = RMWI->getDataLayout();
227 return DL.getTypeStoreSize(Ty: RMWI->getValOperand()->getType());
228}
229
230static unsigned getAtomicOpSize(AtomicCmpXchgInst *CASI) {
231 const DataLayout &DL = CASI->getDataLayout();
232 return DL.getTypeStoreSize(Ty: CASI->getCompareOperand()->getType());
233}
234
235/// Copy metadata that's safe to preserve when widening atomics.
236static void copyMetadataForAtomic(Instruction &Dest,
237 const Instruction &Source) {
238 SmallVector<std::pair<unsigned, MDNode *>, 8> MD;
239 Source.getAllMetadata(MDs&: MD);
240 LLVMContext &Ctx = Dest.getContext();
241 MDBuilder MDB(Ctx);
242
243 for (auto [ID, N] : MD) {
244 switch (ID) {
245 case LLVMContext::MD_dbg:
246 case LLVMContext::MD_tbaa:
247 case LLVMContext::MD_tbaa_struct:
248 case LLVMContext::MD_alias_scope:
249 case LLVMContext::MD_noalias:
250 case LLVMContext::MD_noalias_addrspace:
251 case LLVMContext::MD_access_group:
252 case LLVMContext::MD_mmra:
253 Dest.setMetadata(KindID: ID, Node: N);
254 break;
255 default:
256 if (ID == Ctx.getMDKindID(Name: "amdgpu.no.remote.memory"))
257 Dest.setMetadata(KindID: ID, Node: N);
258 else if (ID == Ctx.getMDKindID(Name: "amdgpu.no.fine.grained.memory"))
259 Dest.setMetadata(KindID: ID, Node: N);
260
261 // Losing atomic.ignore.denormal.mode, but it doesn't matter for current
262 // uses.
263 break;
264 }
265 }
266}
267
268template <typename Inst>
269static bool atomicSizeSupported(const TargetLowering *TLI, Inst *I) {
270 unsigned Size = getAtomicOpSize(I);
271 Align Alignment = I->getAlign();
272 unsigned MaxSize = TLI->getMaxAtomicSizeInBitsSupported() / 8;
273 return Alignment >= Size && Size <= MaxSize;
274}
275
276template <typename Inst>
277static void writeUnsupportedAtomicSizeReason(const TargetLowering *TLI, Inst *I,
278 raw_ostream &OS) {
279 unsigned Size = getAtomicOpSize(I);
280 Align Alignment = I->getAlign();
281 bool NeedSeparator = false;
282
283 if (Alignment < Size) {
284 OS << "instruction alignment " << Alignment.value()
285 << " is smaller than the required " << Size
286 << "-byte alignment for this atomic operation";
287 NeedSeparator = true;
288 }
289
290 unsigned MaxSize = TLI->getMaxAtomicSizeInBitsSupported() / 8;
291 if (Size > MaxSize) {
292 if (NeedSeparator)
293 OS << "; ";
294 OS << "target supports atomics up to " << MaxSize
295 << " bytes, but this atomic accesses " << Size << " bytes";
296 }
297}
298
299template <typename Inst>
300void AtomicExpandImpl::handleUnsupportedAtomicSize(
301 Inst *I, const Twine &AtomicOpName, Instruction *DiagnosticInst) const {
302 assert(!atomicSizeSupported(TLI, I) && "expected unsupported atomic size");
303 SmallString<128> FailureReason;
304 raw_svector_ostream OS(FailureReason);
305 writeUnsupportedAtomicSizeReason(TLI, I, OS);
306 handleFailure(FailedInst&: *I, Msg: Twine("unsupported ") + AtomicOpName + ": " + FailureReason,
307 DiagnosticInst);
308}
309
310bool AtomicExpandImpl::tryInsertTrailingSeqCstFence(Instruction *AtomicI) {
311 if (!TLI->shouldInsertTrailingSeqCstFenceForAtomicStore(I: AtomicI))
312 return false;
313
314 IRBuilder Builder(AtomicI);
315 if (auto *TrailingFence = TLI->emitTrailingFence(
316 Builder, Inst: AtomicI, Ord: AtomicOrdering::SequentiallyConsistent)) {
317 TrailingFence->moveAfter(MovePos: AtomicI);
318 return true;
319 }
320 return false;
321}
322
323template <typename AtomicInst>
324bool AtomicExpandImpl::tryInsertFencesForAtomic(AtomicInst *AtomicI,
325 bool OrderingRequiresFence,
326 AtomicOrdering NewOrdering) {
327 bool ShouldInsertFences = TLI->shouldInsertFencesForAtomic(I: AtomicI);
328 if (OrderingRequiresFence && ShouldInsertFences) {
329 AtomicOrdering FenceOrdering = AtomicI->getOrdering();
330 AtomicI->setOrdering(NewOrdering);
331 return bracketInstWithFences(I: AtomicI, Order: FenceOrdering);
332 }
333 if (!ShouldInsertFences)
334 return tryInsertTrailingSeqCstFence(AtomicI);
335 return false;
336}
337
338/// In a single-threaded environment, atomic operations can be lowered to their
339/// non-atomic equivalents: fences are removed, and atomic loads, stores, RMW,
340/// and cmpxchg become plain memory operations.
341bool AtomicExpandImpl::lowerToNonAtomic(Instruction *I) {
342 if (auto *FI = dyn_cast<FenceInst>(Val: I)) {
343 FI->eraseFromParent();
344 return true;
345 }
346
347 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: I))
348 return lowerAtomicCmpXchgInst(CXI);
349
350 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: I))
351 return lowerAtomicRMWInst(RMWI);
352
353 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
354 if (LI->isAtomic()) {
355 LI->setAtomic(Ordering: AtomicOrdering::NotAtomic);
356 LI->setElementwise(false);
357 return true;
358 }
359
360 return false;
361 }
362
363 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
364 if (SI->isAtomic()) {
365 SI->setAtomic(Ordering: AtomicOrdering::NotAtomic);
366 SI->setElementwise(false);
367 return true;
368 }
369
370 return false;
371 }
372
373 return false;
374}
375
376bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
377 if (SingleThreaded)
378 return lowerToNonAtomic(I);
379
380 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
381 if (!LI->isAtomic())
382 return false;
383
384 if (!atomicSizeSupported(TLI, I: LI)) {
385 expandAtomicLoadToLibcall(LI);
386 return true;
387 }
388
389 bool MadeChange = false;
390 if (TLI->shouldCastAtomicLoadInIR(LI) ==
391 TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
392 LI = convertAtomicLoadToIntegerType(LI);
393 MadeChange = true;
394 }
395
396 MadeChange |= tryInsertFencesForAtomic(
397 AtomicI: LI, OrderingRequiresFence: isAcquireOrStronger(AO: LI->getOrdering()), NewOrdering: AtomicOrdering::Monotonic);
398
399 MadeChange |= tryExpandAtomicLoad(LI);
400 return MadeChange;
401 }
402
403 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
404 if (!SI->isAtomic())
405 return false;
406
407 if (!atomicSizeSupported(TLI, I: SI)) {
408 expandAtomicStoreToLibcall(LI: SI);
409 return true;
410 }
411
412 bool MadeChange = false;
413 if (TLI->shouldCastAtomicStoreInIR(SI) ==
414 TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
415 SI = convertAtomicStoreToIntegerType(SI);
416 MadeChange = true;
417 }
418
419 MadeChange |= tryInsertFencesForAtomic(
420 AtomicI: SI, OrderingRequiresFence: isReleaseOrStronger(AO: SI->getOrdering()), NewOrdering: AtomicOrdering::Monotonic);
421
422 MadeChange |= tryExpandAtomicStore(SI);
423 return MadeChange;
424 }
425
426 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: I)) {
427 if (!atomicSizeSupported(TLI, I: RMWI)) {
428 expandAtomicRMWToLibcall(I: RMWI);
429 return true;
430 }
431
432 bool MadeChange = false;
433 if (TLI->shouldCastAtomicRMWIInIR(RMWI) ==
434 TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
435 RMWI = convertAtomicXchgToIntegerType(RMWI);
436 MadeChange = true;
437 }
438
439 MadeChange |= tryInsertFencesForAtomic(
440 AtomicI: RMWI,
441 OrderingRequiresFence: isReleaseOrStronger(AO: RMWI->getOrdering()) ||
442 isAcquireOrStronger(AO: RMWI->getOrdering()),
443 NewOrdering: TLI->atomicOperationOrderAfterFenceSplit(I: RMWI));
444
445 // There are two different ways of expanding RMW instructions:
446 // - into a load if it is idempotent
447 // - into a Cmpxchg/LL-SC loop otherwise
448 // we try them in that order.
449 MadeChange |= (isIdempotentRMW(RMWI) && simplifyIdempotentRMW(RMWI)) ||
450 tryExpandAtomicRMW(AI: RMWI);
451 return MadeChange;
452 }
453
454 if (auto *CASI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
455 if (!atomicSizeSupported(TLI, I: CASI)) {
456 expandAtomicCASToLibcall(I: CASI);
457 return true;
458 }
459
460 // TODO: when we're ready to make the change at the IR level, we can
461 // extend convertCmpXchgToInteger for floating point too.
462 bool MadeChange = false;
463 if (CASI->getCompareOperand()->getType()->isPointerTy()) {
464 // TODO: add a TLI hook to control this so that each target can
465 // convert to lowering the original type one at a time.
466 CASI = convertCmpXchgToIntegerType(CI: CASI);
467 MadeChange = true;
468 }
469
470 auto CmpXchgExpansion = TLI->shouldExpandAtomicCmpXchgInIR(AI: CASI);
471 if (TLI->shouldInsertFencesForAtomic(I: CASI)) {
472 if (CmpXchgExpansion == TargetLoweringBase::AtomicExpansionKind::None &&
473 (isReleaseOrStronger(AO: CASI->getSuccessOrdering()) ||
474 isAcquireOrStronger(AO: CASI->getSuccessOrdering()) ||
475 isAcquireOrStronger(AO: CASI->getFailureOrdering()))) {
476 // If a compare and swap is lowered to LL/SC, we can do smarter fence
477 // insertion, with a stronger one on the success path than on the
478 // failure path. As a result, fence insertion is directly done by
479 // expandAtomicCmpXchg in that case.
480 AtomicOrdering FenceOrdering = CASI->getMergedOrdering();
481 AtomicOrdering CASOrdering =
482 TLI->atomicOperationOrderAfterFenceSplit(I: CASI);
483 CASI->setSuccessOrdering(CASOrdering);
484 CASI->setFailureOrdering(CASOrdering);
485 MadeChange |= bracketInstWithFences(I: CASI, Order: FenceOrdering);
486 }
487 } else if (CmpXchgExpansion !=
488 TargetLoweringBase::AtomicExpansionKind::LLSC) {
489 // CmpXchg LLSC is handled in expandAtomicCmpXchg().
490 MadeChange |= tryInsertTrailingSeqCstFence(AtomicI: CASI);
491 }
492
493 MadeChange |= tryExpandAtomicCmpXchg(CI: CASI);
494 return MadeChange;
495 }
496
497 return false;
498}
499
500bool AtomicExpandImpl::run(Function &F,
501 const ModuleLibcallLoweringInfo &LibcallResult,
502 const TargetMachine *TM) {
503 SingleThreaded = F.getParent()->getThreadModel() == ThreadModel::Single;
504
505 const auto *Subtarget = TM->getSubtargetImpl(F);
506 // In a single-threaded environment atomics are lowered to non-atomic form
507 if (!SingleThreaded && !Subtarget->enableAtomicExpand())
508 return false;
509 TLI = Subtarget->getTargetLowering();
510 LibcallLowering = &getLibcallLowering(ModuleInfo: LibcallResult, Subtarget: *Subtarget);
511 DL = &F.getDataLayout();
512
513 bool MadeChange = false;
514
515 for (Function::iterator BBI = F.begin(), BBE = F.end(); BBI != BBE; ++BBI) {
516 BasicBlock *BB = &*BBI;
517
518 BasicBlock::reverse_iterator Next;
519
520 for (BasicBlock::reverse_iterator I = BB->rbegin(), E = BB->rend(); I != E;
521 I = Next) {
522 Instruction &Inst = *I;
523 Next = std::next(x: I);
524
525 if (processAtomicInstr(I: &Inst)) {
526 MadeChange = true;
527
528 // New blocks may have been inserted.
529 BBE = F.end();
530 }
531 }
532 }
533
534 return MadeChange;
535}
536
537bool AtomicExpandLegacy::runOnFunction(Function &F) {
538
539 auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
540 if (!TPC)
541 return false;
542 auto *TM = &TPC->getTM<TargetMachine>();
543
544 const ModuleLibcallLoweringInfo &LibcallResult =
545 getAnalysis<LibcallLoweringInfoWrapper>().getResult(M: *F.getParent());
546 AtomicExpandImpl AE;
547 return AE.run(F, LibcallResult, TM);
548}
549
550FunctionPass *llvm::createAtomicExpandLegacyPass() {
551 return new AtomicExpandLegacy();
552}
553
554PreservedAnalyses AtomicExpandPass::run(Function &F,
555 FunctionAnalysisManager &FAM) {
556 auto &MAMProxy = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: F);
557
558 const ModuleLibcallLoweringInfo *LibcallResult =
559 MAMProxy.getCachedResult<LibcallLoweringModuleAnalysis>(IR&: *F.getParent());
560
561 if (!LibcallResult) {
562 F.getContext().emitError(ErrorStr: "'" + LibcallLoweringModuleAnalysis::name() +
563 "' analysis required");
564 return PreservedAnalyses::all();
565 }
566
567 AtomicExpandImpl AE;
568
569 bool Changed = AE.run(F, LibcallResult: *LibcallResult, TM);
570 if (!Changed)
571 return PreservedAnalyses::all();
572
573 return PreservedAnalyses::none();
574}
575
576bool AtomicExpandImpl::bracketInstWithFences(Instruction *I,
577 AtomicOrdering Order) {
578 ReplacementIRBuilder Builder(I, *DL);
579
580 auto LeadingFence = TLI->emitLeadingFence(Builder, Inst: I, Ord: Order);
581
582 auto TrailingFence = TLI->emitTrailingFence(Builder, Inst: I, Ord: Order);
583 // We have a guard here because not every atomic operation generates a
584 // trailing fence.
585 if (TrailingFence)
586 TrailingFence->moveAfter(MovePos: I);
587
588 return (LeadingFence || TrailingFence);
589}
590
591/// Get the iX type with the same bitwidth as T.
592IntegerType *
593AtomicExpandImpl::getCorrespondingIntegerType(Type *T, const DataLayout &DL) {
594 EVT VT = TLI->getMemValueType(DL, Ty: T);
595 unsigned BitWidth = VT.getStoreSizeInBits();
596 assert(BitWidth == VT.getSizeInBits() && "must be a power of two");
597 return IntegerType::get(C&: T->getContext(), NumBits: BitWidth);
598}
599
600/// Convert an atomic load of a non-integral type to an integer load of the
601/// equivalent bitwidth. See the function comment on
602/// convertAtomicStoreToIntegerType for background.
603LoadInst *AtomicExpandImpl::convertAtomicLoadToIntegerType(LoadInst *LI) {
604 auto *M = LI->getModule();
605 Type *NewTy = getCorrespondingIntegerType(T: LI->getType(), DL: M->getDataLayout());
606
607 ReplacementIRBuilder Builder(LI, *DL);
608
609 Value *Addr = LI->getPointerOperand();
610
611 auto *NewLI = Builder.CreateLoad(Ty: NewTy, Ptr: Addr, Props: LI->getProperties());
612 LLVM_DEBUG(dbgs() << "Replaced " << *LI << " with " << *NewLI << "\n");
613
614 Value *NewVal = LI->getType()->isPtrOrPtrVectorTy()
615 ? Builder.CreateIntToPtr(V: NewLI, DestTy: LI->getType())
616 : Builder.CreateBitCast(V: NewLI, DestTy: LI->getType());
617 LI->replaceAllUsesWith(V: NewVal);
618 LI->eraseFromParent();
619 return NewLI;
620}
621
622AtomicRMWInst *
623AtomicExpandImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
624 assert(RMWI->getOperation() == AtomicRMWInst::Xchg);
625
626 auto *M = RMWI->getModule();
627 Type *NewTy =
628 getCorrespondingIntegerType(T: RMWI->getType(), DL: M->getDataLayout());
629
630 ReplacementIRBuilder Builder(RMWI, *DL);
631
632 Value *Addr = RMWI->getPointerOperand();
633 Value *Val = RMWI->getValOperand();
634 Value *NewVal = Builder.CreateBitPreservingCastChain(DL: *DL, V: Val, NewTy);
635
636 auto *NewRMWI = Builder.CreateAtomicRMW(Op: AtomicRMWInst::Xchg, Ptr: Addr, Val: NewVal,
637 Align: RMWI->getAlign(), Ordering: RMWI->getOrdering(),
638 SSID: RMWI->getSyncScopeID());
639 NewRMWI->setVolatile(RMWI->isVolatile());
640 copyMetadataForAtomic(Dest&: *NewRMWI, Source: *RMWI);
641 LLVM_DEBUG(dbgs() << "Replaced " << *RMWI << " with " << *NewRMWI << "\n");
642
643 Value *NewRVal =
644 Builder.CreateBitPreservingCastChain(DL: *DL, V: NewRMWI, NewTy: RMWI->getType());
645 RMWI->replaceAllUsesWith(V: NewRVal);
646 RMWI->eraseFromParent();
647 return NewRMWI;
648}
649
650bool AtomicExpandImpl::tryExpandAtomicLoad(LoadInst *LI) {
651 switch (TLI->shouldExpandAtomicLoadInIR(LI)) {
652 case TargetLoweringBase::AtomicExpansionKind::None:
653 return false;
654 case TargetLoweringBase::AtomicExpansionKind::LLSC:
655 expandAtomicOpToLLSC(
656 I: LI, ResultTy: LI->getType(), Addr: LI->getPointerOperand(), AddrAlign: LI->getAlign(),
657 MemOpOrder: LI->getOrdering(),
658 PerformOp: [](IRBuilderBase &Builder, Value *Loaded) { return Loaded; });
659 return true;
660 case TargetLoweringBase::AtomicExpansionKind::LLOnly:
661 return expandAtomicLoadToLL(LI);
662 case TargetLoweringBase::AtomicExpansionKind::CmpXChg:
663 return expandAtomicLoadToCmpXchg(LI);
664 case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
665 LI->setAtomic(Ordering: AtomicOrdering::NotAtomic);
666 return true;
667 case TargetLoweringBase::AtomicExpansionKind::CustomExpand:
668 TLI->emitExpandAtomicLoad(LI);
669 return true;
670 default:
671 llvm_unreachable("Unhandled case in tryExpandAtomicLoad");
672 }
673}
674
675bool AtomicExpandImpl::tryExpandAtomicStore(StoreInst *SI) {
676 switch (TLI->shouldExpandAtomicStoreInIR(SI)) {
677 case TargetLoweringBase::AtomicExpansionKind::None:
678 return false;
679 case TargetLoweringBase::AtomicExpansionKind::CustomExpand:
680 TLI->emitExpandAtomicStore(SI);
681 return true;
682 case TargetLoweringBase::AtomicExpansionKind::Expand:
683 expandAtomicStoreToXChg(SI);
684 return true;
685 case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
686 SI->setAtomic(Ordering: AtomicOrdering::NotAtomic);
687 return true;
688 default:
689 llvm_unreachable("Unhandled case in tryExpandAtomicStore");
690 }
691}
692
693bool AtomicExpandImpl::expandAtomicLoadToLL(LoadInst *LI) {
694 ReplacementIRBuilder Builder(LI, *DL);
695
696 // On some architectures, load-linked instructions are atomic for larger
697 // sizes than normal loads. For example, the only 64-bit load guaranteed
698 // to be single-copy atomic by ARM is an ldrexd (A3.5.3).
699 Value *Val = TLI->emitLoadLinked(Builder, ValueTy: LI->getType(),
700 Addr: LI->getPointerOperand(), Ord: LI->getOrdering());
701 TLI->emitAtomicCmpXchgNoStoreLLBalance(Builder);
702
703 LI->replaceAllUsesWith(V: Val);
704 LI->eraseFromParent();
705
706 return true;
707}
708
709bool AtomicExpandImpl::expandAtomicLoadToCmpXchg(LoadInst *LI) {
710 ReplacementIRBuilder Builder(LI, *DL);
711 AtomicOrdering Order = LI->getOrdering();
712 if (Order == AtomicOrdering::Unordered)
713 Order = AtomicOrdering::Monotonic;
714
715 Value *Addr = LI->getPointerOperand();
716 Type *Ty = LI->getType();
717
718 // cmpxchg supports only integer and pointer operands. If the load type is
719 // FP or vector, run the cmpxchg on the same-sized integer and bitcast the
720 // result back; mirrors createCmpXchgInstFun.
721 bool NeedBitcast = Ty->isFloatingPointTy() || Ty->isVectorTy();
722 Type *CmpXchgTy = Ty;
723 if (NeedBitcast)
724 CmpXchgTy = Builder.getIntNTy(N: Ty->getPrimitiveSizeInBits());
725 Constant *DummyVal = Constant::getNullValue(Ty: CmpXchgTy);
726
727 AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
728 Ptr: Addr, Cmp: DummyVal, New: DummyVal, Align: LI->getAlign(), SuccessOrdering: Order,
729 FailureOrdering: AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: Order),
730 SSID: LI->getSyncScopeID());
731 Pair->setVolatile(LI->isVolatile());
732 Value *Loaded = Builder.CreateExtractValue(Agg: Pair, Idxs: 0, Name: "loaded");
733 if (NeedBitcast)
734 Loaded = Builder.CreateBitCast(V: Loaded, DestTy: Ty);
735
736 LI->replaceAllUsesWith(V: Loaded);
737 LI->eraseFromParent();
738
739 return true;
740}
741
742/// Convert an atomic store of a non-integral type to an integer store of the
743/// equivalent bitwidth. We used to not support floating point or vector
744/// atomics in the IR at all. The backends learned to deal with the bitcast
745/// idiom because that was the only way of expressing the notion of a atomic
746/// float or vector store. The long term plan is to teach each backend to
747/// instruction select from the original atomic store, but as a migration
748/// mechanism, we convert back to the old format which the backends understand.
749/// Each backend will need individual work to recognize the new format.
750StoreInst *AtomicExpandImpl::convertAtomicStoreToIntegerType(StoreInst *SI) {
751 ReplacementIRBuilder Builder(SI, *DL);
752 auto *M = SI->getModule();
753 Type *NewTy = getCorrespondingIntegerType(T: SI->getValueOperand()->getType(),
754 DL: M->getDataLayout());
755 Value *NewVal = SI->getValueOperand()->getType()->isPtrOrPtrVectorTy()
756 ? Builder.CreatePtrToInt(V: SI->getValueOperand(), DestTy: NewTy)
757 : Builder.CreateBitCast(V: SI->getValueOperand(), DestTy: NewTy);
758
759 Value *Addr = SI->getPointerOperand();
760
761 StoreInst *NewSI = Builder.CreateStore(Val: NewVal, Ptr: Addr, Props: SI->getProperties());
762 LLVM_DEBUG(dbgs() << "Replaced " << *SI << " with " << *NewSI << "\n");
763 SI->eraseFromParent();
764 return NewSI;
765}
766
767void AtomicExpandImpl::expandAtomicStoreToXChg(StoreInst *SI) {
768 // This function is only called on atomic stores that are too large to be
769 // atomic if implemented as a native store. So we replace them by an
770 // atomic swap, that can be implemented for example as a ldrex/strex on ARM
771 // or lock cmpxchg8/16b on X86, as these are atomic for larger sizes.
772 // It is the responsibility of the target to only signal expansion via
773 // shouldExpandAtomicRMW in cases where this is required and possible.
774 ReplacementIRBuilder Builder(SI, *DL);
775 AtomicOrdering Ordering = SI->getOrdering();
776 assert(Ordering != AtomicOrdering::NotAtomic);
777 AtomicOrdering RMWOrdering = Ordering == AtomicOrdering::Unordered
778 ? AtomicOrdering::Monotonic
779 : Ordering;
780 AtomicRMWInst *AI = Builder.CreateAtomicRMW(
781 Op: AtomicRMWInst::Xchg, Ptr: SI->getPointerOperand(), Val: SI->getValueOperand(),
782 Align: SI->getAlign(), Ordering: RMWOrdering, SSID: SI->getSyncScopeID());
783 AI->setVolatile(SI->isVolatile());
784 SI->eraseFromParent();
785
786 // Now we have an appropriate swap instruction, lower it as usual.
787 tryExpandAtomicRMW(AI);
788}
789
790static void createCmpXchgInstFun(IRBuilderBase &Builder, Value *Addr,
791 Value *Loaded, Value *NewVal, Align AddrAlign,
792 AtomicOrdering MemOpOrder, SyncScope::ID SSID,
793 bool IsVolatile, Value *&Success,
794 Value *&NewLoaded, Instruction *MetadataSrc) {
795 Type *OrigTy = NewVal->getType();
796
797 // This code can go away when cmpxchg supports FP and vector types.
798 assert(!OrigTy->isPointerTy());
799 bool NeedBitcast = OrigTy->isFloatingPointTy() || OrigTy->isVectorTy();
800 if (NeedBitcast) {
801 IntegerType *IntTy = Builder.getIntNTy(N: OrigTy->getPrimitiveSizeInBits());
802 NewVal = Builder.CreateBitCast(V: NewVal, DestTy: IntTy);
803 Loaded = Builder.CreateBitCast(V: Loaded, DestTy: IntTy);
804 }
805
806 AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
807 Ptr: Addr, Cmp: Loaded, New: NewVal, Align: AddrAlign, SuccessOrdering: MemOpOrder,
808 FailureOrdering: AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: MemOpOrder), SSID);
809 Pair->setVolatile(IsVolatile);
810 if (MetadataSrc)
811 copyMetadataForAtomic(Dest&: *Pair, Source: *MetadataSrc);
812
813 Success = Builder.CreateExtractValue(Agg: Pair, Idxs: 1, Name: "success");
814 NewLoaded = Builder.CreateExtractValue(Agg: Pair, Idxs: 0, Name: "newloaded");
815
816 if (NeedBitcast)
817 NewLoaded = Builder.CreateBitCast(V: NewLoaded, DestTy: OrigTy);
818}
819
820bool AtomicExpandImpl::tryExpandAtomicRMW(AtomicRMWInst *AI) {
821 LLVMContext &Ctx = AI->getModule()->getContext();
822 TargetLowering::AtomicExpansionKind Kind = TLI->shouldExpandAtomicRMWInIR(RMW: AI);
823 switch (Kind) {
824 case TargetLoweringBase::AtomicExpansionKind::None:
825 return false;
826 case TargetLoweringBase::AtomicExpansionKind::LLSC: {
827 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
828 unsigned ValueSize = getAtomicOpSize(RMWI: AI);
829 if (ValueSize < MinCASSize) {
830 expandPartwordAtomicRMW(I: AI,
831 ExpansionKind: TargetLoweringBase::AtomicExpansionKind::LLSC);
832 } else {
833 auto PerformOp = [&](IRBuilderBase &Builder, Value *Loaded) {
834 return buildAtomicRMWValue(Op: AI->getOperation(), Builder, Loaded,
835 Val: AI->getValOperand());
836 };
837 expandAtomicOpToLLSC(I: AI, ResultTy: AI->getType(), Addr: AI->getPointerOperand(),
838 AddrAlign: AI->getAlign(), MemOpOrder: AI->getOrdering(), PerformOp);
839 }
840 return true;
841 }
842 case TargetLoweringBase::AtomicExpansionKind::CmpXChg: {
843 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
844 unsigned ValueSize = getAtomicOpSize(RMWI: AI);
845 if (ValueSize < MinCASSize) {
846 expandPartwordAtomicRMW(I: AI,
847 ExpansionKind: TargetLoweringBase::AtomicExpansionKind::CmpXChg);
848 } else {
849 SmallVector<StringRef> SSNs;
850 Ctx.getSyncScopeNames(SSNs);
851 auto MemScope = SSNs[AI->getSyncScopeID()].empty()
852 ? "system"
853 : SSNs[AI->getSyncScopeID()];
854 OptimizationRemarkEmitter ORE(AI->getFunction());
855 ORE.emit(RemarkBuilder: [&]() {
856 return OptimizationRemark(DEBUG_TYPE, "Passed", AI)
857 << "A compare and swap loop was generated for an atomic "
858 << AI->getOperationName(Op: AI->getOperation()) << " operation at "
859 << MemScope << " memory scope";
860 });
861 expandAtomicRMWToCmpXchg(AI, CreateCmpXchg: createCmpXchgInstFun);
862 }
863 return true;
864 }
865 case TargetLoweringBase::AtomicExpansionKind::MaskedIntrinsic: {
866 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
867 unsigned ValueSize = getAtomicOpSize(RMWI: AI);
868 if (ValueSize < MinCASSize) {
869 AtomicRMWInst::BinOp Op = AI->getOperation();
870 // Widen And/Or/Xor and give the target another chance at expanding it.
871 if (Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||
872 Op == AtomicRMWInst::And) {
873 tryExpandAtomicRMW(AI: widenPartwordAtomicRMW(AI));
874 return true;
875 }
876 }
877 expandAtomicRMWToMaskedIntrinsic(AI);
878 return true;
879 }
880 case TargetLoweringBase::AtomicExpansionKind::BitTestIntrinsic: {
881 TLI->emitBitTestAtomicRMWIntrinsic(AI);
882 return true;
883 }
884 case TargetLoweringBase::AtomicExpansionKind::CmpArithIntrinsic: {
885 TLI->emitCmpArithAtomicRMWIntrinsic(AI);
886 return true;
887 }
888 case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
889 return lowerAtomicRMWInst(RMWI: AI);
890 case TargetLoweringBase::AtomicExpansionKind::CustomExpand:
891 TLI->emitExpandAtomicRMW(AI);
892 return true;
893 default:
894 llvm_unreachable("Unhandled case in tryExpandAtomicRMW");
895 }
896}
897
898namespace {
899
900struct PartwordMaskValues {
901 // These three fields are guaranteed to be set by createMaskInstrs.
902 Type *WordType = nullptr;
903 Type *ValueType = nullptr;
904 Type *IntValueType = nullptr;
905 Value *AlignedAddr = nullptr;
906 Align AlignedAddrAlignment;
907 // The remaining fields can be null.
908 Value *ShiftAmt = nullptr;
909 Value *Mask = nullptr;
910 Value *Inv_Mask = nullptr;
911};
912
913[[maybe_unused]]
914raw_ostream &operator<<(raw_ostream &O, const PartwordMaskValues &PMV) {
915 auto PrintObj = [&O](auto *V) {
916 if (V)
917 O << *V;
918 else
919 O << "nullptr";
920 O << '\n';
921 };
922 O << "PartwordMaskValues {\n";
923 O << " WordType: ";
924 PrintObj(PMV.WordType);
925 O << " ValueType: ";
926 PrintObj(PMV.ValueType);
927 O << " AlignedAddr: ";
928 PrintObj(PMV.AlignedAddr);
929 O << " AlignedAddrAlignment: " << PMV.AlignedAddrAlignment.value() << '\n';
930 O << " ShiftAmt: ";
931 PrintObj(PMV.ShiftAmt);
932 O << " Mask: ";
933 PrintObj(PMV.Mask);
934 O << " Inv_Mask: ";
935 PrintObj(PMV.Inv_Mask);
936 O << "}\n";
937 return O;
938}
939
940} // end anonymous namespace
941
942/// This is a helper function which builds instructions to provide
943/// values necessary for partword atomic operations. It takes an
944/// incoming address, Addr, and ValueType, and constructs the address,
945/// shift-amounts and masks needed to work with a larger value of size
946/// WordSize.
947///
948/// AlignedAddr: Addr rounded down to a multiple of WordSize
949///
950/// ShiftAmt: Number of bits to right-shift a WordSize value loaded
951/// from AlignAddr for it to have the same value as if
952/// ValueType was loaded from Addr.
953///
954/// Mask: Value to mask with the value loaded from AlignAddr to
955/// include only the part that would've been loaded from Addr.
956///
957/// Inv_Mask: The inverse of Mask.
958static PartwordMaskValues createMaskInstrs(IRBuilderBase &Builder,
959 Instruction *I, Type *ValueType,
960 Value *Addr, Align AddrAlign,
961 unsigned MinWordSize) {
962 PartwordMaskValues PMV;
963
964 Module *M = I->getModule();
965 LLVMContext &Ctx = M->getContext();
966 const DataLayout &DL = M->getDataLayout();
967 unsigned ValueSize = DL.getTypeStoreSize(Ty: ValueType);
968
969 PMV.ValueType = PMV.IntValueType = ValueType;
970 if (PMV.ValueType->isFloatingPointTy() || PMV.ValueType->isVectorTy())
971 PMV.IntValueType =
972 Type::getIntNTy(C&: Ctx, N: ValueType->getPrimitiveSizeInBits());
973
974 PMV.WordType = MinWordSize > ValueSize ? Type::getIntNTy(C&: Ctx, N: MinWordSize * 8)
975 : ValueType;
976 if (PMV.ValueType == PMV.WordType) {
977 PMV.AlignedAddr = Addr;
978 PMV.AlignedAddrAlignment = AddrAlign;
979 PMV.ShiftAmt = ConstantInt::get(Ty: PMV.ValueType, V: 0);
980 PMV.Mask = ConstantInt::get(Ty: PMV.ValueType, V: ~0, /*isSigned*/ IsSigned: true);
981 return PMV;
982 }
983
984 PMV.AlignedAddrAlignment = Align(MinWordSize);
985
986 assert(ValueSize < MinWordSize);
987
988 PointerType *PtrTy = cast<PointerType>(Val: Addr->getType());
989 IntegerType *IntTy = DL.getIndexType(C&: Ctx, AddressSpace: PtrTy->getAddressSpace());
990 Value *PtrLSB;
991
992 if (AddrAlign < MinWordSize) {
993 PMV.AlignedAddr = Builder.CreateIntrinsic(
994 ID: Intrinsic::ptrmask, OverloadTypes: {PtrTy, IntTy},
995 Args: {Addr, ConstantInt::getSigned(Ty: IntTy, V: ~(uint64_t)(MinWordSize - 1))},
996 FMFSource: nullptr, Name: "AlignedAddr");
997
998 Value *AddrInt = Builder.CreatePtrToInt(V: Addr, DestTy: IntTy);
999 PtrLSB = Builder.CreateAnd(LHS: AddrInt, RHS: MinWordSize - 1, Name: "PtrLSB");
1000 } else {
1001 // If the alignment is high enough, the LSB are known 0.
1002 PMV.AlignedAddr = Addr;
1003 PtrLSB = ConstantInt::getNullValue(Ty: IntTy);
1004 }
1005
1006 if (DL.isLittleEndian()) {
1007 // turn bytes into bits
1008 PMV.ShiftAmt = Builder.CreateShl(LHS: PtrLSB, RHS: 3);
1009 } else {
1010 // turn bytes into bits, and count from the other side.
1011 PMV.ShiftAmt = Builder.CreateShl(
1012 LHS: Builder.CreateXor(LHS: PtrLSB, RHS: MinWordSize - ValueSize), RHS: 3);
1013 }
1014
1015 PMV.ShiftAmt = Builder.CreateTrunc(V: PMV.ShiftAmt, DestTy: PMV.WordType, Name: "ShiftAmt");
1016 PMV.Mask = Builder.CreateShl(
1017 LHS: ConstantInt::get(Ty: PMV.WordType, V: (1 << (ValueSize * 8)) - 1), RHS: PMV.ShiftAmt,
1018 Name: "Mask");
1019
1020 PMV.Inv_Mask = Builder.CreateNot(V: PMV.Mask, Name: "Inv_Mask");
1021
1022 return PMV;
1023}
1024
1025static Value *extractMaskedValue(IRBuilderBase &Builder, Value *WideWord,
1026 const PartwordMaskValues &PMV) {
1027 assert(WideWord->getType() == PMV.WordType && "Widened type mismatch");
1028 if (PMV.WordType == PMV.ValueType)
1029 return WideWord;
1030
1031 Value *Shift = Builder.CreateLShr(LHS: WideWord, RHS: PMV.ShiftAmt, Name: "shifted");
1032 Value *Trunc = Builder.CreateTrunc(V: Shift, DestTy: PMV.IntValueType, Name: "extracted");
1033 return Builder.CreateBitCast(V: Trunc, DestTy: PMV.ValueType);
1034}
1035
1036static Value *insertMaskedValue(IRBuilderBase &Builder, Value *WideWord,
1037 Value *Updated, const PartwordMaskValues &PMV) {
1038 assert(WideWord->getType() == PMV.WordType && "Widened type mismatch");
1039 assert(Updated->getType() == PMV.ValueType && "Value type mismatch");
1040 if (PMV.WordType == PMV.ValueType)
1041 return Updated;
1042
1043 Updated = Builder.CreateBitCast(V: Updated, DestTy: PMV.IntValueType);
1044
1045 Value *ZExt = Builder.CreateZExt(V: Updated, DestTy: PMV.WordType, Name: "extended");
1046 Value *Shift =
1047 Builder.CreateShl(LHS: ZExt, RHS: PMV.ShiftAmt, Name: "shifted", /*HasNUW*/ true);
1048 Value *And = Builder.CreateAnd(LHS: WideWord, RHS: PMV.Inv_Mask, Name: "unmasked");
1049 Value *Or = Builder.CreateOr(LHS: And, RHS: Shift, Name: "inserted");
1050 return Or;
1051}
1052
1053/// Emit IR to implement a masked version of a given atomicrmw
1054/// operation. (That is, only the bits under the Mask should be
1055/// affected by the operation)
1056static Value *performMaskedAtomicOp(AtomicRMWInst::BinOp Op,
1057 IRBuilderBase &Builder, Value *Loaded,
1058 Value *ValOperand_Shifted, Value *Inc,
1059 const PartwordMaskValues &PMV) {
1060 // TODO: update to use
1061 // https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge in order
1062 // to merge bits from two values without requiring PMV.Inv_Mask.
1063
1064 assert(Op != AtomicRMWInst::Or && Op != AtomicRMWInst::Xor &&
1065 Op != AtomicRMWInst::And &&
1066 "Or/Xor/And handled by widenPartwordAtomicRMW");
1067
1068 if (Op == AtomicRMWInst::Xchg) {
1069 // Clear all the bits we are exchanging out. These are the bits under the
1070 // mask. We can clear them with an `and` of the inverse mask.
1071 Value *Loaded_MaskOut = Builder.CreateAnd(LHS: Loaded, RHS: PMV.Inv_Mask);
1072 // Now that the prevous bits are cleared, we can swap in the new value with
1073 // an `or`.
1074 Value *FinalVal = Builder.CreateOr(LHS: Loaded_MaskOut, RHS: ValOperand_Shifted);
1075 return FinalVal;
1076 }
1077
1078 if (Op == AtomicRMWInst::Nand ||
1079 (!PMV.ValueType->isVectorTy() &&
1080 (Op == AtomicRMWInst::Add || Op == AtomicRMWInst::Sub))) {
1081 // For `Nand` and non-vector `Add` and `Sub`, we can perform the operation
1082 // on the entire word because the extra bits in the unmasked region don't
1083 // affect the computation in the masked region. The operation might still
1084 // overwrite the unmasked region (e.g. from integer overflow or underflow),
1085 // so we have to reapply the unmasked region afterwards.
1086 //
1087 // This trick doesn't work for vector `Add` and `Sub` because we use a
1088 // scalar operation on the entire word. Scalarizing vector `Add` and `Sub`
1089 // isn't legal because the vector versions may have element-wise overflows.
1090 // TODO: For these, can we use a wider vector op with additional lanes?
1091
1092 // Atomic operation across the entire word.
1093 Value *NewVal =
1094 buildAtomicRMWValue(Op, Builder, Loaded, Val: ValOperand_Shifted);
1095 // Reapply the bits in the unmasked region.
1096 Value *NewVal_Masked = Builder.CreateAnd(LHS: NewVal, RHS: PMV.Mask);
1097 Value *Loaded_MaskOut = Builder.CreateAnd(LHS: Loaded, RHS: PMV.Inv_Mask);
1098 Value *FinalVal = Builder.CreateOr(LHS: Loaded_MaskOut, RHS: NewVal_Masked);
1099 return FinalVal;
1100 }
1101
1102 // All other ops operate on the sub-word size. Truncate down to the
1103 // original size, and expand out again after doing the operation. Bitcasts
1104 // will be inserted for FP values.
1105 assert(!ValOperand_Shifted);
1106 Value *Loaded_Extract = extractMaskedValue(Builder, WideWord: Loaded, PMV);
1107 Value *NewVal = buildAtomicRMWValue(Op, Builder, Loaded: Loaded_Extract, Val: Inc);
1108 Value *FinalVal = insertMaskedValue(Builder, WideWord: Loaded, Updated: NewVal, PMV);
1109 return FinalVal;
1110}
1111
1112/// Expand a sub-word atomicrmw operation into an appropriate
1113/// word-sized operation.
1114///
1115/// It will create an LL/SC or cmpxchg loop, as appropriate, the same
1116/// way as a typical atomicrmw expansion. The only difference here is
1117/// that the operation inside of the loop may operate upon only a
1118/// part of the value.
1119void AtomicExpandImpl::expandPartwordAtomicRMW(
1120 AtomicRMWInst *AI, TargetLoweringBase::AtomicExpansionKind ExpansionKind) {
1121 // Widen And/Or/Xor and give the target another chance at expanding it.
1122 AtomicRMWInst::BinOp Op = AI->getOperation();
1123 if (Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||
1124 Op == AtomicRMWInst::And) {
1125 tryExpandAtomicRMW(AI: widenPartwordAtomicRMW(AI));
1126 return;
1127 }
1128 AtomicOrdering MemOpOrder = AI->getOrdering();
1129 SyncScope::ID SSID = AI->getSyncScopeID();
1130
1131 ReplacementIRBuilder Builder(AI, *DL);
1132
1133 PartwordMaskValues PMV =
1134 createMaskInstrs(Builder, I: AI, ValueType: AI->getType(), Addr: AI->getPointerOperand(),
1135 AddrAlign: AI->getAlign(), MinWordSize: TLI->getMinCmpXchgSizeInBits() / 8);
1136
1137 Value *ValOperand_Shifted = nullptr;
1138 bool NeedsShiftedOperand =
1139 Op == AtomicRMWInst::Xchg || Op == AtomicRMWInst::Nand ||
1140 (!PMV.ValueType->isVectorTy() &&
1141 (Op == AtomicRMWInst::Add || Op == AtomicRMWInst::Sub));
1142
1143 if (NeedsShiftedOperand) {
1144 Value *ValOp = Builder.CreateBitCast(V: AI->getValOperand(), DestTy: PMV.IntValueType);
1145 ValOperand_Shifted =
1146 Builder.CreateShl(LHS: Builder.CreateZExt(V: ValOp, DestTy: PMV.WordType), RHS: PMV.ShiftAmt,
1147 Name: "ValOperand_Shifted");
1148 }
1149
1150 auto PerformPartwordOp = [&](IRBuilderBase &Builder, Value *Loaded) {
1151 return performMaskedAtomicOp(Op, Builder, Loaded, ValOperand_Shifted,
1152 Inc: AI->getValOperand(), PMV);
1153 };
1154
1155 Value *OldResult;
1156 if (ExpansionKind == TargetLoweringBase::AtomicExpansionKind::CmpXChg) {
1157 OldResult = insertRMWCmpXchgLoop(Builder, ResultType: PMV.WordType, Addr: PMV.AlignedAddr,
1158 AddrAlign: PMV.AlignedAddrAlignment, MemOpOrder, SSID,
1159 IsVolatile: AI->isVolatile(), PerformOp: PerformPartwordOp,
1160 CreateCmpXchg: createCmpXchgInstFun, MetadataSrc: AI);
1161 } else {
1162 assert(ExpansionKind == TargetLoweringBase::AtomicExpansionKind::LLSC);
1163 OldResult = insertRMWLLSCLoop(Builder, ResultTy: PMV.WordType, Addr: PMV.AlignedAddr,
1164 AddrAlign: PMV.AlignedAddrAlignment, MemOpOrder,
1165 PerformOp: PerformPartwordOp);
1166 }
1167
1168 Value *FinalOldResult = extractMaskedValue(Builder, WideWord: OldResult, PMV);
1169 AI->replaceAllUsesWith(V: FinalOldResult);
1170 AI->eraseFromParent();
1171}
1172
1173// Widen the bitwise atomicrmw (or/xor/and) to the minimum supported width.
1174AtomicRMWInst *AtomicExpandImpl::widenPartwordAtomicRMW(AtomicRMWInst *AI) {
1175 ReplacementIRBuilder Builder(AI, *DL);
1176 AtomicRMWInst::BinOp Op = AI->getOperation();
1177
1178 assert((Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||
1179 Op == AtomicRMWInst::And) &&
1180 "Unable to widen operation");
1181
1182 PartwordMaskValues PMV =
1183 createMaskInstrs(Builder, I: AI, ValueType: AI->getType(), Addr: AI->getPointerOperand(),
1184 AddrAlign: AI->getAlign(), MinWordSize: TLI->getMinCmpXchgSizeInBits() / 8);
1185
1186 Value *ValOp = AI->getValOperand();
1187 if (ValOp->getType()->isVectorTy())
1188 // For vectors, bitcast to the integer type before extending. Note that
1189 // or/xor/and on vectors are equivalent to the same operation on an integer
1190 // that spans the vector, so we can use the integer type for the operation.
1191 ValOp = Builder.CreateBitCast(V: ValOp, DestTy: PMV.IntValueType);
1192 Value *ValOperand_Shifted =
1193 Builder.CreateShl(LHS: Builder.CreateZExt(V: ValOp, DestTy: PMV.WordType), RHS: PMV.ShiftAmt,
1194 Name: "ValOperand_Shifted");
1195
1196 Value *NewOperand;
1197
1198 if (Op == AtomicRMWInst::And)
1199 NewOperand =
1200 Builder.CreateOr(LHS: ValOperand_Shifted, RHS: PMV.Inv_Mask, Name: "AndOperand");
1201 else
1202 NewOperand = ValOperand_Shifted;
1203
1204 AtomicRMWInst *NewAI = Builder.CreateAtomicRMW(
1205 Op, Ptr: PMV.AlignedAddr, Val: NewOperand, Align: PMV.AlignedAddrAlignment,
1206 Ordering: AI->getOrdering(), SSID: AI->getSyncScopeID());
1207
1208 NewAI->setVolatile(AI->isVolatile());
1209 copyMetadataForAtomic(Dest&: *NewAI, Source: *AI);
1210
1211 Value *FinalOldResult = extractMaskedValue(Builder, WideWord: NewAI, PMV);
1212 AI->replaceAllUsesWith(V: FinalOldResult);
1213 AI->eraseFromParent();
1214 return NewAI;
1215}
1216
1217bool AtomicExpandImpl::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
1218 // The basic idea here is that we're expanding a cmpxchg of a
1219 // smaller memory size up to a word-sized cmpxchg. To do this, we
1220 // need to add a retry-loop for strong cmpxchg, so that
1221 // modifications to other parts of the word don't cause a spurious
1222 // failure.
1223
1224 // This generates code like the following:
1225 // [[Setup mask values PMV.*]]
1226 // %NewVal_Shifted = shl i32 %NewVal, %PMV.ShiftAmt
1227 // %Cmp_Shifted = shl i32 %Cmp, %PMV.ShiftAmt
1228 // %InitLoaded = load i32* %addr
1229 // %InitLoaded_MaskOut = and i32 %InitLoaded, %PMV.Inv_Mask
1230 // br partword.cmpxchg.loop
1231 // partword.cmpxchg.loop:
1232 // %Loaded_MaskOut = phi i32 [ %InitLoaded_MaskOut, %entry ],
1233 // [ %OldVal_MaskOut, %partword.cmpxchg.failure ]
1234 // %FullWord_NewVal = or i32 %Loaded_MaskOut, %NewVal_Shifted
1235 // %FullWord_Cmp = or i32 %Loaded_MaskOut, %Cmp_Shifted
1236 // %NewCI = cmpxchg i32* %PMV.AlignedAddr, i32 %FullWord_Cmp,
1237 // i32 %FullWord_NewVal success_ordering failure_ordering
1238 // %OldVal = extractvalue { i32, i1 } %NewCI, 0
1239 // %Success = extractvalue { i32, i1 } %NewCI, 1
1240 // br i1 %Success, label %partword.cmpxchg.end,
1241 // label %partword.cmpxchg.failure
1242 // partword.cmpxchg.failure:
1243 // %OldVal_MaskOut = and i32 %OldVal, %PMV.Inv_Mask
1244 // %ShouldContinue = icmp ne i32 %Loaded_MaskOut, %OldVal_MaskOut
1245 // br i1 %ShouldContinue, label %partword.cmpxchg.loop,
1246 // label %partword.cmpxchg.end
1247 // partword.cmpxchg.end:
1248 // %tmp1 = lshr i32 %OldVal, %PMV.ShiftAmt
1249 // %FinalOldVal = trunc i32 %tmp1 to i8
1250 // %tmp2 = insertvalue { i8, i1 } undef, i8 %FinalOldVal, 0
1251 // %Res = insertvalue { i8, i1 } %25, i1 %Success, 1
1252
1253 Value *Addr = CI->getPointerOperand();
1254 Value *Cmp = CI->getCompareOperand();
1255 Value *NewVal = CI->getNewValOperand();
1256
1257 BasicBlock *BB = CI->getParent();
1258 Function *F = BB->getParent();
1259 ReplacementIRBuilder Builder(CI, *DL);
1260 LLVMContext &Ctx = Builder.getContext();
1261
1262 BasicBlock *EndBB =
1263 BB->splitBasicBlock(I: CI->getIterator(), BBName: "partword.cmpxchg.end");
1264 auto FailureBB =
1265 BasicBlock::Create(Context&: Ctx, Name: "partword.cmpxchg.failure", Parent: F, InsertBefore: EndBB);
1266 auto LoopBB = BasicBlock::Create(Context&: Ctx, Name: "partword.cmpxchg.loop", Parent: F, InsertBefore: FailureBB);
1267
1268 // The split call above "helpfully" added a branch at the end of BB
1269 // (to the wrong place).
1270 std::prev(x: BB->end())->eraseFromParent();
1271 Builder.SetInsertPoint(BB);
1272
1273 PartwordMaskValues PMV =
1274 createMaskInstrs(Builder, I: CI, ValueType: CI->getCompareOperand()->getType(), Addr,
1275 AddrAlign: CI->getAlign(), MinWordSize: TLI->getMinCmpXchgSizeInBits() / 8);
1276
1277 // Shift the incoming values over, into the right location in the word.
1278 Value *NewVal_Shifted =
1279 Builder.CreateShl(LHS: Builder.CreateZExt(V: NewVal, DestTy: PMV.WordType), RHS: PMV.ShiftAmt);
1280 Value *Cmp_Shifted =
1281 Builder.CreateShl(LHS: Builder.CreateZExt(V: Cmp, DestTy: PMV.WordType), RHS: PMV.ShiftAmt);
1282
1283 // Load the entire current word, and mask into place the expected and new
1284 // values
1285 LoadInst *InitLoaded = Builder.CreateLoad(Ty: PMV.WordType, Ptr: PMV.AlignedAddr);
1286 Value *InitLoaded_MaskOut = Builder.CreateAnd(LHS: InitLoaded, RHS: PMV.Inv_Mask);
1287 Builder.CreateBr(Dest: LoopBB);
1288
1289 // partword.cmpxchg.loop:
1290 Builder.SetInsertPoint(LoopBB);
1291 PHINode *Loaded_MaskOut = Builder.CreatePHI(Ty: PMV.WordType, NumReservedValues: 2);
1292 Loaded_MaskOut->addIncoming(V: InitLoaded_MaskOut, BB);
1293
1294 // The initial load must be atomic with the same synchronization scope
1295 // to avoid a data race with concurrent stores. If the instruction being
1296 // emulated is volatile, issue a volatile load.
1297 // addIncoming is done first so that any replaceAllUsesWith calls during
1298 // normalization correctly update the PHI incoming value.
1299 InitLoaded->setVolatile(CI->isVolatile());
1300 if (TLI->shouldIssueAtomicLoadForAtomicEmulationLoop()) {
1301 InitLoaded->setAtomic(Ordering: AtomicOrdering::Monotonic, SSID: CI->getSyncScopeID());
1302 // The newly created load might need to be lowered further. Because it is
1303 // created in the same block as the atomicrmw, the AtomicExpand loop will
1304 // not process it again.
1305 processAtomicInstr(I: InitLoaded);
1306 }
1307
1308 // Mask/Or the expected and new values into place in the loaded word.
1309 Value *FullWord_NewVal = Builder.CreateOr(LHS: Loaded_MaskOut, RHS: NewVal_Shifted);
1310 Value *FullWord_Cmp = Builder.CreateOr(LHS: Loaded_MaskOut, RHS: Cmp_Shifted);
1311 AtomicCmpXchgInst *NewCI = Builder.CreateAtomicCmpXchg(
1312 Ptr: PMV.AlignedAddr, Cmp: FullWord_Cmp, New: FullWord_NewVal, Align: PMV.AlignedAddrAlignment,
1313 SuccessOrdering: CI->getSuccessOrdering(), FailureOrdering: CI->getFailureOrdering(), SSID: CI->getSyncScopeID());
1314 NewCI->setVolatile(CI->isVolatile());
1315 // When we're building a strong cmpxchg, we need a loop, so you
1316 // might think we could use a weak cmpxchg inside. But, using strong
1317 // allows the below comparison for ShouldContinue, and we're
1318 // expecting the underlying cmpxchg to be a machine instruction,
1319 // which is strong anyways.
1320 NewCI->setWeak(CI->isWeak());
1321
1322 Value *OldVal = Builder.CreateExtractValue(Agg: NewCI, Idxs: 0);
1323 Value *Success = Builder.CreateExtractValue(Agg: NewCI, Idxs: 1);
1324
1325 if (CI->isWeak())
1326 Builder.CreateBr(Dest: EndBB);
1327 else
1328 Builder.CreateCondBr(Cond: Success, True: EndBB, False: FailureBB);
1329
1330 // partword.cmpxchg.failure:
1331 Builder.SetInsertPoint(FailureBB);
1332 // Upon failure, verify that the masked-out part of the loaded value
1333 // has been modified. If it didn't, abort the cmpxchg, since the
1334 // masked-in part must've.
1335 Value *OldVal_MaskOut = Builder.CreateAnd(LHS: OldVal, RHS: PMV.Inv_Mask);
1336 Value *ShouldContinue = Builder.CreateICmpNE(LHS: Loaded_MaskOut, RHS: OldVal_MaskOut);
1337 Builder.CreateCondBr(Cond: ShouldContinue, True: LoopBB, False: EndBB);
1338
1339 // Add the second value to the phi from above
1340 Loaded_MaskOut->addIncoming(V: OldVal_MaskOut, BB: FailureBB);
1341
1342 // partword.cmpxchg.end:
1343 Builder.SetInsertPoint(CI);
1344
1345 Value *FinalOldVal = extractMaskedValue(Builder, WideWord: OldVal, PMV);
1346 Value *Res = PoisonValue::get(T: CI->getType());
1347 Res = Builder.CreateInsertValue(Agg: Res, Val: FinalOldVal, Idxs: 0);
1348 Res = Builder.CreateInsertValue(Agg: Res, Val: Success, Idxs: 1);
1349
1350 CI->replaceAllUsesWith(V: Res);
1351 CI->eraseFromParent();
1352 return true;
1353}
1354
1355void AtomicExpandImpl::expandAtomicOpToLLSC(
1356 Instruction *I, Type *ResultType, Value *Addr, Align AddrAlign,
1357 AtomicOrdering MemOpOrder,
1358 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp) {
1359 ReplacementIRBuilder Builder(I, *DL);
1360 Value *Loaded = insertRMWLLSCLoop(Builder, ResultTy: ResultType, Addr, AddrAlign,
1361 MemOpOrder, PerformOp);
1362
1363 I->replaceAllUsesWith(V: Loaded);
1364 I->eraseFromParent();
1365}
1366
1367void AtomicExpandImpl::expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI) {
1368 ReplacementIRBuilder Builder(AI, *DL);
1369
1370 PartwordMaskValues PMV =
1371 createMaskInstrs(Builder, I: AI, ValueType: AI->getType(), Addr: AI->getPointerOperand(),
1372 AddrAlign: AI->getAlign(), MinWordSize: TLI->getMinCmpXchgSizeInBits() / 8);
1373
1374 // The value operand must be sign-extended for signed min/max so that the
1375 // target's signed comparison instructions can be used. Otherwise, just
1376 // zero-ext.
1377 Instruction::CastOps CastOp = Instruction::ZExt;
1378 AtomicRMWInst::BinOp RMWOp = AI->getOperation();
1379 if (RMWOp == AtomicRMWInst::Max || RMWOp == AtomicRMWInst::Min)
1380 CastOp = Instruction::SExt;
1381
1382 Value *ValOperand_Shifted = Builder.CreateShl(
1383 LHS: Builder.CreateCast(Op: CastOp, V: AI->getValOperand(), DestTy: PMV.WordType),
1384 RHS: PMV.ShiftAmt, Name: "ValOperand_Shifted");
1385 Value *OldResult = TLI->emitMaskedAtomicRMWIntrinsic(
1386 Builder, AI, AlignedAddr: PMV.AlignedAddr, Incr: ValOperand_Shifted, Mask: PMV.Mask, ShiftAmt: PMV.ShiftAmt,
1387 Ord: AI->getOrdering());
1388 Value *FinalOldResult = extractMaskedValue(Builder, WideWord: OldResult, PMV);
1389 AI->replaceAllUsesWith(V: FinalOldResult);
1390 AI->eraseFromParent();
1391}
1392
1393void AtomicExpandImpl::expandAtomicCmpXchgToMaskedIntrinsic(
1394 AtomicCmpXchgInst *CI) {
1395 ReplacementIRBuilder Builder(CI, *DL);
1396
1397 PartwordMaskValues PMV = createMaskInstrs(
1398 Builder, I: CI, ValueType: CI->getCompareOperand()->getType(), Addr: CI->getPointerOperand(),
1399 AddrAlign: CI->getAlign(), MinWordSize: TLI->getMinCmpXchgSizeInBits() / 8);
1400
1401 Value *CmpVal_Shifted = Builder.CreateShl(
1402 LHS: Builder.CreateZExt(V: CI->getCompareOperand(), DestTy: PMV.WordType), RHS: PMV.ShiftAmt,
1403 Name: "CmpVal_Shifted");
1404 Value *NewVal_Shifted = Builder.CreateShl(
1405 LHS: Builder.CreateZExt(V: CI->getNewValOperand(), DestTy: PMV.WordType), RHS: PMV.ShiftAmt,
1406 Name: "NewVal_Shifted");
1407 Value *OldVal = TLI->emitMaskedAtomicCmpXchgIntrinsic(
1408 Builder, CI, AlignedAddr: PMV.AlignedAddr, CmpVal: CmpVal_Shifted, NewVal: NewVal_Shifted, Mask: PMV.Mask,
1409 Ord: CI->getMergedOrdering());
1410 Value *FinalOldVal = extractMaskedValue(Builder, WideWord: OldVal, PMV);
1411 Value *Res = PoisonValue::get(T: CI->getType());
1412 Res = Builder.CreateInsertValue(Agg: Res, Val: FinalOldVal, Idxs: 0);
1413 Value *Success = Builder.CreateICmpEQ(
1414 LHS: CmpVal_Shifted, RHS: Builder.CreateAnd(LHS: OldVal, RHS: PMV.Mask), Name: "Success");
1415 Res = Builder.CreateInsertValue(Agg: Res, Val: Success, Idxs: 1);
1416
1417 CI->replaceAllUsesWith(V: Res);
1418 CI->eraseFromParent();
1419}
1420
1421Value *AtomicExpandImpl::insertRMWLLSCLoop(
1422 IRBuilderBase &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
1423 AtomicOrdering MemOpOrder,
1424 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp) {
1425 LLVMContext &Ctx = Builder.getContext();
1426 BasicBlock *BB = Builder.GetInsertBlock();
1427 Function *F = BB->getParent();
1428
1429 assert(AddrAlign >= F->getDataLayout().getTypeStoreSize(ResultTy) &&
1430 "Expected at least natural alignment at this point.");
1431
1432 // Given: atomicrmw some_op iN* %addr, iN %incr ordering
1433 //
1434 // The standard expansion we produce is:
1435 // [...]
1436 // atomicrmw.start:
1437 // %loaded = @load.linked(%addr)
1438 // %new = some_op iN %loaded, %incr
1439 // %stored = @store_conditional(%new, %addr)
1440 // %try_again = icmp i32 ne %stored, 0
1441 // br i1 %try_again, label %loop, label %atomicrmw.end
1442 // atomicrmw.end:
1443 // [...]
1444 BasicBlock *ExitBB =
1445 BB->splitBasicBlock(I: Builder.GetInsertPoint(), BBName: "atomicrmw.end");
1446 BasicBlock *LoopBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.start", Parent: F, InsertBefore: ExitBB);
1447
1448 // The split call above "helpfully" added a branch at the end of BB (to the
1449 // wrong place).
1450 std::prev(x: BB->end())->eraseFromParent();
1451 Builder.SetInsertPoint(BB);
1452 Builder.CreateBr(Dest: LoopBB);
1453
1454 // Start the main loop block now that we've taken care of the preliminaries.
1455 Builder.SetInsertPoint(LoopBB);
1456 Value *Loaded = TLI->emitLoadLinked(Builder, ValueTy: ResultTy, Addr, Ord: MemOpOrder);
1457
1458 Value *NewVal = PerformOp(Builder, Loaded);
1459
1460 Value *StoreSuccess =
1461 TLI->emitStoreConditional(Builder, Val: NewVal, Addr, Ord: MemOpOrder);
1462 Value *TryAgain = Builder.CreateICmpNE(
1463 LHS: StoreSuccess, RHS: ConstantInt::get(Ty: IntegerType::get(C&: Ctx, NumBits: 32), V: 0), Name: "tryagain");
1464
1465 Instruction *CondBr = Builder.CreateCondBr(Cond: TryAgain, True: LoopBB, False: ExitBB);
1466
1467 // Atomic RMW expands to a Load-linked / Store-Conditional loop, because it is
1468 // hard to predict precise branch weigths we mark the branch as "unknown"
1469 // (50/50) to prevent misleading optimizations.
1470 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *CondBr, DEBUG_TYPE);
1471
1472 Builder.SetInsertPoint(TheBB: ExitBB, IP: ExitBB->begin());
1473 return Loaded;
1474}
1475
1476/// Convert an atomic cmpxchg of a non-integral type to an integer cmpxchg of
1477/// the equivalent bitwidth. We used to not support pointer cmpxchg in the
1478/// IR. As a migration step, we convert back to what use to be the standard
1479/// way to represent a pointer cmpxchg so that we can update backends one by
1480/// one.
1481AtomicCmpXchgInst *
1482AtomicExpandImpl::convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI) {
1483 auto *M = CI->getModule();
1484 Type *NewTy = getCorrespondingIntegerType(T: CI->getCompareOperand()->getType(),
1485 DL: M->getDataLayout());
1486
1487 ReplacementIRBuilder Builder(CI, *DL);
1488
1489 Value *Addr = CI->getPointerOperand();
1490
1491 Value *NewCmp = Builder.CreatePtrToInt(V: CI->getCompareOperand(), DestTy: NewTy);
1492 Value *NewNewVal = Builder.CreatePtrToInt(V: CI->getNewValOperand(), DestTy: NewTy);
1493
1494 auto *NewCI = Builder.CreateAtomicCmpXchg(
1495 Ptr: Addr, Cmp: NewCmp, New: NewNewVal, Align: CI->getAlign(), SuccessOrdering: CI->getSuccessOrdering(),
1496 FailureOrdering: CI->getFailureOrdering(), SSID: CI->getSyncScopeID());
1497 NewCI->setVolatile(CI->isVolatile());
1498 NewCI->setWeak(CI->isWeak());
1499 LLVM_DEBUG(dbgs() << "Replaced " << *CI << " with " << *NewCI << "\n");
1500
1501 Value *OldVal = Builder.CreateExtractValue(Agg: NewCI, Idxs: 0);
1502 Value *Succ = Builder.CreateExtractValue(Agg: NewCI, Idxs: 1);
1503
1504 OldVal = Builder.CreateIntToPtr(V: OldVal, DestTy: CI->getCompareOperand()->getType());
1505
1506 Value *Res = PoisonValue::get(T: CI->getType());
1507 Res = Builder.CreateInsertValue(Agg: Res, Val: OldVal, Idxs: 0);
1508 Res = Builder.CreateInsertValue(Agg: Res, Val: Succ, Idxs: 1);
1509
1510 CI->replaceAllUsesWith(V: Res);
1511 CI->eraseFromParent();
1512 return NewCI;
1513}
1514
1515bool AtomicExpandImpl::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
1516 AtomicOrdering SuccessOrder = CI->getSuccessOrdering();
1517 AtomicOrdering FailureOrder = CI->getFailureOrdering();
1518 Value *Addr = CI->getPointerOperand();
1519 BasicBlock *BB = CI->getParent();
1520 Function *F = BB->getParent();
1521 LLVMContext &Ctx = F->getContext();
1522 // If shouldInsertFencesForAtomic() returns true, then the target does not
1523 // want to deal with memory orders, and emitLeading/TrailingFence should take
1524 // care of everything. Otherwise, emitLeading/TrailingFence are no-op and we
1525 // should preserve the ordering.
1526 bool ShouldInsertFencesForAtomic = TLI->shouldInsertFencesForAtomic(I: CI);
1527 AtomicOrdering MemOpOrder = ShouldInsertFencesForAtomic
1528 ? AtomicOrdering::Monotonic
1529 : CI->getMergedOrdering();
1530
1531 // In implementations which use a barrier to achieve release semantics, we can
1532 // delay emitting this barrier until we know a store is actually going to be
1533 // attempted. The cost of this delay is that we need 2 copies of the block
1534 // emitting the load-linked, affecting code size.
1535 //
1536 // Ideally, this logic would be unconditional except for the minsize check
1537 // since in other cases the extra blocks naturally collapse down to the
1538 // minimal loop. Unfortunately, this puts too much stress on later
1539 // optimisations so we avoid emitting the extra logic in those cases too.
1540 bool HasReleasedLoadBB = !CI->isWeak() && ShouldInsertFencesForAtomic &&
1541 SuccessOrder != AtomicOrdering::Monotonic &&
1542 SuccessOrder != AtomicOrdering::Acquire &&
1543 !F->hasMinSize();
1544
1545 // There's no overhead for sinking the release barrier in a weak cmpxchg, so
1546 // do it even on minsize.
1547 bool UseUnconditionalReleaseBarrier = F->hasMinSize() && !CI->isWeak();
1548
1549 // Given: cmpxchg some_op iN* %addr, iN %desired, iN %new success_ord fail_ord
1550 //
1551 // The full expansion we produce is:
1552 // [...]
1553 // %aligned.addr = ...
1554 // cmpxchg.start:
1555 // %unreleasedload = @load.linked(%aligned.addr)
1556 // %unreleasedload.extract = extract value from %unreleasedload
1557 // %should_store = icmp eq %unreleasedload.extract, %desired
1558 // br i1 %should_store, label %cmpxchg.releasingstore,
1559 // label %cmpxchg.nostore
1560 // cmpxchg.releasingstore:
1561 // fence?
1562 // br label cmpxchg.trystore
1563 // cmpxchg.trystore:
1564 // %loaded.trystore = phi [%unreleasedload, %cmpxchg.releasingstore],
1565 // [%releasedload, %cmpxchg.releasedload]
1566 // %updated.new = insert %new into %loaded.trystore
1567 // %stored = @store_conditional(%updated.new, %aligned.addr)
1568 // %success = icmp eq i32 %stored, 0
1569 // br i1 %success, label %cmpxchg.success,
1570 // label %cmpxchg.releasedload/%cmpxchg.failure
1571 // cmpxchg.releasedload:
1572 // %releasedload = @load.linked(%aligned.addr)
1573 // %releasedload.extract = extract value from %releasedload
1574 // %should_store = icmp eq %releasedload.extract, %desired
1575 // br i1 %should_store, label %cmpxchg.trystore,
1576 // label %cmpxchg.failure
1577 // cmpxchg.success:
1578 // fence?
1579 // br label %cmpxchg.end
1580 // cmpxchg.nostore:
1581 // %loaded.nostore = phi [%unreleasedload, %cmpxchg.start],
1582 // [%releasedload,
1583 // %cmpxchg.releasedload/%cmpxchg.trystore]
1584 // @load_linked_fail_balance()?
1585 // br label %cmpxchg.failure
1586 // cmpxchg.failure:
1587 // fence?
1588 // br label %cmpxchg.end
1589 // cmpxchg.end:
1590 // %loaded.exit = phi [%loaded.nostore, %cmpxchg.failure],
1591 // [%loaded.trystore, %cmpxchg.trystore]
1592 // %success = phi i1 [true, %cmpxchg.success], [false, %cmpxchg.failure]
1593 // %loaded = extract value from %loaded.exit
1594 // %restmp = insertvalue { iN, i1 } undef, iN %loaded, 0
1595 // %res = insertvalue { iN, i1 } %restmp, i1 %success, 1
1596 // [...]
1597 BasicBlock *ExitBB = BB->splitBasicBlock(I: CI->getIterator(), BBName: "cmpxchg.end");
1598 auto FailureBB = BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.failure", Parent: F, InsertBefore: ExitBB);
1599 auto NoStoreBB = BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.nostore", Parent: F, InsertBefore: FailureBB);
1600 auto SuccessBB = BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.success", Parent: F, InsertBefore: NoStoreBB);
1601 auto ReleasedLoadBB =
1602 BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.releasedload", Parent: F, InsertBefore: SuccessBB);
1603 auto TryStoreBB =
1604 BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.trystore", Parent: F, InsertBefore: ReleasedLoadBB);
1605 auto ReleasingStoreBB =
1606 BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.fencedstore", Parent: F, InsertBefore: TryStoreBB);
1607 auto StartBB = BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.start", Parent: F, InsertBefore: ReleasingStoreBB);
1608
1609 ReplacementIRBuilder Builder(CI, *DL);
1610
1611 // The split call above "helpfully" added a branch at the end of BB (to the
1612 // wrong place), but we might want a fence too. It's easiest to just remove
1613 // the branch entirely.
1614 std::prev(x: BB->end())->eraseFromParent();
1615 Builder.SetInsertPoint(BB);
1616 if (ShouldInsertFencesForAtomic && UseUnconditionalReleaseBarrier)
1617 TLI->emitLeadingFence(Builder, Inst: CI, Ord: SuccessOrder);
1618
1619 PartwordMaskValues PMV =
1620 createMaskInstrs(Builder, I: CI, ValueType: CI->getCompareOperand()->getType(), Addr,
1621 AddrAlign: CI->getAlign(), MinWordSize: TLI->getMinCmpXchgSizeInBits() / 8);
1622 Builder.CreateBr(Dest: StartBB);
1623
1624 // Start the main loop block now that we've taken care of the preliminaries.
1625 Builder.SetInsertPoint(StartBB);
1626 Value *UnreleasedLoad =
1627 TLI->emitLoadLinked(Builder, ValueTy: PMV.WordType, Addr: PMV.AlignedAddr, Ord: MemOpOrder);
1628 Value *UnreleasedLoadExtract =
1629 extractMaskedValue(Builder, WideWord: UnreleasedLoad, PMV);
1630 Value *ShouldStore = Builder.CreateICmpEQ(
1631 LHS: UnreleasedLoadExtract, RHS: CI->getCompareOperand(), Name: "should_store");
1632
1633 // If the cmpxchg doesn't actually need any ordering when it fails, we can
1634 // jump straight past that fence instruction (if it exists).
1635 Builder.CreateCondBr(Cond: ShouldStore, True: ReleasingStoreBB, False: NoStoreBB,
1636 BranchWeights: MDBuilder(F->getContext()).createLikelyBranchWeights());
1637
1638 Builder.SetInsertPoint(ReleasingStoreBB);
1639 if (ShouldInsertFencesForAtomic && !UseUnconditionalReleaseBarrier)
1640 TLI->emitLeadingFence(Builder, Inst: CI, Ord: SuccessOrder);
1641 Builder.CreateBr(Dest: TryStoreBB);
1642
1643 Builder.SetInsertPoint(TryStoreBB);
1644 PHINode *LoadedTryStore =
1645 Builder.CreatePHI(Ty: PMV.WordType, NumReservedValues: 2, Name: "loaded.trystore");
1646 LoadedTryStore->addIncoming(V: UnreleasedLoad, BB: ReleasingStoreBB);
1647 Value *NewValueInsert =
1648 insertMaskedValue(Builder, WideWord: LoadedTryStore, Updated: CI->getNewValOperand(), PMV);
1649 Value *StoreSuccess = TLI->emitStoreConditional(Builder, Val: NewValueInsert,
1650 Addr: PMV.AlignedAddr, Ord: MemOpOrder);
1651 StoreSuccess = Builder.CreateICmpEQ(
1652 LHS: StoreSuccess, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: 0), Name: "success");
1653 BasicBlock *RetryBB = HasReleasedLoadBB ? ReleasedLoadBB : StartBB;
1654 Builder.CreateCondBr(Cond: StoreSuccess, True: SuccessBB,
1655 False: CI->isWeak() ? FailureBB : RetryBB,
1656 BranchWeights: MDBuilder(F->getContext()).createLikelyBranchWeights());
1657
1658 Builder.SetInsertPoint(ReleasedLoadBB);
1659 Value *SecondLoad;
1660 if (HasReleasedLoadBB) {
1661 SecondLoad =
1662 TLI->emitLoadLinked(Builder, ValueTy: PMV.WordType, Addr: PMV.AlignedAddr, Ord: MemOpOrder);
1663 Value *SecondLoadExtract = extractMaskedValue(Builder, WideWord: SecondLoad, PMV);
1664 ShouldStore = Builder.CreateICmpEQ(LHS: SecondLoadExtract,
1665 RHS: CI->getCompareOperand(), Name: "should_store");
1666
1667 // If the cmpxchg doesn't actually need any ordering when it fails, we can
1668 // jump straight past that fence instruction (if it exists).
1669 Builder.CreateCondBr(
1670 Cond: ShouldStore, True: TryStoreBB, False: NoStoreBB,
1671 BranchWeights: MDBuilder(F->getContext()).createLikelyBranchWeights());
1672 // Update PHI node in TryStoreBB.
1673 LoadedTryStore->addIncoming(V: SecondLoad, BB: ReleasedLoadBB);
1674 } else
1675 Builder.CreateUnreachable();
1676
1677 // Make sure later instructions don't get reordered with a fence if
1678 // necessary.
1679 Builder.SetInsertPoint(SuccessBB);
1680 if (ShouldInsertFencesForAtomic ||
1681 TLI->shouldInsertTrailingSeqCstFenceForAtomicStore(I: CI))
1682 TLI->emitTrailingFence(Builder, Inst: CI, Ord: SuccessOrder);
1683 Builder.CreateBr(Dest: ExitBB);
1684
1685 Builder.SetInsertPoint(NoStoreBB);
1686 PHINode *LoadedNoStore =
1687 Builder.CreatePHI(Ty: UnreleasedLoad->getType(), NumReservedValues: 2, Name: "loaded.nostore");
1688 LoadedNoStore->addIncoming(V: UnreleasedLoad, BB: StartBB);
1689 if (HasReleasedLoadBB)
1690 LoadedNoStore->addIncoming(V: SecondLoad, BB: ReleasedLoadBB);
1691
1692 // In the failing case, where we don't execute the store-conditional, the
1693 // target might want to balance out the load-linked with a dedicated
1694 // instruction (e.g., on ARM, clearing the exclusive monitor).
1695 TLI->emitAtomicCmpXchgNoStoreLLBalance(Builder);
1696 Builder.CreateBr(Dest: FailureBB);
1697
1698 Builder.SetInsertPoint(FailureBB);
1699 PHINode *LoadedFailure =
1700 Builder.CreatePHI(Ty: UnreleasedLoad->getType(), NumReservedValues: 2, Name: "loaded.failure");
1701 LoadedFailure->addIncoming(V: LoadedNoStore, BB: NoStoreBB);
1702 if (CI->isWeak())
1703 LoadedFailure->addIncoming(V: LoadedTryStore, BB: TryStoreBB);
1704 if (ShouldInsertFencesForAtomic)
1705 TLI->emitTrailingFence(Builder, Inst: CI, Ord: FailureOrder);
1706 Builder.CreateBr(Dest: ExitBB);
1707
1708 // Finally, we have control-flow based knowledge of whether the cmpxchg
1709 // succeeded or not. We expose this to later passes by converting any
1710 // subsequent "icmp eq/ne %loaded, %oldval" into a use of an appropriate
1711 // PHI.
1712 Builder.SetInsertPoint(TheBB: ExitBB, IP: ExitBB->begin());
1713 PHINode *LoadedExit =
1714 Builder.CreatePHI(Ty: UnreleasedLoad->getType(), NumReservedValues: 2, Name: "loaded.exit");
1715 LoadedExit->addIncoming(V: LoadedTryStore, BB: SuccessBB);
1716 LoadedExit->addIncoming(V: LoadedFailure, BB: FailureBB);
1717 PHINode *Success = Builder.CreatePHI(Ty: Type::getInt1Ty(C&: Ctx), NumReservedValues: 2, Name: "success");
1718 Success->addIncoming(V: ConstantInt::getTrue(Context&: Ctx), BB: SuccessBB);
1719 Success->addIncoming(V: ConstantInt::getFalse(Context&: Ctx), BB: FailureBB);
1720
1721 // This is the "exit value" from the cmpxchg expansion. It may be of
1722 // a type wider than the one in the cmpxchg instruction.
1723 Value *LoadedFull = LoadedExit;
1724
1725 Builder.SetInsertPoint(TheBB: ExitBB, IP: std::next(x: Success->getIterator()));
1726 Value *Loaded = extractMaskedValue(Builder, WideWord: LoadedFull, PMV);
1727
1728 // Look for any users of the cmpxchg that are just comparing the loaded value
1729 // against the desired one, and replace them with the CFG-derived version.
1730 SmallVector<ExtractValueInst *, 2> PrunedInsts;
1731 for (auto *User : CI->users()) {
1732 ExtractValueInst *EV = dyn_cast<ExtractValueInst>(Val: User);
1733 if (!EV)
1734 continue;
1735
1736 assert(EV->getNumIndices() == 1 && EV->getIndices()[0] <= 1 &&
1737 "weird extraction from { iN, i1 }");
1738
1739 if (EV->getIndices()[0] == 0)
1740 EV->replaceAllUsesWith(V: Loaded);
1741 else
1742 EV->replaceAllUsesWith(V: Success);
1743
1744 PrunedInsts.push_back(Elt: EV);
1745 }
1746
1747 // We can remove the instructions now we're no longer iterating through them.
1748 for (auto *EV : PrunedInsts)
1749 EV->eraseFromParent();
1750
1751 if (!CI->use_empty()) {
1752 // Some use of the full struct return that we don't understand has happened,
1753 // so we've got to reconstruct it properly.
1754 Value *Res;
1755 Res = Builder.CreateInsertValue(Agg: PoisonValue::get(T: CI->getType()), Val: Loaded, Idxs: 0);
1756 Res = Builder.CreateInsertValue(Agg: Res, Val: Success, Idxs: 1);
1757
1758 CI->replaceAllUsesWith(V: Res);
1759 }
1760
1761 CI->eraseFromParent();
1762 return true;
1763}
1764
1765bool AtomicExpandImpl::isIdempotentRMW(AtomicRMWInst *RMWI) {
1766 if (RMWI->isVolatile())
1767 return false;
1768 // TODO: Add floating point support.
1769 auto C = dyn_cast<ConstantInt>(Val: RMWI->getValOperand());
1770 if (!C)
1771 return false;
1772
1773 switch (RMWI->getOperation()) {
1774 case AtomicRMWInst::Add:
1775 case AtomicRMWInst::Sub:
1776 case AtomicRMWInst::Or:
1777 case AtomicRMWInst::Xor:
1778 return C->isZero();
1779 case AtomicRMWInst::And:
1780 return C->isMinusOne();
1781 case AtomicRMWInst::Min:
1782 return C->isMaxValue(IsSigned: true);
1783 case AtomicRMWInst::Max:
1784 return C->isMinValue(IsSigned: true);
1785 case AtomicRMWInst::UMin:
1786 return C->isMaxValue(IsSigned: false);
1787 case AtomicRMWInst::UMax:
1788 return C->isMinValue(IsSigned: false);
1789 default:
1790 return false;
1791 }
1792}
1793
1794bool AtomicExpandImpl::simplifyIdempotentRMW(AtomicRMWInst *RMWI) {
1795 if (auto ResultingLoad = TLI->lowerIdempotentRMWIntoFencedLoad(RMWI)) {
1796 tryExpandAtomicLoad(LI: ResultingLoad);
1797 return true;
1798 }
1799 return false;
1800}
1801
1802Value *AtomicExpandImpl::insertRMWCmpXchgLoop(
1803 IRBuilderBase &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
1804 AtomicOrdering MemOpOrder, SyncScope::ID SSID, bool IsVolatile,
1805 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp,
1806 CreateCmpXchgInstFun CreateCmpXchg, Instruction *MetadataSrc) {
1807 LLVMContext &Ctx = Builder.getContext();
1808 BasicBlock *BB = Builder.GetInsertBlock();
1809 Function *F = BB->getParent();
1810
1811 // Given: atomicrmw some_op iN* %addr, iN %incr ordering
1812 //
1813 // The standard expansion we produce is:
1814 // [...]
1815 // %init_loaded = load atomic iN* %addr
1816 // br label %loop
1817 // loop:
1818 // %loaded = phi iN [ %init_loaded, %entry ], [ %new_loaded, %loop ]
1819 // %new = some_op iN %loaded, %incr
1820 // %pair = cmpxchg iN* %addr, iN %loaded, iN %new
1821 // %new_loaded = extractvalue { iN, i1 } %pair, 0
1822 // %success = extractvalue { iN, i1 } %pair, 1
1823 // br i1 %success, label %atomicrmw.end, label %loop
1824 // atomicrmw.end:
1825 // [...]
1826 BasicBlock *ExitBB =
1827 BB->splitBasicBlock(I: Builder.GetInsertPoint(), BBName: "atomicrmw.end");
1828 BasicBlock *LoopBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.start", Parent: F, InsertBefore: ExitBB);
1829
1830 // The split call above "helpfully" added a branch at the end of BB (to the
1831 // wrong place), but we want a load. It's easiest to just remove
1832 // the branch entirely.
1833 std::prev(x: BB->end())->eraseFromParent();
1834 Builder.SetInsertPoint(BB);
1835 LoadInst *InitLoaded = Builder.CreateAlignedLoad(Ty: ResultTy, Ptr: Addr, Align: AddrAlign);
1836 Builder.CreateBr(Dest: LoopBB);
1837
1838 // Start the main loop block now that we've taken care of the preliminaries.
1839 Builder.SetInsertPoint(LoopBB);
1840 PHINode *Loaded = Builder.CreatePHI(Ty: ResultTy, NumReservedValues: 2, Name: "loaded");
1841 Loaded->addIncoming(V: InitLoaded, BB);
1842
1843 // The initial load must be atomic with the same synchronization scope
1844 // to avoid a data race with concurrent stores. If the instruction being
1845 // emulated is volatile, issue a volatile load.
1846 // addIncoming is done first so that any replaceAllUsesWith calls during
1847 // normalization correctly update the PHI incoming value.
1848 InitLoaded->setVolatile(IsVolatile);
1849 if (TLI->shouldIssueAtomicLoadForAtomicEmulationLoop()) {
1850 InitLoaded->setAtomic(Ordering: AtomicOrdering::Monotonic, SSID);
1851 // The newly created load might need to be lowered further. Because it is
1852 // created in the same block as the atomicrmw, the AtomicExpand loop will
1853 // not process it again.
1854 processAtomicInstr(I: InitLoaded);
1855 }
1856
1857 Value *NewVal = PerformOp(Builder, Loaded);
1858
1859 Value *NewLoaded = nullptr;
1860 Value *Success = nullptr;
1861
1862 CreateCmpXchg(Builder, Addr, Loaded, NewVal, AddrAlign,
1863 MemOpOrder == AtomicOrdering::Unordered
1864 ? AtomicOrdering::Monotonic
1865 : MemOpOrder,
1866 SSID, IsVolatile, Success, NewLoaded, MetadataSrc);
1867 assert(Success && NewLoaded);
1868
1869 Loaded->addIncoming(V: NewLoaded, BB: LoopBB);
1870
1871 Instruction *CondBr = Builder.CreateCondBr(Cond: Success, True: ExitBB, False: LoopBB);
1872
1873 // Atomic RMW expands to a cmpxchg loop, Since precise branch weights
1874 // cannot be easily determined here, we mark the branch as "unknown" (50/50)
1875 // to prevent misleading optimizations.
1876 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *CondBr, DEBUG_TYPE);
1877
1878 Builder.SetInsertPoint(TheBB: ExitBB, IP: ExitBB->begin());
1879 return NewLoaded;
1880}
1881
1882bool AtomicExpandImpl::tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
1883 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
1884 unsigned ValueSize = getAtomicOpSize(CASI: CI);
1885
1886 switch (TLI->shouldExpandAtomicCmpXchgInIR(AI: CI)) {
1887 default:
1888 llvm_unreachable("Unhandled case in tryExpandAtomicCmpXchg");
1889 case TargetLoweringBase::AtomicExpansionKind::None:
1890 if (ValueSize < MinCASSize)
1891 return expandPartwordCmpXchg(CI);
1892 return false;
1893 case TargetLoweringBase::AtomicExpansionKind::LLSC: {
1894 return expandAtomicCmpXchg(CI);
1895 }
1896 case TargetLoweringBase::AtomicExpansionKind::MaskedIntrinsic:
1897 expandAtomicCmpXchgToMaskedIntrinsic(CI);
1898 return true;
1899 case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
1900 return lowerAtomicCmpXchgInst(CXI: CI);
1901 case TargetLoweringBase::AtomicExpansionKind::CustomExpand: {
1902 TLI->emitExpandAtomicCmpXchg(CI);
1903 return true;
1904 }
1905 }
1906}
1907
1908bool AtomicExpandImpl::expandAtomicRMWToCmpXchg(
1909 AtomicRMWInst *AI, CreateCmpXchgInstFun CreateCmpXchg) {
1910 ReplacementIRBuilder Builder(AI, AI->getDataLayout());
1911 Builder.setIsFPConstrained(
1912 AI->getFunction()->hasFnAttribute(Kind: Attribute::StrictFP));
1913
1914 // FIXME: If FP exceptions are observable, we should force them off for the
1915 // loop for the FP atomics.
1916 Value *Loaded = AtomicExpandImpl::insertRMWCmpXchgLoop(
1917 Builder, ResultTy: AI->getType(), Addr: AI->getPointerOperand(), AddrAlign: AI->getAlign(),
1918 MemOpOrder: AI->getOrdering(), SSID: AI->getSyncScopeID(), IsVolatile: AI->isVolatile(),
1919 PerformOp: [&](IRBuilderBase &Builder, Value *Loaded) {
1920 return buildAtomicRMWValue(Op: AI->getOperation(), Builder, Loaded,
1921 Val: AI->getValOperand());
1922 },
1923 CreateCmpXchg, /*MetadataSrc=*/AI);
1924
1925 AI->replaceAllUsesWith(V: Loaded);
1926 AI->eraseFromParent();
1927 return true;
1928}
1929
1930// In order to use one of the sized library calls such as
1931// __atomic_fetch_add_4, the alignment must be sufficient, the size
1932// must be one of the potentially-specialized sizes, and the value
1933// type must actually exist in C on the target (otherwise, the
1934// function wouldn't actually be defined.)
1935static bool canUseSizedAtomicCall(unsigned Size, Align Alignment,
1936 const DataLayout &DL) {
1937 // TODO: "LargestSize" is an approximation for "largest type that
1938 // you can express in C". It seems to be the case that int128 is
1939 // supported on all 64-bit platforms, otherwise only up to 64-bit
1940 // integers are supported. If we get this wrong, then we'll try to
1941 // call a sized libcall that doesn't actually exist. There should
1942 // really be some more reliable way in LLVM of determining integer
1943 // sizes which are valid in the target's C ABI...
1944 unsigned LargestSize = DL.getLargestLegalIntTypeSizeInBits() >= 64 ? 16 : 8;
1945 return Alignment >= Size &&
1946 (Size == 1 || Size == 2 || Size == 4 || Size == 8 || Size == 16) &&
1947 Size <= LargestSize;
1948}
1949
1950void AtomicExpandImpl::expandAtomicLoadToLibcall(LoadInst *I) {
1951 static const RTLIB::Libcall Libcalls[6] = {
1952 RTLIB::ATOMIC_LOAD, RTLIB::ATOMIC_LOAD_1, RTLIB::ATOMIC_LOAD_2,
1953 RTLIB::ATOMIC_LOAD_4, RTLIB::ATOMIC_LOAD_8, RTLIB::ATOMIC_LOAD_16};
1954 unsigned Size = getAtomicOpSize(LI: I);
1955
1956 bool Expanded = expandAtomicOpToLibcall(
1957 I, Size, Alignment: I->getAlign(), PointerOperand: I->getPointerOperand(), ValueOperand: nullptr, CASExpected: nullptr,
1958 Ordering: I->getOrdering(), Ordering2: AtomicOrdering::NotAtomic, Libcalls);
1959 if (!Expanded)
1960 handleUnsupportedAtomicSize(I, AtomicOpName: "atomic load");
1961}
1962
1963void AtomicExpandImpl::expandAtomicStoreToLibcall(StoreInst *I) {
1964 static const RTLIB::Libcall Libcalls[6] = {
1965 RTLIB::ATOMIC_STORE, RTLIB::ATOMIC_STORE_1, RTLIB::ATOMIC_STORE_2,
1966 RTLIB::ATOMIC_STORE_4, RTLIB::ATOMIC_STORE_8, RTLIB::ATOMIC_STORE_16};
1967 unsigned Size = getAtomicOpSize(SI: I);
1968
1969 bool Expanded = expandAtomicOpToLibcall(
1970 I, Size, Alignment: I->getAlign(), PointerOperand: I->getPointerOperand(), ValueOperand: I->getValueOperand(),
1971 CASExpected: nullptr, Ordering: I->getOrdering(), Ordering2: AtomicOrdering::NotAtomic, Libcalls);
1972 if (!Expanded)
1973 handleUnsupportedAtomicSize(I, AtomicOpName: "atomic store");
1974}
1975
1976void AtomicExpandImpl::expandAtomicCASToLibcall(AtomicCmpXchgInst *I,
1977 const Twine &AtomicOpName,
1978 Instruction *DiagnosticInst) {
1979 static const RTLIB::Libcall Libcalls[6] = {
1980 RTLIB::ATOMIC_COMPARE_EXCHANGE, RTLIB::ATOMIC_COMPARE_EXCHANGE_1,
1981 RTLIB::ATOMIC_COMPARE_EXCHANGE_2, RTLIB::ATOMIC_COMPARE_EXCHANGE_4,
1982 RTLIB::ATOMIC_COMPARE_EXCHANGE_8, RTLIB::ATOMIC_COMPARE_EXCHANGE_16};
1983 unsigned Size = getAtomicOpSize(CASI: I);
1984
1985 bool Expanded = expandAtomicOpToLibcall(
1986 I, Size, Alignment: I->getAlign(), PointerOperand: I->getPointerOperand(), ValueOperand: I->getNewValOperand(),
1987 CASExpected: I->getCompareOperand(), Ordering: I->getSuccessOrdering(), Ordering2: I->getFailureOrdering(),
1988 Libcalls);
1989 if (!Expanded)
1990 handleUnsupportedAtomicSize(I, AtomicOpName, DiagnosticInst);
1991}
1992
1993static ArrayRef<RTLIB::Libcall> GetRMWLibcall(AtomicRMWInst::BinOp Op) {
1994 static const RTLIB::Libcall LibcallsXchg[6] = {
1995 RTLIB::ATOMIC_EXCHANGE, RTLIB::ATOMIC_EXCHANGE_1,
1996 RTLIB::ATOMIC_EXCHANGE_2, RTLIB::ATOMIC_EXCHANGE_4,
1997 RTLIB::ATOMIC_EXCHANGE_8, RTLIB::ATOMIC_EXCHANGE_16};
1998 static const RTLIB::Libcall LibcallsAdd[6] = {
1999 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_ADD_1,
2000 RTLIB::ATOMIC_FETCH_ADD_2, RTLIB::ATOMIC_FETCH_ADD_4,
2001 RTLIB::ATOMIC_FETCH_ADD_8, RTLIB::ATOMIC_FETCH_ADD_16};
2002 static const RTLIB::Libcall LibcallsSub[6] = {
2003 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_SUB_1,
2004 RTLIB::ATOMIC_FETCH_SUB_2, RTLIB::ATOMIC_FETCH_SUB_4,
2005 RTLIB::ATOMIC_FETCH_SUB_8, RTLIB::ATOMIC_FETCH_SUB_16};
2006 static const RTLIB::Libcall LibcallsAnd[6] = {
2007 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_AND_1,
2008 RTLIB::ATOMIC_FETCH_AND_2, RTLIB::ATOMIC_FETCH_AND_4,
2009 RTLIB::ATOMIC_FETCH_AND_8, RTLIB::ATOMIC_FETCH_AND_16};
2010 static const RTLIB::Libcall LibcallsOr[6] = {
2011 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_OR_1,
2012 RTLIB::ATOMIC_FETCH_OR_2, RTLIB::ATOMIC_FETCH_OR_4,
2013 RTLIB::ATOMIC_FETCH_OR_8, RTLIB::ATOMIC_FETCH_OR_16};
2014 static const RTLIB::Libcall LibcallsXor[6] = {
2015 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_XOR_1,
2016 RTLIB::ATOMIC_FETCH_XOR_2, RTLIB::ATOMIC_FETCH_XOR_4,
2017 RTLIB::ATOMIC_FETCH_XOR_8, RTLIB::ATOMIC_FETCH_XOR_16};
2018 static const RTLIB::Libcall LibcallsNand[6] = {
2019 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_NAND_1,
2020 RTLIB::ATOMIC_FETCH_NAND_2, RTLIB::ATOMIC_FETCH_NAND_4,
2021 RTLIB::ATOMIC_FETCH_NAND_8, RTLIB::ATOMIC_FETCH_NAND_16};
2022
2023 switch (Op) {
2024 case AtomicRMWInst::BAD_BINOP:
2025 llvm_unreachable("Should not have BAD_BINOP.");
2026 case AtomicRMWInst::Xchg:
2027 return ArrayRef(LibcallsXchg);
2028 case AtomicRMWInst::Add:
2029 return ArrayRef(LibcallsAdd);
2030 case AtomicRMWInst::Sub:
2031 return ArrayRef(LibcallsSub);
2032 case AtomicRMWInst::And:
2033 return ArrayRef(LibcallsAnd);
2034 case AtomicRMWInst::Or:
2035 return ArrayRef(LibcallsOr);
2036 case AtomicRMWInst::Xor:
2037 return ArrayRef(LibcallsXor);
2038 case AtomicRMWInst::Nand:
2039 return ArrayRef(LibcallsNand);
2040 case AtomicRMWInst::Max:
2041 case AtomicRMWInst::Min:
2042 case AtomicRMWInst::UMax:
2043 case AtomicRMWInst::UMin:
2044 case AtomicRMWInst::FMax:
2045 case AtomicRMWInst::FMin:
2046 case AtomicRMWInst::FMaximum:
2047 case AtomicRMWInst::FMinimum:
2048 case AtomicRMWInst::FMaximumNum:
2049 case AtomicRMWInst::FMinimumNum:
2050 case AtomicRMWInst::FAdd:
2051 case AtomicRMWInst::FSub:
2052 case AtomicRMWInst::UIncWrap:
2053 case AtomicRMWInst::UDecWrap:
2054 case AtomicRMWInst::USubCond:
2055 case AtomicRMWInst::USubSat:
2056 // No atomic libcalls are available for these.
2057 return {};
2058 }
2059 llvm_unreachable("Unexpected AtomicRMW operation.");
2060}
2061
2062void AtomicExpandImpl::expandAtomicRMWToLibcall(AtomicRMWInst *I) {
2063 ArrayRef<RTLIB::Libcall> Libcalls = GetRMWLibcall(Op: I->getOperation());
2064
2065 unsigned Size = getAtomicOpSize(RMWI: I);
2066
2067 bool Success = false;
2068 if (!Libcalls.empty())
2069 Success = expandAtomicOpToLibcall(
2070 I, Size, Alignment: I->getAlign(), PointerOperand: I->getPointerOperand(), ValueOperand: I->getValOperand(),
2071 CASExpected: nullptr, Ordering: I->getOrdering(), Ordering2: AtomicOrdering::NotAtomic, Libcalls);
2072
2073 // The expansion failed: either there were no libcalls at all for
2074 // the operation (min/max), or there were only size-specialized
2075 // libcalls (add/sub/etc) and we needed a generic. So, expand to a
2076 // CAS libcall, via a CAS loop, instead.
2077 if (!Success) {
2078 expandAtomicRMWToCmpXchg(
2079 AI: I, CreateCmpXchg: [this, I](IRBuilderBase &Builder, Value *Addr, Value *Loaded,
2080 Value *NewVal, Align Alignment, AtomicOrdering MemOpOrder,
2081 SyncScope::ID SSID, bool IsVolatile, Value *&Success,
2082 Value *&NewLoaded, Instruction *MetadataSrc) {
2083 // Create the CAS instruction normally...
2084 AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
2085 Ptr: Addr, Cmp: Loaded, New: NewVal, Align: Alignment, SuccessOrdering: MemOpOrder,
2086 FailureOrdering: AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: MemOpOrder), SSID);
2087 Pair->setVolatile(IsVolatile);
2088 if (MetadataSrc)
2089 copyMetadataForAtomic(Dest&: *Pair, Source: *MetadataSrc);
2090
2091 Success = Builder.CreateExtractValue(Agg: Pair, Idxs: 1, Name: "success");
2092 NewLoaded = Builder.CreateExtractValue(Agg: Pair, Idxs: 0, Name: "newloaded");
2093
2094 // ...and then expand the CAS into a libcall.
2095 expandAtomicCASToLibcall(
2096 I: Pair,
2097 AtomicOpName: "atomicrmw " + AtomicRMWInst::getOperationName(Op: I->getOperation()),
2098 DiagnosticInst: MetadataSrc);
2099 });
2100 }
2101}
2102
2103// A helper routine for the above expandAtomic*ToLibcall functions.
2104//
2105// 'Libcalls' contains an array of enum values for the particular
2106// ATOMIC libcalls to be emitted. All of the other arguments besides
2107// 'I' are extracted from the Instruction subclass by the
2108// caller. Depending on the particular call, some will be null.
2109bool AtomicExpandImpl::expandAtomicOpToLibcall(
2110 Instruction *I, unsigned Size, Align Alignment, Value *PointerOperand,
2111 Value *ValueOperand, Value *CASExpected, AtomicOrdering Ordering,
2112 AtomicOrdering Ordering2, ArrayRef<RTLIB::Libcall> Libcalls) {
2113 assert(Libcalls.size() == 6);
2114
2115 LLVMContext &Ctx = I->getContext();
2116 Module *M = I->getModule();
2117 const DataLayout &DL = M->getDataLayout();
2118 IRBuilder<> Builder(I);
2119 IRBuilder<> AllocaBuilder(&I->getFunction()->getEntryBlock().front());
2120
2121 bool UseSizedLibcall = canUseSizedAtomicCall(Size, Alignment, DL);
2122 Type *SizedIntTy = Type::getIntNTy(C&: Ctx, N: Size * 8);
2123
2124 if (M->getTargetTriple().isOSWindows() && M->getTargetTriple().isX86_64() &&
2125 Size == 16) {
2126 // x86_64 Windows passes i128 as an XMM vector; on return, it is in
2127 // XMM0, and as a parameter, it is passed indirectly. The generic lowering
2128 // rules handles this correctly if we pass it as a v2i64 rather than
2129 // i128. This is what Clang does in the frontend for such types as well
2130 // (see WinX86_64ABIInfo::classify in Clang).
2131 SizedIntTy = FixedVectorType::get(ElementType: Type::getInt64Ty(C&: Ctx), NumElts: 2);
2132 }
2133
2134 const Align AllocaAlignment = DL.getPrefTypeAlign(Ty: SizedIntTy);
2135
2136 // TODO: the "order" argument type is "int", not int32. So
2137 // getInt32Ty may be wrong if the arch uses e.g. 16-bit ints.
2138 assert(Ordering != AtomicOrdering::NotAtomic && "expect atomic MO");
2139 Constant *OrderingVal =
2140 ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: (int)toCABI(AO: Ordering));
2141 Constant *Ordering2Val = nullptr;
2142 if (CASExpected) {
2143 assert(Ordering2 != AtomicOrdering::NotAtomic && "expect atomic MO");
2144 Ordering2Val =
2145 ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: (int)toCABI(AO: Ordering2));
2146 }
2147 bool HasResult = I->getType() != Type::getVoidTy(C&: Ctx);
2148
2149 RTLIB::Libcall RTLibType;
2150 if (UseSizedLibcall) {
2151 switch (Size) {
2152 case 1:
2153 RTLibType = Libcalls[1];
2154 break;
2155 case 2:
2156 RTLibType = Libcalls[2];
2157 break;
2158 case 4:
2159 RTLibType = Libcalls[3];
2160 break;
2161 case 8:
2162 RTLibType = Libcalls[4];
2163 break;
2164 case 16:
2165 RTLibType = Libcalls[5];
2166 break;
2167 }
2168 } else if (Libcalls[0] != RTLIB::UNKNOWN_LIBCALL) {
2169 RTLibType = Libcalls[0];
2170 } else {
2171 // Can't use sized function, and there's no generic for this
2172 // operation, so give up.
2173 return false;
2174 }
2175
2176 RTLIB::LibcallImpl LibcallImpl = LibcallLowering->getLibcallImpl(Call: RTLibType);
2177 if (LibcallImpl == RTLIB::Unsupported) {
2178 // This target does not implement the requested atomic libcall so give up.
2179 return false;
2180 }
2181
2182 // Build up the function call. There's two kinds. First, the sized
2183 // variants. These calls are going to be one of the following (with
2184 // N=1,2,4,8,16):
2185 // iN __atomic_load_N(iN *ptr, int ordering)
2186 // void __atomic_store_N(iN *ptr, iN val, int ordering)
2187 // iN __atomic_{exchange|fetch_*}_N(iN *ptr, iN val, int ordering)
2188 // bool __atomic_compare_exchange_N(iN *ptr, iN *expected, iN desired,
2189 // int success_order, int failure_order)
2190 //
2191 // Note that these functions can be used for non-integer atomic
2192 // operations, the values just need to be bitcast to integers on the
2193 // way in and out.
2194 //
2195 // And, then, the generic variants. They look like the following:
2196 // void __atomic_load(size_t size, void *ptr, void *ret, int ordering)
2197 // void __atomic_store(size_t size, void *ptr, void *val, int ordering)
2198 // void __atomic_exchange(size_t size, void *ptr, void *val, void *ret,
2199 // int ordering)
2200 // bool __atomic_compare_exchange(size_t size, void *ptr, void *expected,
2201 // void *desired, int success_order,
2202 // int failure_order)
2203 //
2204 // The different signatures are built up depending on the
2205 // 'UseSizedLibcall', 'CASExpected', 'ValueOperand', and 'HasResult'
2206 // variables.
2207
2208 AllocaInst *AllocaCASExpected = nullptr;
2209 AllocaInst *AllocaValue = nullptr;
2210 AllocaInst *AllocaResult = nullptr;
2211
2212 Type *ResultTy;
2213 SmallVector<Value *, 6> Args;
2214 AttributeList Attr;
2215
2216 // 'size' argument.
2217 if (!UseSizedLibcall) {
2218 // Note, getIntPtrType is assumed equivalent to size_t.
2219 Args.push_back(Elt: ConstantInt::get(Ty: DL.getIntPtrType(C&: Ctx), V: Size));
2220 }
2221
2222 // 'ptr' argument.
2223 // note: This assumes all address spaces share a common libfunc
2224 // implementation and that addresses are convertable. For systems without
2225 // that property, we'd need to extend this mechanism to support AS-specific
2226 // families of atomic intrinsics.
2227 Value *PtrVal = PointerOperand;
2228 PtrVal = Builder.CreateAddrSpaceCast(V: PtrVal, DestTy: PointerType::getUnqual(C&: Ctx));
2229 Args.push_back(Elt: PtrVal);
2230
2231 // 'expected' argument, if present.
2232 if (CASExpected) {
2233 AllocaCASExpected = AllocaBuilder.CreateAlloca(Ty: CASExpected->getType());
2234 AllocaCASExpected->setAlignment(AllocaAlignment);
2235 Builder.CreateLifetimeStart(Ptr: AllocaCASExpected);
2236 Builder.CreateAlignedStore(Val: CASExpected, Ptr: AllocaCASExpected, Align: AllocaAlignment);
2237 Args.push_back(Elt: AllocaCASExpected);
2238 }
2239
2240 // 'val' argument ('desired' for cas), if present.
2241 if (ValueOperand) {
2242 if (UseSizedLibcall) {
2243 Value *IntValue =
2244 Builder.CreateBitPreservingCastChain(DL, V: ValueOperand, NewTy: SizedIntTy);
2245 Args.push_back(Elt: IntValue);
2246 } else {
2247 AllocaValue = AllocaBuilder.CreateAlloca(Ty: ValueOperand->getType());
2248 AllocaValue->setAlignment(AllocaAlignment);
2249 Builder.CreateLifetimeStart(Ptr: AllocaValue);
2250 Builder.CreateAlignedStore(Val: ValueOperand, Ptr: AllocaValue, Align: AllocaAlignment);
2251 Args.push_back(Elt: AllocaValue);
2252 }
2253 }
2254
2255 // 'ret' argument.
2256 if (!CASExpected && HasResult && !UseSizedLibcall) {
2257 AllocaResult = AllocaBuilder.CreateAlloca(Ty: I->getType());
2258 AllocaResult->setAlignment(AllocaAlignment);
2259 Builder.CreateLifetimeStart(Ptr: AllocaResult);
2260 Args.push_back(Elt: AllocaResult);
2261 }
2262
2263 // 'ordering' ('success_order' for cas) argument.
2264 Args.push_back(Elt: OrderingVal);
2265
2266 // 'failure_order' argument, if present.
2267 if (Ordering2Val)
2268 Args.push_back(Elt: Ordering2Val);
2269
2270 // Now, the return type.
2271 if (CASExpected) {
2272 ResultTy = Type::getInt1Ty(C&: Ctx);
2273 Attr = Attr.addRetAttribute(C&: Ctx, Kind: Attribute::ZExt);
2274 } else if (HasResult && UseSizedLibcall)
2275 ResultTy = SizedIntTy;
2276 else
2277 ResultTy = Type::getVoidTy(C&: Ctx);
2278
2279 // Done with setting up arguments and return types, create the call:
2280 SmallVector<Type *, 6> ArgTys;
2281 for (Value *Arg : Args)
2282 ArgTys.push_back(Elt: Arg->getType());
2283 FunctionType *FnType = FunctionType::get(Result: ResultTy, Params: ArgTys, isVarArg: false);
2284 FunctionCallee LibcallFn = M->getOrInsertFunction(
2285 Name: RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: LibcallImpl), T: FnType,
2286 AttributeList: Attr);
2287 CallInst *Call = Builder.CreateCall(Callee: LibcallFn, Args);
2288 Call->setAttributes(Attr);
2289 Value *Result = Call;
2290
2291 // And then, extract the results...
2292 if (ValueOperand && !UseSizedLibcall)
2293 Builder.CreateLifetimeEnd(Ptr: AllocaValue);
2294
2295 if (CASExpected) {
2296 // The final result from the CAS is {load of 'expected' alloca, bool result
2297 // from call}
2298 Type *FinalResultTy = I->getType();
2299 Value *V = PoisonValue::get(T: FinalResultTy);
2300 Value *ExpectedOut = Builder.CreateAlignedLoad(
2301 Ty: CASExpected->getType(), Ptr: AllocaCASExpected, Align: AllocaAlignment);
2302 Builder.CreateLifetimeEnd(Ptr: AllocaCASExpected);
2303 V = Builder.CreateInsertValue(Agg: V, Val: ExpectedOut, Idxs: 0);
2304 V = Builder.CreateInsertValue(Agg: V, Val: Result, Idxs: 1);
2305 I->replaceAllUsesWith(V);
2306 } else if (HasResult) {
2307 Value *V;
2308 if (UseSizedLibcall) {
2309 // Add bitcasts from Result's scalar type to I's <n x ptr> vector type
2310 auto *PtrTy = dyn_cast<PointerType>(Val: I->getType()->getScalarType());
2311 auto *VTy = dyn_cast<VectorType>(Val: I->getType());
2312 if (VTy && PtrTy && !Result->getType()->isVectorTy()) {
2313 unsigned AS = PtrTy->getAddressSpace();
2314 Value *BC = Builder.CreateBitCast(
2315 V: Result, DestTy: VTy->getWithNewType(EltTy: DL.getIntPtrType(C&: Ctx, AddressSpace: AS)));
2316 V = Builder.CreateIntToPtr(V: BC, DestTy: I->getType());
2317 } else
2318 V = Builder.CreateBitOrPointerCast(V: Result, DestTy: I->getType());
2319 } else {
2320 V = Builder.CreateAlignedLoad(Ty: I->getType(), Ptr: AllocaResult,
2321 Align: AllocaAlignment);
2322 Builder.CreateLifetimeEnd(Ptr: AllocaResult);
2323 }
2324 I->replaceAllUsesWith(V);
2325 }
2326 I->eraseFromParent();
2327 return true;
2328}
2329