1//===- AtomicExpandPass.cpp - Expand atomic instructions ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass (at IR level) to replace atomic instructions with
10// __atomic_* library calls, or target specific instruction which implement the
11// same semantics in a way which better fits the target backend. This can
12// include the use of (intrinsic-based) load-linked/store-conditional loops,
13// AtomicCmpXchg, or type coercions.
14//
15//===----------------------------------------------------------------------===//
16
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/STLFunctionalExtras.h"
19#include "llvm/ADT/SmallString.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/Analysis/InstSimplifyFolder.h"
22#include "llvm/Analysis/OptimizationRemarkEmitter.h"
23#include "llvm/CodeGen/AtomicExpand.h"
24#include "llvm/CodeGen/TargetLowering.h"
25#include "llvm/CodeGen/TargetPassConfig.h"
26#include "llvm/CodeGen/TargetSubtargetInfo.h"
27#include "llvm/CodeGen/ValueTypes.h"
28#include "llvm/IR/Attributes.h"
29#include "llvm/IR/BasicBlock.h"
30#include "llvm/IR/Constant.h"
31#include "llvm/IR/Constants.h"
32#include "llvm/IR/DataLayout.h"
33#include "llvm/IR/DerivedTypes.h"
34#include "llvm/IR/Function.h"
35#include "llvm/IR/IRBuilder.h"
36#include "llvm/IR/Instruction.h"
37#include "llvm/IR/Instructions.h"
38#include "llvm/IR/MDBuilder.h"
39#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
40#include "llvm/IR/Module.h"
41#include "llvm/IR/ProfDataUtils.h"
42#include "llvm/IR/Type.h"
43#include "llvm/IR/User.h"
44#include "llvm/IR/Value.h"
45#include "llvm/InitializePasses.h"
46#include "llvm/Pass.h"
47#include "llvm/Support/AtomicOrdering.h"
48#include "llvm/Support/Casting.h"
49#include "llvm/Support/Debug.h"
50#include "llvm/Support/ErrorHandling.h"
51#include "llvm/Support/raw_ostream.h"
52#include "llvm/Target/TargetMachine.h"
53#include "llvm/Transforms/Utils/LowerAtomic.h"
54#include <cassert>
55#include <cstdint>
56#include <iterator>
57
58using namespace llvm;
59
60#define DEBUG_TYPE "atomic-expand"
61
62namespace {
63
64class AtomicExpandImpl {
65 const TargetLowering *TLI = nullptr;
66 const LibcallLoweringInfo *LibcallLowering = nullptr;
67 const DataLayout *DL = nullptr;
68
69private:
70 /// Callback type for emitting a cmpxchg instruction during RMW expansion.
71 /// Parameters: (Builder, Addr, Loaded, NewVal, AddrAlign, MemOpOrder,
72 /// SSID, IsVolatile, /* OUT */ Success, /* OUT */ NewLoaded,
73 /// MetadataSrc)
74 using CreateCmpXchgInstFun = function_ref<void(
75 IRBuilderBase &, Value *, Value *, Value *, Align, AtomicOrdering,
76 SyncScope::ID, bool, Value *&, Value *&, Instruction *)>;
77
78 void handleFailure(Instruction &FailedInst, const Twine &Msg,
79 Instruction *DiagnosticInst = nullptr) const {
80 LLVMContext &Ctx = FailedInst.getContext();
81
82 // TODO: Do not use generic error type.
83 Ctx.emitError(I: DiagnosticInst ? DiagnosticInst : &FailedInst, ErrorStr: Msg);
84
85 if (!FailedInst.getType()->isVoidTy())
86 FailedInst.replaceAllUsesWith(V: PoisonValue::get(T: FailedInst.getType()));
87 FailedInst.eraseFromParent();
88 }
89
90 template <typename Inst>
91 void handleUnsupportedAtomicSize(Inst *I, const Twine &AtomicOpName,
92 Instruction *DiagnosticInst = nullptr) const;
93
94 bool bracketInstWithFences(Instruction *I, AtomicOrdering Order);
95 bool tryInsertTrailingSeqCstFence(Instruction *AtomicI);
96 template <typename AtomicInst>
97 bool tryInsertFencesForAtomic(AtomicInst *AtomicI, bool OrderingRequiresFence,
98 AtomicOrdering NewOrdering);
99 IntegerType *getCorrespondingIntegerType(Type *T, const DataLayout &DL);
100 LoadInst *convertAtomicLoadToIntegerType(LoadInst *LI);
101 bool tryExpandAtomicLoad(LoadInst *LI);
102 bool expandAtomicLoadToLL(LoadInst *LI);
103 bool expandAtomicLoadToCmpXchg(LoadInst *LI);
104 StoreInst *convertAtomicStoreToIntegerType(StoreInst *SI);
105 bool tryExpandAtomicStore(StoreInst *SI);
106 void expandAtomicStoreToXChg(StoreInst *SI);
107 bool tryExpandAtomicRMW(AtomicRMWInst *AI);
108 AtomicRMWInst *convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI);
109 Value *
110 insertRMWLLSCLoop(IRBuilderBase &Builder, Type *ResultTy, Value *Addr,
111 Align AddrAlign, AtomicOrdering MemOpOrder,
112 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp);
113 void expandAtomicOpToLLSC(
114 Instruction *I, Type *ResultTy, Value *Addr, Align AddrAlign,
115 AtomicOrdering MemOpOrder,
116 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp);
117 void expandPartwordAtomicRMW(
118 AtomicRMWInst *I, TargetLoweringBase::AtomicExpansionKind ExpansionKind);
119 AtomicRMWInst *widenPartwordAtomicRMW(AtomicRMWInst *AI);
120 bool expandPartwordCmpXchg(AtomicCmpXchgInst *I);
121 void expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI);
122 void expandAtomicCmpXchgToMaskedIntrinsic(AtomicCmpXchgInst *CI);
123
124 AtomicCmpXchgInst *convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI);
125 Value *insertRMWCmpXchgLoop(
126 IRBuilderBase &Builder, Type *ResultType, Value *Addr, Align AddrAlign,
127 AtomicOrdering MemOpOrder, SyncScope::ID SSID, bool IsVolatile,
128 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp,
129 CreateCmpXchgInstFun CreateCmpXchg, Instruction *MetadataSrc);
130 bool tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI);
131
132 bool expandAtomicCmpXchg(AtomicCmpXchgInst *CI);
133 bool isIdempotentRMW(AtomicRMWInst *RMWI);
134 bool simplifyIdempotentRMW(AtomicRMWInst *RMWI);
135
136 bool expandAtomicOpToLibcall(Instruction *I, unsigned Size, Align Alignment,
137 Value *PointerOperand, Value *ValueOperand,
138 Value *CASExpected, AtomicOrdering Ordering,
139 AtomicOrdering Ordering2,
140 ArrayRef<RTLIB::Libcall> Libcalls);
141 void expandAtomicLoadToLibcall(LoadInst *LI);
142 void expandAtomicStoreToLibcall(StoreInst *LI);
143 void expandAtomicRMWToLibcall(AtomicRMWInst *I);
144 void expandAtomicCASToLibcall(AtomicCmpXchgInst *I,
145 const Twine &AtomicOpName = "cmpxchg",
146 Instruction *DiagnosticInst = nullptr);
147
148 bool expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
149 CreateCmpXchgInstFun CreateCmpXchg);
150
151 bool processAtomicInstr(Instruction *I);
152
153public:
154 bool run(Function &F, const ModuleLibcallLoweringInfo &LibcallResult,
155 const TargetMachine *TM);
156};
157
158class AtomicExpandLegacy : public FunctionPass {
159public:
160 static char ID; // Pass identification, replacement for typeid
161
162 AtomicExpandLegacy() : FunctionPass(ID) {}
163
164 void getAnalysisUsage(AnalysisUsage &AU) const override {
165 AU.addRequired<LibcallLoweringInfoWrapper>();
166 FunctionPass::getAnalysisUsage(AU);
167 }
168
169 bool runOnFunction(Function &F) override;
170};
171
172// IRBuilder to be used for replacement atomic instructions.
173struct ReplacementIRBuilder
174 : IRBuilder<InstSimplifyFolder, IRBuilderCallbackInserter> {
175 MDNode *MMRAMD = nullptr;
176 MDNode *PCSectionsMD = nullptr;
177
178 // Preserves the DebugLoc from I, and preserves still valid metadata.
179 // Enable StrictFP builder mode when appropriate.
180 explicit ReplacementIRBuilder(Instruction *I, const DataLayout &DL)
181 : IRBuilder(
182 I->getContext(), InstSimplifyFolder(DL),
183 IRBuilderCallbackInserter([this](Instruction *I) { addMD(I); })) {
184 SetInsertPoint(I);
185 if (BB->getParent()->getAttributes().hasFnAttr(Kind: Attribute::StrictFP))
186 this->setIsFPConstrained(true);
187
188 MMRAMD = I->getMetadata(KindID: LLVMContext::MD_mmra);
189 PCSectionsMD = I->getMetadata(KindID: LLVMContext::MD_pcsections);
190 }
191
192 void addMD(Instruction *I) {
193 if (canInstructionHaveMMRAs(I: *I))
194 I->setMetadata(KindID: LLVMContext::MD_mmra, Node: MMRAMD);
195 I->setMetadata(KindID: LLVMContext::MD_pcsections, Node: PCSectionsMD);
196 }
197};
198
199} // end anonymous namespace
200
201char AtomicExpandLegacy::ID = 0;
202
203char &llvm::AtomicExpandID = AtomicExpandLegacy::ID;
204
205INITIALIZE_PASS_BEGIN(AtomicExpandLegacy, DEBUG_TYPE,
206 "Expand Atomic instructions", false, false)
207INITIALIZE_PASS_DEPENDENCY(LibcallLoweringInfoWrapper)
208INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
209INITIALIZE_PASS_END(AtomicExpandLegacy, DEBUG_TYPE,
210 "Expand Atomic instructions", false, false)
211
212// Helper functions to retrieve the size of atomic instructions.
213static unsigned getAtomicOpSize(LoadInst *LI) {
214 const DataLayout &DL = LI->getDataLayout();
215 return DL.getTypeStoreSize(Ty: LI->getType());
216}
217
218static unsigned getAtomicOpSize(StoreInst *SI) {
219 const DataLayout &DL = SI->getDataLayout();
220 return DL.getTypeStoreSize(Ty: SI->getValueOperand()->getType());
221}
222
223static unsigned getAtomicOpSize(AtomicRMWInst *RMWI) {
224 const DataLayout &DL = RMWI->getDataLayout();
225 return DL.getTypeStoreSize(Ty: RMWI->getValOperand()->getType());
226}
227
228static unsigned getAtomicOpSize(AtomicCmpXchgInst *CASI) {
229 const DataLayout &DL = CASI->getDataLayout();
230 return DL.getTypeStoreSize(Ty: CASI->getCompareOperand()->getType());
231}
232
233/// Copy metadata that's safe to preserve when widening atomics.
234static void copyMetadataForAtomic(Instruction &Dest,
235 const Instruction &Source) {
236 SmallVector<std::pair<unsigned, MDNode *>, 8> MD;
237 Source.getAllMetadata(MDs&: MD);
238 LLVMContext &Ctx = Dest.getContext();
239 MDBuilder MDB(Ctx);
240
241 for (auto [ID, N] : MD) {
242 switch (ID) {
243 case LLVMContext::MD_dbg:
244 case LLVMContext::MD_tbaa:
245 case LLVMContext::MD_tbaa_struct:
246 case LLVMContext::MD_alias_scope:
247 case LLVMContext::MD_noalias:
248 case LLVMContext::MD_noalias_addrspace:
249 case LLVMContext::MD_access_group:
250 case LLVMContext::MD_mmra:
251 Dest.setMetadata(KindID: ID, Node: N);
252 break;
253 default:
254 if (ID == Ctx.getMDKindID(Name: "amdgpu.no.remote.memory"))
255 Dest.setMetadata(KindID: ID, Node: N);
256 else if (ID == Ctx.getMDKindID(Name: "amdgpu.no.fine.grained.memory"))
257 Dest.setMetadata(KindID: ID, Node: N);
258
259 // Losing amdgpu.ignore.denormal.mode, but it doesn't matter for current
260 // uses.
261 break;
262 }
263 }
264}
265
266template <typename Inst>
267static bool atomicSizeSupported(const TargetLowering *TLI, Inst *I) {
268 unsigned Size = getAtomicOpSize(I);
269 Align Alignment = I->getAlign();
270 unsigned MaxSize = TLI->getMaxAtomicSizeInBitsSupported() / 8;
271 return Alignment >= Size && Size <= MaxSize;
272}
273
274template <typename Inst>
275static void writeUnsupportedAtomicSizeReason(const TargetLowering *TLI, Inst *I,
276 raw_ostream &OS) {
277 unsigned Size = getAtomicOpSize(I);
278 Align Alignment = I->getAlign();
279 bool NeedSeparator = false;
280
281 if (Alignment < Size) {
282 OS << "instruction alignment " << Alignment.value()
283 << " is smaller than the required " << Size
284 << "-byte alignment for this atomic operation";
285 NeedSeparator = true;
286 }
287
288 unsigned MaxSize = TLI->getMaxAtomicSizeInBitsSupported() / 8;
289 if (Size > MaxSize) {
290 if (NeedSeparator)
291 OS << "; ";
292 OS << "target supports atomics up to " << MaxSize
293 << " bytes, but this atomic accesses " << Size << " bytes";
294 }
295}
296
297template <typename Inst>
298void AtomicExpandImpl::handleUnsupportedAtomicSize(
299 Inst *I, const Twine &AtomicOpName, Instruction *DiagnosticInst) const {
300 assert(!atomicSizeSupported(TLI, I) && "expected unsupported atomic size");
301 SmallString<128> FailureReason;
302 raw_svector_ostream OS(FailureReason);
303 writeUnsupportedAtomicSizeReason(TLI, I, OS);
304 handleFailure(FailedInst&: *I, Msg: Twine("unsupported ") + AtomicOpName + ": " + FailureReason,
305 DiagnosticInst);
306}
307
308bool AtomicExpandImpl::tryInsertTrailingSeqCstFence(Instruction *AtomicI) {
309 if (!TLI->shouldInsertTrailingSeqCstFenceForAtomicStore(I: AtomicI))
310 return false;
311
312 IRBuilder Builder(AtomicI);
313 if (auto *TrailingFence = TLI->emitTrailingFence(
314 Builder, Inst: AtomicI, Ord: AtomicOrdering::SequentiallyConsistent)) {
315 TrailingFence->moveAfter(MovePos: AtomicI);
316 return true;
317 }
318 return false;
319}
320
321template <typename AtomicInst>
322bool AtomicExpandImpl::tryInsertFencesForAtomic(AtomicInst *AtomicI,
323 bool OrderingRequiresFence,
324 AtomicOrdering NewOrdering) {
325 bool ShouldInsertFences = TLI->shouldInsertFencesForAtomic(I: AtomicI);
326 if (OrderingRequiresFence && ShouldInsertFences) {
327 AtomicOrdering FenceOrdering = AtomicI->getOrdering();
328 AtomicI->setOrdering(NewOrdering);
329 return bracketInstWithFences(I: AtomicI, Order: FenceOrdering);
330 }
331 if (!ShouldInsertFences)
332 return tryInsertTrailingSeqCstFence(AtomicI);
333 return false;
334}
335
336bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
337 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
338 if (!LI->isAtomic())
339 return false;
340
341 if (!atomicSizeSupported(TLI, I: LI)) {
342 expandAtomicLoadToLibcall(LI);
343 return true;
344 }
345
346 bool MadeChange = false;
347 if (TLI->shouldCastAtomicLoadInIR(LI) ==
348 TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
349 LI = convertAtomicLoadToIntegerType(LI);
350 MadeChange = true;
351 }
352
353 MadeChange |= tryInsertFencesForAtomic(
354 AtomicI: LI, OrderingRequiresFence: isAcquireOrStronger(AO: LI->getOrdering()), NewOrdering: AtomicOrdering::Monotonic);
355
356 MadeChange |= tryExpandAtomicLoad(LI);
357 return MadeChange;
358 }
359
360 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
361 if (!SI->isAtomic())
362 return false;
363
364 if (!atomicSizeSupported(TLI, I: SI)) {
365 expandAtomicStoreToLibcall(LI: SI);
366 return true;
367 }
368
369 bool MadeChange = false;
370 if (TLI->shouldCastAtomicStoreInIR(SI) ==
371 TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
372 SI = convertAtomicStoreToIntegerType(SI);
373 MadeChange = true;
374 }
375
376 MadeChange |= tryInsertFencesForAtomic(
377 AtomicI: SI, OrderingRequiresFence: isReleaseOrStronger(AO: SI->getOrdering()), NewOrdering: AtomicOrdering::Monotonic);
378
379 MadeChange |= tryExpandAtomicStore(SI);
380 return MadeChange;
381 }
382
383 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: I)) {
384 if (!atomicSizeSupported(TLI, I: RMWI)) {
385 expandAtomicRMWToLibcall(I: RMWI);
386 return true;
387 }
388
389 bool MadeChange = false;
390 if (TLI->shouldCastAtomicRMWIInIR(RMWI) ==
391 TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
392 RMWI = convertAtomicXchgToIntegerType(RMWI);
393 MadeChange = true;
394 }
395
396 MadeChange |= tryInsertFencesForAtomic(
397 AtomicI: RMWI,
398 OrderingRequiresFence: isReleaseOrStronger(AO: RMWI->getOrdering()) ||
399 isAcquireOrStronger(AO: RMWI->getOrdering()),
400 NewOrdering: TLI->atomicOperationOrderAfterFenceSplit(I: RMWI));
401
402 // There are two different ways of expanding RMW instructions:
403 // - into a load if it is idempotent
404 // - into a Cmpxchg/LL-SC loop otherwise
405 // we try them in that order.
406 MadeChange |= (isIdempotentRMW(RMWI) && simplifyIdempotentRMW(RMWI)) ||
407 tryExpandAtomicRMW(AI: RMWI);
408 return MadeChange;
409 }
410
411 if (auto *CASI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
412 if (!atomicSizeSupported(TLI, I: CASI)) {
413 expandAtomicCASToLibcall(I: CASI);
414 return true;
415 }
416
417 // TODO: when we're ready to make the change at the IR level, we can
418 // extend convertCmpXchgToInteger for floating point too.
419 bool MadeChange = false;
420 if (CASI->getCompareOperand()->getType()->isPointerTy()) {
421 // TODO: add a TLI hook to control this so that each target can
422 // convert to lowering the original type one at a time.
423 CASI = convertCmpXchgToIntegerType(CI: CASI);
424 MadeChange = true;
425 }
426
427 auto CmpXchgExpansion = TLI->shouldExpandAtomicCmpXchgInIR(AI: CASI);
428 if (TLI->shouldInsertFencesForAtomic(I: CASI)) {
429 if (CmpXchgExpansion == TargetLoweringBase::AtomicExpansionKind::None &&
430 (isReleaseOrStronger(AO: CASI->getSuccessOrdering()) ||
431 isAcquireOrStronger(AO: CASI->getSuccessOrdering()) ||
432 isAcquireOrStronger(AO: CASI->getFailureOrdering()))) {
433 // If a compare and swap is lowered to LL/SC, we can do smarter fence
434 // insertion, with a stronger one on the success path than on the
435 // failure path. As a result, fence insertion is directly done by
436 // expandAtomicCmpXchg in that case.
437 AtomicOrdering FenceOrdering = CASI->getMergedOrdering();
438 AtomicOrdering CASOrdering =
439 TLI->atomicOperationOrderAfterFenceSplit(I: CASI);
440 CASI->setSuccessOrdering(CASOrdering);
441 CASI->setFailureOrdering(CASOrdering);
442 MadeChange |= bracketInstWithFences(I: CASI, Order: FenceOrdering);
443 }
444 } else if (CmpXchgExpansion !=
445 TargetLoweringBase::AtomicExpansionKind::LLSC) {
446 // CmpXchg LLSC is handled in expandAtomicCmpXchg().
447 MadeChange |= tryInsertTrailingSeqCstFence(AtomicI: CASI);
448 }
449
450 MadeChange |= tryExpandAtomicCmpXchg(CI: CASI);
451 return MadeChange;
452 }
453
454 return false;
455}
456
457bool AtomicExpandImpl::run(Function &F,
458 const ModuleLibcallLoweringInfo &LibcallResult,
459 const TargetMachine *TM) {
460 const auto *Subtarget = TM->getSubtargetImpl(F);
461 if (!Subtarget->enableAtomicExpand())
462 return false;
463 TLI = Subtarget->getTargetLowering();
464 LibcallLowering = &LibcallResult.getLibcallLowering(Subtarget: *Subtarget);
465 DL = &F.getDataLayout();
466
467 bool MadeChange = false;
468
469 for (Function::iterator BBI = F.begin(), BBE = F.end(); BBI != BBE; ++BBI) {
470 BasicBlock *BB = &*BBI;
471
472 BasicBlock::reverse_iterator Next;
473
474 for (BasicBlock::reverse_iterator I = BB->rbegin(), E = BB->rend(); I != E;
475 I = Next) {
476 Instruction &Inst = *I;
477 Next = std::next(x: I);
478
479 if (processAtomicInstr(I: &Inst)) {
480 MadeChange = true;
481
482 // New blocks may have been inserted.
483 BBE = F.end();
484 }
485 }
486 }
487
488 return MadeChange;
489}
490
491bool AtomicExpandLegacy::runOnFunction(Function &F) {
492
493 auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
494 if (!TPC)
495 return false;
496 auto *TM = &TPC->getTM<TargetMachine>();
497
498 const ModuleLibcallLoweringInfo &LibcallResult =
499 getAnalysis<LibcallLoweringInfoWrapper>().getResult(M: *F.getParent());
500 AtomicExpandImpl AE;
501 return AE.run(F, LibcallResult, TM);
502}
503
504FunctionPass *llvm::createAtomicExpandLegacyPass() {
505 return new AtomicExpandLegacy();
506}
507
508PreservedAnalyses AtomicExpandPass::run(Function &F,
509 FunctionAnalysisManager &FAM) {
510 auto &MAMProxy = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: F);
511
512 const ModuleLibcallLoweringInfo *LibcallResult =
513 MAMProxy.getCachedResult<LibcallLoweringModuleAnalysis>(IR&: *F.getParent());
514
515 if (!LibcallResult) {
516 F.getContext().emitError(ErrorStr: "'" + LibcallLoweringModuleAnalysis::name() +
517 "' analysis required");
518 return PreservedAnalyses::all();
519 }
520
521 AtomicExpandImpl AE;
522
523 bool Changed = AE.run(F, LibcallResult: *LibcallResult, TM);
524 if (!Changed)
525 return PreservedAnalyses::all();
526
527 return PreservedAnalyses::none();
528}
529
530bool AtomicExpandImpl::bracketInstWithFences(Instruction *I,
531 AtomicOrdering Order) {
532 ReplacementIRBuilder Builder(I, *DL);
533
534 auto LeadingFence = TLI->emitLeadingFence(Builder, Inst: I, Ord: Order);
535
536 auto TrailingFence = TLI->emitTrailingFence(Builder, Inst: I, Ord: Order);
537 // We have a guard here because not every atomic operation generates a
538 // trailing fence.
539 if (TrailingFence)
540 TrailingFence->moveAfter(MovePos: I);
541
542 return (LeadingFence || TrailingFence);
543}
544
545/// Get the iX type with the same bitwidth as T.
546IntegerType *
547AtomicExpandImpl::getCorrespondingIntegerType(Type *T, const DataLayout &DL) {
548 EVT VT = TLI->getMemValueType(DL, Ty: T);
549 unsigned BitWidth = VT.getStoreSizeInBits();
550 assert(BitWidth == VT.getSizeInBits() && "must be a power of two");
551 return IntegerType::get(C&: T->getContext(), NumBits: BitWidth);
552}
553
554/// Convert an atomic load of a non-integral type to an integer load of the
555/// equivalent bitwidth. See the function comment on
556/// convertAtomicStoreToIntegerType for background.
557LoadInst *AtomicExpandImpl::convertAtomicLoadToIntegerType(LoadInst *LI) {
558 auto *M = LI->getModule();
559 Type *NewTy = getCorrespondingIntegerType(T: LI->getType(), DL: M->getDataLayout());
560
561 ReplacementIRBuilder Builder(LI, *DL);
562
563 Value *Addr = LI->getPointerOperand();
564
565 auto *NewLI = Builder.CreateLoad(Ty: NewTy, Ptr: Addr, Props: LI->getProperties());
566 LLVM_DEBUG(dbgs() << "Replaced " << *LI << " with " << *NewLI << "\n");
567
568 Value *NewVal = LI->getType()->isPtrOrPtrVectorTy()
569 ? Builder.CreateIntToPtr(V: NewLI, DestTy: LI->getType())
570 : Builder.CreateBitCast(V: NewLI, DestTy: LI->getType());
571 LI->replaceAllUsesWith(V: NewVal);
572 LI->eraseFromParent();
573 return NewLI;
574}
575
576AtomicRMWInst *
577AtomicExpandImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
578 assert(RMWI->getOperation() == AtomicRMWInst::Xchg);
579
580 auto *M = RMWI->getModule();
581 Type *NewTy =
582 getCorrespondingIntegerType(T: RMWI->getType(), DL: M->getDataLayout());
583
584 ReplacementIRBuilder Builder(RMWI, *DL);
585
586 Value *Addr = RMWI->getPointerOperand();
587 Value *Val = RMWI->getValOperand();
588 Value *NewVal = Val->getType()->isPointerTy()
589 ? Builder.CreatePtrToInt(V: Val, DestTy: NewTy)
590 : Builder.CreateBitCast(V: Val, DestTy: NewTy);
591
592 auto *NewRMWI = Builder.CreateAtomicRMW(Op: AtomicRMWInst::Xchg, Ptr: Addr, Val: NewVal,
593 Align: RMWI->getAlign(), Ordering: RMWI->getOrdering(),
594 SSID: RMWI->getSyncScopeID());
595 NewRMWI->setVolatile(RMWI->isVolatile());
596 copyMetadataForAtomic(Dest&: *NewRMWI, Source: *RMWI);
597 LLVM_DEBUG(dbgs() << "Replaced " << *RMWI << " with " << *NewRMWI << "\n");
598
599 Value *NewRVal = RMWI->getType()->isPointerTy()
600 ? Builder.CreateIntToPtr(V: NewRMWI, DestTy: RMWI->getType())
601 : Builder.CreateBitCast(V: NewRMWI, DestTy: RMWI->getType());
602 RMWI->replaceAllUsesWith(V: NewRVal);
603 RMWI->eraseFromParent();
604 return NewRMWI;
605}
606
607bool AtomicExpandImpl::tryExpandAtomicLoad(LoadInst *LI) {
608 switch (TLI->shouldExpandAtomicLoadInIR(LI)) {
609 case TargetLoweringBase::AtomicExpansionKind::None:
610 return false;
611 case TargetLoweringBase::AtomicExpansionKind::LLSC:
612 expandAtomicOpToLLSC(
613 I: LI, ResultTy: LI->getType(), Addr: LI->getPointerOperand(), AddrAlign: LI->getAlign(),
614 MemOpOrder: LI->getOrdering(),
615 PerformOp: [](IRBuilderBase &Builder, Value *Loaded) { return Loaded; });
616 return true;
617 case TargetLoweringBase::AtomicExpansionKind::LLOnly:
618 return expandAtomicLoadToLL(LI);
619 case TargetLoweringBase::AtomicExpansionKind::CmpXChg:
620 return expandAtomicLoadToCmpXchg(LI);
621 case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
622 LI->setAtomic(Ordering: AtomicOrdering::NotAtomic);
623 return true;
624 case TargetLoweringBase::AtomicExpansionKind::CustomExpand:
625 TLI->emitExpandAtomicLoad(LI);
626 return true;
627 default:
628 llvm_unreachable("Unhandled case in tryExpandAtomicLoad");
629 }
630}
631
632bool AtomicExpandImpl::tryExpandAtomicStore(StoreInst *SI) {
633 switch (TLI->shouldExpandAtomicStoreInIR(SI)) {
634 case TargetLoweringBase::AtomicExpansionKind::None:
635 return false;
636 case TargetLoweringBase::AtomicExpansionKind::CustomExpand:
637 TLI->emitExpandAtomicStore(SI);
638 return true;
639 case TargetLoweringBase::AtomicExpansionKind::Expand:
640 expandAtomicStoreToXChg(SI);
641 return true;
642 case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
643 SI->setAtomic(Ordering: AtomicOrdering::NotAtomic);
644 return true;
645 default:
646 llvm_unreachable("Unhandled case in tryExpandAtomicStore");
647 }
648}
649
650bool AtomicExpandImpl::expandAtomicLoadToLL(LoadInst *LI) {
651 ReplacementIRBuilder Builder(LI, *DL);
652
653 // On some architectures, load-linked instructions are atomic for larger
654 // sizes than normal loads. For example, the only 64-bit load guaranteed
655 // to be single-copy atomic by ARM is an ldrexd (A3.5.3).
656 Value *Val = TLI->emitLoadLinked(Builder, ValueTy: LI->getType(),
657 Addr: LI->getPointerOperand(), Ord: LI->getOrdering());
658 TLI->emitAtomicCmpXchgNoStoreLLBalance(Builder);
659
660 LI->replaceAllUsesWith(V: Val);
661 LI->eraseFromParent();
662
663 return true;
664}
665
666bool AtomicExpandImpl::expandAtomicLoadToCmpXchg(LoadInst *LI) {
667 ReplacementIRBuilder Builder(LI, *DL);
668 AtomicOrdering Order = LI->getOrdering();
669 if (Order == AtomicOrdering::Unordered)
670 Order = AtomicOrdering::Monotonic;
671
672 Value *Addr = LI->getPointerOperand();
673 Type *Ty = LI->getType();
674
675 // cmpxchg supports only integer and pointer operands. If the load type is
676 // FP or vector, run the cmpxchg on the same-sized integer and bitcast the
677 // result back; mirrors createCmpXchgInstFun.
678 bool NeedBitcast = Ty->isFloatingPointTy() || Ty->isVectorTy();
679 Type *CmpXchgTy = Ty;
680 if (NeedBitcast)
681 CmpXchgTy = Builder.getIntNTy(N: Ty->getPrimitiveSizeInBits());
682 Constant *DummyVal = Constant::getNullValue(Ty: CmpXchgTy);
683
684 AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
685 Ptr: Addr, Cmp: DummyVal, New: DummyVal, Align: LI->getAlign(), SuccessOrdering: Order,
686 FailureOrdering: AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: Order),
687 SSID: LI->getSyncScopeID());
688 Pair->setVolatile(LI->isVolatile());
689 Value *Loaded = Builder.CreateExtractValue(Agg: Pair, Idxs: 0, Name: "loaded");
690 if (NeedBitcast)
691 Loaded = Builder.CreateBitCast(V: Loaded, DestTy: Ty);
692
693 LI->replaceAllUsesWith(V: Loaded);
694 LI->eraseFromParent();
695
696 return true;
697}
698
699/// Convert an atomic store of a non-integral type to an integer store of the
700/// equivalent bitwidth. We used to not support floating point or vector
701/// atomics in the IR at all. The backends learned to deal with the bitcast
702/// idiom because that was the only way of expressing the notion of a atomic
703/// float or vector store. The long term plan is to teach each backend to
704/// instruction select from the original atomic store, but as a migration
705/// mechanism, we convert back to the old format which the backends understand.
706/// Each backend will need individual work to recognize the new format.
707StoreInst *AtomicExpandImpl::convertAtomicStoreToIntegerType(StoreInst *SI) {
708 ReplacementIRBuilder Builder(SI, *DL);
709 auto *M = SI->getModule();
710 Type *NewTy = getCorrespondingIntegerType(T: SI->getValueOperand()->getType(),
711 DL: M->getDataLayout());
712 Value *NewVal = SI->getValueOperand()->getType()->isPtrOrPtrVectorTy()
713 ? Builder.CreatePtrToInt(V: SI->getValueOperand(), DestTy: NewTy)
714 : Builder.CreateBitCast(V: SI->getValueOperand(), DestTy: NewTy);
715
716 Value *Addr = SI->getPointerOperand();
717
718 StoreInst *NewSI = Builder.CreateStore(Val: NewVal, Ptr: Addr, Props: SI->getProperties());
719 LLVM_DEBUG(dbgs() << "Replaced " << *SI << " with " << *NewSI << "\n");
720 SI->eraseFromParent();
721 return NewSI;
722}
723
724void AtomicExpandImpl::expandAtomicStoreToXChg(StoreInst *SI) {
725 // This function is only called on atomic stores that are too large to be
726 // atomic if implemented as a native store. So we replace them by an
727 // atomic swap, that can be implemented for example as a ldrex/strex on ARM
728 // or lock cmpxchg8/16b on X86, as these are atomic for larger sizes.
729 // It is the responsibility of the target to only signal expansion via
730 // shouldExpandAtomicRMW in cases where this is required and possible.
731 ReplacementIRBuilder Builder(SI, *DL);
732 AtomicOrdering Ordering = SI->getOrdering();
733 assert(Ordering != AtomicOrdering::NotAtomic);
734 AtomicOrdering RMWOrdering = Ordering == AtomicOrdering::Unordered
735 ? AtomicOrdering::Monotonic
736 : Ordering;
737 AtomicRMWInst *AI = Builder.CreateAtomicRMW(
738 Op: AtomicRMWInst::Xchg, Ptr: SI->getPointerOperand(), Val: SI->getValueOperand(),
739 Align: SI->getAlign(), Ordering: RMWOrdering, SSID: SI->getSyncScopeID());
740 AI->setVolatile(SI->isVolatile());
741 SI->eraseFromParent();
742
743 // Now we have an appropriate swap instruction, lower it as usual.
744 tryExpandAtomicRMW(AI);
745}
746
747static void createCmpXchgInstFun(IRBuilderBase &Builder, Value *Addr,
748 Value *Loaded, Value *NewVal, Align AddrAlign,
749 AtomicOrdering MemOpOrder, SyncScope::ID SSID,
750 bool IsVolatile, Value *&Success,
751 Value *&NewLoaded, Instruction *MetadataSrc) {
752 Type *OrigTy = NewVal->getType();
753
754 // This code can go away when cmpxchg supports FP and vector types.
755 assert(!OrigTy->isPointerTy());
756 bool NeedBitcast = OrigTy->isFloatingPointTy() || OrigTy->isVectorTy();
757 if (NeedBitcast) {
758 IntegerType *IntTy = Builder.getIntNTy(N: OrigTy->getPrimitiveSizeInBits());
759 NewVal = Builder.CreateBitCast(V: NewVal, DestTy: IntTy);
760 Loaded = Builder.CreateBitCast(V: Loaded, DestTy: IntTy);
761 }
762
763 AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
764 Ptr: Addr, Cmp: Loaded, New: NewVal, Align: AddrAlign, SuccessOrdering: MemOpOrder,
765 FailureOrdering: AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: MemOpOrder), SSID);
766 Pair->setVolatile(IsVolatile);
767 if (MetadataSrc)
768 copyMetadataForAtomic(Dest&: *Pair, Source: *MetadataSrc);
769
770 Success = Builder.CreateExtractValue(Agg: Pair, Idxs: 1, Name: "success");
771 NewLoaded = Builder.CreateExtractValue(Agg: Pair, Idxs: 0, Name: "newloaded");
772
773 if (NeedBitcast)
774 NewLoaded = Builder.CreateBitCast(V: NewLoaded, DestTy: OrigTy);
775}
776
777bool AtomicExpandImpl::tryExpandAtomicRMW(AtomicRMWInst *AI) {
778 LLVMContext &Ctx = AI->getModule()->getContext();
779 TargetLowering::AtomicExpansionKind Kind = TLI->shouldExpandAtomicRMWInIR(RMW: AI);
780 switch (Kind) {
781 case TargetLoweringBase::AtomicExpansionKind::None:
782 return false;
783 case TargetLoweringBase::AtomicExpansionKind::LLSC: {
784 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
785 unsigned ValueSize = getAtomicOpSize(RMWI: AI);
786 if (ValueSize < MinCASSize) {
787 expandPartwordAtomicRMW(I: AI,
788 ExpansionKind: TargetLoweringBase::AtomicExpansionKind::LLSC);
789 } else {
790 auto PerformOp = [&](IRBuilderBase &Builder, Value *Loaded) {
791 return buildAtomicRMWValue(Op: AI->getOperation(), Builder, Loaded,
792 Val: AI->getValOperand());
793 };
794 expandAtomicOpToLLSC(I: AI, ResultTy: AI->getType(), Addr: AI->getPointerOperand(),
795 AddrAlign: AI->getAlign(), MemOpOrder: AI->getOrdering(), PerformOp);
796 }
797 return true;
798 }
799 case TargetLoweringBase::AtomicExpansionKind::CmpXChg: {
800 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
801 unsigned ValueSize = getAtomicOpSize(RMWI: AI);
802 if (ValueSize < MinCASSize) {
803 expandPartwordAtomicRMW(I: AI,
804 ExpansionKind: TargetLoweringBase::AtomicExpansionKind::CmpXChg);
805 } else {
806 SmallVector<StringRef> SSNs;
807 Ctx.getSyncScopeNames(SSNs);
808 auto MemScope = SSNs[AI->getSyncScopeID()].empty()
809 ? "system"
810 : SSNs[AI->getSyncScopeID()];
811 OptimizationRemarkEmitter ORE(AI->getFunction());
812 ORE.emit(RemarkBuilder: [&]() {
813 return OptimizationRemark(DEBUG_TYPE, "Passed", AI)
814 << "A compare and swap loop was generated for an atomic "
815 << AI->getOperationName(Op: AI->getOperation()) << " operation at "
816 << MemScope << " memory scope";
817 });
818 expandAtomicRMWToCmpXchg(AI, CreateCmpXchg: createCmpXchgInstFun);
819 }
820 return true;
821 }
822 case TargetLoweringBase::AtomicExpansionKind::MaskedIntrinsic: {
823 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
824 unsigned ValueSize = getAtomicOpSize(RMWI: AI);
825 if (ValueSize < MinCASSize) {
826 AtomicRMWInst::BinOp Op = AI->getOperation();
827 // Widen And/Or/Xor and give the target another chance at expanding it.
828 if (Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||
829 Op == AtomicRMWInst::And) {
830 tryExpandAtomicRMW(AI: widenPartwordAtomicRMW(AI));
831 return true;
832 }
833 }
834 expandAtomicRMWToMaskedIntrinsic(AI);
835 return true;
836 }
837 case TargetLoweringBase::AtomicExpansionKind::BitTestIntrinsic: {
838 TLI->emitBitTestAtomicRMWIntrinsic(AI);
839 return true;
840 }
841 case TargetLoweringBase::AtomicExpansionKind::CmpArithIntrinsic: {
842 TLI->emitCmpArithAtomicRMWIntrinsic(AI);
843 return true;
844 }
845 case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
846 return lowerAtomicRMWInst(RMWI: AI);
847 case TargetLoweringBase::AtomicExpansionKind::CustomExpand:
848 TLI->emitExpandAtomicRMW(AI);
849 return true;
850 default:
851 llvm_unreachable("Unhandled case in tryExpandAtomicRMW");
852 }
853}
854
855namespace {
856
857struct PartwordMaskValues {
858 // These three fields are guaranteed to be set by createMaskInstrs.
859 Type *WordType = nullptr;
860 Type *ValueType = nullptr;
861 Type *IntValueType = nullptr;
862 Value *AlignedAddr = nullptr;
863 Align AlignedAddrAlignment;
864 // The remaining fields can be null.
865 Value *ShiftAmt = nullptr;
866 Value *Mask = nullptr;
867 Value *Inv_Mask = nullptr;
868};
869
870[[maybe_unused]]
871raw_ostream &operator<<(raw_ostream &O, const PartwordMaskValues &PMV) {
872 auto PrintObj = [&O](auto *V) {
873 if (V)
874 O << *V;
875 else
876 O << "nullptr";
877 O << '\n';
878 };
879 O << "PartwordMaskValues {\n";
880 O << " WordType: ";
881 PrintObj(PMV.WordType);
882 O << " ValueType: ";
883 PrintObj(PMV.ValueType);
884 O << " AlignedAddr: ";
885 PrintObj(PMV.AlignedAddr);
886 O << " AlignedAddrAlignment: " << PMV.AlignedAddrAlignment.value() << '\n';
887 O << " ShiftAmt: ";
888 PrintObj(PMV.ShiftAmt);
889 O << " Mask: ";
890 PrintObj(PMV.Mask);
891 O << " Inv_Mask: ";
892 PrintObj(PMV.Inv_Mask);
893 O << "}\n";
894 return O;
895}
896
897} // end anonymous namespace
898
899/// This is a helper function which builds instructions to provide
900/// values necessary for partword atomic operations. It takes an
901/// incoming address, Addr, and ValueType, and constructs the address,
902/// shift-amounts and masks needed to work with a larger value of size
903/// WordSize.
904///
905/// AlignedAddr: Addr rounded down to a multiple of WordSize
906///
907/// ShiftAmt: Number of bits to right-shift a WordSize value loaded
908/// from AlignAddr for it to have the same value as if
909/// ValueType was loaded from Addr.
910///
911/// Mask: Value to mask with the value loaded from AlignAddr to
912/// include only the part that would've been loaded from Addr.
913///
914/// Inv_Mask: The inverse of Mask.
915static PartwordMaskValues createMaskInstrs(IRBuilderBase &Builder,
916 Instruction *I, Type *ValueType,
917 Value *Addr, Align AddrAlign,
918 unsigned MinWordSize) {
919 PartwordMaskValues PMV;
920
921 Module *M = I->getModule();
922 LLVMContext &Ctx = M->getContext();
923 const DataLayout &DL = M->getDataLayout();
924 unsigned ValueSize = DL.getTypeStoreSize(Ty: ValueType);
925
926 PMV.ValueType = PMV.IntValueType = ValueType;
927 if (PMV.ValueType->isFloatingPointTy() || PMV.ValueType->isVectorTy())
928 PMV.IntValueType =
929 Type::getIntNTy(C&: Ctx, N: ValueType->getPrimitiveSizeInBits());
930
931 PMV.WordType = MinWordSize > ValueSize ? Type::getIntNTy(C&: Ctx, N: MinWordSize * 8)
932 : ValueType;
933 if (PMV.ValueType == PMV.WordType) {
934 PMV.AlignedAddr = Addr;
935 PMV.AlignedAddrAlignment = AddrAlign;
936 PMV.ShiftAmt = ConstantInt::get(Ty: PMV.ValueType, V: 0);
937 PMV.Mask = ConstantInt::get(Ty: PMV.ValueType, V: ~0, /*isSigned*/ IsSigned: true);
938 return PMV;
939 }
940
941 PMV.AlignedAddrAlignment = Align(MinWordSize);
942
943 assert(ValueSize < MinWordSize);
944
945 PointerType *PtrTy = cast<PointerType>(Val: Addr->getType());
946 IntegerType *IntTy = DL.getIndexType(C&: Ctx, AddressSpace: PtrTy->getAddressSpace());
947 Value *PtrLSB;
948
949 if (AddrAlign < MinWordSize) {
950 PMV.AlignedAddr = Builder.CreateIntrinsic(
951 ID: Intrinsic::ptrmask, OverloadTypes: {PtrTy, IntTy},
952 Args: {Addr, ConstantInt::getSigned(Ty: IntTy, V: ~(uint64_t)(MinWordSize - 1))},
953 FMFSource: nullptr, Name: "AlignedAddr");
954
955 Value *AddrInt = Builder.CreatePtrToInt(V: Addr, DestTy: IntTy);
956 PtrLSB = Builder.CreateAnd(LHS: AddrInt, RHS: MinWordSize - 1, Name: "PtrLSB");
957 } else {
958 // If the alignment is high enough, the LSB are known 0.
959 PMV.AlignedAddr = Addr;
960 PtrLSB = ConstantInt::getNullValue(Ty: IntTy);
961 }
962
963 if (DL.isLittleEndian()) {
964 // turn bytes into bits
965 PMV.ShiftAmt = Builder.CreateShl(LHS: PtrLSB, RHS: 3);
966 } else {
967 // turn bytes into bits, and count from the other side.
968 PMV.ShiftAmt = Builder.CreateShl(
969 LHS: Builder.CreateXor(LHS: PtrLSB, RHS: MinWordSize - ValueSize), RHS: 3);
970 }
971
972 PMV.ShiftAmt = Builder.CreateTrunc(V: PMV.ShiftAmt, DestTy: PMV.WordType, Name: "ShiftAmt");
973 PMV.Mask = Builder.CreateShl(
974 LHS: ConstantInt::get(Ty: PMV.WordType, V: (1 << (ValueSize * 8)) - 1), RHS: PMV.ShiftAmt,
975 Name: "Mask");
976
977 PMV.Inv_Mask = Builder.CreateNot(V: PMV.Mask, Name: "Inv_Mask");
978
979 return PMV;
980}
981
982static Value *extractMaskedValue(IRBuilderBase &Builder, Value *WideWord,
983 const PartwordMaskValues &PMV) {
984 assert(WideWord->getType() == PMV.WordType && "Widened type mismatch");
985 if (PMV.WordType == PMV.ValueType)
986 return WideWord;
987
988 Value *Shift = Builder.CreateLShr(LHS: WideWord, RHS: PMV.ShiftAmt, Name: "shifted");
989 Value *Trunc = Builder.CreateTrunc(V: Shift, DestTy: PMV.IntValueType, Name: "extracted");
990 return Builder.CreateBitCast(V: Trunc, DestTy: PMV.ValueType);
991}
992
993static Value *insertMaskedValue(IRBuilderBase &Builder, Value *WideWord,
994 Value *Updated, const PartwordMaskValues &PMV) {
995 assert(WideWord->getType() == PMV.WordType && "Widened type mismatch");
996 assert(Updated->getType() == PMV.ValueType && "Value type mismatch");
997 if (PMV.WordType == PMV.ValueType)
998 return Updated;
999
1000 Updated = Builder.CreateBitCast(V: Updated, DestTy: PMV.IntValueType);
1001
1002 Value *ZExt = Builder.CreateZExt(V: Updated, DestTy: PMV.WordType, Name: "extended");
1003 Value *Shift =
1004 Builder.CreateShl(LHS: ZExt, RHS: PMV.ShiftAmt, Name: "shifted", /*HasNUW*/ true);
1005 Value *And = Builder.CreateAnd(LHS: WideWord, RHS: PMV.Inv_Mask, Name: "unmasked");
1006 Value *Or = Builder.CreateOr(LHS: And, RHS: Shift, Name: "inserted");
1007 return Or;
1008}
1009
1010/// Emit IR to implement a masked version of a given atomicrmw
1011/// operation. (That is, only the bits under the Mask should be
1012/// affected by the operation)
1013static Value *performMaskedAtomicOp(AtomicRMWInst::BinOp Op,
1014 IRBuilderBase &Builder, Value *Loaded,
1015 Value *Shifted_Inc, Value *Inc,
1016 const PartwordMaskValues &PMV) {
1017 // TODO: update to use
1018 // https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge in order
1019 // to merge bits from two values without requiring PMV.Inv_Mask.
1020 switch (Op) {
1021 case AtomicRMWInst::Xchg: {
1022 Value *Loaded_MaskOut = Builder.CreateAnd(LHS: Loaded, RHS: PMV.Inv_Mask);
1023 Value *FinalVal = Builder.CreateOr(LHS: Loaded_MaskOut, RHS: Shifted_Inc);
1024 return FinalVal;
1025 }
1026 case AtomicRMWInst::Or:
1027 case AtomicRMWInst::Xor:
1028 case AtomicRMWInst::And:
1029 llvm_unreachable("Or/Xor/And handled by widenPartwordAtomicRMW");
1030 case AtomicRMWInst::Add:
1031 case AtomicRMWInst::Sub:
1032 case AtomicRMWInst::Nand: {
1033 // The other arithmetic ops need to be masked into place.
1034 Value *NewVal = buildAtomicRMWValue(Op, Builder, Loaded, Val: Shifted_Inc);
1035 Value *NewVal_Masked = Builder.CreateAnd(LHS: NewVal, RHS: PMV.Mask);
1036 Value *Loaded_MaskOut = Builder.CreateAnd(LHS: Loaded, RHS: PMV.Inv_Mask);
1037 Value *FinalVal = Builder.CreateOr(LHS: Loaded_MaskOut, RHS: NewVal_Masked);
1038 return FinalVal;
1039 }
1040 case AtomicRMWInst::Max:
1041 case AtomicRMWInst::Min:
1042 case AtomicRMWInst::UMax:
1043 case AtomicRMWInst::UMin:
1044 case AtomicRMWInst::FAdd:
1045 case AtomicRMWInst::FSub:
1046 case AtomicRMWInst::FMin:
1047 case AtomicRMWInst::FMax:
1048 case AtomicRMWInst::FMaximum:
1049 case AtomicRMWInst::FMinimum:
1050 case AtomicRMWInst::FMaximumNum:
1051 case AtomicRMWInst::FMinimumNum:
1052 case AtomicRMWInst::UIncWrap:
1053 case AtomicRMWInst::UDecWrap:
1054 case AtomicRMWInst::USubCond:
1055 case AtomicRMWInst::USubSat: {
1056 // Finally, other ops will operate on the full value, so truncate down to
1057 // the original size, and expand out again after doing the
1058 // operation. Bitcasts will be inserted for FP values.
1059 Value *Loaded_Extract = extractMaskedValue(Builder, WideWord: Loaded, PMV);
1060 Value *NewVal = buildAtomicRMWValue(Op, Builder, Loaded: Loaded_Extract, Val: Inc);
1061 Value *FinalVal = insertMaskedValue(Builder, WideWord: Loaded, Updated: NewVal, PMV);
1062 return FinalVal;
1063 }
1064 default:
1065 llvm_unreachable("Unknown atomic op");
1066 }
1067}
1068
1069/// Expand a sub-word atomicrmw operation into an appropriate
1070/// word-sized operation.
1071///
1072/// It will create an LL/SC or cmpxchg loop, as appropriate, the same
1073/// way as a typical atomicrmw expansion. The only difference here is
1074/// that the operation inside of the loop may operate upon only a
1075/// part of the value.
1076void AtomicExpandImpl::expandPartwordAtomicRMW(
1077 AtomicRMWInst *AI, TargetLoweringBase::AtomicExpansionKind ExpansionKind) {
1078 // Widen And/Or/Xor and give the target another chance at expanding it.
1079 AtomicRMWInst::BinOp Op = AI->getOperation();
1080 if (Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||
1081 Op == AtomicRMWInst::And) {
1082 tryExpandAtomicRMW(AI: widenPartwordAtomicRMW(AI));
1083 return;
1084 }
1085 AtomicOrdering MemOpOrder = AI->getOrdering();
1086 SyncScope::ID SSID = AI->getSyncScopeID();
1087
1088 ReplacementIRBuilder Builder(AI, *DL);
1089
1090 PartwordMaskValues PMV =
1091 createMaskInstrs(Builder, I: AI, ValueType: AI->getType(), Addr: AI->getPointerOperand(),
1092 AddrAlign: AI->getAlign(), MinWordSize: TLI->getMinCmpXchgSizeInBits() / 8);
1093
1094 Value *ValOperand_Shifted = nullptr;
1095 if (Op == AtomicRMWInst::Xchg || Op == AtomicRMWInst::Add ||
1096 Op == AtomicRMWInst::Sub || Op == AtomicRMWInst::Nand) {
1097 Value *ValOp = Builder.CreateBitCast(V: AI->getValOperand(), DestTy: PMV.IntValueType);
1098 ValOperand_Shifted =
1099 Builder.CreateShl(LHS: Builder.CreateZExt(V: ValOp, DestTy: PMV.WordType), RHS: PMV.ShiftAmt,
1100 Name: "ValOperand_Shifted");
1101 }
1102
1103 auto PerformPartwordOp = [&](IRBuilderBase &Builder, Value *Loaded) {
1104 return performMaskedAtomicOp(Op, Builder, Loaded, Shifted_Inc: ValOperand_Shifted,
1105 Inc: AI->getValOperand(), PMV);
1106 };
1107
1108 Value *OldResult;
1109 if (ExpansionKind == TargetLoweringBase::AtomicExpansionKind::CmpXChg) {
1110 OldResult = insertRMWCmpXchgLoop(Builder, ResultType: PMV.WordType, Addr: PMV.AlignedAddr,
1111 AddrAlign: PMV.AlignedAddrAlignment, MemOpOrder, SSID,
1112 IsVolatile: AI->isVolatile(), PerformOp: PerformPartwordOp,
1113 CreateCmpXchg: createCmpXchgInstFun, MetadataSrc: AI);
1114 } else {
1115 assert(ExpansionKind == TargetLoweringBase::AtomicExpansionKind::LLSC);
1116 OldResult = insertRMWLLSCLoop(Builder, ResultTy: PMV.WordType, Addr: PMV.AlignedAddr,
1117 AddrAlign: PMV.AlignedAddrAlignment, MemOpOrder,
1118 PerformOp: PerformPartwordOp);
1119 }
1120
1121 Value *FinalOldResult = extractMaskedValue(Builder, WideWord: OldResult, PMV);
1122 AI->replaceAllUsesWith(V: FinalOldResult);
1123 AI->eraseFromParent();
1124}
1125
1126// Widen the bitwise atomicrmw (or/xor/and) to the minimum supported width.
1127AtomicRMWInst *AtomicExpandImpl::widenPartwordAtomicRMW(AtomicRMWInst *AI) {
1128 ReplacementIRBuilder Builder(AI, *DL);
1129 AtomicRMWInst::BinOp Op = AI->getOperation();
1130
1131 assert((Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||
1132 Op == AtomicRMWInst::And) &&
1133 "Unable to widen operation");
1134
1135 PartwordMaskValues PMV =
1136 createMaskInstrs(Builder, I: AI, ValueType: AI->getType(), Addr: AI->getPointerOperand(),
1137 AddrAlign: AI->getAlign(), MinWordSize: TLI->getMinCmpXchgSizeInBits() / 8);
1138
1139 Value *ValOperand_Shifted =
1140 Builder.CreateShl(LHS: Builder.CreateZExt(V: AI->getValOperand(), DestTy: PMV.WordType),
1141 RHS: PMV.ShiftAmt, Name: "ValOperand_Shifted");
1142
1143 Value *NewOperand;
1144
1145 if (Op == AtomicRMWInst::And)
1146 NewOperand =
1147 Builder.CreateOr(LHS: ValOperand_Shifted, RHS: PMV.Inv_Mask, Name: "AndOperand");
1148 else
1149 NewOperand = ValOperand_Shifted;
1150
1151 AtomicRMWInst *NewAI = Builder.CreateAtomicRMW(
1152 Op, Ptr: PMV.AlignedAddr, Val: NewOperand, Align: PMV.AlignedAddrAlignment,
1153 Ordering: AI->getOrdering(), SSID: AI->getSyncScopeID());
1154
1155 NewAI->setVolatile(AI->isVolatile());
1156 copyMetadataForAtomic(Dest&: *NewAI, Source: *AI);
1157
1158 Value *FinalOldResult = extractMaskedValue(Builder, WideWord: NewAI, PMV);
1159 AI->replaceAllUsesWith(V: FinalOldResult);
1160 AI->eraseFromParent();
1161 return NewAI;
1162}
1163
1164bool AtomicExpandImpl::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
1165 // The basic idea here is that we're expanding a cmpxchg of a
1166 // smaller memory size up to a word-sized cmpxchg. To do this, we
1167 // need to add a retry-loop for strong cmpxchg, so that
1168 // modifications to other parts of the word don't cause a spurious
1169 // failure.
1170
1171 // This generates code like the following:
1172 // [[Setup mask values PMV.*]]
1173 // %NewVal_Shifted = shl i32 %NewVal, %PMV.ShiftAmt
1174 // %Cmp_Shifted = shl i32 %Cmp, %PMV.ShiftAmt
1175 // %InitLoaded = load i32* %addr
1176 // %InitLoaded_MaskOut = and i32 %InitLoaded, %PMV.Inv_Mask
1177 // br partword.cmpxchg.loop
1178 // partword.cmpxchg.loop:
1179 // %Loaded_MaskOut = phi i32 [ %InitLoaded_MaskOut, %entry ],
1180 // [ %OldVal_MaskOut, %partword.cmpxchg.failure ]
1181 // %FullWord_NewVal = or i32 %Loaded_MaskOut, %NewVal_Shifted
1182 // %FullWord_Cmp = or i32 %Loaded_MaskOut, %Cmp_Shifted
1183 // %NewCI = cmpxchg i32* %PMV.AlignedAddr, i32 %FullWord_Cmp,
1184 // i32 %FullWord_NewVal success_ordering failure_ordering
1185 // %OldVal = extractvalue { i32, i1 } %NewCI, 0
1186 // %Success = extractvalue { i32, i1 } %NewCI, 1
1187 // br i1 %Success, label %partword.cmpxchg.end,
1188 // label %partword.cmpxchg.failure
1189 // partword.cmpxchg.failure:
1190 // %OldVal_MaskOut = and i32 %OldVal, %PMV.Inv_Mask
1191 // %ShouldContinue = icmp ne i32 %Loaded_MaskOut, %OldVal_MaskOut
1192 // br i1 %ShouldContinue, label %partword.cmpxchg.loop,
1193 // label %partword.cmpxchg.end
1194 // partword.cmpxchg.end:
1195 // %tmp1 = lshr i32 %OldVal, %PMV.ShiftAmt
1196 // %FinalOldVal = trunc i32 %tmp1 to i8
1197 // %tmp2 = insertvalue { i8, i1 } undef, i8 %FinalOldVal, 0
1198 // %Res = insertvalue { i8, i1 } %25, i1 %Success, 1
1199
1200 Value *Addr = CI->getPointerOperand();
1201 Value *Cmp = CI->getCompareOperand();
1202 Value *NewVal = CI->getNewValOperand();
1203
1204 BasicBlock *BB = CI->getParent();
1205 Function *F = BB->getParent();
1206 ReplacementIRBuilder Builder(CI, *DL);
1207 LLVMContext &Ctx = Builder.getContext();
1208
1209 BasicBlock *EndBB =
1210 BB->splitBasicBlock(I: CI->getIterator(), BBName: "partword.cmpxchg.end");
1211 auto FailureBB =
1212 BasicBlock::Create(Context&: Ctx, Name: "partword.cmpxchg.failure", Parent: F, InsertBefore: EndBB);
1213 auto LoopBB = BasicBlock::Create(Context&: Ctx, Name: "partword.cmpxchg.loop", Parent: F, InsertBefore: FailureBB);
1214
1215 // The split call above "helpfully" added a branch at the end of BB
1216 // (to the wrong place).
1217 std::prev(x: BB->end())->eraseFromParent();
1218 Builder.SetInsertPoint(BB);
1219
1220 PartwordMaskValues PMV =
1221 createMaskInstrs(Builder, I: CI, ValueType: CI->getCompareOperand()->getType(), Addr,
1222 AddrAlign: CI->getAlign(), MinWordSize: TLI->getMinCmpXchgSizeInBits() / 8);
1223
1224 // Shift the incoming values over, into the right location in the word.
1225 Value *NewVal_Shifted =
1226 Builder.CreateShl(LHS: Builder.CreateZExt(V: NewVal, DestTy: PMV.WordType), RHS: PMV.ShiftAmt);
1227 Value *Cmp_Shifted =
1228 Builder.CreateShl(LHS: Builder.CreateZExt(V: Cmp, DestTy: PMV.WordType), RHS: PMV.ShiftAmt);
1229
1230 // Load the entire current word, and mask into place the expected and new
1231 // values
1232 LoadInst *InitLoaded = Builder.CreateLoad(Ty: PMV.WordType, Ptr: PMV.AlignedAddr);
1233 Value *InitLoaded_MaskOut = Builder.CreateAnd(LHS: InitLoaded, RHS: PMV.Inv_Mask);
1234 Builder.CreateBr(Dest: LoopBB);
1235
1236 // partword.cmpxchg.loop:
1237 Builder.SetInsertPoint(LoopBB);
1238 PHINode *Loaded_MaskOut = Builder.CreatePHI(Ty: PMV.WordType, NumReservedValues: 2);
1239 Loaded_MaskOut->addIncoming(V: InitLoaded_MaskOut, BB);
1240
1241 // The initial load must be atomic with the same synchronization scope
1242 // to avoid a data race with concurrent stores. If the instruction being
1243 // emulated is volatile, issue a volatile load.
1244 // addIncoming is done first so that any replaceAllUsesWith calls during
1245 // normalization correctly update the PHI incoming value.
1246 InitLoaded->setVolatile(CI->isVolatile());
1247 if (TLI->shouldIssueAtomicLoadForAtomicEmulationLoop()) {
1248 InitLoaded->setAtomic(Ordering: AtomicOrdering::Monotonic, SSID: CI->getSyncScopeID());
1249 // The newly created load might need to be lowered further. Because it is
1250 // created in the same block as the atomicrmw, the AtomicExpand loop will
1251 // not process it again.
1252 processAtomicInstr(I: InitLoaded);
1253 }
1254
1255 // Mask/Or the expected and new values into place in the loaded word.
1256 Value *FullWord_NewVal = Builder.CreateOr(LHS: Loaded_MaskOut, RHS: NewVal_Shifted);
1257 Value *FullWord_Cmp = Builder.CreateOr(LHS: Loaded_MaskOut, RHS: Cmp_Shifted);
1258 AtomicCmpXchgInst *NewCI = Builder.CreateAtomicCmpXchg(
1259 Ptr: PMV.AlignedAddr, Cmp: FullWord_Cmp, New: FullWord_NewVal, Align: PMV.AlignedAddrAlignment,
1260 SuccessOrdering: CI->getSuccessOrdering(), FailureOrdering: CI->getFailureOrdering(), SSID: CI->getSyncScopeID());
1261 NewCI->setVolatile(CI->isVolatile());
1262 // When we're building a strong cmpxchg, we need a loop, so you
1263 // might think we could use a weak cmpxchg inside. But, using strong
1264 // allows the below comparison for ShouldContinue, and we're
1265 // expecting the underlying cmpxchg to be a machine instruction,
1266 // which is strong anyways.
1267 NewCI->setWeak(CI->isWeak());
1268
1269 Value *OldVal = Builder.CreateExtractValue(Agg: NewCI, Idxs: 0);
1270 Value *Success = Builder.CreateExtractValue(Agg: NewCI, Idxs: 1);
1271
1272 if (CI->isWeak())
1273 Builder.CreateBr(Dest: EndBB);
1274 else
1275 Builder.CreateCondBr(Cond: Success, True: EndBB, False: FailureBB);
1276
1277 // partword.cmpxchg.failure:
1278 Builder.SetInsertPoint(FailureBB);
1279 // Upon failure, verify that the masked-out part of the loaded value
1280 // has been modified. If it didn't, abort the cmpxchg, since the
1281 // masked-in part must've.
1282 Value *OldVal_MaskOut = Builder.CreateAnd(LHS: OldVal, RHS: PMV.Inv_Mask);
1283 Value *ShouldContinue = Builder.CreateICmpNE(LHS: Loaded_MaskOut, RHS: OldVal_MaskOut);
1284 Builder.CreateCondBr(Cond: ShouldContinue, True: LoopBB, False: EndBB);
1285
1286 // Add the second value to the phi from above
1287 Loaded_MaskOut->addIncoming(V: OldVal_MaskOut, BB: FailureBB);
1288
1289 // partword.cmpxchg.end:
1290 Builder.SetInsertPoint(CI);
1291
1292 Value *FinalOldVal = extractMaskedValue(Builder, WideWord: OldVal, PMV);
1293 Value *Res = PoisonValue::get(T: CI->getType());
1294 Res = Builder.CreateInsertValue(Agg: Res, Val: FinalOldVal, Idxs: 0);
1295 Res = Builder.CreateInsertValue(Agg: Res, Val: Success, Idxs: 1);
1296
1297 CI->replaceAllUsesWith(V: Res);
1298 CI->eraseFromParent();
1299 return true;
1300}
1301
1302void AtomicExpandImpl::expandAtomicOpToLLSC(
1303 Instruction *I, Type *ResultType, Value *Addr, Align AddrAlign,
1304 AtomicOrdering MemOpOrder,
1305 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp) {
1306 ReplacementIRBuilder Builder(I, *DL);
1307 Value *Loaded = insertRMWLLSCLoop(Builder, ResultTy: ResultType, Addr, AddrAlign,
1308 MemOpOrder, PerformOp);
1309
1310 I->replaceAllUsesWith(V: Loaded);
1311 I->eraseFromParent();
1312}
1313
1314void AtomicExpandImpl::expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI) {
1315 ReplacementIRBuilder Builder(AI, *DL);
1316
1317 PartwordMaskValues PMV =
1318 createMaskInstrs(Builder, I: AI, ValueType: AI->getType(), Addr: AI->getPointerOperand(),
1319 AddrAlign: AI->getAlign(), MinWordSize: TLI->getMinCmpXchgSizeInBits() / 8);
1320
1321 // The value operand must be sign-extended for signed min/max so that the
1322 // target's signed comparison instructions can be used. Otherwise, just
1323 // zero-ext.
1324 Instruction::CastOps CastOp = Instruction::ZExt;
1325 AtomicRMWInst::BinOp RMWOp = AI->getOperation();
1326 if (RMWOp == AtomicRMWInst::Max || RMWOp == AtomicRMWInst::Min)
1327 CastOp = Instruction::SExt;
1328
1329 Value *ValOperand_Shifted = Builder.CreateShl(
1330 LHS: Builder.CreateCast(Op: CastOp, V: AI->getValOperand(), DestTy: PMV.WordType),
1331 RHS: PMV.ShiftAmt, Name: "ValOperand_Shifted");
1332 Value *OldResult = TLI->emitMaskedAtomicRMWIntrinsic(
1333 Builder, AI, AlignedAddr: PMV.AlignedAddr, Incr: ValOperand_Shifted, Mask: PMV.Mask, ShiftAmt: PMV.ShiftAmt,
1334 Ord: AI->getOrdering());
1335 Value *FinalOldResult = extractMaskedValue(Builder, WideWord: OldResult, PMV);
1336 AI->replaceAllUsesWith(V: FinalOldResult);
1337 AI->eraseFromParent();
1338}
1339
1340void AtomicExpandImpl::expandAtomicCmpXchgToMaskedIntrinsic(
1341 AtomicCmpXchgInst *CI) {
1342 ReplacementIRBuilder Builder(CI, *DL);
1343
1344 PartwordMaskValues PMV = createMaskInstrs(
1345 Builder, I: CI, ValueType: CI->getCompareOperand()->getType(), Addr: CI->getPointerOperand(),
1346 AddrAlign: CI->getAlign(), MinWordSize: TLI->getMinCmpXchgSizeInBits() / 8);
1347
1348 Value *CmpVal_Shifted = Builder.CreateShl(
1349 LHS: Builder.CreateZExt(V: CI->getCompareOperand(), DestTy: PMV.WordType), RHS: PMV.ShiftAmt,
1350 Name: "CmpVal_Shifted");
1351 Value *NewVal_Shifted = Builder.CreateShl(
1352 LHS: Builder.CreateZExt(V: CI->getNewValOperand(), DestTy: PMV.WordType), RHS: PMV.ShiftAmt,
1353 Name: "NewVal_Shifted");
1354 Value *OldVal = TLI->emitMaskedAtomicCmpXchgIntrinsic(
1355 Builder, CI, AlignedAddr: PMV.AlignedAddr, CmpVal: CmpVal_Shifted, NewVal: NewVal_Shifted, Mask: PMV.Mask,
1356 Ord: CI->getMergedOrdering());
1357 Value *FinalOldVal = extractMaskedValue(Builder, WideWord: OldVal, PMV);
1358 Value *Res = PoisonValue::get(T: CI->getType());
1359 Res = Builder.CreateInsertValue(Agg: Res, Val: FinalOldVal, Idxs: 0);
1360 Value *Success = Builder.CreateICmpEQ(
1361 LHS: CmpVal_Shifted, RHS: Builder.CreateAnd(LHS: OldVal, RHS: PMV.Mask), Name: "Success");
1362 Res = Builder.CreateInsertValue(Agg: Res, Val: Success, Idxs: 1);
1363
1364 CI->replaceAllUsesWith(V: Res);
1365 CI->eraseFromParent();
1366}
1367
1368Value *AtomicExpandImpl::insertRMWLLSCLoop(
1369 IRBuilderBase &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
1370 AtomicOrdering MemOpOrder,
1371 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp) {
1372 LLVMContext &Ctx = Builder.getContext();
1373 BasicBlock *BB = Builder.GetInsertBlock();
1374 Function *F = BB->getParent();
1375
1376 assert(AddrAlign >= F->getDataLayout().getTypeStoreSize(ResultTy) &&
1377 "Expected at least natural alignment at this point.");
1378
1379 // Given: atomicrmw some_op iN* %addr, iN %incr ordering
1380 //
1381 // The standard expansion we produce is:
1382 // [...]
1383 // atomicrmw.start:
1384 // %loaded = @load.linked(%addr)
1385 // %new = some_op iN %loaded, %incr
1386 // %stored = @store_conditional(%new, %addr)
1387 // %try_again = icmp i32 ne %stored, 0
1388 // br i1 %try_again, label %loop, label %atomicrmw.end
1389 // atomicrmw.end:
1390 // [...]
1391 BasicBlock *ExitBB =
1392 BB->splitBasicBlock(I: Builder.GetInsertPoint(), BBName: "atomicrmw.end");
1393 BasicBlock *LoopBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.start", Parent: F, InsertBefore: ExitBB);
1394
1395 // The split call above "helpfully" added a branch at the end of BB (to the
1396 // wrong place).
1397 std::prev(x: BB->end())->eraseFromParent();
1398 Builder.SetInsertPoint(BB);
1399 Builder.CreateBr(Dest: LoopBB);
1400
1401 // Start the main loop block now that we've taken care of the preliminaries.
1402 Builder.SetInsertPoint(LoopBB);
1403 Value *Loaded = TLI->emitLoadLinked(Builder, ValueTy: ResultTy, Addr, Ord: MemOpOrder);
1404
1405 Value *NewVal = PerformOp(Builder, Loaded);
1406
1407 Value *StoreSuccess =
1408 TLI->emitStoreConditional(Builder, Val: NewVal, Addr, Ord: MemOpOrder);
1409 Value *TryAgain = Builder.CreateICmpNE(
1410 LHS: StoreSuccess, RHS: ConstantInt::get(Ty: IntegerType::get(C&: Ctx, NumBits: 32), V: 0), Name: "tryagain");
1411
1412 Instruction *CondBr = Builder.CreateCondBr(Cond: TryAgain, True: LoopBB, False: ExitBB);
1413
1414 // Atomic RMW expands to a Load-linked / Store-Conditional loop, because it is
1415 // hard to predict precise branch weigths we mark the branch as "unknown"
1416 // (50/50) to prevent misleading optimizations.
1417 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *CondBr, DEBUG_TYPE);
1418
1419 Builder.SetInsertPoint(TheBB: ExitBB, IP: ExitBB->begin());
1420 return Loaded;
1421}
1422
1423/// Convert an atomic cmpxchg of a non-integral type to an integer cmpxchg of
1424/// the equivalent bitwidth. We used to not support pointer cmpxchg in the
1425/// IR. As a migration step, we convert back to what use to be the standard
1426/// way to represent a pointer cmpxchg so that we can update backends one by
1427/// one.
1428AtomicCmpXchgInst *
1429AtomicExpandImpl::convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI) {
1430 auto *M = CI->getModule();
1431 Type *NewTy = getCorrespondingIntegerType(T: CI->getCompareOperand()->getType(),
1432 DL: M->getDataLayout());
1433
1434 ReplacementIRBuilder Builder(CI, *DL);
1435
1436 Value *Addr = CI->getPointerOperand();
1437
1438 Value *NewCmp = Builder.CreatePtrToInt(V: CI->getCompareOperand(), DestTy: NewTy);
1439 Value *NewNewVal = Builder.CreatePtrToInt(V: CI->getNewValOperand(), DestTy: NewTy);
1440
1441 auto *NewCI = Builder.CreateAtomicCmpXchg(
1442 Ptr: Addr, Cmp: NewCmp, New: NewNewVal, Align: CI->getAlign(), SuccessOrdering: CI->getSuccessOrdering(),
1443 FailureOrdering: CI->getFailureOrdering(), SSID: CI->getSyncScopeID());
1444 NewCI->setVolatile(CI->isVolatile());
1445 NewCI->setWeak(CI->isWeak());
1446 LLVM_DEBUG(dbgs() << "Replaced " << *CI << " with " << *NewCI << "\n");
1447
1448 Value *OldVal = Builder.CreateExtractValue(Agg: NewCI, Idxs: 0);
1449 Value *Succ = Builder.CreateExtractValue(Agg: NewCI, Idxs: 1);
1450
1451 OldVal = Builder.CreateIntToPtr(V: OldVal, DestTy: CI->getCompareOperand()->getType());
1452
1453 Value *Res = PoisonValue::get(T: CI->getType());
1454 Res = Builder.CreateInsertValue(Agg: Res, Val: OldVal, Idxs: 0);
1455 Res = Builder.CreateInsertValue(Agg: Res, Val: Succ, Idxs: 1);
1456
1457 CI->replaceAllUsesWith(V: Res);
1458 CI->eraseFromParent();
1459 return NewCI;
1460}
1461
1462bool AtomicExpandImpl::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
1463 AtomicOrdering SuccessOrder = CI->getSuccessOrdering();
1464 AtomicOrdering FailureOrder = CI->getFailureOrdering();
1465 Value *Addr = CI->getPointerOperand();
1466 BasicBlock *BB = CI->getParent();
1467 Function *F = BB->getParent();
1468 LLVMContext &Ctx = F->getContext();
1469 // If shouldInsertFencesForAtomic() returns true, then the target does not
1470 // want to deal with memory orders, and emitLeading/TrailingFence should take
1471 // care of everything. Otherwise, emitLeading/TrailingFence are no-op and we
1472 // should preserve the ordering.
1473 bool ShouldInsertFencesForAtomic = TLI->shouldInsertFencesForAtomic(I: CI);
1474 AtomicOrdering MemOpOrder = ShouldInsertFencesForAtomic
1475 ? AtomicOrdering::Monotonic
1476 : CI->getMergedOrdering();
1477
1478 // In implementations which use a barrier to achieve release semantics, we can
1479 // delay emitting this barrier until we know a store is actually going to be
1480 // attempted. The cost of this delay is that we need 2 copies of the block
1481 // emitting the load-linked, affecting code size.
1482 //
1483 // Ideally, this logic would be unconditional except for the minsize check
1484 // since in other cases the extra blocks naturally collapse down to the
1485 // minimal loop. Unfortunately, this puts too much stress on later
1486 // optimisations so we avoid emitting the extra logic in those cases too.
1487 bool HasReleasedLoadBB = !CI->isWeak() && ShouldInsertFencesForAtomic &&
1488 SuccessOrder != AtomicOrdering::Monotonic &&
1489 SuccessOrder != AtomicOrdering::Acquire &&
1490 !F->hasMinSize();
1491
1492 // There's no overhead for sinking the release barrier in a weak cmpxchg, so
1493 // do it even on minsize.
1494 bool UseUnconditionalReleaseBarrier = F->hasMinSize() && !CI->isWeak();
1495
1496 // Given: cmpxchg some_op iN* %addr, iN %desired, iN %new success_ord fail_ord
1497 //
1498 // The full expansion we produce is:
1499 // [...]
1500 // %aligned.addr = ...
1501 // cmpxchg.start:
1502 // %unreleasedload = @load.linked(%aligned.addr)
1503 // %unreleasedload.extract = extract value from %unreleasedload
1504 // %should_store = icmp eq %unreleasedload.extract, %desired
1505 // br i1 %should_store, label %cmpxchg.releasingstore,
1506 // label %cmpxchg.nostore
1507 // cmpxchg.releasingstore:
1508 // fence?
1509 // br label cmpxchg.trystore
1510 // cmpxchg.trystore:
1511 // %loaded.trystore = phi [%unreleasedload, %cmpxchg.releasingstore],
1512 // [%releasedload, %cmpxchg.releasedload]
1513 // %updated.new = insert %new into %loaded.trystore
1514 // %stored = @store_conditional(%updated.new, %aligned.addr)
1515 // %success = icmp eq i32 %stored, 0
1516 // br i1 %success, label %cmpxchg.success,
1517 // label %cmpxchg.releasedload/%cmpxchg.failure
1518 // cmpxchg.releasedload:
1519 // %releasedload = @load.linked(%aligned.addr)
1520 // %releasedload.extract = extract value from %releasedload
1521 // %should_store = icmp eq %releasedload.extract, %desired
1522 // br i1 %should_store, label %cmpxchg.trystore,
1523 // label %cmpxchg.failure
1524 // cmpxchg.success:
1525 // fence?
1526 // br label %cmpxchg.end
1527 // cmpxchg.nostore:
1528 // %loaded.nostore = phi [%unreleasedload, %cmpxchg.start],
1529 // [%releasedload,
1530 // %cmpxchg.releasedload/%cmpxchg.trystore]
1531 // @load_linked_fail_balance()?
1532 // br label %cmpxchg.failure
1533 // cmpxchg.failure:
1534 // fence?
1535 // br label %cmpxchg.end
1536 // cmpxchg.end:
1537 // %loaded.exit = phi [%loaded.nostore, %cmpxchg.failure],
1538 // [%loaded.trystore, %cmpxchg.trystore]
1539 // %success = phi i1 [true, %cmpxchg.success], [false, %cmpxchg.failure]
1540 // %loaded = extract value from %loaded.exit
1541 // %restmp = insertvalue { iN, i1 } undef, iN %loaded, 0
1542 // %res = insertvalue { iN, i1 } %restmp, i1 %success, 1
1543 // [...]
1544 BasicBlock *ExitBB = BB->splitBasicBlock(I: CI->getIterator(), BBName: "cmpxchg.end");
1545 auto FailureBB = BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.failure", Parent: F, InsertBefore: ExitBB);
1546 auto NoStoreBB = BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.nostore", Parent: F, InsertBefore: FailureBB);
1547 auto SuccessBB = BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.success", Parent: F, InsertBefore: NoStoreBB);
1548 auto ReleasedLoadBB =
1549 BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.releasedload", Parent: F, InsertBefore: SuccessBB);
1550 auto TryStoreBB =
1551 BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.trystore", Parent: F, InsertBefore: ReleasedLoadBB);
1552 auto ReleasingStoreBB =
1553 BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.fencedstore", Parent: F, InsertBefore: TryStoreBB);
1554 auto StartBB = BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.start", Parent: F, InsertBefore: ReleasingStoreBB);
1555
1556 ReplacementIRBuilder Builder(CI, *DL);
1557
1558 // The split call above "helpfully" added a branch at the end of BB (to the
1559 // wrong place), but we might want a fence too. It's easiest to just remove
1560 // the branch entirely.
1561 std::prev(x: BB->end())->eraseFromParent();
1562 Builder.SetInsertPoint(BB);
1563 if (ShouldInsertFencesForAtomic && UseUnconditionalReleaseBarrier)
1564 TLI->emitLeadingFence(Builder, Inst: CI, Ord: SuccessOrder);
1565
1566 PartwordMaskValues PMV =
1567 createMaskInstrs(Builder, I: CI, ValueType: CI->getCompareOperand()->getType(), Addr,
1568 AddrAlign: CI->getAlign(), MinWordSize: TLI->getMinCmpXchgSizeInBits() / 8);
1569 Builder.CreateBr(Dest: StartBB);
1570
1571 // Start the main loop block now that we've taken care of the preliminaries.
1572 Builder.SetInsertPoint(StartBB);
1573 Value *UnreleasedLoad =
1574 TLI->emitLoadLinked(Builder, ValueTy: PMV.WordType, Addr: PMV.AlignedAddr, Ord: MemOpOrder);
1575 Value *UnreleasedLoadExtract =
1576 extractMaskedValue(Builder, WideWord: UnreleasedLoad, PMV);
1577 Value *ShouldStore = Builder.CreateICmpEQ(
1578 LHS: UnreleasedLoadExtract, RHS: CI->getCompareOperand(), Name: "should_store");
1579
1580 // If the cmpxchg doesn't actually need any ordering when it fails, we can
1581 // jump straight past that fence instruction (if it exists).
1582 Builder.CreateCondBr(Cond: ShouldStore, True: ReleasingStoreBB, False: NoStoreBB,
1583 BranchWeights: MDBuilder(F->getContext()).createLikelyBranchWeights());
1584
1585 Builder.SetInsertPoint(ReleasingStoreBB);
1586 if (ShouldInsertFencesForAtomic && !UseUnconditionalReleaseBarrier)
1587 TLI->emitLeadingFence(Builder, Inst: CI, Ord: SuccessOrder);
1588 Builder.CreateBr(Dest: TryStoreBB);
1589
1590 Builder.SetInsertPoint(TryStoreBB);
1591 PHINode *LoadedTryStore =
1592 Builder.CreatePHI(Ty: PMV.WordType, NumReservedValues: 2, Name: "loaded.trystore");
1593 LoadedTryStore->addIncoming(V: UnreleasedLoad, BB: ReleasingStoreBB);
1594 Value *NewValueInsert =
1595 insertMaskedValue(Builder, WideWord: LoadedTryStore, Updated: CI->getNewValOperand(), PMV);
1596 Value *StoreSuccess = TLI->emitStoreConditional(Builder, Val: NewValueInsert,
1597 Addr: PMV.AlignedAddr, Ord: MemOpOrder);
1598 StoreSuccess = Builder.CreateICmpEQ(
1599 LHS: StoreSuccess, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: 0), Name: "success");
1600 BasicBlock *RetryBB = HasReleasedLoadBB ? ReleasedLoadBB : StartBB;
1601 Builder.CreateCondBr(Cond: StoreSuccess, True: SuccessBB,
1602 False: CI->isWeak() ? FailureBB : RetryBB,
1603 BranchWeights: MDBuilder(F->getContext()).createLikelyBranchWeights());
1604
1605 Builder.SetInsertPoint(ReleasedLoadBB);
1606 Value *SecondLoad;
1607 if (HasReleasedLoadBB) {
1608 SecondLoad =
1609 TLI->emitLoadLinked(Builder, ValueTy: PMV.WordType, Addr: PMV.AlignedAddr, Ord: MemOpOrder);
1610 Value *SecondLoadExtract = extractMaskedValue(Builder, WideWord: SecondLoad, PMV);
1611 ShouldStore = Builder.CreateICmpEQ(LHS: SecondLoadExtract,
1612 RHS: CI->getCompareOperand(), Name: "should_store");
1613
1614 // If the cmpxchg doesn't actually need any ordering when it fails, we can
1615 // jump straight past that fence instruction (if it exists).
1616 Builder.CreateCondBr(
1617 Cond: ShouldStore, True: TryStoreBB, False: NoStoreBB,
1618 BranchWeights: MDBuilder(F->getContext()).createLikelyBranchWeights());
1619 // Update PHI node in TryStoreBB.
1620 LoadedTryStore->addIncoming(V: SecondLoad, BB: ReleasedLoadBB);
1621 } else
1622 Builder.CreateUnreachable();
1623
1624 // Make sure later instructions don't get reordered with a fence if
1625 // necessary.
1626 Builder.SetInsertPoint(SuccessBB);
1627 if (ShouldInsertFencesForAtomic ||
1628 TLI->shouldInsertTrailingSeqCstFenceForAtomicStore(I: CI))
1629 TLI->emitTrailingFence(Builder, Inst: CI, Ord: SuccessOrder);
1630 Builder.CreateBr(Dest: ExitBB);
1631
1632 Builder.SetInsertPoint(NoStoreBB);
1633 PHINode *LoadedNoStore =
1634 Builder.CreatePHI(Ty: UnreleasedLoad->getType(), NumReservedValues: 2, Name: "loaded.nostore");
1635 LoadedNoStore->addIncoming(V: UnreleasedLoad, BB: StartBB);
1636 if (HasReleasedLoadBB)
1637 LoadedNoStore->addIncoming(V: SecondLoad, BB: ReleasedLoadBB);
1638
1639 // In the failing case, where we don't execute the store-conditional, the
1640 // target might want to balance out the load-linked with a dedicated
1641 // instruction (e.g., on ARM, clearing the exclusive monitor).
1642 TLI->emitAtomicCmpXchgNoStoreLLBalance(Builder);
1643 Builder.CreateBr(Dest: FailureBB);
1644
1645 Builder.SetInsertPoint(FailureBB);
1646 PHINode *LoadedFailure =
1647 Builder.CreatePHI(Ty: UnreleasedLoad->getType(), NumReservedValues: 2, Name: "loaded.failure");
1648 LoadedFailure->addIncoming(V: LoadedNoStore, BB: NoStoreBB);
1649 if (CI->isWeak())
1650 LoadedFailure->addIncoming(V: LoadedTryStore, BB: TryStoreBB);
1651 if (ShouldInsertFencesForAtomic)
1652 TLI->emitTrailingFence(Builder, Inst: CI, Ord: FailureOrder);
1653 Builder.CreateBr(Dest: ExitBB);
1654
1655 // Finally, we have control-flow based knowledge of whether the cmpxchg
1656 // succeeded or not. We expose this to later passes by converting any
1657 // subsequent "icmp eq/ne %loaded, %oldval" into a use of an appropriate
1658 // PHI.
1659 Builder.SetInsertPoint(TheBB: ExitBB, IP: ExitBB->begin());
1660 PHINode *LoadedExit =
1661 Builder.CreatePHI(Ty: UnreleasedLoad->getType(), NumReservedValues: 2, Name: "loaded.exit");
1662 LoadedExit->addIncoming(V: LoadedTryStore, BB: SuccessBB);
1663 LoadedExit->addIncoming(V: LoadedFailure, BB: FailureBB);
1664 PHINode *Success = Builder.CreatePHI(Ty: Type::getInt1Ty(C&: Ctx), NumReservedValues: 2, Name: "success");
1665 Success->addIncoming(V: ConstantInt::getTrue(Context&: Ctx), BB: SuccessBB);
1666 Success->addIncoming(V: ConstantInt::getFalse(Context&: Ctx), BB: FailureBB);
1667
1668 // This is the "exit value" from the cmpxchg expansion. It may be of
1669 // a type wider than the one in the cmpxchg instruction.
1670 Value *LoadedFull = LoadedExit;
1671
1672 Builder.SetInsertPoint(TheBB: ExitBB, IP: std::next(x: Success->getIterator()));
1673 Value *Loaded = extractMaskedValue(Builder, WideWord: LoadedFull, PMV);
1674
1675 // Look for any users of the cmpxchg that are just comparing the loaded value
1676 // against the desired one, and replace them with the CFG-derived version.
1677 SmallVector<ExtractValueInst *, 2> PrunedInsts;
1678 for (auto *User : CI->users()) {
1679 ExtractValueInst *EV = dyn_cast<ExtractValueInst>(Val: User);
1680 if (!EV)
1681 continue;
1682
1683 assert(EV->getNumIndices() == 1 && EV->getIndices()[0] <= 1 &&
1684 "weird extraction from { iN, i1 }");
1685
1686 if (EV->getIndices()[0] == 0)
1687 EV->replaceAllUsesWith(V: Loaded);
1688 else
1689 EV->replaceAllUsesWith(V: Success);
1690
1691 PrunedInsts.push_back(Elt: EV);
1692 }
1693
1694 // We can remove the instructions now we're no longer iterating through them.
1695 for (auto *EV : PrunedInsts)
1696 EV->eraseFromParent();
1697
1698 if (!CI->use_empty()) {
1699 // Some use of the full struct return that we don't understand has happened,
1700 // so we've got to reconstruct it properly.
1701 Value *Res;
1702 Res = Builder.CreateInsertValue(Agg: PoisonValue::get(T: CI->getType()), Val: Loaded, Idxs: 0);
1703 Res = Builder.CreateInsertValue(Agg: Res, Val: Success, Idxs: 1);
1704
1705 CI->replaceAllUsesWith(V: Res);
1706 }
1707
1708 CI->eraseFromParent();
1709 return true;
1710}
1711
1712bool AtomicExpandImpl::isIdempotentRMW(AtomicRMWInst *RMWI) {
1713 if (RMWI->isVolatile())
1714 return false;
1715 // TODO: Add floating point support.
1716 auto C = dyn_cast<ConstantInt>(Val: RMWI->getValOperand());
1717 if (!C)
1718 return false;
1719
1720 switch (RMWI->getOperation()) {
1721 case AtomicRMWInst::Add:
1722 case AtomicRMWInst::Sub:
1723 case AtomicRMWInst::Or:
1724 case AtomicRMWInst::Xor:
1725 return C->isZero();
1726 case AtomicRMWInst::And:
1727 return C->isMinusOne();
1728 case AtomicRMWInst::Min:
1729 return C->isMaxValue(IsSigned: true);
1730 case AtomicRMWInst::Max:
1731 return C->isMinValue(IsSigned: true);
1732 case AtomicRMWInst::UMin:
1733 return C->isMaxValue(IsSigned: false);
1734 case AtomicRMWInst::UMax:
1735 return C->isMinValue(IsSigned: false);
1736 default:
1737 return false;
1738 }
1739}
1740
1741bool AtomicExpandImpl::simplifyIdempotentRMW(AtomicRMWInst *RMWI) {
1742 if (auto ResultingLoad = TLI->lowerIdempotentRMWIntoFencedLoad(RMWI)) {
1743 tryExpandAtomicLoad(LI: ResultingLoad);
1744 return true;
1745 }
1746 return false;
1747}
1748
1749Value *AtomicExpandImpl::insertRMWCmpXchgLoop(
1750 IRBuilderBase &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
1751 AtomicOrdering MemOpOrder, SyncScope::ID SSID, bool IsVolatile,
1752 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp,
1753 CreateCmpXchgInstFun CreateCmpXchg, Instruction *MetadataSrc) {
1754 LLVMContext &Ctx = Builder.getContext();
1755 BasicBlock *BB = Builder.GetInsertBlock();
1756 Function *F = BB->getParent();
1757
1758 // Given: atomicrmw some_op iN* %addr, iN %incr ordering
1759 //
1760 // The standard expansion we produce is:
1761 // [...]
1762 // %init_loaded = load atomic iN* %addr
1763 // br label %loop
1764 // loop:
1765 // %loaded = phi iN [ %init_loaded, %entry ], [ %new_loaded, %loop ]
1766 // %new = some_op iN %loaded, %incr
1767 // %pair = cmpxchg iN* %addr, iN %loaded, iN %new
1768 // %new_loaded = extractvalue { iN, i1 } %pair, 0
1769 // %success = extractvalue { iN, i1 } %pair, 1
1770 // br i1 %success, label %atomicrmw.end, label %loop
1771 // atomicrmw.end:
1772 // [...]
1773 BasicBlock *ExitBB =
1774 BB->splitBasicBlock(I: Builder.GetInsertPoint(), BBName: "atomicrmw.end");
1775 BasicBlock *LoopBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.start", Parent: F, InsertBefore: ExitBB);
1776
1777 // The split call above "helpfully" added a branch at the end of BB (to the
1778 // wrong place), but we want a load. It's easiest to just remove
1779 // the branch entirely.
1780 std::prev(x: BB->end())->eraseFromParent();
1781 Builder.SetInsertPoint(BB);
1782 LoadInst *InitLoaded = Builder.CreateAlignedLoad(Ty: ResultTy, Ptr: Addr, Align: AddrAlign);
1783 Builder.CreateBr(Dest: LoopBB);
1784
1785 // Start the main loop block now that we've taken care of the preliminaries.
1786 Builder.SetInsertPoint(LoopBB);
1787 PHINode *Loaded = Builder.CreatePHI(Ty: ResultTy, NumReservedValues: 2, Name: "loaded");
1788 Loaded->addIncoming(V: InitLoaded, BB);
1789
1790 // The initial load must be atomic with the same synchronization scope
1791 // to avoid a data race with concurrent stores. If the instruction being
1792 // emulated is volatile, issue a volatile load.
1793 // addIncoming is done first so that any replaceAllUsesWith calls during
1794 // normalization correctly update the PHI incoming value.
1795 InitLoaded->setVolatile(IsVolatile);
1796 if (TLI->shouldIssueAtomicLoadForAtomicEmulationLoop()) {
1797 InitLoaded->setAtomic(Ordering: AtomicOrdering::Monotonic, SSID);
1798 // The newly created load might need to be lowered further. Because it is
1799 // created in the same block as the atomicrmw, the AtomicExpand loop will
1800 // not process it again.
1801 processAtomicInstr(I: InitLoaded);
1802 }
1803
1804 Value *NewVal = PerformOp(Builder, Loaded);
1805
1806 Value *NewLoaded = nullptr;
1807 Value *Success = nullptr;
1808
1809 CreateCmpXchg(Builder, Addr, Loaded, NewVal, AddrAlign,
1810 MemOpOrder == AtomicOrdering::Unordered
1811 ? AtomicOrdering::Monotonic
1812 : MemOpOrder,
1813 SSID, IsVolatile, Success, NewLoaded, MetadataSrc);
1814 assert(Success && NewLoaded);
1815
1816 Loaded->addIncoming(V: NewLoaded, BB: LoopBB);
1817
1818 Instruction *CondBr = Builder.CreateCondBr(Cond: Success, True: ExitBB, False: LoopBB);
1819
1820 // Atomic RMW expands to a cmpxchg loop, Since precise branch weights
1821 // cannot be easily determined here, we mark the branch as "unknown" (50/50)
1822 // to prevent misleading optimizations.
1823 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *CondBr, DEBUG_TYPE);
1824
1825 Builder.SetInsertPoint(TheBB: ExitBB, IP: ExitBB->begin());
1826 return NewLoaded;
1827}
1828
1829bool AtomicExpandImpl::tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
1830 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
1831 unsigned ValueSize = getAtomicOpSize(CASI: CI);
1832
1833 switch (TLI->shouldExpandAtomicCmpXchgInIR(AI: CI)) {
1834 default:
1835 llvm_unreachable("Unhandled case in tryExpandAtomicCmpXchg");
1836 case TargetLoweringBase::AtomicExpansionKind::None:
1837 if (ValueSize < MinCASSize)
1838 return expandPartwordCmpXchg(CI);
1839 return false;
1840 case TargetLoweringBase::AtomicExpansionKind::LLSC: {
1841 return expandAtomicCmpXchg(CI);
1842 }
1843 case TargetLoweringBase::AtomicExpansionKind::MaskedIntrinsic:
1844 expandAtomicCmpXchgToMaskedIntrinsic(CI);
1845 return true;
1846 case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
1847 return lowerAtomicCmpXchgInst(CXI: CI);
1848 case TargetLoweringBase::AtomicExpansionKind::CustomExpand: {
1849 TLI->emitExpandAtomicCmpXchg(CI);
1850 return true;
1851 }
1852 }
1853}
1854
1855bool AtomicExpandImpl::expandAtomicRMWToCmpXchg(
1856 AtomicRMWInst *AI, CreateCmpXchgInstFun CreateCmpXchg) {
1857 ReplacementIRBuilder Builder(AI, AI->getDataLayout());
1858 Builder.setIsFPConstrained(
1859 AI->getFunction()->hasFnAttribute(Kind: Attribute::StrictFP));
1860
1861 // FIXME: If FP exceptions are observable, we should force them off for the
1862 // loop for the FP atomics.
1863 Value *Loaded = AtomicExpandImpl::insertRMWCmpXchgLoop(
1864 Builder, ResultTy: AI->getType(), Addr: AI->getPointerOperand(), AddrAlign: AI->getAlign(),
1865 MemOpOrder: AI->getOrdering(), SSID: AI->getSyncScopeID(), IsVolatile: AI->isVolatile(),
1866 PerformOp: [&](IRBuilderBase &Builder, Value *Loaded) {
1867 return buildAtomicRMWValue(Op: AI->getOperation(), Builder, Loaded,
1868 Val: AI->getValOperand());
1869 },
1870 CreateCmpXchg, /*MetadataSrc=*/AI);
1871
1872 AI->replaceAllUsesWith(V: Loaded);
1873 AI->eraseFromParent();
1874 return true;
1875}
1876
1877// In order to use one of the sized library calls such as
1878// __atomic_fetch_add_4, the alignment must be sufficient, the size
1879// must be one of the potentially-specialized sizes, and the value
1880// type must actually exist in C on the target (otherwise, the
1881// function wouldn't actually be defined.)
1882static bool canUseSizedAtomicCall(unsigned Size, Align Alignment,
1883 const DataLayout &DL) {
1884 // TODO: "LargestSize" is an approximation for "largest type that
1885 // you can express in C". It seems to be the case that int128 is
1886 // supported on all 64-bit platforms, otherwise only up to 64-bit
1887 // integers are supported. If we get this wrong, then we'll try to
1888 // call a sized libcall that doesn't actually exist. There should
1889 // really be some more reliable way in LLVM of determining integer
1890 // sizes which are valid in the target's C ABI...
1891 unsigned LargestSize = DL.getLargestLegalIntTypeSizeInBits() >= 64 ? 16 : 8;
1892 return Alignment >= Size &&
1893 (Size == 1 || Size == 2 || Size == 4 || Size == 8 || Size == 16) &&
1894 Size <= LargestSize;
1895}
1896
1897void AtomicExpandImpl::expandAtomicLoadToLibcall(LoadInst *I) {
1898 static const RTLIB::Libcall Libcalls[6] = {
1899 RTLIB::ATOMIC_LOAD, RTLIB::ATOMIC_LOAD_1, RTLIB::ATOMIC_LOAD_2,
1900 RTLIB::ATOMIC_LOAD_4, RTLIB::ATOMIC_LOAD_8, RTLIB::ATOMIC_LOAD_16};
1901 unsigned Size = getAtomicOpSize(LI: I);
1902
1903 bool Expanded = expandAtomicOpToLibcall(
1904 I, Size, Alignment: I->getAlign(), PointerOperand: I->getPointerOperand(), ValueOperand: nullptr, CASExpected: nullptr,
1905 Ordering: I->getOrdering(), Ordering2: AtomicOrdering::NotAtomic, Libcalls);
1906 if (!Expanded)
1907 handleUnsupportedAtomicSize(I, AtomicOpName: "atomic load");
1908}
1909
1910void AtomicExpandImpl::expandAtomicStoreToLibcall(StoreInst *I) {
1911 static const RTLIB::Libcall Libcalls[6] = {
1912 RTLIB::ATOMIC_STORE, RTLIB::ATOMIC_STORE_1, RTLIB::ATOMIC_STORE_2,
1913 RTLIB::ATOMIC_STORE_4, RTLIB::ATOMIC_STORE_8, RTLIB::ATOMIC_STORE_16};
1914 unsigned Size = getAtomicOpSize(SI: I);
1915
1916 bool Expanded = expandAtomicOpToLibcall(
1917 I, Size, Alignment: I->getAlign(), PointerOperand: I->getPointerOperand(), ValueOperand: I->getValueOperand(),
1918 CASExpected: nullptr, Ordering: I->getOrdering(), Ordering2: AtomicOrdering::NotAtomic, Libcalls);
1919 if (!Expanded)
1920 handleUnsupportedAtomicSize(I, AtomicOpName: "atomic store");
1921}
1922
1923void AtomicExpandImpl::expandAtomicCASToLibcall(AtomicCmpXchgInst *I,
1924 const Twine &AtomicOpName,
1925 Instruction *DiagnosticInst) {
1926 static const RTLIB::Libcall Libcalls[6] = {
1927 RTLIB::ATOMIC_COMPARE_EXCHANGE, RTLIB::ATOMIC_COMPARE_EXCHANGE_1,
1928 RTLIB::ATOMIC_COMPARE_EXCHANGE_2, RTLIB::ATOMIC_COMPARE_EXCHANGE_4,
1929 RTLIB::ATOMIC_COMPARE_EXCHANGE_8, RTLIB::ATOMIC_COMPARE_EXCHANGE_16};
1930 unsigned Size = getAtomicOpSize(CASI: I);
1931
1932 bool Expanded = expandAtomicOpToLibcall(
1933 I, Size, Alignment: I->getAlign(), PointerOperand: I->getPointerOperand(), ValueOperand: I->getNewValOperand(),
1934 CASExpected: I->getCompareOperand(), Ordering: I->getSuccessOrdering(), Ordering2: I->getFailureOrdering(),
1935 Libcalls);
1936 if (!Expanded)
1937 handleUnsupportedAtomicSize(I, AtomicOpName, DiagnosticInst);
1938}
1939
1940static ArrayRef<RTLIB::Libcall> GetRMWLibcall(AtomicRMWInst::BinOp Op) {
1941 static const RTLIB::Libcall LibcallsXchg[6] = {
1942 RTLIB::ATOMIC_EXCHANGE, RTLIB::ATOMIC_EXCHANGE_1,
1943 RTLIB::ATOMIC_EXCHANGE_2, RTLIB::ATOMIC_EXCHANGE_4,
1944 RTLIB::ATOMIC_EXCHANGE_8, RTLIB::ATOMIC_EXCHANGE_16};
1945 static const RTLIB::Libcall LibcallsAdd[6] = {
1946 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_ADD_1,
1947 RTLIB::ATOMIC_FETCH_ADD_2, RTLIB::ATOMIC_FETCH_ADD_4,
1948 RTLIB::ATOMIC_FETCH_ADD_8, RTLIB::ATOMIC_FETCH_ADD_16};
1949 static const RTLIB::Libcall LibcallsSub[6] = {
1950 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_SUB_1,
1951 RTLIB::ATOMIC_FETCH_SUB_2, RTLIB::ATOMIC_FETCH_SUB_4,
1952 RTLIB::ATOMIC_FETCH_SUB_8, RTLIB::ATOMIC_FETCH_SUB_16};
1953 static const RTLIB::Libcall LibcallsAnd[6] = {
1954 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_AND_1,
1955 RTLIB::ATOMIC_FETCH_AND_2, RTLIB::ATOMIC_FETCH_AND_4,
1956 RTLIB::ATOMIC_FETCH_AND_8, RTLIB::ATOMIC_FETCH_AND_16};
1957 static const RTLIB::Libcall LibcallsOr[6] = {
1958 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_OR_1,
1959 RTLIB::ATOMIC_FETCH_OR_2, RTLIB::ATOMIC_FETCH_OR_4,
1960 RTLIB::ATOMIC_FETCH_OR_8, RTLIB::ATOMIC_FETCH_OR_16};
1961 static const RTLIB::Libcall LibcallsXor[6] = {
1962 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_XOR_1,
1963 RTLIB::ATOMIC_FETCH_XOR_2, RTLIB::ATOMIC_FETCH_XOR_4,
1964 RTLIB::ATOMIC_FETCH_XOR_8, RTLIB::ATOMIC_FETCH_XOR_16};
1965 static const RTLIB::Libcall LibcallsNand[6] = {
1966 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_NAND_1,
1967 RTLIB::ATOMIC_FETCH_NAND_2, RTLIB::ATOMIC_FETCH_NAND_4,
1968 RTLIB::ATOMIC_FETCH_NAND_8, RTLIB::ATOMIC_FETCH_NAND_16};
1969
1970 switch (Op) {
1971 case AtomicRMWInst::BAD_BINOP:
1972 llvm_unreachable("Should not have BAD_BINOP.");
1973 case AtomicRMWInst::Xchg:
1974 return ArrayRef(LibcallsXchg);
1975 case AtomicRMWInst::Add:
1976 return ArrayRef(LibcallsAdd);
1977 case AtomicRMWInst::Sub:
1978 return ArrayRef(LibcallsSub);
1979 case AtomicRMWInst::And:
1980 return ArrayRef(LibcallsAnd);
1981 case AtomicRMWInst::Or:
1982 return ArrayRef(LibcallsOr);
1983 case AtomicRMWInst::Xor:
1984 return ArrayRef(LibcallsXor);
1985 case AtomicRMWInst::Nand:
1986 return ArrayRef(LibcallsNand);
1987 case AtomicRMWInst::Max:
1988 case AtomicRMWInst::Min:
1989 case AtomicRMWInst::UMax:
1990 case AtomicRMWInst::UMin:
1991 case AtomicRMWInst::FMax:
1992 case AtomicRMWInst::FMin:
1993 case AtomicRMWInst::FMaximum:
1994 case AtomicRMWInst::FMinimum:
1995 case AtomicRMWInst::FMaximumNum:
1996 case AtomicRMWInst::FMinimumNum:
1997 case AtomicRMWInst::FAdd:
1998 case AtomicRMWInst::FSub:
1999 case AtomicRMWInst::UIncWrap:
2000 case AtomicRMWInst::UDecWrap:
2001 case AtomicRMWInst::USubCond:
2002 case AtomicRMWInst::USubSat:
2003 // No atomic libcalls are available for these.
2004 return {};
2005 }
2006 llvm_unreachable("Unexpected AtomicRMW operation.");
2007}
2008
2009void AtomicExpandImpl::expandAtomicRMWToLibcall(AtomicRMWInst *I) {
2010 ArrayRef<RTLIB::Libcall> Libcalls = GetRMWLibcall(Op: I->getOperation());
2011
2012 unsigned Size = getAtomicOpSize(RMWI: I);
2013
2014 bool Success = false;
2015 if (!Libcalls.empty())
2016 Success = expandAtomicOpToLibcall(
2017 I, Size, Alignment: I->getAlign(), PointerOperand: I->getPointerOperand(), ValueOperand: I->getValOperand(),
2018 CASExpected: nullptr, Ordering: I->getOrdering(), Ordering2: AtomicOrdering::NotAtomic, Libcalls);
2019
2020 // The expansion failed: either there were no libcalls at all for
2021 // the operation (min/max), or there were only size-specialized
2022 // libcalls (add/sub/etc) and we needed a generic. So, expand to a
2023 // CAS libcall, via a CAS loop, instead.
2024 if (!Success) {
2025 expandAtomicRMWToCmpXchg(
2026 AI: I, CreateCmpXchg: [this, I](IRBuilderBase &Builder, Value *Addr, Value *Loaded,
2027 Value *NewVal, Align Alignment, AtomicOrdering MemOpOrder,
2028 SyncScope::ID SSID, bool IsVolatile, Value *&Success,
2029 Value *&NewLoaded, Instruction *MetadataSrc) {
2030 // Create the CAS instruction normally...
2031 AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
2032 Ptr: Addr, Cmp: Loaded, New: NewVal, Align: Alignment, SuccessOrdering: MemOpOrder,
2033 FailureOrdering: AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: MemOpOrder), SSID);
2034 Pair->setVolatile(IsVolatile);
2035 if (MetadataSrc)
2036 copyMetadataForAtomic(Dest&: *Pair, Source: *MetadataSrc);
2037
2038 Success = Builder.CreateExtractValue(Agg: Pair, Idxs: 1, Name: "success");
2039 NewLoaded = Builder.CreateExtractValue(Agg: Pair, Idxs: 0, Name: "newloaded");
2040
2041 // ...and then expand the CAS into a libcall.
2042 expandAtomicCASToLibcall(
2043 I: Pair,
2044 AtomicOpName: "atomicrmw " + AtomicRMWInst::getOperationName(Op: I->getOperation()),
2045 DiagnosticInst: MetadataSrc);
2046 });
2047 }
2048}
2049
2050// A helper routine for the above expandAtomic*ToLibcall functions.
2051//
2052// 'Libcalls' contains an array of enum values for the particular
2053// ATOMIC libcalls to be emitted. All of the other arguments besides
2054// 'I' are extracted from the Instruction subclass by the
2055// caller. Depending on the particular call, some will be null.
2056bool AtomicExpandImpl::expandAtomicOpToLibcall(
2057 Instruction *I, unsigned Size, Align Alignment, Value *PointerOperand,
2058 Value *ValueOperand, Value *CASExpected, AtomicOrdering Ordering,
2059 AtomicOrdering Ordering2, ArrayRef<RTLIB::Libcall> Libcalls) {
2060 assert(Libcalls.size() == 6);
2061
2062 LLVMContext &Ctx = I->getContext();
2063 Module *M = I->getModule();
2064 const DataLayout &DL = M->getDataLayout();
2065 IRBuilder<> Builder(I);
2066 IRBuilder<> AllocaBuilder(&I->getFunction()->getEntryBlock().front());
2067
2068 bool UseSizedLibcall = canUseSizedAtomicCall(Size, Alignment, DL);
2069 Type *SizedIntTy = Type::getIntNTy(C&: Ctx, N: Size * 8);
2070
2071 if (M->getTargetTriple().isOSWindows() && M->getTargetTriple().isX86_64() &&
2072 Size == 16) {
2073 // x86_64 Windows passes i128 as an XMM vector; on return, it is in
2074 // XMM0, and as a parameter, it is passed indirectly. The generic lowering
2075 // rules handles this correctly if we pass it as a v2i64 rather than
2076 // i128. This is what Clang does in the frontend for such types as well
2077 // (see WinX86_64ABIInfo::classify in Clang).
2078 SizedIntTy = FixedVectorType::get(ElementType: Type::getInt64Ty(C&: Ctx), NumElts: 2);
2079 }
2080
2081 const Align AllocaAlignment = DL.getPrefTypeAlign(Ty: SizedIntTy);
2082
2083 // TODO: the "order" argument type is "int", not int32. So
2084 // getInt32Ty may be wrong if the arch uses e.g. 16-bit ints.
2085 assert(Ordering != AtomicOrdering::NotAtomic && "expect atomic MO");
2086 Constant *OrderingVal =
2087 ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: (int)toCABI(AO: Ordering));
2088 Constant *Ordering2Val = nullptr;
2089 if (CASExpected) {
2090 assert(Ordering2 != AtomicOrdering::NotAtomic && "expect atomic MO");
2091 Ordering2Val =
2092 ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: (int)toCABI(AO: Ordering2));
2093 }
2094 bool HasResult = I->getType() != Type::getVoidTy(C&: Ctx);
2095
2096 RTLIB::Libcall RTLibType;
2097 if (UseSizedLibcall) {
2098 switch (Size) {
2099 case 1:
2100 RTLibType = Libcalls[1];
2101 break;
2102 case 2:
2103 RTLibType = Libcalls[2];
2104 break;
2105 case 4:
2106 RTLibType = Libcalls[3];
2107 break;
2108 case 8:
2109 RTLibType = Libcalls[4];
2110 break;
2111 case 16:
2112 RTLibType = Libcalls[5];
2113 break;
2114 }
2115 } else if (Libcalls[0] != RTLIB::UNKNOWN_LIBCALL) {
2116 RTLibType = Libcalls[0];
2117 } else {
2118 // Can't use sized function, and there's no generic for this
2119 // operation, so give up.
2120 return false;
2121 }
2122
2123 RTLIB::LibcallImpl LibcallImpl = LibcallLowering->getLibcallImpl(Call: RTLibType);
2124 if (LibcallImpl == RTLIB::Unsupported) {
2125 // This target does not implement the requested atomic libcall so give up.
2126 return false;
2127 }
2128
2129 // Build up the function call. There's two kinds. First, the sized
2130 // variants. These calls are going to be one of the following (with
2131 // N=1,2,4,8,16):
2132 // iN __atomic_load_N(iN *ptr, int ordering)
2133 // void __atomic_store_N(iN *ptr, iN val, int ordering)
2134 // iN __atomic_{exchange|fetch_*}_N(iN *ptr, iN val, int ordering)
2135 // bool __atomic_compare_exchange_N(iN *ptr, iN *expected, iN desired,
2136 // int success_order, int failure_order)
2137 //
2138 // Note that these functions can be used for non-integer atomic
2139 // operations, the values just need to be bitcast to integers on the
2140 // way in and out.
2141 //
2142 // And, then, the generic variants. They look like the following:
2143 // void __atomic_load(size_t size, void *ptr, void *ret, int ordering)
2144 // void __atomic_store(size_t size, void *ptr, void *val, int ordering)
2145 // void __atomic_exchange(size_t size, void *ptr, void *val, void *ret,
2146 // int ordering)
2147 // bool __atomic_compare_exchange(size_t size, void *ptr, void *expected,
2148 // void *desired, int success_order,
2149 // int failure_order)
2150 //
2151 // The different signatures are built up depending on the
2152 // 'UseSizedLibcall', 'CASExpected', 'ValueOperand', and 'HasResult'
2153 // variables.
2154
2155 AllocaInst *AllocaCASExpected = nullptr;
2156 AllocaInst *AllocaValue = nullptr;
2157 AllocaInst *AllocaResult = nullptr;
2158
2159 Type *ResultTy;
2160 SmallVector<Value *, 6> Args;
2161 AttributeList Attr;
2162
2163 // 'size' argument.
2164 if (!UseSizedLibcall) {
2165 // Note, getIntPtrType is assumed equivalent to size_t.
2166 Args.push_back(Elt: ConstantInt::get(Ty: DL.getIntPtrType(C&: Ctx), V: Size));
2167 }
2168
2169 // 'ptr' argument.
2170 // note: This assumes all address spaces share a common libfunc
2171 // implementation and that addresses are convertable. For systems without
2172 // that property, we'd need to extend this mechanism to support AS-specific
2173 // families of atomic intrinsics.
2174 Value *PtrVal = PointerOperand;
2175 PtrVal = Builder.CreateAddrSpaceCast(V: PtrVal, DestTy: PointerType::getUnqual(C&: Ctx));
2176 Args.push_back(Elt: PtrVal);
2177
2178 // 'expected' argument, if present.
2179 if (CASExpected) {
2180 AllocaCASExpected = AllocaBuilder.CreateAlloca(Ty: CASExpected->getType());
2181 AllocaCASExpected->setAlignment(AllocaAlignment);
2182 Builder.CreateLifetimeStart(Ptr: AllocaCASExpected);
2183 Builder.CreateAlignedStore(Val: CASExpected, Ptr: AllocaCASExpected, Align: AllocaAlignment);
2184 Args.push_back(Elt: AllocaCASExpected);
2185 }
2186
2187 // 'val' argument ('desired' for cas), if present.
2188 if (ValueOperand) {
2189 if (UseSizedLibcall) {
2190 Value *IntValue =
2191 Builder.CreateBitPreservingCastChain(DL, V: ValueOperand, NewTy: SizedIntTy);
2192 Args.push_back(Elt: IntValue);
2193 } else {
2194 AllocaValue = AllocaBuilder.CreateAlloca(Ty: ValueOperand->getType());
2195 AllocaValue->setAlignment(AllocaAlignment);
2196 Builder.CreateLifetimeStart(Ptr: AllocaValue);
2197 Builder.CreateAlignedStore(Val: ValueOperand, Ptr: AllocaValue, Align: AllocaAlignment);
2198 Args.push_back(Elt: AllocaValue);
2199 }
2200 }
2201
2202 // 'ret' argument.
2203 if (!CASExpected && HasResult && !UseSizedLibcall) {
2204 AllocaResult = AllocaBuilder.CreateAlloca(Ty: I->getType());
2205 AllocaResult->setAlignment(AllocaAlignment);
2206 Builder.CreateLifetimeStart(Ptr: AllocaResult);
2207 Args.push_back(Elt: AllocaResult);
2208 }
2209
2210 // 'ordering' ('success_order' for cas) argument.
2211 Args.push_back(Elt: OrderingVal);
2212
2213 // 'failure_order' argument, if present.
2214 if (Ordering2Val)
2215 Args.push_back(Elt: Ordering2Val);
2216
2217 // Now, the return type.
2218 if (CASExpected) {
2219 ResultTy = Type::getInt1Ty(C&: Ctx);
2220 Attr = Attr.addRetAttribute(C&: Ctx, Kind: Attribute::ZExt);
2221 } else if (HasResult && UseSizedLibcall)
2222 ResultTy = SizedIntTy;
2223 else
2224 ResultTy = Type::getVoidTy(C&: Ctx);
2225
2226 // Done with setting up arguments and return types, create the call:
2227 SmallVector<Type *, 6> ArgTys;
2228 for (Value *Arg : Args)
2229 ArgTys.push_back(Elt: Arg->getType());
2230 FunctionType *FnType = FunctionType::get(Result: ResultTy, Params: ArgTys, isVarArg: false);
2231 FunctionCallee LibcallFn = M->getOrInsertFunction(
2232 Name: RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: LibcallImpl), T: FnType,
2233 AttributeList: Attr);
2234 CallInst *Call = Builder.CreateCall(Callee: LibcallFn, Args);
2235 Call->setAttributes(Attr);
2236 Value *Result = Call;
2237
2238 // And then, extract the results...
2239 if (ValueOperand && !UseSizedLibcall)
2240 Builder.CreateLifetimeEnd(Ptr: AllocaValue);
2241
2242 if (CASExpected) {
2243 // The final result from the CAS is {load of 'expected' alloca, bool result
2244 // from call}
2245 Type *FinalResultTy = I->getType();
2246 Value *V = PoisonValue::get(T: FinalResultTy);
2247 Value *ExpectedOut = Builder.CreateAlignedLoad(
2248 Ty: CASExpected->getType(), Ptr: AllocaCASExpected, Align: AllocaAlignment);
2249 Builder.CreateLifetimeEnd(Ptr: AllocaCASExpected);
2250 V = Builder.CreateInsertValue(Agg: V, Val: ExpectedOut, Idxs: 0);
2251 V = Builder.CreateInsertValue(Agg: V, Val: Result, Idxs: 1);
2252 I->replaceAllUsesWith(V);
2253 } else if (HasResult) {
2254 Value *V;
2255 if (UseSizedLibcall) {
2256 // Add bitcasts from Result's scalar type to I's <n x ptr> vector type
2257 auto *PtrTy = dyn_cast<PointerType>(Val: I->getType()->getScalarType());
2258 auto *VTy = dyn_cast<VectorType>(Val: I->getType());
2259 if (VTy && PtrTy && !Result->getType()->isVectorTy()) {
2260 unsigned AS = PtrTy->getAddressSpace();
2261 Value *BC = Builder.CreateBitCast(
2262 V: Result, DestTy: VTy->getWithNewType(EltTy: DL.getIntPtrType(C&: Ctx, AddressSpace: AS)));
2263 V = Builder.CreateIntToPtr(V: BC, DestTy: I->getType());
2264 } else
2265 V = Builder.CreateBitOrPointerCast(V: Result, DestTy: I->getType());
2266 } else {
2267 V = Builder.CreateAlignedLoad(Ty: I->getType(), Ptr: AllocaResult,
2268 Align: AllocaAlignment);
2269 Builder.CreateLifetimeEnd(Ptr: AllocaResult);
2270 }
2271 I->replaceAllUsesWith(V);
2272 }
2273 I->eraseFromParent();
2274 return true;
2275}
2276