1//===- AtomicExpandPass.cpp - Expand atomic instructions ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass (at IR level) to replace atomic instructions with
10// __atomic_* library calls, or target specific instruction which implement the
11// same semantics in a way which better fits the target backend. This can
12// include the use of (intrinsic-based) load-linked/store-conditional loops,
13// AtomicCmpXchg, or type coercions.
14//
15//===----------------------------------------------------------------------===//
16
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/STLFunctionalExtras.h"
19#include "llvm/ADT/SmallString.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/Analysis/InstSimplifyFolder.h"
22#include "llvm/Analysis/OptimizationRemarkEmitter.h"
23#include "llvm/Analysis/VectorUtils.h"
24#include "llvm/CodeGen/AtomicExpand.h"
25#include "llvm/CodeGen/TargetLowering.h"
26#include "llvm/CodeGen/TargetPassConfig.h"
27#include "llvm/CodeGen/TargetSubtargetInfo.h"
28#include "llvm/CodeGen/ValueTypes.h"
29#include "llvm/IR/Attributes.h"
30#include "llvm/IR/BasicBlock.h"
31#include "llvm/IR/Constant.h"
32#include "llvm/IR/Constants.h"
33#include "llvm/IR/DataLayout.h"
34#include "llvm/IR/DerivedTypes.h"
35#include "llvm/IR/Function.h"
36#include "llvm/IR/IRBuilder.h"
37#include "llvm/IR/Instruction.h"
38#include "llvm/IR/Instructions.h"
39#include "llvm/IR/MDBuilder.h"
40#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
41#include "llvm/IR/Module.h"
42#include "llvm/IR/ProfDataUtils.h"
43#include "llvm/IR/Type.h"
44#include "llvm/IR/User.h"
45#include "llvm/IR/Value.h"
46#include "llvm/InitializePasses.h"
47#include "llvm/Pass.h"
48#include "llvm/Support/Alignment.h"
49#include "llvm/Support/AtomicOrdering.h"
50#include "llvm/Support/Casting.h"
51#include "llvm/Support/Debug.h"
52#include "llvm/Support/ErrorHandling.h"
53#include "llvm/Support/raw_ostream.h"
54#include "llvm/Target/TargetMachine.h"
55#include "llvm/Transforms/Utils/LowerAtomic.h"
56#include <cassert>
57#include <cstdint>
58#include <iterator>
59
60using namespace llvm;
61
62#define DEBUG_TYPE "atomic-expand"
63
64namespace {
65
66class AtomicExpandImpl {
67 const TargetLowering *TLI = nullptr;
68 const LibcallLoweringInfo *LibcallLowering = nullptr;
69 const DataLayout *DL = nullptr;
70 bool SingleThreaded = false;
71
72private:
73 /// Callback type for emitting a cmpxchg instruction during RMW expansion.
74 /// Parameters: (Builder, Addr, Loaded, NewVal, AddrAlign, MemOpOrder,
75 /// SSID, IsVolatile, /* OUT */ Success, /* OUT */ NewLoaded,
76 /// MetadataSrc)
77 using CreateCmpXchgInstFun = function_ref<void(
78 IRBuilderBase &, Value *, Value *, Value *, Align, AtomicOrdering,
79 SyncScope::ID, bool, Value *&, Value *&, Instruction *)>;
80
81 void handleFailure(Instruction &FailedInst, const Twine &Msg,
82 Instruction *DiagnosticInst = nullptr) const {
83 LLVMContext &Ctx = FailedInst.getContext();
84
85 // TODO: Do not use generic error type.
86 Ctx.emitError(I: DiagnosticInst ? DiagnosticInst : &FailedInst, ErrorStr: Msg);
87
88 if (!FailedInst.getType()->isVoidTy())
89 FailedInst.replaceAllUsesWith(V: PoisonValue::get(T: FailedInst.getType()));
90 FailedInst.eraseFromParent();
91 }
92
93 template <typename Inst>
94 void handleUnsupportedAtomicSize(Inst *I, const Twine &AtomicOpName,
95 Instruction *DiagnosticInst = nullptr) const;
96
97 bool bracketInstWithFences(Instruction *I, AtomicOrdering Order);
98 bool tryInsertTrailingSeqCstFence(Instruction *AtomicI);
99 template <typename AtomicInst>
100 bool tryInsertFencesForAtomic(AtomicInst *AtomicI, bool OrderingRequiresFence,
101 AtomicOrdering NewOrdering);
102 IntegerType *getCorrespondingIntegerType(Type *T, const DataLayout &DL);
103 LoadInst *convertAtomicLoadToIntegerType(LoadInst *LI);
104 bool tryExpandAtomicLoad(LoadInst *LI);
105 bool expandAtomicLoadToLL(LoadInst *LI);
106 bool expandAtomicLoadToCmpXchg(LoadInst *LI);
107 StoreInst *convertAtomicStoreToIntegerType(StoreInst *SI);
108 bool tryExpandAtomicStore(StoreInst *SI);
109 void expandAtomicStoreToXChg(StoreInst *SI);
110 bool tryExpandAtomicRMW(AtomicRMWInst *AI);
111 void expandElementwiseAtomicRMW(AtomicRMWInst *AI);
112 void expandAtomicSubToAdd(AtomicRMWInst *AI);
113 AtomicRMWInst *convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI);
114 Value *
115 insertRMWLLSCLoop(IRBuilderBase &Builder, Type *ResultTy, Value *Addr,
116 Align AddrAlign, AtomicOrdering MemOpOrder,
117 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp);
118 void expandAtomicOpToLLSC(
119 Instruction *I, Type *ResultTy, Value *Addr, Align AddrAlign,
120 AtomicOrdering MemOpOrder,
121 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp);
122 void expandPartwordAtomicRMW(
123 AtomicRMWInst *I, TargetLoweringBase::AtomicExpansionKind ExpansionKind);
124 AtomicRMWInst *widenPartwordAtomicRMW(AtomicRMWInst *AI);
125 bool expandPartwordCmpXchg(AtomicCmpXchgInst *I);
126 void expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI);
127 void expandAtomicCmpXchgToMaskedIntrinsic(AtomicCmpXchgInst *CI);
128
129 AtomicCmpXchgInst *convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI);
130 Value *insertRMWCmpXchgLoop(
131 IRBuilderBase &Builder, Type *ResultType, Value *Addr, Align AddrAlign,
132 AtomicOrdering MemOpOrder, SyncScope::ID SSID, bool IsVolatile,
133 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp,
134 CreateCmpXchgInstFun CreateCmpXchg, Instruction *MetadataSrc);
135 bool tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI);
136
137 bool expandAtomicCmpXchg(AtomicCmpXchgInst *CI);
138 bool isIdempotentRMW(AtomicRMWInst *RMWI);
139 bool simplifyIdempotentRMW(AtomicRMWInst *RMWI);
140
141 bool expandAtomicOpToLibcall(Instruction *I, unsigned Size, Align Alignment,
142 Value *PointerOperand, Value *ValueOperand,
143 Value *CASExpected, AtomicOrdering Ordering,
144 AtomicOrdering Ordering2,
145 ArrayRef<RTLIB::Libcall> Libcalls);
146 void expandAtomicLoadToLibcall(LoadInst *LI);
147 void expandAtomicStoreToLibcall(StoreInst *LI);
148 void expandAtomicRMWToLibcall(AtomicRMWInst *I);
149 void expandAtomicCASToLibcall(AtomicCmpXchgInst *I,
150 const Twine &AtomicOpName = "cmpxchg",
151 Instruction *DiagnosticInst = nullptr);
152
153 bool expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
154 CreateCmpXchgInstFun CreateCmpXchg);
155
156 bool lowerToNonAtomic(Instruction *I);
157 bool processAtomicInstr(Instruction *I);
158
159public:
160 bool run(Function &F, const ModuleLibcallLoweringInfo &LibcallResult,
161 const TargetMachine *TM);
162};
163
164class AtomicExpandLegacy : public FunctionPass {
165public:
166 static char ID; // Pass identification, replacement for typeid
167
168 AtomicExpandLegacy() : FunctionPass(ID) {}
169
170 void getAnalysisUsage(AnalysisUsage &AU) const override {
171 AU.addRequired<LibcallLoweringInfoWrapper>();
172 FunctionPass::getAnalysisUsage(AU);
173 }
174
175 bool runOnFunction(Function &F) override;
176};
177
178// IRBuilder to be used for replacement atomic instructions.
179struct ReplacementIRBuilder
180 : IRBuilder<InstSimplifyFolder, IRBuilderCallbackInserter> {
181 MDNode *MMRAMD = nullptr;
182 MDNode *PCSectionsMD = nullptr;
183
184 // Preserves the DebugLoc from I, and preserves still valid metadata.
185 // Enable StrictFP builder mode when appropriate.
186 explicit ReplacementIRBuilder(Instruction *I, const DataLayout &DL)
187 : IRBuilder(
188 I->getIterator(), InstSimplifyFolder(DL),
189 IRBuilderCallbackInserter([this](Instruction *I) { addMD(I); })) {
190 if (BB->getParent()->getAttributes().hasFnAttr(Kind: Attribute::StrictFP))
191 this->setIsFPConstrained(true);
192
193 MMRAMD = I->getMetadata(KindID: LLVMContext::MD_mmra);
194 PCSectionsMD = I->getMetadata(KindID: LLVMContext::MD_pcsections);
195 }
196
197 void addMD(Instruction *I) {
198 if (canInstructionHaveMMRAs(I: *I))
199 I->setMetadata(KindID: LLVMContext::MD_mmra, Node: MMRAMD);
200 I->setMetadata(KindID: LLVMContext::MD_pcsections, Node: PCSectionsMD);
201 }
202};
203
204} // end anonymous namespace
205
206char AtomicExpandLegacy::ID = 0;
207
208char &llvm::AtomicExpandID = AtomicExpandLegacy::ID;
209
210INITIALIZE_PASS_BEGIN(AtomicExpandLegacy, DEBUG_TYPE,
211 "Expand Atomic instructions", false, false)
212INITIALIZE_PASS_DEPENDENCY(LibcallLoweringInfoWrapper)
213INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
214INITIALIZE_PASS_END(AtomicExpandLegacy, DEBUG_TYPE,
215 "Expand Atomic instructions", false, false)
216
217// Helper functions to retrieve the size of atomic instructions.
218static unsigned getAtomicOpSize(LoadInst *LI) {
219 const DataLayout &DL = LI->getDataLayout();
220 return DL.getTypeStoreSize(Ty: LI->getType());
221}
222
223static unsigned getAtomicOpSize(StoreInst *SI) {
224 const DataLayout &DL = SI->getDataLayout();
225 return DL.getTypeStoreSize(Ty: SI->getValueOperand()->getType());
226}
227
228static unsigned getAtomicOpSize(AtomicRMWInst *RMWI) {
229 const DataLayout &DL = RMWI->getDataLayout();
230 return DL.getTypeStoreSize(Ty: RMWI->getValOperand()->getType());
231}
232
233static unsigned getAtomicOpSize(AtomicCmpXchgInst *CASI) {
234 const DataLayout &DL = CASI->getDataLayout();
235 return DL.getTypeStoreSize(Ty: CASI->getCompareOperand()->getType());
236}
237
238/// Copy metadata that's safe to preserve when widening atomics.
239static void copyMetadataForAtomic(Instruction &Dest,
240 const Instruction &Source) {
241 SmallVector<std::pair<unsigned, MDNode *>, 8> MD;
242 Source.getAllMetadata(MDs&: MD);
243 LLVMContext &Ctx = Dest.getContext();
244 MDBuilder MDB(Ctx);
245
246 for (auto [ID, N] : MD) {
247 switch (ID) {
248 case LLVMContext::MD_dbg:
249 case LLVMContext::MD_tbaa:
250 case LLVMContext::MD_tbaa_struct:
251 case LLVMContext::MD_alias_scope:
252 case LLVMContext::MD_mem_cache_hint:
253 case LLVMContext::MD_noalias:
254 case LLVMContext::MD_noalias_addrspace:
255 case LLVMContext::MD_access_group:
256 case LLVMContext::MD_mmra:
257 Dest.setMetadata(KindID: ID, Node: N);
258 break;
259 default:
260 if (ID == Ctx.getMDKindID(Name: "amdgpu.no.remote.memory"))
261 Dest.setMetadata(KindID: ID, Node: N);
262 else if (ID == Ctx.getMDKindID(Name: "amdgpu.no.fine.grained.memory"))
263 Dest.setMetadata(KindID: ID, Node: N);
264
265 // Losing atomic.ignore.denormal.mode, but it doesn't matter for current
266 // uses.
267 break;
268 }
269 }
270}
271
272template <typename Inst>
273static bool atomicSizeSupported(const TargetLowering *TLI, Inst *I) {
274 unsigned Size = getAtomicOpSize(I);
275 Align Alignment = I->getAlign();
276 unsigned MaxSize = TLI->getMaxAtomicSizeInBitsSupported() / 8;
277 return Size <= MaxSize && TLI->isAtomicAlignmentSupported(Alignment, SizeInBytes: Size);
278}
279
280template <typename Inst>
281static void writeUnsupportedAtomicSizeReason(const TargetLowering *TLI, Inst *I,
282 raw_ostream &OS) {
283 unsigned Size = getAtomicOpSize(I);
284 Align Alignment = I->getAlign();
285 bool NeedSeparator = false;
286
287 if (!TLI->isAtomicAlignmentSupported(Alignment, SizeInBytes: Size)) {
288 OS << "instruction alignment " << Alignment.value()
289 << " is smaller than the required " << Size
290 << "-byte alignment for this atomic operation";
291 NeedSeparator = true;
292 }
293
294 unsigned MaxSize = TLI->getMaxAtomicSizeInBitsSupported() / 8;
295 if (Size > MaxSize) {
296 if (NeedSeparator)
297 OS << "; ";
298 OS << "target supports atomics up to " << MaxSize
299 << " bytes, but this atomic accesses " << Size << " bytes";
300 }
301}
302
303template <typename Inst>
304void AtomicExpandImpl::handleUnsupportedAtomicSize(
305 Inst *I, const Twine &AtomicOpName, Instruction *DiagnosticInst) const {
306 assert(!atomicSizeSupported(TLI, I) && "expected unsupported atomic size");
307 SmallString<128> FailureReason;
308 raw_svector_ostream OS(FailureReason);
309 writeUnsupportedAtomicSizeReason(TLI, I, OS);
310 handleFailure(FailedInst&: *I, Msg: Twine("unsupported ") + AtomicOpName + ": " + FailureReason,
311 DiagnosticInst);
312}
313
314bool AtomicExpandImpl::tryInsertTrailingSeqCstFence(Instruction *AtomicI) {
315 if (!TLI->shouldInsertTrailingSeqCstFenceForAtomicStore(I: AtomicI))
316 return false;
317
318 IRBuilder Builder(AtomicI);
319 if (auto *TrailingFence = TLI->emitTrailingFence(
320 Builder, Inst: AtomicI, Ord: AtomicOrdering::SequentiallyConsistent)) {
321 TrailingFence->moveAfter(MovePos: AtomicI);
322 return true;
323 }
324 return false;
325}
326
327template <typename AtomicInst>
328bool AtomicExpandImpl::tryInsertFencesForAtomic(AtomicInst *AtomicI,
329 bool OrderingRequiresFence,
330 AtomicOrdering NewOrdering) {
331 bool ShouldInsertFences = TLI->shouldInsertFencesForAtomic(I: AtomicI);
332 if (OrderingRequiresFence && ShouldInsertFences) {
333 AtomicOrdering FenceOrdering = AtomicI->getOrdering();
334 AtomicI->setOrdering(NewOrdering);
335 return bracketInstWithFences(I: AtomicI, Order: FenceOrdering);
336 }
337 if (!ShouldInsertFences)
338 return tryInsertTrailingSeqCstFence(AtomicI);
339 return false;
340}
341
342/// In a single-threaded environment, atomic operations can be lowered to their
343/// non-atomic equivalents: fences are removed, and atomic loads, stores, RMW,
344/// and cmpxchg become plain memory operations.
345bool AtomicExpandImpl::lowerToNonAtomic(Instruction *I) {
346 if (auto *FI = dyn_cast<FenceInst>(Val: I)) {
347 FI->eraseFromParent();
348 return true;
349 }
350
351 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: I))
352 return lowerAtomicCmpXchgInst(CXI);
353
354 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: I))
355 return lowerAtomicRMWInst(RMWI);
356
357 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
358 if (LI->isAtomic()) {
359 LI->setAtomic(Ordering: AtomicOrdering::NotAtomic);
360 LI->setElementwise(false);
361 return true;
362 }
363
364 return false;
365 }
366
367 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
368 if (SI->isAtomic()) {
369 SI->setAtomic(Ordering: AtomicOrdering::NotAtomic);
370 SI->setElementwise(false);
371 return true;
372 }
373
374 return false;
375 }
376
377 return false;
378}
379
380bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
381 if (SingleThreaded)
382 return lowerToNonAtomic(I);
383
384 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
385 if (!LI->isAtomic())
386 return false;
387
388 if (!atomicSizeSupported(TLI, I: LI)) {
389 expandAtomicLoadToLibcall(LI);
390 return true;
391 }
392
393 bool MadeChange = false;
394 if (TLI->shouldCastAtomicLoadInIR(LI) ==
395 TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
396 LI = convertAtomicLoadToIntegerType(LI);
397 MadeChange = true;
398 }
399
400 MadeChange |= tryInsertFencesForAtomic(
401 AtomicI: LI, OrderingRequiresFence: isAcquireOrStronger(AO: LI->getOrdering()), NewOrdering: AtomicOrdering::Monotonic);
402
403 MadeChange |= tryExpandAtomicLoad(LI);
404 return MadeChange;
405 }
406
407 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
408 if (!SI->isAtomic())
409 return false;
410
411 if (!atomicSizeSupported(TLI, I: SI)) {
412 expandAtomicStoreToLibcall(LI: SI);
413 return true;
414 }
415
416 bool MadeChange = false;
417 if (TLI->shouldCastAtomicStoreInIR(SI) ==
418 TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
419 SI = convertAtomicStoreToIntegerType(SI);
420 MadeChange = true;
421 }
422
423 MadeChange |= tryInsertFencesForAtomic(
424 AtomicI: SI, OrderingRequiresFence: isReleaseOrStronger(AO: SI->getOrdering()), NewOrdering: AtomicOrdering::Monotonic);
425
426 MadeChange |= tryExpandAtomicStore(SI);
427 return MadeChange;
428 }
429
430 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: I)) {
431 bool IsElementwiseExpand =
432 RMWI->isElementwise() &&
433 TLI->shouldExpandAtomicRMWInIR(RMW: RMWI) ==
434 TargetLoweringBase::AtomicExpansionKind::Expand;
435
436 if (!atomicSizeSupported(TLI, I: RMWI)) {
437 // Elementwise expansion may split an atomicrmw into smaller, supported
438 // atomic sizes.
439 if (!IsElementwiseExpand) {
440 expandAtomicRMWToLibcall(I: RMWI);
441 return true;
442 }
443 }
444
445 bool MadeChange = false;
446 if (!IsElementwiseExpand &&
447 TLI->shouldCastAtomicRMWIInIR(RMWI) ==
448 TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
449 RMWI = convertAtomicXchgToIntegerType(RMWI);
450 MadeChange = true;
451 }
452
453 MadeChange |= tryInsertFencesForAtomic(
454 AtomicI: RMWI, OrderingRequiresFence: isStrongerThanMonotonic(AO: RMWI->getOrdering()),
455 NewOrdering: TLI->atomicOperationOrderAfterFenceSplit(I: RMWI));
456
457 // There are two different ways of expanding RMW instructions:
458 // - into a load if it is idempotent
459 // - into a Cmpxchg/LL-SC loop otherwise
460 // we try them in that order.
461 MadeChange |= (isIdempotentRMW(RMWI) && simplifyIdempotentRMW(RMWI)) ||
462 tryExpandAtomicRMW(AI: RMWI);
463 return MadeChange;
464 }
465
466 if (auto *CASI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
467 if (!atomicSizeSupported(TLI, I: CASI)) {
468 expandAtomicCASToLibcall(I: CASI);
469 return true;
470 }
471
472 // TODO: when we're ready to make the change at the IR level, we can
473 // extend convertCmpXchgToInteger for floating point too.
474 bool MadeChange = false;
475 if (CASI->getCompareOperand()->getType()->isPointerTy()) {
476 // TODO: add a TLI hook to control this so that each target can
477 // convert to lowering the original type one at a time.
478 CASI = convertCmpXchgToIntegerType(CI: CASI);
479 MadeChange = true;
480 }
481
482 auto CmpXchgExpansion = TLI->shouldExpandAtomicCmpXchgInIR(AI: CASI);
483 if (TLI->shouldInsertFencesForAtomic(I: CASI)) {
484 if (CmpXchgExpansion == TargetLoweringBase::AtomicExpansionKind::None &&
485 (isReleaseOrStronger(AO: CASI->getSuccessOrdering()) ||
486 isAcquireOrStronger(AO: CASI->getSuccessOrdering()) ||
487 isAcquireOrStronger(AO: CASI->getFailureOrdering()))) {
488 // If a compare and swap is lowered to LL/SC, we can do smarter fence
489 // insertion, with a stronger one on the success path than on the
490 // failure path. As a result, fence insertion is directly done by
491 // expandAtomicCmpXchg in that case.
492 AtomicOrdering FenceOrdering = CASI->getMergedOrdering();
493 AtomicOrdering CASOrdering =
494 TLI->atomicOperationOrderAfterFenceSplit(I: CASI);
495 CASI->setSuccessOrdering(CASOrdering);
496 CASI->setFailureOrdering(CASOrdering);
497 MadeChange |= bracketInstWithFences(I: CASI, Order: FenceOrdering);
498 }
499 } else if (CmpXchgExpansion !=
500 TargetLoweringBase::AtomicExpansionKind::LLSC) {
501 // CmpXchg LLSC is handled in expandAtomicCmpXchg().
502 MadeChange |= tryInsertTrailingSeqCstFence(AtomicI: CASI);
503 }
504
505 MadeChange |= tryExpandAtomicCmpXchg(CI: CASI);
506 return MadeChange;
507 }
508
509 return false;
510}
511
512bool AtomicExpandImpl::run(Function &F,
513 const ModuleLibcallLoweringInfo &LibcallResult,
514 const TargetMachine *TM) {
515 SingleThreaded = F.getParent()->getThreadModel() == ThreadModel::Single;
516
517 const auto *Subtarget = TM->getSubtargetImpl(F);
518 // In a single-threaded environment atomics are lowered to non-atomic form
519 if (!SingleThreaded && !Subtarget->enableAtomicExpand())
520 return false;
521 TLI = Subtarget->getTargetLowering();
522 LibcallLowering = &getLibcallLowering(ModuleInfo: LibcallResult, Subtarget: *Subtarget);
523 DL = &F.getDataLayout();
524
525 bool MadeChange = false;
526
527 for (Function::iterator BBI = F.begin(), BBE = F.end(); BBI != BBE; ++BBI) {
528 BasicBlock *BB = &*BBI;
529
530 BasicBlock::reverse_iterator Next;
531
532 for (BasicBlock::reverse_iterator I = BB->rbegin(), E = BB->rend(); I != E;
533 I = Next) {
534 Instruction &Inst = *I;
535 Next = std::next(x: I);
536
537 if (processAtomicInstr(I: &Inst)) {
538 MadeChange = true;
539
540 // New blocks may have been inserted.
541 BBE = F.end();
542 }
543 }
544 }
545
546 return MadeChange;
547}
548
549bool AtomicExpandLegacy::runOnFunction(Function &F) {
550
551 auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
552 if (!TPC)
553 return false;
554 auto *TM = &TPC->getTM<TargetMachine>();
555
556 const ModuleLibcallLoweringInfo &LibcallResult =
557 getAnalysis<LibcallLoweringInfoWrapper>().getResult(M: *F.getParent());
558 AtomicExpandImpl AE;
559 return AE.run(F, LibcallResult, TM);
560}
561
562FunctionPass *llvm::createAtomicExpandLegacyPass() {
563 return new AtomicExpandLegacy();
564}
565
566PreservedAnalyses AtomicExpandPass::run(Function &F,
567 FunctionAnalysisManager &FAM) {
568 auto &MAMProxy = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: F);
569
570 const ModuleLibcallLoweringInfo *LibcallResult =
571 MAMProxy.getCachedResult<LibcallLoweringModuleAnalysis>(IR&: *F.getParent());
572
573 if (!LibcallResult) {
574 F.getContext().emitError(ErrorStr: "'" + LibcallLoweringModuleAnalysis::name() +
575 "' analysis required");
576 return PreservedAnalyses::all();
577 }
578
579 AtomicExpandImpl AE;
580
581 bool Changed = AE.run(F, LibcallResult: *LibcallResult, TM);
582 if (!Changed)
583 return PreservedAnalyses::all();
584
585 return PreservedAnalyses::none();
586}
587
588bool AtomicExpandImpl::bracketInstWithFences(Instruction *I,
589 AtomicOrdering Order) {
590 ReplacementIRBuilder Builder(I, *DL);
591
592 auto LeadingFence = TLI->emitLeadingFence(Builder, Inst: I, Ord: Order);
593
594 auto TrailingFence = TLI->emitTrailingFence(Builder, Inst: I, Ord: Order);
595 // We have a guard here because not every atomic operation generates a
596 // trailing fence.
597 if (TrailingFence)
598 TrailingFence->moveAfter(MovePos: I);
599
600 return (LeadingFence || TrailingFence);
601}
602
603/// Get the iX type with the same bitwidth as T.
604IntegerType *
605AtomicExpandImpl::getCorrespondingIntegerType(Type *T, const DataLayout &DL) {
606 EVT VT = TLI->getMemValueType(DL, Ty: T);
607 unsigned BitWidth = VT.getStoreSizeInBits();
608 assert(BitWidth == VT.getSizeInBits() && "must be a power of two");
609 return IntegerType::get(C&: T->getContext(), NumBits: BitWidth);
610}
611
612/// Convert an atomic load of a non-integral type to an integer load of the
613/// equivalent bitwidth. See the function comment on
614/// convertAtomicStoreToIntegerType for background.
615LoadInst *AtomicExpandImpl::convertAtomicLoadToIntegerType(LoadInst *LI) {
616 auto *M = LI->getModule();
617 Type *NewTy = getCorrespondingIntegerType(T: LI->getType(), DL: M->getDataLayout());
618
619 ReplacementIRBuilder Builder(LI, *DL);
620
621 Value *Addr = LI->getPointerOperand();
622
623 auto *NewLI = Builder.CreateLoad(Ty: NewTy, Ptr: Addr, Props: LI->getProperties());
624 LLVM_DEBUG(dbgs() << "Replaced " << *LI << " with " << *NewLI << "\n");
625
626 Value *NewVal = LI->getType()->isPtrOrPtrVectorTy()
627 ? Builder.CreateIntToPtr(V: NewLI, DestTy: LI->getType())
628 : Builder.CreateBitCast(V: NewLI, DestTy: LI->getType());
629 LI->replaceAllUsesWith(V: NewVal);
630 LI->eraseFromParent();
631 return NewLI;
632}
633
634AtomicRMWInst *
635AtomicExpandImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
636 assert(RMWI->getOperation() == AtomicRMWInst::Xchg);
637
638 auto *M = RMWI->getModule();
639 Type *NewTy =
640 getCorrespondingIntegerType(T: RMWI->getType(), DL: M->getDataLayout());
641
642 ReplacementIRBuilder Builder(RMWI, *DL);
643
644 Value *Addr = RMWI->getPointerOperand();
645 Value *Val = RMWI->getValOperand();
646 Value *NewVal = Builder.CreateBitPreservingCastChain(DL: *DL, V: Val, NewTy);
647
648 auto *NewRMWI = Builder.CreateAtomicRMW(Op: AtomicRMWInst::Xchg, Ptr: Addr, Val: NewVal,
649 Align: RMWI->getAlign(), Ordering: RMWI->getOrdering(),
650 SSID: RMWI->getSyncScopeID());
651 NewRMWI->setVolatile(RMWI->isVolatile());
652 copyMetadataForAtomic(Dest&: *NewRMWI, Source: *RMWI);
653 LLVM_DEBUG(dbgs() << "Replaced " << *RMWI << " with " << *NewRMWI << "\n");
654
655 Value *NewRVal =
656 Builder.CreateBitPreservingCastChain(DL: *DL, V: NewRMWI, NewTy: RMWI->getType());
657 RMWI->replaceAllUsesWith(V: NewRVal);
658 RMWI->eraseFromParent();
659 return NewRMWI;
660}
661
662/// Generic Expand handling for elementwise vector atomicrmw instructions.
663/// Halve an `atomicrmw elementwise <N x T>` and feed each half back through the
664/// normal atomic expansion pipeline. The target's shouldExpandAtomicRMWInIR()
665/// is re-queried at the halved width and may return any expansion kind there
666/// (e.g. None to preserve as a native vector atomic, Expand to keep halving,
667/// CmpXChg to emit one wide cmpxchg loop at the halved width). Two cases:
668/// N == 1: collapse directly to one scalar atomicrmw T
669/// N > 1: split into two atomicrmw elementwise <N/2 x T> (low at Ptr, high
670/// at gep inbounds <N/2 x T>, Ptr, 1); reassemble via a
671/// shufflevector.
672/// Note that halving works because the vectors must always be a power of two.
673///
674/// Each new atomicrmw inherits the original ordering/syncscope/volatility and
675/// metadata, and is fed back through processAtomicInstr() so further expansion
676/// fires per-half.
677void AtomicExpandImpl::expandElementwiseAtomicRMW(AtomicRMWInst *AI) {
678 assert(AI->isElementwise());
679 auto *VecTy = cast<FixedVectorType>(Val: AI->getType());
680 Type *LaneTy = VecTy->getElementType();
681 const unsigned NumLanes = VecTy->getNumElements();
682 Value *Ptr = AI->getPointerOperand();
683 Value *Val = AI->getValOperand();
684
685 ReplacementIRBuilder Builder(AI, *DL);
686 IntegerType *IdxTy =
687 DL->getIndexType(C&: AI->getContext(), AddressSpace: AI->getPointerAddressSpace());
688
689 auto CreateRMWInstruction = [&](Value *HalfPtr, Value *HalfVal, Align A,
690 bool Elementwise) -> AtomicRMWInst * {
691 auto *NewAI =
692 Builder.CreateAtomicRMW(Op: AI->getOperation(), Ptr: HalfPtr, Val: HalfVal, Align: A,
693 Ordering: AI->getOrdering(), SSID: AI->getSyncScopeID());
694 NewAI->setVolatile(AI->isVolatile());
695 NewAI->setElementwise(Elementwise);
696 copyMetadataForAtomic(Dest&: *NewAI, Source: *AI);
697 return NewAI;
698 };
699
700 Constant *IdxZero = ConstantInt::get(Ty: IdxTy, V: 0);
701 Constant *IdxOne = ConstantInt::get(Ty: IdxTy, V: 1);
702
703 // N == 1: collapse to a single scalar atomicrmw T and wrap the result back
704 // into a <1 x T> vector.
705 if (NumLanes == 1) {
706 Value *LaneVal = Builder.CreateExtractElement(Vec: Val, Idx: IdxZero, Name: "lane.val");
707 AtomicRMWInst *LaneRMW = CreateRMWInstruction(Ptr, LaneVal, AI->getAlign(),
708 /*Elementwise=*/false);
709 Value *Result = Builder.CreateInsertElement(Vec: PoisonValue::get(T: VecTy),
710 NewElt: LaneRMW, Idx: IdxZero, Name: "lane.old");
711 AI->replaceAllUsesWith(V: Result);
712 AI->eraseFromParent();
713 processAtomicInstr(I: LaneRMW);
714 return;
715 }
716
717 // N > 1: split into two <N/2 x T> elementwise atomicrmws and recurse via
718 // processAtomicInstr().
719 assert(isPowerOf2_32(NumLanes) &&
720 "elementwise atomicrmw vector length must be a power of two");
721 const unsigned HalfLanes = NumLanes / 2;
722 auto *HalfVecTy = FixedVectorType::get(ElementType: LaneTy, NumElts: HalfLanes);
723 const uint64_t HalfBytes = DL->getTypeStoreSize(Ty: HalfVecTy).getFixedValue();
724
725 Value *LoVal = Builder.CreateShuffleVector(
726 V: Val, Mask: createSequentialMask(Start: 0, NumInts: HalfLanes, NumUndefs: 0), Name: "lo.val");
727 Value *HiVal = Builder.CreateShuffleVector(
728 V: Val, Mask: createSequentialMask(Start: HalfLanes, NumInts: HalfLanes, NumUndefs: 0), Name: "hi.val");
729
730 Value *HiPtr = Builder.CreateInBoundsGEP(Ty: HalfVecTy, Ptr, IdxList: IdxOne, Name: "hi.ptr");
731 Align LoAlign = AI->getAlign();
732 Align HiAlign = commonAlignment(A: LoAlign, Offset: HalfBytes);
733
734 AtomicRMWInst *LoRMW =
735 CreateRMWInstruction(Ptr, LoVal, LoAlign, /*Elementwise=*/true);
736 AtomicRMWInst *HiRMW =
737 CreateRMWInstruction(HiPtr, HiVal, HiAlign, /*Elementwise=*/true);
738
739 Value *Result = concatenateVectors(Builder, Vecs: {LoRMW, HiRMW});
740 Result->setName("old");
741
742 AI->replaceAllUsesWith(V: Result);
743 AI->eraseFromParent();
744 processAtomicInstr(I: LoRMW);
745 processAtomicInstr(I: HiRMW);
746}
747
748bool AtomicExpandImpl::tryExpandAtomicLoad(LoadInst *LI) {
749 switch (TLI->shouldExpandAtomicLoadInIR(LI)) {
750 case TargetLoweringBase::AtomicExpansionKind::None:
751 return false;
752 case TargetLoweringBase::AtomicExpansionKind::LLSC:
753 expandAtomicOpToLLSC(
754 I: LI, ResultTy: LI->getType(), Addr: LI->getPointerOperand(), AddrAlign: LI->getAlign(),
755 MemOpOrder: LI->getOrdering(),
756 PerformOp: [](IRBuilderBase &Builder, Value *Loaded) { return Loaded; });
757 return true;
758 case TargetLoweringBase::AtomicExpansionKind::LLOnly:
759 return expandAtomicLoadToLL(LI);
760 case TargetLoweringBase::AtomicExpansionKind::CmpXChg:
761 return expandAtomicLoadToCmpXchg(LI);
762 case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
763 LI->setAtomic(Ordering: AtomicOrdering::NotAtomic);
764 return true;
765 case TargetLoweringBase::AtomicExpansionKind::CustomExpand:
766 TLI->emitExpandAtomicLoad(LI);
767 return true;
768 default:
769 llvm_unreachable("Unhandled case in tryExpandAtomicLoad");
770 }
771}
772
773bool AtomicExpandImpl::tryExpandAtomicStore(StoreInst *SI) {
774 switch (TLI->shouldExpandAtomicStoreInIR(SI)) {
775 case TargetLoweringBase::AtomicExpansionKind::None:
776 return false;
777 case TargetLoweringBase::AtomicExpansionKind::CustomExpand:
778 TLI->emitExpandAtomicStore(SI);
779 return true;
780 case TargetLoweringBase::AtomicExpansionKind::Expand:
781 expandAtomicStoreToXChg(SI);
782 return true;
783 case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
784 SI->setAtomic(Ordering: AtomicOrdering::NotAtomic);
785 return true;
786 default:
787 llvm_unreachable("Unhandled case in tryExpandAtomicStore");
788 }
789}
790
791bool AtomicExpandImpl::expandAtomicLoadToLL(LoadInst *LI) {
792 ReplacementIRBuilder Builder(LI, *DL);
793
794 // On some architectures, load-linked instructions are atomic for larger
795 // sizes than normal loads. For example, the only 64-bit load guaranteed
796 // to be single-copy atomic by ARM is an ldrexd (A3.5.3).
797 Value *Val = TLI->emitLoadLinked(Builder, ValueTy: LI->getType(),
798 Addr: LI->getPointerOperand(), Ord: LI->getOrdering());
799 TLI->emitAtomicCmpXchgNoStoreLLBalance(Builder);
800
801 LI->replaceAllUsesWith(V: Val);
802 LI->eraseFromParent();
803
804 return true;
805}
806
807bool AtomicExpandImpl::expandAtomicLoadToCmpXchg(LoadInst *LI) {
808 ReplacementIRBuilder Builder(LI, *DL);
809 AtomicOrdering Order = LI->getOrdering();
810 if (Order == AtomicOrdering::Unordered)
811 Order = AtomicOrdering::Monotonic;
812
813 Value *Addr = LI->getPointerOperand();
814 Type *Ty = LI->getType();
815
816 // cmpxchg supports only integer and pointer operands. If the load type is
817 // FP or vector, run the cmpxchg on the same-sized integer and bitcast the
818 // result back; mirrors createCmpXchgInstFun.
819 bool NeedBitcast = Ty->isFloatingPointTy() || Ty->isVectorTy();
820 Type *CmpXchgTy = Ty;
821 if (NeedBitcast)
822 CmpXchgTy = Builder.getIntNTy(N: Ty->getPrimitiveSizeInBits());
823 Constant *DummyVal = Constant::getNullValue(Ty: CmpXchgTy);
824
825 AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
826 Ptr: Addr, Cmp: DummyVal, New: DummyVal, Align: LI->getAlign(), SuccessOrdering: Order,
827 FailureOrdering: AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: Order),
828 SSID: LI->getSyncScopeID());
829 Pair->setVolatile(LI->isVolatile());
830 Value *Loaded = Builder.CreateExtractValue(Agg: Pair, Idxs: 0, Name: "loaded");
831 if (NeedBitcast)
832 Loaded = Builder.CreateBitCast(V: Loaded, DestTy: Ty);
833
834 LI->replaceAllUsesWith(V: Loaded);
835 LI->eraseFromParent();
836
837 return true;
838}
839
840/// Convert an atomic store of a non-integral type to an integer store of the
841/// equivalent bitwidth. We used to not support floating point or vector
842/// atomics in the IR at all. The backends learned to deal with the bitcast
843/// idiom because that was the only way of expressing the notion of a atomic
844/// float or vector store. The long term plan is to teach each backend to
845/// instruction select from the original atomic store, but as a migration
846/// mechanism, we convert back to the old format which the backends understand.
847/// Each backend will need individual work to recognize the new format.
848StoreInst *AtomicExpandImpl::convertAtomicStoreToIntegerType(StoreInst *SI) {
849 ReplacementIRBuilder Builder(SI, *DL);
850 auto *M = SI->getModule();
851 Type *NewTy = getCorrespondingIntegerType(T: SI->getValueOperand()->getType(),
852 DL: M->getDataLayout());
853 Value *NewVal = SI->getValueOperand()->getType()->isPtrOrPtrVectorTy()
854 ? Builder.CreatePtrToInt(V: SI->getValueOperand(), DestTy: NewTy)
855 : Builder.CreateBitCast(V: SI->getValueOperand(), DestTy: NewTy);
856
857 Value *Addr = SI->getPointerOperand();
858
859 StoreInst *NewSI = Builder.CreateStore(Val: NewVal, Ptr: Addr, Props: SI->getProperties());
860 copyMetadataForAtomic(Dest&: *NewSI, Source: *SI);
861 LLVM_DEBUG(dbgs() << "Replaced " << *SI << " with " << *NewSI << "\n");
862 SI->eraseFromParent();
863 return NewSI;
864}
865
866void AtomicExpandImpl::expandAtomicStoreToXChg(StoreInst *SI) {
867 // This function is only called on atomic stores that are too large to be
868 // atomic if implemented as a native store. So we replace them by an
869 // atomic swap, that can be implemented for example as a ldrex/strex on ARM
870 // or lock cmpxchg8/16b on X86, as these are atomic for larger sizes.
871 // It is the responsibility of the target to only signal expansion via
872 // shouldExpandAtomicRMW in cases where this is required and possible.
873 ReplacementIRBuilder Builder(SI, *DL);
874 AtomicOrdering Ordering = SI->getOrdering();
875 assert(Ordering != AtomicOrdering::NotAtomic);
876 AtomicOrdering RMWOrdering = Ordering == AtomicOrdering::Unordered
877 ? AtomicOrdering::Monotonic
878 : Ordering;
879 AtomicRMWInst *AI = Builder.CreateAtomicRMW(
880 Op: AtomicRMWInst::Xchg, Ptr: SI->getPointerOperand(), Val: SI->getValueOperand(),
881 Align: SI->getAlign(), Ordering: RMWOrdering, SSID: SI->getSyncScopeID());
882 AI->setVolatile(SI->isVolatile());
883 SI->eraseFromParent();
884
885 // Now we have an appropriate swap instruction, lower it as usual.
886 tryExpandAtomicRMW(AI);
887}
888
889static void createCmpXchgInstFun(IRBuilderBase &Builder, Value *Addr,
890 Value *Loaded, Value *NewVal, Align AddrAlign,
891 AtomicOrdering MemOpOrder, SyncScope::ID SSID,
892 bool IsVolatile, Value *&Success,
893 Value *&NewLoaded, Instruction *MetadataSrc) {
894 Type *OrigTy = NewVal->getType();
895
896 // This code can go away when cmpxchg supports FP and vector types.
897 assert(!OrigTy->isPointerTy());
898 bool NeedBitcast = OrigTy->isFloatingPointTy() || OrigTy->isVectorTy();
899 if (NeedBitcast) {
900 IntegerType *IntTy = Builder.getIntNTy(N: OrigTy->getPrimitiveSizeInBits());
901 NewVal = Builder.CreateBitCast(V: NewVal, DestTy: IntTy);
902 Loaded = Builder.CreateBitCast(V: Loaded, DestTy: IntTy);
903 }
904
905 AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
906 Ptr: Addr, Cmp: Loaded, New: NewVal, Align: AddrAlign, SuccessOrdering: MemOpOrder,
907 FailureOrdering: AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: MemOpOrder), SSID);
908 Pair->setVolatile(IsVolatile);
909 if (MetadataSrc)
910 copyMetadataForAtomic(Dest&: *Pair, Source: *MetadataSrc);
911
912 Success = Builder.CreateExtractValue(Agg: Pair, Idxs: 1, Name: "success");
913 NewLoaded = Builder.CreateExtractValue(Agg: Pair, Idxs: 0, Name: "newloaded");
914
915 if (NeedBitcast)
916 NewLoaded = Builder.CreateBitCast(V: NewLoaded, DestTy: OrigTy);
917}
918
919void AtomicExpandImpl::expandAtomicSubToAdd(AtomicRMWInst *AI) {
920 ReplacementIRBuilder Builder(AI, *DL);
921 AtomicRMWInst::BinOp NewOp;
922 Value *NewVal;
923
924 switch (AI->getOperation()) {
925 case AtomicRMWInst::Sub:
926 NewOp = AtomicRMWInst::Add;
927 NewVal = Builder.CreateNeg(V: AI->getValOperand(), Name: "neg");
928 break;
929 case AtomicRMWInst::FSub:
930 NewOp = AtomicRMWInst::FAdd;
931 NewVal = Builder.CreateFNeg(V: AI->getValOperand(), Name: "fneg");
932 break;
933 default:
934 llvm_unreachable("unsupported atomicrmw expansion");
935 }
936
937 AI->setOperation(NewOp);
938 AI->setOperand(i_nocapture: 1, Val_nocapture: NewVal);
939}
940
941bool AtomicExpandImpl::tryExpandAtomicRMW(AtomicRMWInst *AI) {
942 LLVMContext &Ctx = AI->getModule()->getContext();
943 TargetLowering::AtomicExpansionKind Kind = TLI->shouldExpandAtomicRMWInIR(RMW: AI);
944 switch (Kind) {
945 case TargetLoweringBase::AtomicExpansionKind::None:
946 return false;
947 case TargetLoweringBase::AtomicExpansionKind::LLSC: {
948 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
949 unsigned ValueSize = getAtomicOpSize(RMWI: AI);
950 if (ValueSize < MinCASSize) {
951 expandPartwordAtomicRMW(I: AI,
952 ExpansionKind: TargetLoweringBase::AtomicExpansionKind::LLSC);
953 } else {
954 auto PerformOp = [&](IRBuilderBase &Builder, Value *Loaded) {
955 return buildAtomicRMWValue(Op: AI->getOperation(), Builder, Loaded,
956 Val: AI->getValOperand());
957 };
958 expandAtomicOpToLLSC(I: AI, ResultTy: AI->getType(), Addr: AI->getPointerOperand(),
959 AddrAlign: AI->getAlign(), MemOpOrder: AI->getOrdering(), PerformOp);
960 }
961 return true;
962 }
963 case TargetLoweringBase::AtomicExpansionKind::CmpXChg: {
964 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
965 unsigned ValueSize = getAtomicOpSize(RMWI: AI);
966 if (ValueSize < MinCASSize) {
967 expandPartwordAtomicRMW(I: AI,
968 ExpansionKind: TargetLoweringBase::AtomicExpansionKind::CmpXChg);
969 } else {
970 SmallVector<StringRef> SSNs;
971 Ctx.getSyncScopeNames(SSNs);
972 auto MemScope = SSNs[AI->getSyncScopeID()].empty()
973 ? "system"
974 : SSNs[AI->getSyncScopeID()];
975 OptimizationRemarkEmitter ORE(AI->getFunction());
976 ORE.emit(RemarkBuilder: [&]() {
977 return OptimizationRemark(DEBUG_TYPE, "Passed", AI)
978 << "A compare and swap loop was generated for an atomic "
979 << AI->getOperationName(Op: AI->getOperation()) << " operation at "
980 << MemScope << " memory scope";
981 });
982 expandAtomicRMWToCmpXchg(AI, CreateCmpXchg: createCmpXchgInstFun);
983 }
984 return true;
985 }
986 case TargetLoweringBase::AtomicExpansionKind::MaskedIntrinsic: {
987 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
988 unsigned ValueSize = getAtomicOpSize(RMWI: AI);
989 if (ValueSize < MinCASSize) {
990 AtomicRMWInst::BinOp Op = AI->getOperation();
991 // Widen And/Or/Xor and give the target another chance at expanding it.
992 if (Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||
993 Op == AtomicRMWInst::And) {
994 tryExpandAtomicRMW(AI: widenPartwordAtomicRMW(AI));
995 return true;
996 }
997 }
998 expandAtomicRMWToMaskedIntrinsic(AI);
999 return true;
1000 }
1001 case TargetLoweringBase::AtomicExpansionKind::BitTestIntrinsic: {
1002 TLI->emitBitTestAtomicRMWIntrinsic(AI);
1003 return true;
1004 }
1005 case TargetLoweringBase::AtomicExpansionKind::CmpArithIntrinsic: {
1006 TLI->emitCmpArithAtomicRMWIntrinsic(AI);
1007 return true;
1008 }
1009 case TargetLoweringBase::AtomicExpansionKind::Expand:
1010 if (AI->isElementwise()) {
1011 expandElementwiseAtomicRMW(AI);
1012 return true;
1013 }
1014 expandAtomicSubToAdd(AI);
1015 return true;
1016 case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
1017 return lowerAtomicRMWInst(RMWI: AI);
1018 case TargetLoweringBase::AtomicExpansionKind::CustomExpand:
1019 TLI->emitExpandAtomicRMW(AI);
1020 return true;
1021 default:
1022 llvm_unreachable("Unhandled case in tryExpandAtomicRMW");
1023 }
1024}
1025
1026namespace {
1027
1028struct PartwordMaskValues {
1029 // These three fields are guaranteed to be set by createMaskInstrs.
1030 Type *WordType = nullptr;
1031 Type *ValueType = nullptr;
1032 Type *IntValueType = nullptr;
1033 Value *AlignedAddr = nullptr;
1034 Align AlignedAddrAlignment;
1035 // The remaining fields can be null.
1036 Value *ShiftAmt = nullptr;
1037 Value *Mask = nullptr;
1038 Value *Inv_Mask = nullptr;
1039};
1040
1041[[maybe_unused]]
1042raw_ostream &operator<<(raw_ostream &O, const PartwordMaskValues &PMV) {
1043 auto PrintObj = [&O](auto *V) {
1044 if (V)
1045 O << *V;
1046 else
1047 O << "nullptr";
1048 O << '\n';
1049 };
1050 O << "PartwordMaskValues {\n";
1051 O << " WordType: ";
1052 PrintObj(PMV.WordType);
1053 O << " ValueType: ";
1054 PrintObj(PMV.ValueType);
1055 O << " AlignedAddr: ";
1056 PrintObj(PMV.AlignedAddr);
1057 O << " AlignedAddrAlignment: " << PMV.AlignedAddrAlignment.value() << '\n';
1058 O << " ShiftAmt: ";
1059 PrintObj(PMV.ShiftAmt);
1060 O << " Mask: ";
1061 PrintObj(PMV.Mask);
1062 O << " Inv_Mask: ";
1063 PrintObj(PMV.Inv_Mask);
1064 O << "}\n";
1065 return O;
1066}
1067
1068} // end anonymous namespace
1069
1070/// This is a helper function which builds instructions to provide
1071/// values necessary for partword atomic operations. It takes an
1072/// incoming address, Addr, and ValueType, and constructs the address,
1073/// shift-amounts and masks needed to work with a larger value of size
1074/// WordSize.
1075///
1076/// AlignedAddr: Addr rounded down to a multiple of WordSize
1077///
1078/// ShiftAmt: Number of bits to right-shift a WordSize value loaded
1079/// from AlignAddr for it to have the same value as if
1080/// ValueType was loaded from Addr.
1081///
1082/// Mask: Value to mask with the value loaded from AlignAddr to
1083/// include only the part that would've been loaded from Addr.
1084///
1085/// Inv_Mask: The inverse of Mask.
1086static PartwordMaskValues createMaskInstrs(IRBuilderBase &Builder,
1087 Instruction *I, Type *ValueType,
1088 Value *Addr, Align AddrAlign,
1089 unsigned MinWordSize) {
1090 PartwordMaskValues PMV;
1091
1092 Module *M = I->getModule();
1093 LLVMContext &Ctx = M->getContext();
1094 const DataLayout &DL = M->getDataLayout();
1095 unsigned ValueSize = DL.getTypeStoreSize(Ty: ValueType);
1096
1097 PMV.ValueType = PMV.IntValueType = ValueType;
1098 if (PMV.ValueType->isFloatingPointTy() || PMV.ValueType->isVectorTy())
1099 PMV.IntValueType =
1100 Type::getIntNTy(C&: Ctx, N: ValueType->getPrimitiveSizeInBits());
1101
1102 PMV.WordType = MinWordSize > ValueSize ? Type::getIntNTy(C&: Ctx, N: MinWordSize * 8)
1103 : ValueType;
1104 if (PMV.ValueType == PMV.WordType) {
1105 PMV.AlignedAddr = Addr;
1106 PMV.AlignedAddrAlignment = AddrAlign;
1107 PMV.ShiftAmt = ConstantInt::get(Ty: PMV.ValueType, V: 0);
1108 PMV.Mask = ConstantInt::get(Ty: PMV.ValueType, V: ~0, /*isSigned*/ IsSigned: true);
1109 return PMV;
1110 }
1111
1112 PMV.AlignedAddrAlignment = Align(MinWordSize);
1113
1114 assert(ValueSize < MinWordSize);
1115
1116 PointerType *PtrTy = cast<PointerType>(Val: Addr->getType());
1117 IntegerType *IntTy = DL.getIndexType(C&: Ctx, AddressSpace: PtrTy->getAddressSpace());
1118 Value *PtrLSB;
1119
1120 if (AddrAlign < MinWordSize) {
1121 PMV.AlignedAddr = Builder.CreateIntrinsic(
1122 ID: Intrinsic::ptrmask, OverloadTypes: {PtrTy, IntTy},
1123 Args: {Addr, ConstantInt::getSigned(Ty: IntTy, V: ~(uint64_t)(MinWordSize - 1))},
1124 FMFSource: nullptr, Name: "AlignedAddr");
1125
1126 Value *AddrInt = Builder.CreatePtrToInt(V: Addr, DestTy: IntTy);
1127 PtrLSB = Builder.CreateAnd(LHS: AddrInt, RHS: MinWordSize - 1, Name: "PtrLSB");
1128 } else {
1129 // If the alignment is high enough, the LSB are known 0.
1130 PMV.AlignedAddr = Addr;
1131 PtrLSB = ConstantInt::getNullValue(Ty: IntTy);
1132 }
1133
1134 if (DL.isLittleEndian()) {
1135 // turn bytes into bits
1136 PMV.ShiftAmt = Builder.CreateShl(LHS: PtrLSB, RHS: 3);
1137 } else {
1138 // turn bytes into bits, and count from the other side.
1139 PMV.ShiftAmt = Builder.CreateShl(
1140 LHS: Builder.CreateXor(LHS: PtrLSB, RHS: MinWordSize - ValueSize), RHS: 3);
1141 }
1142
1143 PMV.ShiftAmt = Builder.CreateTrunc(V: PMV.ShiftAmt, DestTy: PMV.WordType, Name: "ShiftAmt");
1144 PMV.Mask = Builder.CreateShl(
1145 LHS: ConstantInt::get(Ty: PMV.WordType, V: (1 << (ValueSize * 8)) - 1), RHS: PMV.ShiftAmt,
1146 Name: "Mask");
1147
1148 PMV.Inv_Mask = Builder.CreateNot(V: PMV.Mask, Name: "Inv_Mask");
1149
1150 return PMV;
1151}
1152
1153static Value *extractMaskedValue(IRBuilderBase &Builder, Value *WideWord,
1154 const PartwordMaskValues &PMV) {
1155 assert(WideWord->getType() == PMV.WordType && "Widened type mismatch");
1156 if (PMV.WordType == PMV.ValueType)
1157 return WideWord;
1158
1159 Value *Shift = Builder.CreateLShr(LHS: WideWord, RHS: PMV.ShiftAmt, Name: "shifted");
1160 Value *Trunc = Builder.CreateTrunc(V: Shift, DestTy: PMV.IntValueType, Name: "extracted");
1161 return Builder.CreateBitCast(V: Trunc, DestTy: PMV.ValueType);
1162}
1163
1164static Value *insertMaskedValue(IRBuilderBase &Builder, Value *WideWord,
1165 Value *Updated, const PartwordMaskValues &PMV) {
1166 assert(WideWord->getType() == PMV.WordType && "Widened type mismatch");
1167 assert(Updated->getType() == PMV.ValueType && "Value type mismatch");
1168 if (PMV.WordType == PMV.ValueType)
1169 return Updated;
1170
1171 Updated = Builder.CreateBitCast(V: Updated, DestTy: PMV.IntValueType);
1172
1173 Value *ZExt = Builder.CreateZExt(V: Updated, DestTy: PMV.WordType, Name: "extended");
1174 Value *Shift =
1175 Builder.CreateShl(LHS: ZExt, RHS: PMV.ShiftAmt, Name: "shifted", /*HasNUW*/ true);
1176 Value *And = Builder.CreateAnd(LHS: WideWord, RHS: PMV.Inv_Mask, Name: "unmasked");
1177 Value *Or = Builder.CreateOr(LHS: And, RHS: Shift, Name: "inserted");
1178 return Or;
1179}
1180
1181/// Emit IR to implement a masked version of a given atomicrmw
1182/// operation. (That is, only the bits under the Mask should be
1183/// affected by the operation)
1184static Value *performMaskedAtomicOp(AtomicRMWInst::BinOp Op,
1185 IRBuilderBase &Builder, Value *Loaded,
1186 Value *ValOperand_Shifted, Value *Inc,
1187 const PartwordMaskValues &PMV) {
1188 // TODO: update to use
1189 // https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge in order
1190 // to merge bits from two values without requiring PMV.Inv_Mask.
1191
1192 assert(Op != AtomicRMWInst::Or && Op != AtomicRMWInst::Xor &&
1193 Op != AtomicRMWInst::And &&
1194 "Or/Xor/And handled by widenPartwordAtomicRMW");
1195
1196 if (Op == AtomicRMWInst::Xchg) {
1197 // Clear all the bits we are exchanging out. These are the bits under the
1198 // mask. We can clear them with an `and` of the inverse mask.
1199 Value *Loaded_MaskOut = Builder.CreateAnd(LHS: Loaded, RHS: PMV.Inv_Mask);
1200 // Now that the prevous bits are cleared, we can swap in the new value with
1201 // an `or`.
1202 Value *FinalVal = Builder.CreateOr(LHS: Loaded_MaskOut, RHS: ValOperand_Shifted);
1203 return FinalVal;
1204 }
1205
1206 if (Op == AtomicRMWInst::Nand ||
1207 (!PMV.ValueType->isVectorTy() &&
1208 (Op == AtomicRMWInst::Add || Op == AtomicRMWInst::Sub))) {
1209 // For `Nand` and non-vector `Add` and `Sub`, we can perform the operation
1210 // on the entire word because the extra bits in the unmasked region don't
1211 // affect the computation in the masked region. The operation might still
1212 // overwrite the unmasked region (e.g. from integer overflow or underflow),
1213 // so we have to reapply the unmasked region afterwards.
1214 //
1215 // This trick doesn't work for vector `Add` and `Sub` because we use a
1216 // scalar operation on the entire word. Scalarizing vector `Add` and `Sub`
1217 // isn't legal because the vector versions may have element-wise overflows.
1218 // TODO: For these, can we use a wider vector op with additional lanes?
1219
1220 // Atomic operation across the entire word.
1221 Value *NewVal =
1222 buildAtomicRMWValue(Op, Builder, Loaded, Val: ValOperand_Shifted);
1223 // Reapply the bits in the unmasked region.
1224 Value *NewVal_Masked = Builder.CreateAnd(LHS: NewVal, RHS: PMV.Mask);
1225 Value *Loaded_MaskOut = Builder.CreateAnd(LHS: Loaded, RHS: PMV.Inv_Mask);
1226 Value *FinalVal = Builder.CreateOr(LHS: Loaded_MaskOut, RHS: NewVal_Masked);
1227 return FinalVal;
1228 }
1229
1230 // All other ops operate on the sub-word size. Truncate down to the
1231 // original size, and expand out again after doing the operation. Bitcasts
1232 // will be inserted for FP values.
1233 assert(!ValOperand_Shifted);
1234 Value *Loaded_Extract = extractMaskedValue(Builder, WideWord: Loaded, PMV);
1235 Value *NewVal = buildAtomicRMWValue(Op, Builder, Loaded: Loaded_Extract, Val: Inc);
1236 Value *FinalVal = insertMaskedValue(Builder, WideWord: Loaded, Updated: NewVal, PMV);
1237 return FinalVal;
1238}
1239
1240/// Expand a sub-word atomicrmw operation into an appropriate
1241/// word-sized operation.
1242///
1243/// It will create an LL/SC or cmpxchg loop, as appropriate, the same
1244/// way as a typical atomicrmw expansion. The only difference here is
1245/// that the operation inside of the loop may operate upon only a
1246/// part of the value.
1247void AtomicExpandImpl::expandPartwordAtomicRMW(
1248 AtomicRMWInst *AI, TargetLoweringBase::AtomicExpansionKind ExpansionKind) {
1249 // Widen And/Or/Xor and give the target another chance at expanding it.
1250 AtomicRMWInst::BinOp Op = AI->getOperation();
1251 if (Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||
1252 Op == AtomicRMWInst::And) {
1253 tryExpandAtomicRMW(AI: widenPartwordAtomicRMW(AI));
1254 return;
1255 }
1256 AtomicOrdering MemOpOrder = AI->getOrdering();
1257 SyncScope::ID SSID = AI->getSyncScopeID();
1258
1259 ReplacementIRBuilder Builder(AI, *DL);
1260
1261 PartwordMaskValues PMV =
1262 createMaskInstrs(Builder, I: AI, ValueType: AI->getType(), Addr: AI->getPointerOperand(),
1263 AddrAlign: AI->getAlign(), MinWordSize: TLI->getMinCmpXchgSizeInBits() / 8);
1264
1265 Value *ValOperand_Shifted = nullptr;
1266 bool NeedsShiftedOperand =
1267 Op == AtomicRMWInst::Xchg || Op == AtomicRMWInst::Nand ||
1268 (!PMV.ValueType->isVectorTy() &&
1269 (Op == AtomicRMWInst::Add || Op == AtomicRMWInst::Sub));
1270
1271 if (NeedsShiftedOperand) {
1272 Value *ValOp = Builder.CreateBitCast(V: AI->getValOperand(), DestTy: PMV.IntValueType);
1273 ValOperand_Shifted =
1274 Builder.CreateShl(LHS: Builder.CreateZExt(V: ValOp, DestTy: PMV.WordType), RHS: PMV.ShiftAmt,
1275 Name: "ValOperand_Shifted");
1276 }
1277
1278 auto PerformPartwordOp = [&](IRBuilderBase &Builder, Value *Loaded) {
1279 return performMaskedAtomicOp(Op, Builder, Loaded, ValOperand_Shifted,
1280 Inc: AI->getValOperand(), PMV);
1281 };
1282
1283 Value *OldResult;
1284 if (ExpansionKind == TargetLoweringBase::AtomicExpansionKind::CmpXChg) {
1285 OldResult = insertRMWCmpXchgLoop(Builder, ResultType: PMV.WordType, Addr: PMV.AlignedAddr,
1286 AddrAlign: PMV.AlignedAddrAlignment, MemOpOrder, SSID,
1287 IsVolatile: AI->isVolatile(), PerformOp: PerformPartwordOp,
1288 CreateCmpXchg: createCmpXchgInstFun, MetadataSrc: AI);
1289 } else {
1290 assert(ExpansionKind == TargetLoweringBase::AtomicExpansionKind::LLSC);
1291 OldResult = insertRMWLLSCLoop(Builder, ResultTy: PMV.WordType, Addr: PMV.AlignedAddr,
1292 AddrAlign: PMV.AlignedAddrAlignment, MemOpOrder,
1293 PerformOp: PerformPartwordOp);
1294 }
1295
1296 Value *FinalOldResult = extractMaskedValue(Builder, WideWord: OldResult, PMV);
1297 AI->replaceAllUsesWith(V: FinalOldResult);
1298 AI->eraseFromParent();
1299}
1300
1301// Widen the bitwise atomicrmw (or/xor/and) to the minimum supported width.
1302AtomicRMWInst *AtomicExpandImpl::widenPartwordAtomicRMW(AtomicRMWInst *AI) {
1303 ReplacementIRBuilder Builder(AI, *DL);
1304 AtomicRMWInst::BinOp Op = AI->getOperation();
1305
1306 assert((Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||
1307 Op == AtomicRMWInst::And) &&
1308 "Unable to widen operation");
1309
1310 PartwordMaskValues PMV =
1311 createMaskInstrs(Builder, I: AI, ValueType: AI->getType(), Addr: AI->getPointerOperand(),
1312 AddrAlign: AI->getAlign(), MinWordSize: TLI->getMinCmpXchgSizeInBits() / 8);
1313
1314 Value *ValOp = AI->getValOperand();
1315 if (ValOp->getType()->isVectorTy())
1316 // For vectors, bitcast to the integer type before extending. Note that
1317 // or/xor/and on vectors are equivalent to the same operation on an integer
1318 // that spans the vector, so we can use the integer type for the operation.
1319 ValOp = Builder.CreateBitCast(V: ValOp, DestTy: PMV.IntValueType);
1320 Value *ValOperand_Shifted =
1321 Builder.CreateShl(LHS: Builder.CreateZExt(V: ValOp, DestTy: PMV.WordType), RHS: PMV.ShiftAmt,
1322 Name: "ValOperand_Shifted");
1323
1324 Value *NewOperand;
1325
1326 if (Op == AtomicRMWInst::And)
1327 NewOperand =
1328 Builder.CreateOr(LHS: ValOperand_Shifted, RHS: PMV.Inv_Mask, Name: "AndOperand");
1329 else
1330 NewOperand = ValOperand_Shifted;
1331
1332 AtomicRMWInst *NewAI = Builder.CreateAtomicRMW(
1333 Op, Ptr: PMV.AlignedAddr, Val: NewOperand, Align: PMV.AlignedAddrAlignment,
1334 Ordering: AI->getOrdering(), SSID: AI->getSyncScopeID());
1335
1336 NewAI->setVolatile(AI->isVolatile());
1337 copyMetadataForAtomic(Dest&: *NewAI, Source: *AI);
1338
1339 Value *FinalOldResult = extractMaskedValue(Builder, WideWord: NewAI, PMV);
1340 AI->replaceAllUsesWith(V: FinalOldResult);
1341 AI->eraseFromParent();
1342 return NewAI;
1343}
1344
1345bool AtomicExpandImpl::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
1346 // The basic idea here is that we're expanding a cmpxchg of a
1347 // smaller memory size up to a word-sized cmpxchg. To do this, we
1348 // need to add a retry-loop for strong cmpxchg, so that
1349 // modifications to other parts of the word don't cause a spurious
1350 // failure.
1351
1352 // This generates code like the following:
1353 // [[Setup mask values PMV.*]]
1354 // %NewVal_Shifted = shl i32 %NewVal, %PMV.ShiftAmt
1355 // %Cmp_Shifted = shl i32 %Cmp, %PMV.ShiftAmt
1356 // %InitLoaded = load i32* %addr
1357 // %InitLoaded_MaskOut = and i32 %InitLoaded, %PMV.Inv_Mask
1358 // br partword.cmpxchg.loop
1359 // partword.cmpxchg.loop:
1360 // %Loaded_MaskOut = phi i32 [ %InitLoaded_MaskOut, %entry ],
1361 // [ %OldVal_MaskOut, %partword.cmpxchg.failure ]
1362 // %FullWord_NewVal = or i32 %Loaded_MaskOut, %NewVal_Shifted
1363 // %FullWord_Cmp = or i32 %Loaded_MaskOut, %Cmp_Shifted
1364 // %NewCI = cmpxchg i32* %PMV.AlignedAddr, i32 %FullWord_Cmp,
1365 // i32 %FullWord_NewVal success_ordering failure_ordering
1366 // %OldVal = extractvalue { i32, i1 } %NewCI, 0
1367 // %Success = extractvalue { i32, i1 } %NewCI, 1
1368 // br i1 %Success, label %partword.cmpxchg.end,
1369 // label %partword.cmpxchg.failure
1370 // partword.cmpxchg.failure:
1371 // %OldVal_MaskOut = and i32 %OldVal, %PMV.Inv_Mask
1372 // %ShouldContinue = icmp ne i32 %Loaded_MaskOut, %OldVal_MaskOut
1373 // br i1 %ShouldContinue, label %partword.cmpxchg.loop,
1374 // label %partword.cmpxchg.end
1375 // partword.cmpxchg.end:
1376 // %tmp1 = lshr i32 %OldVal, %PMV.ShiftAmt
1377 // %FinalOldVal = trunc i32 %tmp1 to i8
1378 // %tmp2 = insertvalue { i8, i1 } undef, i8 %FinalOldVal, 0
1379 // %Res = insertvalue { i8, i1 } %25, i1 %Success, 1
1380
1381 Value *Addr = CI->getPointerOperand();
1382 Value *Cmp = CI->getCompareOperand();
1383 Value *NewVal = CI->getNewValOperand();
1384
1385 BasicBlock *BB = CI->getParent();
1386 Function *F = BB->getParent();
1387 ReplacementIRBuilder Builder(CI, *DL);
1388 LLVMContext &Ctx = Builder.getContext();
1389
1390 BasicBlock *EndBB =
1391 BB->splitBasicBlock(I: CI->getIterator(), BBName: "partword.cmpxchg.end");
1392 auto FailureBB =
1393 BasicBlock::Create(Context&: Ctx, Name: "partword.cmpxchg.failure", Parent: F, InsertBefore: EndBB);
1394 auto LoopBB = BasicBlock::Create(Context&: Ctx, Name: "partword.cmpxchg.loop", Parent: F, InsertBefore: FailureBB);
1395
1396 // The split call above "helpfully" added a branch at the end of BB
1397 // (to the wrong place).
1398 std::prev(x: BB->end())->eraseFromParent();
1399 Builder.SetInsertPoint(BB);
1400
1401 PartwordMaskValues PMV =
1402 createMaskInstrs(Builder, I: CI, ValueType: CI->getCompareOperand()->getType(), Addr,
1403 AddrAlign: CI->getAlign(), MinWordSize: TLI->getMinCmpXchgSizeInBits() / 8);
1404
1405 // Shift the incoming values over, into the right location in the word.
1406 Value *NewVal_Shifted =
1407 Builder.CreateShl(LHS: Builder.CreateZExt(V: NewVal, DestTy: PMV.WordType), RHS: PMV.ShiftAmt);
1408 Value *Cmp_Shifted =
1409 Builder.CreateShl(LHS: Builder.CreateZExt(V: Cmp, DestTy: PMV.WordType), RHS: PMV.ShiftAmt);
1410
1411 // Load the entire current word, and mask into place the expected and new
1412 // values
1413 LoadInst *InitLoaded = Builder.CreateLoad(Ty: PMV.WordType, Ptr: PMV.AlignedAddr);
1414 Value *InitLoaded_MaskOut = Builder.CreateAnd(LHS: InitLoaded, RHS: PMV.Inv_Mask);
1415 Builder.CreateBr(Dest: LoopBB);
1416
1417 // partword.cmpxchg.loop:
1418 Builder.SetInsertPoint(LoopBB);
1419 PHINode *Loaded_MaskOut = Builder.CreatePHI(Ty: PMV.WordType, NumReservedValues: 2);
1420 Loaded_MaskOut->addIncoming(V: InitLoaded_MaskOut, BB);
1421
1422 // The initial load must be atomic with the same synchronization scope
1423 // to avoid a data race with concurrent stores. If the instruction being
1424 // emulated is volatile, issue a volatile load.
1425 // addIncoming is done first so that any replaceAllUsesWith calls during
1426 // normalization correctly update the PHI incoming value.
1427 InitLoaded->setVolatile(CI->isVolatile());
1428 if (TLI->shouldIssueAtomicLoadForAtomicEmulationLoop()) {
1429 InitLoaded->setAtomic(Ordering: AtomicOrdering::Monotonic, SSID: CI->getSyncScopeID());
1430 // The newly created load might need to be lowered further. Because it is
1431 // created in the same block as the atomicrmw, the AtomicExpand loop will
1432 // not process it again.
1433 processAtomicInstr(I: InitLoaded);
1434 }
1435
1436 // Mask/Or the expected and new values into place in the loaded word.
1437 Value *FullWord_NewVal = Builder.CreateOr(LHS: Loaded_MaskOut, RHS: NewVal_Shifted);
1438 Value *FullWord_Cmp = Builder.CreateOr(LHS: Loaded_MaskOut, RHS: Cmp_Shifted);
1439 AtomicCmpXchgInst *NewCI = Builder.CreateAtomicCmpXchg(
1440 Ptr: PMV.AlignedAddr, Cmp: FullWord_Cmp, New: FullWord_NewVal, Align: PMV.AlignedAddrAlignment,
1441 SuccessOrdering: CI->getSuccessOrdering(), FailureOrdering: CI->getFailureOrdering(), SSID: CI->getSyncScopeID());
1442 NewCI->setVolatile(CI->isVolatile());
1443 // When we're building a strong cmpxchg, we need a loop, so you
1444 // might think we could use a weak cmpxchg inside. But, using strong
1445 // allows the below comparison for ShouldContinue, and we're
1446 // expecting the underlying cmpxchg to be a machine instruction,
1447 // which is strong anyways.
1448 NewCI->setWeak(CI->isWeak());
1449
1450 Value *OldVal = Builder.CreateExtractValue(Agg: NewCI, Idxs: 0);
1451 Value *Success = Builder.CreateExtractValue(Agg: NewCI, Idxs: 1);
1452
1453 if (CI->isWeak())
1454 Builder.CreateBr(Dest: EndBB);
1455 else
1456 Builder.CreateCondBr(Cond: Success, True: EndBB, False: FailureBB);
1457
1458 // partword.cmpxchg.failure:
1459 Builder.SetInsertPoint(FailureBB);
1460 // Upon failure, verify that the masked-out part of the loaded value
1461 // has been modified. If it didn't, abort the cmpxchg, since the
1462 // masked-in part must've.
1463 Value *OldVal_MaskOut = Builder.CreateAnd(LHS: OldVal, RHS: PMV.Inv_Mask);
1464 Value *ShouldContinue = Builder.CreateICmpNE(LHS: Loaded_MaskOut, RHS: OldVal_MaskOut);
1465 Builder.CreateCondBr(Cond: ShouldContinue, True: LoopBB, False: EndBB);
1466
1467 // Add the second value to the phi from above
1468 Loaded_MaskOut->addIncoming(V: OldVal_MaskOut, BB: FailureBB);
1469
1470 // partword.cmpxchg.end:
1471 Builder.SetInsertPoint(CI);
1472
1473 Value *FinalOldVal = extractMaskedValue(Builder, WideWord: OldVal, PMV);
1474 Value *Res = PoisonValue::get(T: CI->getType());
1475 Res = Builder.CreateInsertValue(Agg: Res, Val: FinalOldVal, Idxs: 0);
1476 Res = Builder.CreateInsertValue(Agg: Res, Val: Success, Idxs: 1);
1477
1478 CI->replaceAllUsesWith(V: Res);
1479 CI->eraseFromParent();
1480 return true;
1481}
1482
1483void AtomicExpandImpl::expandAtomicOpToLLSC(
1484 Instruction *I, Type *ResultType, Value *Addr, Align AddrAlign,
1485 AtomicOrdering MemOpOrder,
1486 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp) {
1487 ReplacementIRBuilder Builder(I, *DL);
1488 Value *Loaded = insertRMWLLSCLoop(Builder, ResultTy: ResultType, Addr, AddrAlign,
1489 MemOpOrder, PerformOp);
1490
1491 I->replaceAllUsesWith(V: Loaded);
1492 I->eraseFromParent();
1493}
1494
1495void AtomicExpandImpl::expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI) {
1496 ReplacementIRBuilder Builder(AI, *DL);
1497
1498 PartwordMaskValues PMV =
1499 createMaskInstrs(Builder, I: AI, ValueType: AI->getType(), Addr: AI->getPointerOperand(),
1500 AddrAlign: AI->getAlign(), MinWordSize: TLI->getMinCmpXchgSizeInBits() / 8);
1501
1502 // The value operand must be sign-extended for signed min/max so that the
1503 // target's signed comparison instructions can be used. Otherwise, just
1504 // zero-ext.
1505 Instruction::CastOps CastOp = Instruction::ZExt;
1506 AtomicRMWInst::BinOp RMWOp = AI->getOperation();
1507 if (RMWOp == AtomicRMWInst::Max || RMWOp == AtomicRMWInst::Min)
1508 CastOp = Instruction::SExt;
1509
1510 Value *ValOperand_Shifted = Builder.CreateShl(
1511 LHS: Builder.CreateCast(Op: CastOp, V: AI->getValOperand(), DestTy: PMV.WordType),
1512 RHS: PMV.ShiftAmt, Name: "ValOperand_Shifted");
1513 Value *OldResult = TLI->emitMaskedAtomicRMWIntrinsic(
1514 Builder, AI, AlignedAddr: PMV.AlignedAddr, Incr: ValOperand_Shifted, Mask: PMV.Mask, ShiftAmt: PMV.ShiftAmt,
1515 Ord: AI->getOrdering());
1516 Value *FinalOldResult = extractMaskedValue(Builder, WideWord: OldResult, PMV);
1517 AI->replaceAllUsesWith(V: FinalOldResult);
1518 AI->eraseFromParent();
1519}
1520
1521void AtomicExpandImpl::expandAtomicCmpXchgToMaskedIntrinsic(
1522 AtomicCmpXchgInst *CI) {
1523 ReplacementIRBuilder Builder(CI, *DL);
1524
1525 PartwordMaskValues PMV = createMaskInstrs(
1526 Builder, I: CI, ValueType: CI->getCompareOperand()->getType(), Addr: CI->getPointerOperand(),
1527 AddrAlign: CI->getAlign(), MinWordSize: TLI->getMinCmpXchgSizeInBits() / 8);
1528
1529 Value *CmpVal_Shifted = Builder.CreateShl(
1530 LHS: Builder.CreateZExt(V: CI->getCompareOperand(), DestTy: PMV.WordType), RHS: PMV.ShiftAmt,
1531 Name: "CmpVal_Shifted");
1532 Value *NewVal_Shifted = Builder.CreateShl(
1533 LHS: Builder.CreateZExt(V: CI->getNewValOperand(), DestTy: PMV.WordType), RHS: PMV.ShiftAmt,
1534 Name: "NewVal_Shifted");
1535 Value *OldVal = TLI->emitMaskedAtomicCmpXchgIntrinsic(
1536 Builder, CI, AlignedAddr: PMV.AlignedAddr, CmpVal: CmpVal_Shifted, NewVal: NewVal_Shifted, Mask: PMV.Mask,
1537 Ord: CI->getMergedOrdering());
1538 Value *FinalOldVal = extractMaskedValue(Builder, WideWord: OldVal, PMV);
1539 Value *Res = PoisonValue::get(T: CI->getType());
1540 Res = Builder.CreateInsertValue(Agg: Res, Val: FinalOldVal, Idxs: 0);
1541 Value *Success = Builder.CreateICmpEQ(
1542 LHS: CmpVal_Shifted, RHS: Builder.CreateAnd(LHS: OldVal, RHS: PMV.Mask), Name: "Success");
1543 Res = Builder.CreateInsertValue(Agg: Res, Val: Success, Idxs: 1);
1544
1545 CI->replaceAllUsesWith(V: Res);
1546 CI->eraseFromParent();
1547}
1548
1549Value *AtomicExpandImpl::insertRMWLLSCLoop(
1550 IRBuilderBase &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
1551 AtomicOrdering MemOpOrder,
1552 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp) {
1553 LLVMContext &Ctx = Builder.getContext();
1554 BasicBlock *BB = Builder.GetInsertBlock();
1555 Function *F = BB->getParent();
1556
1557 assert(AddrAlign >= F->getDataLayout().getTypeStoreSize(ResultTy) &&
1558 "Expected at least natural alignment at this point.");
1559
1560 // Given: atomicrmw some_op iN* %addr, iN %incr ordering
1561 //
1562 // The standard expansion we produce is:
1563 // [...]
1564 // atomicrmw.start:
1565 // %loaded = @load.linked(%addr)
1566 // %new = some_op iN %loaded, %incr
1567 // %stored = @store_conditional(%new, %addr)
1568 // %try_again = icmp i32 ne %stored, 0
1569 // br i1 %try_again, label %loop, label %atomicrmw.end
1570 // atomicrmw.end:
1571 // [...]
1572 BasicBlock *ExitBB =
1573 BB->splitBasicBlock(I: Builder.GetInsertPoint(), BBName: "atomicrmw.end");
1574 BasicBlock *LoopBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.start", Parent: F, InsertBefore: ExitBB);
1575
1576 // The split call above "helpfully" added a branch at the end of BB (to the
1577 // wrong place).
1578 std::prev(x: BB->end())->eraseFromParent();
1579 Builder.SetInsertPoint(BB);
1580 Builder.CreateBr(Dest: LoopBB);
1581
1582 // Start the main loop block now that we've taken care of the preliminaries.
1583 Builder.SetInsertPoint(LoopBB);
1584 Value *Loaded = TLI->emitLoadLinked(Builder, ValueTy: ResultTy, Addr, Ord: MemOpOrder);
1585
1586 Value *NewVal = PerformOp(Builder, Loaded);
1587
1588 Value *StoreSuccess =
1589 TLI->emitStoreConditional(Builder, Val: NewVal, Addr, Ord: MemOpOrder);
1590 Value *TryAgain = Builder.CreateICmpNE(
1591 LHS: StoreSuccess, RHS: ConstantInt::get(Ty: IntegerType::get(C&: Ctx, NumBits: 32), V: 0), Name: "tryagain");
1592
1593 Instruction *CondBr = Builder.CreateCondBr(Cond: TryAgain, True: LoopBB, False: ExitBB);
1594
1595 // Atomic RMW expands to a Load-linked / Store-Conditional loop, because it is
1596 // hard to predict precise branch weigths we mark the branch as "unknown"
1597 // (50/50) to prevent misleading optimizations.
1598 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *CondBr, DEBUG_TYPE);
1599
1600 Builder.SetInsertPoint(ExitBB->begin());
1601 return Loaded;
1602}
1603
1604/// Convert an atomic cmpxchg of a non-integral type to an integer cmpxchg of
1605/// the equivalent bitwidth. We used to not support pointer cmpxchg in the
1606/// IR. As a migration step, we convert back to what use to be the standard
1607/// way to represent a pointer cmpxchg so that we can update backends one by
1608/// one.
1609AtomicCmpXchgInst *
1610AtomicExpandImpl::convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI) {
1611 auto *M = CI->getModule();
1612 Type *NewTy = getCorrespondingIntegerType(T: CI->getCompareOperand()->getType(),
1613 DL: M->getDataLayout());
1614
1615 ReplacementIRBuilder Builder(CI, *DL);
1616
1617 Value *Addr = CI->getPointerOperand();
1618
1619 Value *NewCmp = Builder.CreatePtrToInt(V: CI->getCompareOperand(), DestTy: NewTy);
1620 Value *NewNewVal = Builder.CreatePtrToInt(V: CI->getNewValOperand(), DestTy: NewTy);
1621
1622 auto *NewCI = Builder.CreateAtomicCmpXchg(
1623 Ptr: Addr, Cmp: NewCmp, New: NewNewVal, Align: CI->getAlign(), SuccessOrdering: CI->getSuccessOrdering(),
1624 FailureOrdering: CI->getFailureOrdering(), SSID: CI->getSyncScopeID());
1625 NewCI->setVolatile(CI->isVolatile());
1626 NewCI->setWeak(CI->isWeak());
1627 LLVM_DEBUG(dbgs() << "Replaced " << *CI << " with " << *NewCI << "\n");
1628
1629 Value *OldVal = Builder.CreateExtractValue(Agg: NewCI, Idxs: 0);
1630 Value *Succ = Builder.CreateExtractValue(Agg: NewCI, Idxs: 1);
1631
1632 OldVal = Builder.CreateIntToPtr(V: OldVal, DestTy: CI->getCompareOperand()->getType());
1633
1634 Value *Res = PoisonValue::get(T: CI->getType());
1635 Res = Builder.CreateInsertValue(Agg: Res, Val: OldVal, Idxs: 0);
1636 Res = Builder.CreateInsertValue(Agg: Res, Val: Succ, Idxs: 1);
1637
1638 CI->replaceAllUsesWith(V: Res);
1639 CI->eraseFromParent();
1640 return NewCI;
1641}
1642
1643bool AtomicExpandImpl::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
1644 AtomicOrdering SuccessOrder = CI->getSuccessOrdering();
1645 AtomicOrdering FailureOrder = CI->getFailureOrdering();
1646 Value *Addr = CI->getPointerOperand();
1647 BasicBlock *BB = CI->getParent();
1648 Function *F = BB->getParent();
1649 LLVMContext &Ctx = F->getContext();
1650 // If shouldInsertFencesForAtomic() returns true, then the target does not
1651 // want to deal with memory orders, and emitLeading/TrailingFence should take
1652 // care of everything. Otherwise, emitLeading/TrailingFence are no-op and we
1653 // should preserve the ordering.
1654 bool ShouldInsertFencesForAtomic = TLI->shouldInsertFencesForAtomic(I: CI);
1655 AtomicOrdering MemOpOrder = ShouldInsertFencesForAtomic
1656 ? AtomicOrdering::Monotonic
1657 : CI->getMergedOrdering();
1658
1659 // In implementations which use a barrier to achieve release semantics, we can
1660 // delay emitting this barrier until we know a store is actually going to be
1661 // attempted. The cost of this delay is that we need 2 copies of the block
1662 // emitting the load-linked, affecting code size.
1663 //
1664 // Ideally, this logic would be unconditional except for the minsize check
1665 // since in other cases the extra blocks naturally collapse down to the
1666 // minimal loop. Unfortunately, this puts too much stress on later
1667 // optimisations so we avoid emitting the extra logic in those cases too.
1668 bool HasReleasedLoadBB = !CI->isWeak() && ShouldInsertFencesForAtomic &&
1669 SuccessOrder != AtomicOrdering::Monotonic &&
1670 SuccessOrder != AtomicOrdering::Acquire &&
1671 !F->hasMinSize();
1672
1673 // There's no overhead for sinking the release barrier in a weak cmpxchg, so
1674 // do it even on minsize.
1675 bool UseUnconditionalReleaseBarrier = F->hasMinSize() && !CI->isWeak();
1676
1677 // Given: cmpxchg some_op iN* %addr, iN %desired, iN %new success_ord fail_ord
1678 //
1679 // The full expansion we produce is:
1680 // [...]
1681 // %aligned.addr = ...
1682 // cmpxchg.start:
1683 // %unreleasedload = @load.linked(%aligned.addr)
1684 // %unreleasedload.extract = extract value from %unreleasedload
1685 // %should_store = icmp eq %unreleasedload.extract, %desired
1686 // br i1 %should_store, label %cmpxchg.releasingstore,
1687 // label %cmpxchg.nostore
1688 // cmpxchg.releasingstore:
1689 // fence?
1690 // br label cmpxchg.trystore
1691 // cmpxchg.trystore:
1692 // %loaded.trystore = phi [%unreleasedload, %cmpxchg.releasingstore],
1693 // [%releasedload, %cmpxchg.releasedload]
1694 // %updated.new = insert %new into %loaded.trystore
1695 // %stored = @store_conditional(%updated.new, %aligned.addr)
1696 // %success = icmp eq i32 %stored, 0
1697 // br i1 %success, label %cmpxchg.success,
1698 // label %cmpxchg.releasedload/%cmpxchg.failure
1699 // cmpxchg.releasedload:
1700 // %releasedload = @load.linked(%aligned.addr)
1701 // %releasedload.extract = extract value from %releasedload
1702 // %should_store = icmp eq %releasedload.extract, %desired
1703 // br i1 %should_store, label %cmpxchg.trystore,
1704 // label %cmpxchg.failure
1705 // cmpxchg.success:
1706 // fence?
1707 // br label %cmpxchg.end
1708 // cmpxchg.nostore:
1709 // %loaded.nostore = phi [%unreleasedload, %cmpxchg.start],
1710 // [%releasedload,
1711 // %cmpxchg.releasedload/%cmpxchg.trystore]
1712 // @load_linked_fail_balance()?
1713 // br label %cmpxchg.failure
1714 // cmpxchg.failure:
1715 // fence?
1716 // br label %cmpxchg.end
1717 // cmpxchg.end:
1718 // %loaded.exit = phi [%loaded.nostore, %cmpxchg.failure],
1719 // [%loaded.trystore, %cmpxchg.trystore]
1720 // %success = phi i1 [true, %cmpxchg.success], [false, %cmpxchg.failure]
1721 // %loaded = extract value from %loaded.exit
1722 // %restmp = insertvalue { iN, i1 } undef, iN %loaded, 0
1723 // %res = insertvalue { iN, i1 } %restmp, i1 %success, 1
1724 // [...]
1725 BasicBlock *ExitBB = BB->splitBasicBlock(I: CI->getIterator(), BBName: "cmpxchg.end");
1726 auto FailureBB = BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.failure", Parent: F, InsertBefore: ExitBB);
1727 auto NoStoreBB = BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.nostore", Parent: F, InsertBefore: FailureBB);
1728 auto SuccessBB = BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.success", Parent: F, InsertBefore: NoStoreBB);
1729 auto ReleasedLoadBB =
1730 BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.releasedload", Parent: F, InsertBefore: SuccessBB);
1731 auto TryStoreBB =
1732 BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.trystore", Parent: F, InsertBefore: ReleasedLoadBB);
1733 auto ReleasingStoreBB =
1734 BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.fencedstore", Parent: F, InsertBefore: TryStoreBB);
1735 auto StartBB = BasicBlock::Create(Context&: Ctx, Name: "cmpxchg.start", Parent: F, InsertBefore: ReleasingStoreBB);
1736
1737 ReplacementIRBuilder Builder(CI, *DL);
1738
1739 // The split call above "helpfully" added a branch at the end of BB (to the
1740 // wrong place), but we might want a fence too. It's easiest to just remove
1741 // the branch entirely.
1742 std::prev(x: BB->end())->eraseFromParent();
1743 Builder.SetInsertPoint(BB);
1744 if (ShouldInsertFencesForAtomic && UseUnconditionalReleaseBarrier)
1745 TLI->emitLeadingFence(Builder, Inst: CI, Ord: SuccessOrder);
1746
1747 PartwordMaskValues PMV =
1748 createMaskInstrs(Builder, I: CI, ValueType: CI->getCompareOperand()->getType(), Addr,
1749 AddrAlign: CI->getAlign(), MinWordSize: TLI->getMinCmpXchgSizeInBits() / 8);
1750 Builder.CreateBr(Dest: StartBB);
1751
1752 // Start the main loop block now that we've taken care of the preliminaries.
1753 Builder.SetInsertPoint(StartBB);
1754 Value *UnreleasedLoad =
1755 TLI->emitLoadLinked(Builder, ValueTy: PMV.WordType, Addr: PMV.AlignedAddr, Ord: MemOpOrder);
1756 Value *UnreleasedLoadExtract =
1757 extractMaskedValue(Builder, WideWord: UnreleasedLoad, PMV);
1758 Value *ShouldStore = Builder.CreateICmpEQ(
1759 LHS: UnreleasedLoadExtract, RHS: CI->getCompareOperand(), Name: "should_store");
1760
1761 // If the cmpxchg doesn't actually need any ordering when it fails, we can
1762 // jump straight past that fence instruction (if it exists).
1763 Builder.CreateCondBr(Cond: ShouldStore, True: ReleasingStoreBB, False: NoStoreBB,
1764 BranchWeights: MDBuilder(F->getContext()).createLikelyBranchWeights());
1765
1766 Builder.SetInsertPoint(ReleasingStoreBB);
1767 if (ShouldInsertFencesForAtomic && !UseUnconditionalReleaseBarrier)
1768 TLI->emitLeadingFence(Builder, Inst: CI, Ord: SuccessOrder);
1769 Builder.CreateBr(Dest: TryStoreBB);
1770
1771 Builder.SetInsertPoint(TryStoreBB);
1772 PHINode *LoadedTryStore =
1773 Builder.CreatePHI(Ty: PMV.WordType, NumReservedValues: 2, Name: "loaded.trystore");
1774 LoadedTryStore->addIncoming(V: UnreleasedLoad, BB: ReleasingStoreBB);
1775 Value *NewValueInsert =
1776 insertMaskedValue(Builder, WideWord: LoadedTryStore, Updated: CI->getNewValOperand(), PMV);
1777 Value *StoreSuccess = TLI->emitStoreConditional(Builder, Val: NewValueInsert,
1778 Addr: PMV.AlignedAddr, Ord: MemOpOrder);
1779 StoreSuccess = Builder.CreateICmpEQ(
1780 LHS: StoreSuccess, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: 0), Name: "success");
1781 BasicBlock *RetryBB = HasReleasedLoadBB ? ReleasedLoadBB : StartBB;
1782 Builder.CreateCondBr(Cond: StoreSuccess, True: SuccessBB,
1783 False: CI->isWeak() ? FailureBB : RetryBB,
1784 BranchWeights: MDBuilder(F->getContext()).createLikelyBranchWeights());
1785
1786 Builder.SetInsertPoint(ReleasedLoadBB);
1787 Value *SecondLoad;
1788 if (HasReleasedLoadBB) {
1789 SecondLoad =
1790 TLI->emitLoadLinked(Builder, ValueTy: PMV.WordType, Addr: PMV.AlignedAddr, Ord: MemOpOrder);
1791 Value *SecondLoadExtract = extractMaskedValue(Builder, WideWord: SecondLoad, PMV);
1792 ShouldStore = Builder.CreateICmpEQ(LHS: SecondLoadExtract,
1793 RHS: CI->getCompareOperand(), Name: "should_store");
1794
1795 // If the cmpxchg doesn't actually need any ordering when it fails, we can
1796 // jump straight past that fence instruction (if it exists).
1797 Builder.CreateCondBr(
1798 Cond: ShouldStore, True: TryStoreBB, False: NoStoreBB,
1799 BranchWeights: MDBuilder(F->getContext()).createLikelyBranchWeights());
1800 // Update PHI node in TryStoreBB.
1801 LoadedTryStore->addIncoming(V: SecondLoad, BB: ReleasedLoadBB);
1802 } else
1803 Builder.CreateUnreachable();
1804
1805 // Make sure later instructions don't get reordered with a fence if
1806 // necessary.
1807 Builder.SetInsertPoint(SuccessBB);
1808 if (ShouldInsertFencesForAtomic ||
1809 TLI->shouldInsertTrailingSeqCstFenceForAtomicStore(I: CI))
1810 TLI->emitTrailingFence(Builder, Inst: CI, Ord: SuccessOrder);
1811 Builder.CreateBr(Dest: ExitBB);
1812
1813 Builder.SetInsertPoint(NoStoreBB);
1814 PHINode *LoadedNoStore =
1815 Builder.CreatePHI(Ty: UnreleasedLoad->getType(), NumReservedValues: 2, Name: "loaded.nostore");
1816 LoadedNoStore->addIncoming(V: UnreleasedLoad, BB: StartBB);
1817 if (HasReleasedLoadBB)
1818 LoadedNoStore->addIncoming(V: SecondLoad, BB: ReleasedLoadBB);
1819
1820 // In the failing case, where we don't execute the store-conditional, the
1821 // target might want to balance out the load-linked with a dedicated
1822 // instruction (e.g., on ARM, clearing the exclusive monitor).
1823 TLI->emitAtomicCmpXchgNoStoreLLBalance(Builder);
1824 Builder.CreateBr(Dest: FailureBB);
1825
1826 Builder.SetInsertPoint(FailureBB);
1827 PHINode *LoadedFailure =
1828 Builder.CreatePHI(Ty: UnreleasedLoad->getType(), NumReservedValues: 2, Name: "loaded.failure");
1829 LoadedFailure->addIncoming(V: LoadedNoStore, BB: NoStoreBB);
1830 if (CI->isWeak())
1831 LoadedFailure->addIncoming(V: LoadedTryStore, BB: TryStoreBB);
1832 if (ShouldInsertFencesForAtomic)
1833 TLI->emitTrailingFence(Builder, Inst: CI, Ord: FailureOrder);
1834 Builder.CreateBr(Dest: ExitBB);
1835
1836 // Finally, we have control-flow based knowledge of whether the cmpxchg
1837 // succeeded or not. We expose this to later passes by converting any
1838 // subsequent "icmp eq/ne %loaded, %oldval" into a use of an appropriate
1839 // PHI.
1840 Builder.SetInsertPoint(ExitBB->begin());
1841 PHINode *LoadedExit =
1842 Builder.CreatePHI(Ty: UnreleasedLoad->getType(), NumReservedValues: 2, Name: "loaded.exit");
1843 LoadedExit->addIncoming(V: LoadedTryStore, BB: SuccessBB);
1844 LoadedExit->addIncoming(V: LoadedFailure, BB: FailureBB);
1845 PHINode *Success = Builder.CreatePHI(Ty: Type::getInt1Ty(C&: Ctx), NumReservedValues: 2, Name: "success");
1846 Success->addIncoming(V: ConstantInt::getTrue(Context&: Ctx), BB: SuccessBB);
1847 Success->addIncoming(V: ConstantInt::getFalse(Context&: Ctx), BB: FailureBB);
1848
1849 // This is the "exit value" from the cmpxchg expansion. It may be of
1850 // a type wider than the one in the cmpxchg instruction.
1851 Value *LoadedFull = LoadedExit;
1852
1853 Builder.SetInsertPoint(std::next(x: Success->getIterator()));
1854 Value *Loaded = extractMaskedValue(Builder, WideWord: LoadedFull, PMV);
1855
1856 // Look for any users of the cmpxchg that are just comparing the loaded value
1857 // against the desired one, and replace them with the CFG-derived version.
1858 SmallVector<ExtractValueInst *, 2> PrunedInsts;
1859 for (auto *User : CI->users()) {
1860 ExtractValueInst *EV = dyn_cast<ExtractValueInst>(Val: User);
1861 if (!EV)
1862 continue;
1863
1864 assert(EV->getNumIndices() == 1 && EV->getIndices()[0] <= 1 &&
1865 "weird extraction from { iN, i1 }");
1866
1867 if (EV->getIndices()[0] == 0)
1868 EV->replaceAllUsesWith(V: Loaded);
1869 else
1870 EV->replaceAllUsesWith(V: Success);
1871
1872 PrunedInsts.push_back(Elt: EV);
1873 }
1874
1875 // We can remove the instructions now we're no longer iterating through them.
1876 for (auto *EV : PrunedInsts)
1877 EV->eraseFromParent();
1878
1879 if (!CI->use_empty()) {
1880 // Some use of the full struct return that we don't understand has happened,
1881 // so we've got to reconstruct it properly.
1882 Value *Res;
1883 Res = Builder.CreateInsertValue(Agg: PoisonValue::get(T: CI->getType()), Val: Loaded, Idxs: 0);
1884 Res = Builder.CreateInsertValue(Agg: Res, Val: Success, Idxs: 1);
1885
1886 CI->replaceAllUsesWith(V: Res);
1887 }
1888
1889 CI->eraseFromParent();
1890 return true;
1891}
1892
1893bool AtomicExpandImpl::isIdempotentRMW(AtomicRMWInst *RMWI) {
1894 if (RMWI->isVolatile())
1895 return false;
1896 // TODO: Add floating point support.
1897 auto C = dyn_cast<ConstantInt>(Val: RMWI->getValOperand());
1898 if (!C)
1899 return false;
1900
1901 switch (RMWI->getOperation()) {
1902 case AtomicRMWInst::Add:
1903 case AtomicRMWInst::Sub:
1904 case AtomicRMWInst::Or:
1905 case AtomicRMWInst::Xor:
1906 return C->isZero();
1907 case AtomicRMWInst::And:
1908 return C->isMinusOne();
1909 case AtomicRMWInst::Min:
1910 return C->isMaxValue(IsSigned: true);
1911 case AtomicRMWInst::Max:
1912 return C->isMinValue(IsSigned: true);
1913 case AtomicRMWInst::UMin:
1914 return C->isMaxValue(IsSigned: false);
1915 case AtomicRMWInst::UMax:
1916 return C->isMinValue(IsSigned: false);
1917 default:
1918 return false;
1919 }
1920}
1921
1922bool AtomicExpandImpl::simplifyIdempotentRMW(AtomicRMWInst *RMWI) {
1923 if (auto ResultingLoad = TLI->lowerIdempotentRMWIntoFencedLoad(RMWI)) {
1924 tryExpandAtomicLoad(LI: ResultingLoad);
1925 return true;
1926 }
1927 return false;
1928}
1929
1930Value *AtomicExpandImpl::insertRMWCmpXchgLoop(
1931 IRBuilderBase &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
1932 AtomicOrdering MemOpOrder, SyncScope::ID SSID, bool IsVolatile,
1933 function_ref<Value *(IRBuilderBase &, Value *)> PerformOp,
1934 CreateCmpXchgInstFun CreateCmpXchg, Instruction *MetadataSrc) {
1935 LLVMContext &Ctx = Builder.getContext();
1936 BasicBlock *BB = Builder.GetInsertBlock();
1937 Function *F = BB->getParent();
1938
1939 // Given: atomicrmw some_op iN* %addr, iN %incr ordering
1940 //
1941 // The standard expansion we produce is:
1942 // [...]
1943 // %init_loaded = load atomic iN* %addr
1944 // br label %loop
1945 // loop:
1946 // %loaded = phi iN [ %init_loaded, %entry ], [ %new_loaded, %loop ]
1947 // %new = some_op iN %loaded, %incr
1948 // %pair = cmpxchg iN* %addr, iN %loaded, iN %new
1949 // %new_loaded = extractvalue { iN, i1 } %pair, 0
1950 // %success = extractvalue { iN, i1 } %pair, 1
1951 // br i1 %success, label %atomicrmw.end, label %loop
1952 // atomicrmw.end:
1953 // [...]
1954 BasicBlock *ExitBB =
1955 BB->splitBasicBlock(I: Builder.GetInsertPoint(), BBName: "atomicrmw.end");
1956 BasicBlock *LoopBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.start", Parent: F, InsertBefore: ExitBB);
1957
1958 // The split call above "helpfully" added a branch at the end of BB (to the
1959 // wrong place), but we want a load. It's easiest to just remove
1960 // the branch entirely.
1961 std::prev(x: BB->end())->eraseFromParent();
1962 Builder.SetInsertPoint(BB);
1963 LoadInst *InitLoaded = Builder.CreateAlignedLoad(Ty: ResultTy, Ptr: Addr, Align: AddrAlign);
1964 Builder.CreateBr(Dest: LoopBB);
1965
1966 // Start the main loop block now that we've taken care of the preliminaries.
1967 Builder.SetInsertPoint(LoopBB);
1968 PHINode *Loaded = Builder.CreatePHI(Ty: ResultTy, NumReservedValues: 2, Name: "loaded");
1969 Loaded->addIncoming(V: InitLoaded, BB);
1970
1971 // The initial load must be atomic with the same synchronization scope
1972 // to avoid a data race with concurrent stores. If the instruction being
1973 // emulated is volatile, issue a volatile load.
1974 // addIncoming is done first so that any replaceAllUsesWith calls during
1975 // normalization correctly update the PHI incoming value.
1976 InitLoaded->setVolatile(IsVolatile);
1977 if (TLI->shouldIssueAtomicLoadForAtomicEmulationLoop()) {
1978 InitLoaded->setAtomic(Ordering: AtomicOrdering::Monotonic, SSID);
1979 // The newly created load might need to be lowered further. Because it is
1980 // created in the same block as the atomicrmw, the AtomicExpand loop will
1981 // not process it again.
1982 processAtomicInstr(I: InitLoaded);
1983 }
1984
1985 Value *NewVal = PerformOp(Builder, Loaded);
1986
1987 Value *NewLoaded = nullptr;
1988 Value *Success = nullptr;
1989
1990 CreateCmpXchg(Builder, Addr, Loaded, NewVal, AddrAlign,
1991 MemOpOrder == AtomicOrdering::Unordered
1992 ? AtomicOrdering::Monotonic
1993 : MemOpOrder,
1994 SSID, IsVolatile, Success, NewLoaded, MetadataSrc);
1995 assert(Success && NewLoaded);
1996
1997 Loaded->addIncoming(V: NewLoaded, BB: LoopBB);
1998
1999 Instruction *CondBr = Builder.CreateCondBr(Cond: Success, True: ExitBB, False: LoopBB);
2000
2001 // Atomic RMW expands to a cmpxchg loop, Since precise branch weights
2002 // cannot be easily determined here, we mark the branch as "unknown" (50/50)
2003 // to prevent misleading optimizations.
2004 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *CondBr, DEBUG_TYPE);
2005
2006 Builder.SetInsertPoint(ExitBB->begin());
2007 return NewLoaded;
2008}
2009
2010bool AtomicExpandImpl::tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
2011 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
2012 unsigned ValueSize = getAtomicOpSize(CASI: CI);
2013
2014 switch (TLI->shouldExpandAtomicCmpXchgInIR(AI: CI)) {
2015 default:
2016 llvm_unreachable("Unhandled case in tryExpandAtomicCmpXchg");
2017 case TargetLoweringBase::AtomicExpansionKind::None:
2018 if (ValueSize < MinCASSize)
2019 return expandPartwordCmpXchg(CI);
2020 return false;
2021 case TargetLoweringBase::AtomicExpansionKind::LLSC: {
2022 return expandAtomicCmpXchg(CI);
2023 }
2024 case TargetLoweringBase::AtomicExpansionKind::MaskedIntrinsic:
2025 expandAtomicCmpXchgToMaskedIntrinsic(CI);
2026 return true;
2027 case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
2028 return lowerAtomicCmpXchgInst(CXI: CI);
2029 case TargetLoweringBase::AtomicExpansionKind::CustomExpand: {
2030 TLI->emitExpandAtomicCmpXchg(CI);
2031 return true;
2032 }
2033 }
2034}
2035
2036bool AtomicExpandImpl::expandAtomicRMWToCmpXchg(
2037 AtomicRMWInst *AI, CreateCmpXchgInstFun CreateCmpXchg) {
2038 ReplacementIRBuilder Builder(AI, AI->getDataLayout());
2039 Builder.setIsFPConstrained(
2040 AI->getFunction()->hasFnAttribute(Kind: Attribute::StrictFP));
2041
2042 // FIXME: If FP exceptions are observable, we should force them off for the
2043 // loop for the FP atomics.
2044 Value *Loaded = AtomicExpandImpl::insertRMWCmpXchgLoop(
2045 Builder, ResultTy: AI->getType(), Addr: AI->getPointerOperand(), AddrAlign: AI->getAlign(),
2046 MemOpOrder: AI->getOrdering(), SSID: AI->getSyncScopeID(), IsVolatile: AI->isVolatile(),
2047 PerformOp: [&](IRBuilderBase &Builder, Value *Loaded) {
2048 return buildAtomicRMWValue(Op: AI->getOperation(), Builder, Loaded,
2049 Val: AI->getValOperand());
2050 },
2051 CreateCmpXchg, /*MetadataSrc=*/AI);
2052
2053 AI->replaceAllUsesWith(V: Loaded);
2054 AI->eraseFromParent();
2055 return true;
2056}
2057
2058// In order to use one of the sized library calls such as
2059// __atomic_fetch_add_4, the alignment must be sufficient, the size
2060// must be one of the potentially-specialized sizes, and the value
2061// type must actually exist in C on the target (otherwise, the
2062// function wouldn't actually be defined.)
2063static bool canUseSizedAtomicCall(unsigned Size, Align Alignment,
2064 const DataLayout &DL) {
2065 // TODO: "LargestSize" is an approximation for "largest type that
2066 // you can express in C". It seems to be the case that int128 is
2067 // supported on all 64-bit platforms, otherwise only up to 64-bit
2068 // integers are supported. If we get this wrong, then we'll try to
2069 // call a sized libcall that doesn't actually exist. There should
2070 // really be some more reliable way in LLVM of determining integer
2071 // sizes which are valid in the target's C ABI...
2072 unsigned LargestSize = DL.getLargestLegalIntTypeSizeInBits() >= 64 ? 16 : 8;
2073 return Alignment >= Size &&
2074 (Size == 1 || Size == 2 || Size == 4 || Size == 8 || Size == 16) &&
2075 Size <= LargestSize;
2076}
2077
2078void AtomicExpandImpl::expandAtomicLoadToLibcall(LoadInst *I) {
2079 static const RTLIB::Libcall Libcalls[6] = {
2080 RTLIB::ATOMIC_LOAD, RTLIB::ATOMIC_LOAD_1, RTLIB::ATOMIC_LOAD_2,
2081 RTLIB::ATOMIC_LOAD_4, RTLIB::ATOMIC_LOAD_8, RTLIB::ATOMIC_LOAD_16};
2082 unsigned Size = getAtomicOpSize(LI: I);
2083
2084 bool Expanded = expandAtomicOpToLibcall(
2085 I, Size, Alignment: I->getAlign(), PointerOperand: I->getPointerOperand(), ValueOperand: nullptr, CASExpected: nullptr,
2086 Ordering: I->getOrdering(), Ordering2: AtomicOrdering::NotAtomic, Libcalls);
2087 if (!Expanded)
2088 handleUnsupportedAtomicSize(I, AtomicOpName: "atomic load");
2089}
2090
2091void AtomicExpandImpl::expandAtomicStoreToLibcall(StoreInst *I) {
2092 static const RTLIB::Libcall Libcalls[6] = {
2093 RTLIB::ATOMIC_STORE, RTLIB::ATOMIC_STORE_1, RTLIB::ATOMIC_STORE_2,
2094 RTLIB::ATOMIC_STORE_4, RTLIB::ATOMIC_STORE_8, RTLIB::ATOMIC_STORE_16};
2095 unsigned Size = getAtomicOpSize(SI: I);
2096
2097 bool Expanded = expandAtomicOpToLibcall(
2098 I, Size, Alignment: I->getAlign(), PointerOperand: I->getPointerOperand(), ValueOperand: I->getValueOperand(),
2099 CASExpected: nullptr, Ordering: I->getOrdering(), Ordering2: AtomicOrdering::NotAtomic, Libcalls);
2100 if (!Expanded)
2101 handleUnsupportedAtomicSize(I, AtomicOpName: "atomic store");
2102}
2103
2104void AtomicExpandImpl::expandAtomicCASToLibcall(AtomicCmpXchgInst *I,
2105 const Twine &AtomicOpName,
2106 Instruction *DiagnosticInst) {
2107 static const RTLIB::Libcall Libcalls[6] = {
2108 RTLIB::ATOMIC_COMPARE_EXCHANGE, RTLIB::ATOMIC_COMPARE_EXCHANGE_1,
2109 RTLIB::ATOMIC_COMPARE_EXCHANGE_2, RTLIB::ATOMIC_COMPARE_EXCHANGE_4,
2110 RTLIB::ATOMIC_COMPARE_EXCHANGE_8, RTLIB::ATOMIC_COMPARE_EXCHANGE_16};
2111 unsigned Size = getAtomicOpSize(CASI: I);
2112
2113 bool Expanded = expandAtomicOpToLibcall(
2114 I, Size, Alignment: I->getAlign(), PointerOperand: I->getPointerOperand(), ValueOperand: I->getNewValOperand(),
2115 CASExpected: I->getCompareOperand(), Ordering: I->getSuccessOrdering(), Ordering2: I->getFailureOrdering(),
2116 Libcalls);
2117 if (!Expanded)
2118 handleUnsupportedAtomicSize(I, AtomicOpName, DiagnosticInst);
2119}
2120
2121static ArrayRef<RTLIB::Libcall> GetRMWLibcall(AtomicRMWInst::BinOp Op) {
2122 static const RTLIB::Libcall LibcallsXchg[6] = {
2123 RTLIB::ATOMIC_EXCHANGE, RTLIB::ATOMIC_EXCHANGE_1,
2124 RTLIB::ATOMIC_EXCHANGE_2, RTLIB::ATOMIC_EXCHANGE_4,
2125 RTLIB::ATOMIC_EXCHANGE_8, RTLIB::ATOMIC_EXCHANGE_16};
2126 static const RTLIB::Libcall LibcallsAdd[6] = {
2127 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_ADD_1,
2128 RTLIB::ATOMIC_FETCH_ADD_2, RTLIB::ATOMIC_FETCH_ADD_4,
2129 RTLIB::ATOMIC_FETCH_ADD_8, RTLIB::ATOMIC_FETCH_ADD_16};
2130 static const RTLIB::Libcall LibcallsSub[6] = {
2131 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_SUB_1,
2132 RTLIB::ATOMIC_FETCH_SUB_2, RTLIB::ATOMIC_FETCH_SUB_4,
2133 RTLIB::ATOMIC_FETCH_SUB_8, RTLIB::ATOMIC_FETCH_SUB_16};
2134 static const RTLIB::Libcall LibcallsAnd[6] = {
2135 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_AND_1,
2136 RTLIB::ATOMIC_FETCH_AND_2, RTLIB::ATOMIC_FETCH_AND_4,
2137 RTLIB::ATOMIC_FETCH_AND_8, RTLIB::ATOMIC_FETCH_AND_16};
2138 static const RTLIB::Libcall LibcallsOr[6] = {
2139 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_OR_1,
2140 RTLIB::ATOMIC_FETCH_OR_2, RTLIB::ATOMIC_FETCH_OR_4,
2141 RTLIB::ATOMIC_FETCH_OR_8, RTLIB::ATOMIC_FETCH_OR_16};
2142 static const RTLIB::Libcall LibcallsXor[6] = {
2143 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_XOR_1,
2144 RTLIB::ATOMIC_FETCH_XOR_2, RTLIB::ATOMIC_FETCH_XOR_4,
2145 RTLIB::ATOMIC_FETCH_XOR_8, RTLIB::ATOMIC_FETCH_XOR_16};
2146 static const RTLIB::Libcall LibcallsNand[6] = {
2147 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_NAND_1,
2148 RTLIB::ATOMIC_FETCH_NAND_2, RTLIB::ATOMIC_FETCH_NAND_4,
2149 RTLIB::ATOMIC_FETCH_NAND_8, RTLIB::ATOMIC_FETCH_NAND_16};
2150
2151 switch (Op) {
2152 case AtomicRMWInst::BAD_BINOP:
2153 llvm_unreachable("Should not have BAD_BINOP.");
2154 case AtomicRMWInst::Xchg:
2155 return ArrayRef(LibcallsXchg);
2156 case AtomicRMWInst::Add:
2157 return ArrayRef(LibcallsAdd);
2158 case AtomicRMWInst::Sub:
2159 return ArrayRef(LibcallsSub);
2160 case AtomicRMWInst::And:
2161 return ArrayRef(LibcallsAnd);
2162 case AtomicRMWInst::Or:
2163 return ArrayRef(LibcallsOr);
2164 case AtomicRMWInst::Xor:
2165 return ArrayRef(LibcallsXor);
2166 case AtomicRMWInst::Nand:
2167 return ArrayRef(LibcallsNand);
2168 case AtomicRMWInst::Max:
2169 case AtomicRMWInst::Min:
2170 case AtomicRMWInst::UMax:
2171 case AtomicRMWInst::UMin:
2172 case AtomicRMWInst::FMax:
2173 case AtomicRMWInst::FMin:
2174 case AtomicRMWInst::FMaximum:
2175 case AtomicRMWInst::FMinimum:
2176 case AtomicRMWInst::FMaximumNum:
2177 case AtomicRMWInst::FMinimumNum:
2178 case AtomicRMWInst::FAdd:
2179 case AtomicRMWInst::FSub:
2180 case AtomicRMWInst::UIncWrap:
2181 case AtomicRMWInst::UDecWrap:
2182 case AtomicRMWInst::USubCond:
2183 case AtomicRMWInst::USubSat:
2184 // No atomic libcalls are available for these.
2185 return {};
2186 }
2187 llvm_unreachable("Unexpected AtomicRMW operation.");
2188}
2189
2190void AtomicExpandImpl::expandAtomicRMWToLibcall(AtomicRMWInst *I) {
2191 ArrayRef<RTLIB::Libcall> Libcalls = GetRMWLibcall(Op: I->getOperation());
2192
2193 unsigned Size = getAtomicOpSize(RMWI: I);
2194
2195 bool Success = false;
2196 if (!Libcalls.empty())
2197 Success = expandAtomicOpToLibcall(
2198 I, Size, Alignment: I->getAlign(), PointerOperand: I->getPointerOperand(), ValueOperand: I->getValOperand(),
2199 CASExpected: nullptr, Ordering: I->getOrdering(), Ordering2: AtomicOrdering::NotAtomic, Libcalls);
2200
2201 // The expansion failed: either there were no libcalls at all for
2202 // the operation (min/max), or there were only size-specialized
2203 // libcalls (add/sub/etc) and we needed a generic. So, expand to a
2204 // CAS libcall, via a CAS loop, instead.
2205 if (!Success) {
2206 expandAtomicRMWToCmpXchg(
2207 AI: I, CreateCmpXchg: [this, I](IRBuilderBase &Builder, Value *Addr, Value *Loaded,
2208 Value *NewVal, Align Alignment, AtomicOrdering MemOpOrder,
2209 SyncScope::ID SSID, bool IsVolatile, Value *&Success,
2210 Value *&NewLoaded, Instruction *MetadataSrc) {
2211 // Create the CAS instruction normally...
2212 AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
2213 Ptr: Addr, Cmp: Loaded, New: NewVal, Align: Alignment, SuccessOrdering: MemOpOrder,
2214 FailureOrdering: AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: MemOpOrder), SSID);
2215 Pair->setVolatile(IsVolatile);
2216 if (MetadataSrc)
2217 copyMetadataForAtomic(Dest&: *Pair, Source: *MetadataSrc);
2218
2219 Success = Builder.CreateExtractValue(Agg: Pair, Idxs: 1, Name: "success");
2220 NewLoaded = Builder.CreateExtractValue(Agg: Pair, Idxs: 0, Name: "newloaded");
2221
2222 // ...and then expand the CAS into a libcall.
2223 expandAtomicCASToLibcall(
2224 I: Pair,
2225 AtomicOpName: "atomicrmw " + AtomicRMWInst::getOperationName(Op: I->getOperation()),
2226 DiagnosticInst: MetadataSrc);
2227 });
2228 }
2229}
2230
2231// A helper routine for the above expandAtomic*ToLibcall functions.
2232//
2233// 'Libcalls' contains an array of enum values for the particular
2234// ATOMIC libcalls to be emitted. All of the other arguments besides
2235// 'I' are extracted from the Instruction subclass by the
2236// caller. Depending on the particular call, some will be null.
2237bool AtomicExpandImpl::expandAtomicOpToLibcall(
2238 Instruction *I, unsigned Size, Align Alignment, Value *PointerOperand,
2239 Value *ValueOperand, Value *CASExpected, AtomicOrdering Ordering,
2240 AtomicOrdering Ordering2, ArrayRef<RTLIB::Libcall> Libcalls) {
2241 assert(Libcalls.size() == 6);
2242
2243 LLVMContext &Ctx = I->getContext();
2244 Module *M = I->getModule();
2245 const DataLayout &DL = M->getDataLayout();
2246 IRBuilder<> Builder(I);
2247 IRBuilder<> AllocaBuilder(&I->getFunction()->getEntryBlock().front());
2248
2249 bool UseSizedLibcall = canUseSizedAtomicCall(Size, Alignment, DL);
2250 Type *SizedIntTy = Type::getIntNTy(C&: Ctx, N: Size * 8);
2251
2252 if (M->getTargetTriple().isOSWindows() && M->getTargetTriple().isX86_64() &&
2253 Size == 16) {
2254 // x86_64 Windows passes i128 as an XMM vector; on return, it is in
2255 // XMM0, and as a parameter, it is passed indirectly. The generic lowering
2256 // rules handles this correctly if we pass it as a v2i64 rather than
2257 // i128. This is what Clang does in the frontend for such types as well
2258 // (see WinX86_64ABIInfo::classify in Clang).
2259 SizedIntTy = FixedVectorType::get(ElementType: Type::getInt64Ty(C&: Ctx), NumElts: 2);
2260 }
2261
2262 const Align AllocaAlignment = DL.getPrefTypeAlign(Ty: SizedIntTy);
2263
2264 // TODO: the "order" argument type is "int", not int32. So
2265 // getInt32Ty may be wrong if the arch uses e.g. 16-bit ints.
2266 assert(Ordering != AtomicOrdering::NotAtomic && "expect atomic MO");
2267 Constant *OrderingVal =
2268 ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: (int)toCABI(AO: Ordering));
2269 Constant *Ordering2Val = nullptr;
2270 if (CASExpected) {
2271 assert(Ordering2 != AtomicOrdering::NotAtomic && "expect atomic MO");
2272 Ordering2Val =
2273 ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: (int)toCABI(AO: Ordering2));
2274 }
2275 bool HasResult = I->getType() != Type::getVoidTy(C&: Ctx);
2276
2277 RTLIB::Libcall RTLibType;
2278 if (UseSizedLibcall) {
2279 switch (Size) {
2280 case 1:
2281 RTLibType = Libcalls[1];
2282 break;
2283 case 2:
2284 RTLibType = Libcalls[2];
2285 break;
2286 case 4:
2287 RTLibType = Libcalls[3];
2288 break;
2289 case 8:
2290 RTLibType = Libcalls[4];
2291 break;
2292 case 16:
2293 RTLibType = Libcalls[5];
2294 break;
2295 }
2296 } else if (Libcalls[0] != RTLIB::UNKNOWN_LIBCALL) {
2297 RTLibType = Libcalls[0];
2298 } else {
2299 // Can't use sized function, and there's no generic for this
2300 // operation, so give up.
2301 return false;
2302 }
2303
2304 RTLIB::LibcallImpl LibcallImpl = LibcallLowering->getLibcallImpl(Call: RTLibType);
2305 if (LibcallImpl == RTLIB::Unsupported) {
2306 // This target does not implement the requested atomic libcall so give up.
2307 return false;
2308 }
2309
2310 // Build up the function call. There's two kinds. First, the sized
2311 // variants. These calls are going to be one of the following (with
2312 // N=1,2,4,8,16):
2313 // iN __atomic_load_N(iN *ptr, int ordering)
2314 // void __atomic_store_N(iN *ptr, iN val, int ordering)
2315 // iN __atomic_{exchange|fetch_*}_N(iN *ptr, iN val, int ordering)
2316 // bool __atomic_compare_exchange_N(iN *ptr, iN *expected, iN desired,
2317 // int success_order, int failure_order)
2318 //
2319 // Note that these functions can be used for non-integer atomic
2320 // operations, the values just need to be bitcast to integers on the
2321 // way in and out.
2322 //
2323 // And, then, the generic variants. They look like the following:
2324 // void __atomic_load(size_t size, void *ptr, void *ret, int ordering)
2325 // void __atomic_store(size_t size, void *ptr, void *val, int ordering)
2326 // void __atomic_exchange(size_t size, void *ptr, void *val, void *ret,
2327 // int ordering)
2328 // bool __atomic_compare_exchange(size_t size, void *ptr, void *expected,
2329 // void *desired, int success_order,
2330 // int failure_order)
2331 //
2332 // The different signatures are built up depending on the
2333 // 'UseSizedLibcall', 'CASExpected', 'ValueOperand', and 'HasResult'
2334 // variables.
2335
2336 AllocaInst *AllocaCASExpected = nullptr;
2337 AllocaInst *AllocaValue = nullptr;
2338 AllocaInst *AllocaResult = nullptr;
2339
2340 Type *ResultTy;
2341 SmallVector<Value *, 6> Args;
2342 AttributeList Attr;
2343
2344 // 'size' argument.
2345 if (!UseSizedLibcall) {
2346 // Note, getIntPtrType is assumed equivalent to size_t.
2347 Args.push_back(Elt: ConstantInt::get(Ty: DL.getIntPtrType(C&: Ctx), V: Size));
2348 }
2349
2350 // 'ptr' argument.
2351 // note: This assumes all address spaces share a common libfunc
2352 // implementation and that addresses are convertable. For systems without
2353 // that property, we'd need to extend this mechanism to support AS-specific
2354 // families of atomic intrinsics.
2355 Value *PtrVal = PointerOperand;
2356 PtrVal = Builder.CreateAddrSpaceCast(V: PtrVal, DestTy: PointerType::getUnqual(C&: Ctx));
2357 Args.push_back(Elt: PtrVal);
2358
2359 // 'expected' argument, if present.
2360 if (CASExpected) {
2361 AllocaCASExpected = AllocaBuilder.CreateAlloca(Ty: CASExpected->getType());
2362 AllocaCASExpected->setAlignment(AllocaAlignment);
2363 Builder.CreateLifetimeStart(Ptr: AllocaCASExpected);
2364 Builder.CreateAlignedStore(Val: CASExpected, Ptr: AllocaCASExpected, Align: AllocaAlignment);
2365 Args.push_back(Elt: AllocaCASExpected);
2366 }
2367
2368 // 'val' argument ('desired' for cas), if present.
2369 if (ValueOperand) {
2370 if (UseSizedLibcall) {
2371 Value *IntValue =
2372 Builder.CreateBitPreservingCastChain(DL, V: ValueOperand, NewTy: SizedIntTy);
2373 Args.push_back(Elt: IntValue);
2374 } else {
2375 AllocaValue = AllocaBuilder.CreateAlloca(Ty: ValueOperand->getType());
2376 AllocaValue->setAlignment(AllocaAlignment);
2377 Builder.CreateLifetimeStart(Ptr: AllocaValue);
2378 Builder.CreateAlignedStore(Val: ValueOperand, Ptr: AllocaValue, Align: AllocaAlignment);
2379 Args.push_back(Elt: AllocaValue);
2380 }
2381 }
2382
2383 // 'ret' argument.
2384 if (!CASExpected && HasResult && !UseSizedLibcall) {
2385 AllocaResult = AllocaBuilder.CreateAlloca(Ty: I->getType());
2386 AllocaResult->setAlignment(AllocaAlignment);
2387 Builder.CreateLifetimeStart(Ptr: AllocaResult);
2388 Args.push_back(Elt: AllocaResult);
2389 }
2390
2391 // 'ordering' ('success_order' for cas) argument.
2392 Args.push_back(Elt: OrderingVal);
2393
2394 // 'failure_order' argument, if present.
2395 if (Ordering2Val)
2396 Args.push_back(Elt: Ordering2Val);
2397
2398 // Now, the return type.
2399 if (CASExpected) {
2400 ResultTy = Type::getInt1Ty(C&: Ctx);
2401 Attr = Attr.addRetAttribute(C&: Ctx, Kind: Attribute::ZExt);
2402 } else if (HasResult && UseSizedLibcall)
2403 ResultTy = SizedIntTy;
2404 else
2405 ResultTy = Type::getVoidTy(C&: Ctx);
2406
2407 // Done with setting up arguments and return types, create the call:
2408 SmallVector<Type *, 6> ArgTys;
2409 for (Value *Arg : Args)
2410 ArgTys.push_back(Elt: Arg->getType());
2411 FunctionType *FnType = FunctionType::get(Result: ResultTy, Params: ArgTys, isVarArg: false);
2412 FunctionCallee LibcallFn = M->getOrInsertFunction(
2413 Name: RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: LibcallImpl), T: FnType,
2414 AttributeList: Attr);
2415 CallInst *Call = Builder.CreateCall(Callee: LibcallFn, Args);
2416 Call->setAttributes(Attr);
2417 Value *Result = Call;
2418
2419 // And then, extract the results...
2420 if (ValueOperand && !UseSizedLibcall)
2421 Builder.CreateLifetimeEnd(Ptr: AllocaValue);
2422
2423 if (CASExpected) {
2424 // The final result from the CAS is {load of 'expected' alloca, bool result
2425 // from call}
2426 Type *FinalResultTy = I->getType();
2427 Value *V = PoisonValue::get(T: FinalResultTy);
2428 Value *ExpectedOut = Builder.CreateAlignedLoad(
2429 Ty: CASExpected->getType(), Ptr: AllocaCASExpected, Align: AllocaAlignment);
2430 Builder.CreateLifetimeEnd(Ptr: AllocaCASExpected);
2431 V = Builder.CreateInsertValue(Agg: V, Val: ExpectedOut, Idxs: 0);
2432 V = Builder.CreateInsertValue(Agg: V, Val: Result, Idxs: 1);
2433 I->replaceAllUsesWith(V);
2434 } else if (HasResult) {
2435 Value *V;
2436 if (UseSizedLibcall) {
2437 // Add bitcasts from Result's scalar type to I's <n x ptr> vector type
2438 auto *PtrTy = dyn_cast<PointerType>(Val: I->getType()->getScalarType());
2439 auto *VTy = dyn_cast<VectorType>(Val: I->getType());
2440 if (VTy && PtrTy && !Result->getType()->isVectorTy()) {
2441 unsigned AS = PtrTy->getAddressSpace();
2442 Value *BC = Builder.CreateBitCast(
2443 V: Result, DestTy: VTy->getWithNewType(EltTy: DL.getIntPtrType(C&: Ctx, AddressSpace: AS)));
2444 V = Builder.CreateIntToPtr(V: BC, DestTy: I->getType());
2445 } else
2446 V = Builder.CreateBitOrPointerCast(V: Result, DestTy: I->getType());
2447 } else {
2448 V = Builder.CreateAlignedLoad(Ty: I->getType(), Ptr: AllocaResult,
2449 Align: AllocaAlignment);
2450 Builder.CreateLifetimeEnd(Ptr: AllocaResult);
2451 }
2452 I->replaceAllUsesWith(V);
2453 }
2454 I->eraseFromParent();
2455 return true;
2456}
2457