1//===--- Atomic.cpp - Codegen of atomic operations ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/Frontend/Atomic/Atomic.h"
10#include "llvm/IR/DerivedTypes.h"
11#include "llvm/IR/IRBuilder.h"
12#include <utility>
13
14using namespace llvm;
15
16bool AtomicInfo::shouldCastToInt(Type *ValTy, bool CmpXchg) {
17 if (ValTy->isFloatingPointTy())
18 return ValTy->isX86_FP80Ty() || CmpXchg;
19 return !ValTy->isIntegerTy() && !ValTy->isPointerTy();
20}
21
22Value *AtomicInfo::EmitAtomicLoadOp(AtomicOrdering AO, bool IsVolatile,
23 bool CmpXchg) {
24 Value *Ptr = getAtomicPointer();
25 Type *AtomicTy = Ty;
26 if (shouldCastToInt(ValTy: Ty, CmpXchg))
27 AtomicTy = IntegerType::get(C&: getLLVMContext(), NumBits: AtomicSizeInBits);
28 LoadInst *Load =
29 Builder->CreateAlignedLoad(Ty: AtomicTy, Ptr, Align: AtomicAlign, Name: "atomic-load");
30 Load->setAtomic(Ordering: AO);
31 if (IsVolatile)
32 Load->setVolatile(true);
33 decorateWithTBAA(I: Load);
34 return Load;
35}
36
37CallInst *AtomicInfo::EmitAtomicLibcall(StringRef fnName, Type *ResultType,
38 ArrayRef<Value *> Args) {
39 LLVMContext &ctx = Builder->getContext();
40 SmallVector<Type *, 6> ArgTys;
41 for (Value *Arg : Args)
42 ArgTys.push_back(Elt: Arg->getType());
43 FunctionType *FnType = FunctionType::get(Result: ResultType, Params: ArgTys, isVarArg: false);
44 Module *M = Builder->GetInsertBlock()->getModule();
45
46 // TODO: Use llvm::TargetLowering for Libcall ABI
47 AttrBuilder fnAttrBuilder(ctx);
48 fnAttrBuilder.addAttribute(Val: Attribute::NoUnwind);
49 fnAttrBuilder.addAttribute(Val: Attribute::WillReturn);
50 AttributeList fnAttrs =
51 AttributeList::get(C&: ctx, Index: AttributeList::FunctionIndex, B: fnAttrBuilder);
52 FunctionCallee LibcallFn = M->getOrInsertFunction(Name: fnName, T: FnType, AttributeList: fnAttrs);
53 CallInst *Call = Builder->CreateCall(Callee: LibcallFn, Args);
54 return Call;
55}
56
57std::pair<Value *, Value *> AtomicInfo::EmitAtomicCompareExchangeLibcall(
58 Value *ExpectedVal, Value *DesiredVal, AtomicOrdering Success,
59 AtomicOrdering Failure) {
60 LLVMContext &ctx = getLLVMContext();
61
62 // __atomic_compare_exchange's expected and desired are passed by pointers
63 // FIXME: types
64
65 // TODO: Get from llvm::TargetMachine / clang::TargetInfo
66 // if clang shares this codegen in future
67 constexpr uint64_t IntBits = 32;
68
69 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
70 // void *desired, int success, int failure);
71
72 Value *Args[6] = {
73 getAtomicSizeValue(),
74 getAtomicPointer(),
75 ExpectedVal,
76 DesiredVal,
77 Constant::getIntegerValue(Ty: IntegerType::get(C&: ctx, NumBits: IntBits),
78 V: APInt(IntBits, static_cast<uint64_t>(Success),
79 /*signed=*/true)),
80 Constant::getIntegerValue(Ty: IntegerType::get(C&: ctx, NumBits: IntBits),
81 V: APInt(IntBits, static_cast<uint64_t>(Failure),
82 /*signed=*/true)),
83 };
84 auto Result = EmitAtomicLibcall(fnName: "__atomic_compare_exchange",
85 ResultType: IntegerType::getInt1Ty(C&: ctx), Args);
86 return std::make_pair(x&: ExpectedVal, y&: Result);
87}
88
89std::pair<Value *, Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
90 Value *ExpectedVal, Value *DesiredVal, AtomicOrdering Success,
91 AtomicOrdering Failure, bool IsVolatile, bool IsWeak) {
92 // Do the atomic store.
93 Value *Addr = getAtomicAddressAsAtomicIntPointer();
94 auto *Inst = Builder->CreateAtomicCmpXchg(Ptr: Addr, Cmp: ExpectedVal, New: DesiredVal,
95 Align: getAtomicAlignment(), SuccessOrdering: Success,
96 FailureOrdering: Failure, SSID: SyncScope::System);
97
98 // Other decoration.
99 Inst->setVolatile(IsVolatile);
100 Inst->setWeak(IsWeak);
101 auto *PreviousVal = Builder->CreateExtractValue(Agg: Inst, /*Idxs=*/0);
102 auto *SuccessFailureVal = Builder->CreateExtractValue(Agg: Inst, /*Idxs=*/1);
103 return std::make_pair(x&: PreviousVal, y&: SuccessFailureVal);
104}
105
106std::pair<LoadInst *, AllocaInst *>
107AtomicInfo::EmitAtomicLoadLibcall(AtomicOrdering AO) {
108 LLVMContext &Ctx = getLLVMContext();
109 Type *SizedIntTy = Type::getIntNTy(C&: Ctx, N: getAtomicSizeInBits());
110 Type *ResultTy;
111 SmallVector<Value *, 6> Args;
112 AttributeList Attr;
113 Module *M = Builder->GetInsertBlock()->getModule();
114 const DataLayout &DL = M->getDataLayout();
115 Args.push_back(
116 Elt: ConstantInt::get(Ty: DL.getIntPtrType(C&: Ctx), V: this->getAtomicSizeInBits() / 8));
117
118 Value *PtrVal = getAtomicPointer();
119 PtrVal = Builder->CreateAddrSpaceCast(V: PtrVal, DestTy: PointerType::getUnqual(C&: Ctx));
120 Args.push_back(Elt: PtrVal);
121
122 auto CurrentIP = Builder->saveIP();
123 Builder->restoreIP(IP: AllocaIP);
124 AllocaInst *AllocaResult =
125 CreateAlloca(Ty, Name: getAtomicPointer()->getName() + "atomic.temp.load");
126 Builder->restoreIP(IP: CurrentIP);
127 const Align AllocaAlignment = DL.getPrefTypeAlign(Ty: SizedIntTy);
128 AllocaResult->setAlignment(AllocaAlignment);
129 Args.push_back(Elt: AllocaResult);
130 Constant *OrderingVal =
131 ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: (int)toCABI(AO));
132 Args.push_back(Elt: OrderingVal);
133
134 ResultTy = Type::getVoidTy(C&: Ctx);
135 SmallVector<Type *, 6> ArgTys;
136 for (Value *Arg : Args)
137 ArgTys.push_back(Elt: Arg->getType());
138 FunctionType *FnType = FunctionType::get(Result: ResultTy, Params: ArgTys, isVarArg: false);
139 FunctionCallee LibcallFn =
140 M->getOrInsertFunction(Name: "__atomic_load", T: FnType, AttributeList: Attr);
141 CallInst *Call = Builder->CreateCall(Callee: LibcallFn, Args);
142 Call->setAttributes(Attr);
143 return std::make_pair(
144 x: Builder->CreateAlignedLoad(Ty, Ptr: AllocaResult, Align: AllocaAlignment),
145 y&: AllocaResult);
146}
147
148void AtomicInfo::EmitAtomicStoreLibcall(AtomicOrdering AO, Value *Source) {
149 LLVMContext &Ctx = getLLVMContext();
150 SmallVector<Value *, 6> Args;
151 AttributeList Attr;
152 Module *M = Builder->GetInsertBlock()->getModule();
153 const DataLayout &DL = M->getDataLayout();
154 Args.push_back(
155 Elt: ConstantInt::get(Ty: DL.getIntPtrType(C&: Ctx), V: this->getAtomicSizeInBits() / 8));
156
157 Value *PtrVal = getAtomicPointer();
158 PtrVal = Builder->CreateAddrSpaceCast(V: PtrVal, DestTy: PointerType::getUnqual(C&: Ctx));
159 Args.push_back(Elt: PtrVal);
160
161 auto CurrentIP = Builder->saveIP();
162 Builder->restoreIP(IP: AllocaIP);
163 Value *SourceAlloca = Builder->CreateAlloca(Ty: Source->getType());
164 Builder->restoreIP(IP: CurrentIP);
165 Builder->CreateStore(Val: Source, Ptr: SourceAlloca);
166 SourceAlloca = Builder->CreatePointerBitCastOrAddrSpaceCast(
167 V: SourceAlloca, DestTy: Builder->getPtrTy());
168 Args.push_back(Elt: SourceAlloca);
169
170 Constant *OrderingVal =
171 ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: (int)toCABI(AO));
172 Args.push_back(Elt: OrderingVal);
173
174 SmallVector<Type *, 6> ArgTys;
175 for (Value *Arg : Args)
176 ArgTys.push_back(Elt: Arg->getType());
177 FunctionType *FnType = FunctionType::get(Result: Type::getVoidTy(C&: Ctx), Params: ArgTys, isVarArg: false);
178 FunctionCallee LibcallFn =
179 M->getOrInsertFunction(Name: "__atomic_store", T: FnType, AttributeList: Attr);
180 CallInst *Call = Builder->CreateCall(Callee: LibcallFn, Args);
181 Call->setAttributes(Attr);
182}
183
184std::pair<Value *, Value *> AtomicInfo::EmitAtomicCompareExchange(
185 Value *ExpectedVal, Value *DesiredVal, AtomicOrdering Success,
186 AtomicOrdering Failure, bool IsVolatile, bool IsWeak) {
187 if (shouldUseLibcall())
188 return EmitAtomicCompareExchangeLibcall(ExpectedVal, DesiredVal, Success,
189 Failure);
190
191 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
192 Failure, IsVolatile, IsWeak);
193 return Res;
194}
195