1//===--- Atomic.cpp - Codegen of atomic operations ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/Frontend/Atomic/Atomic.h"
10#include "llvm/IR/DerivedTypes.h"
11#include "llvm/IR/IRBuilder.h"
12#include <utility>
13
14using namespace llvm;
15
16bool AtomicInfo::shouldCastToInt(Type *ValTy, bool CmpXchg) {
17 if (ValTy->isFloatingPointTy())
18 return ValTy->isX86_FP80Ty() || CmpXchg;
19 return !ValTy->isIntegerTy() && !ValTy->isPointerTy();
20}
21
22Value *AtomicInfo::EmitAtomicLoadOp(AtomicOrdering AO, bool IsVolatile,
23 bool CmpXchg) {
24 Value *Ptr = getAtomicPointer();
25 Type *AtomicTy = Ty;
26 if (shouldCastToInt(ValTy: Ty, CmpXchg))
27 AtomicTy = IntegerType::get(C&: getLLVMContext(), NumBits: AtomicSizeInBits);
28 LoadInst *Load =
29 Builder->CreateAlignedLoad(Ty: AtomicTy, Ptr, Align: AtomicAlign, Name: "atomic-load");
30 Load->setAtomic(Ordering: AO);
31 if (IsVolatile)
32 Load->setVolatile(true);
33 decorateWithTBAA(I: Load);
34 return Load;
35}
36
37CallInst *AtomicInfo::EmitAtomicLibcall(StringRef fnName, Type *ResultType,
38 ArrayRef<Value *> Args) {
39 LLVMContext &ctx = Builder->getContext();
40 SmallVector<Type *, 6> ArgTys;
41 for (Value *Arg : Args)
42 ArgTys.push_back(Elt: Arg->getType());
43 FunctionType *FnType = FunctionType::get(Result: ResultType, Params: ArgTys, isVarArg: false);
44 Module *M = Builder->GetInsertBlock()->getModule();
45
46 // TODO: Use llvm::TargetLowering for Libcall ABI
47 AttrBuilder fnAttrBuilder(ctx);
48 fnAttrBuilder.addAttribute(Val: Attribute::NoUnwind);
49 fnAttrBuilder.addAttribute(Val: Attribute::WillReturn);
50 AttributeList fnAttrs =
51 AttributeList::get(C&: ctx, Index: AttributeList::FunctionIndex, B: fnAttrBuilder);
52 FunctionCallee LibcallFn = M->getOrInsertFunction(Name: fnName, T: FnType, AttributeList: fnAttrs);
53 CallInst *Call = Builder->CreateCall(Callee: LibcallFn, Args);
54 return Call;
55}
56
57std::pair<Value *, Value *> AtomicInfo::EmitAtomicCompareExchangeLibcall(
58 Value *ExpectedVal, Value *DesiredVal, AtomicOrdering Success,
59 AtomicOrdering Failure) {
60 LLVMContext &ctx = getLLVMContext();
61
62 // __atomic_compare_exchange's expected and desired are passed by pointers
63 // FIXME: types
64
65 // TODO: Get from llvm::TargetMachine / clang::TargetInfo
66 // if clang shares this codegen in future
67 constexpr uint64_t IntBits = 32;
68
69 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
70 // void *desired, int success, int failure);
71
72 Value *Args[6] = {
73 getAtomicSizeValue(),
74 getAtomicPointer(),
75 ExpectedVal,
76 DesiredVal,
77 Constant::getIntegerValue(Ty: IntegerType::get(C&: ctx, NumBits: IntBits),
78 V: APInt(IntBits,
79 static_cast<uint64_t>(toCABI(AO: Success)),
80 /*signed=*/true)),
81 Constant::getIntegerValue(Ty: IntegerType::get(C&: ctx, NumBits: IntBits),
82 V: APInt(IntBits,
83 static_cast<uint64_t>(toCABI(AO: Failure)),
84 /*signed=*/true)),
85 };
86 auto Result = EmitAtomicLibcall(fnName: "__atomic_compare_exchange",
87 ResultType: IntegerType::getInt1Ty(C&: ctx), Args);
88 return std::make_pair(x&: ExpectedVal, y&: Result);
89}
90
91std::pair<Value *, Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
92 Value *ExpectedVal, Value *DesiredVal, AtomicOrdering Success,
93 AtomicOrdering Failure, bool IsVolatile, bool IsWeak) {
94 // Do the atomic store.
95 Value *Addr = getAtomicAddressAsAtomicIntPointer();
96 auto *Inst = Builder->CreateAtomicCmpXchg(Ptr: Addr, Cmp: ExpectedVal, New: DesiredVal,
97 Align: getAtomicAlignment(), SuccessOrdering: Success,
98 FailureOrdering: Failure, SSID: SyncScope::System);
99
100 // Other decoration.
101 Inst->setVolatile(IsVolatile);
102 Inst->setWeak(IsWeak);
103 auto *PreviousVal = Builder->CreateExtractValue(Agg: Inst, /*Idxs=*/0);
104 auto *SuccessFailureVal = Builder->CreateExtractValue(Agg: Inst, /*Idxs=*/1);
105 return std::make_pair(x&: PreviousVal, y&: SuccessFailureVal);
106}
107
108std::pair<LoadInst *, AllocaInst *>
109AtomicInfo::EmitAtomicLoadLibcall(AtomicOrdering AO) {
110 LLVMContext &Ctx = getLLVMContext();
111 Type *SizedIntTy = Type::getIntNTy(C&: Ctx, N: getAtomicSizeInBits());
112 Type *ResultTy;
113 SmallVector<Value *, 6> Args;
114 AttributeList Attr;
115 Module *M = Builder->GetInsertBlock()->getModule();
116 const DataLayout &DL = M->getDataLayout();
117 Args.push_back(
118 Elt: ConstantInt::get(Ty: DL.getIntPtrType(C&: Ctx), V: this->getAtomicSizeInBits() / 8));
119
120 Value *PtrVal = getAtomicPointer();
121 PtrVal = Builder->CreateAddrSpaceCast(V: PtrVal, DestTy: PointerType::getUnqual(C&: Ctx));
122 Args.push_back(Elt: PtrVal);
123
124 auto CurrentIP = Builder->saveIP();
125 Builder->restoreIP(IP: AllocaIP);
126 AllocaInst *AllocaResult =
127 CreateAlloca(Ty, Name: getAtomicPointer()->getName() + "atomic.temp.load");
128 Builder->restoreIP(IP: CurrentIP);
129 const Align AllocaAlignment = DL.getPrefTypeAlign(Ty: SizedIntTy);
130 AllocaResult->setAlignment(AllocaAlignment);
131 Args.push_back(Elt: AllocaResult);
132 Constant *OrderingVal =
133 ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: (int)toCABI(AO));
134 Args.push_back(Elt: OrderingVal);
135
136 ResultTy = Type::getVoidTy(C&: Ctx);
137 SmallVector<Type *, 6> ArgTys;
138 for (Value *Arg : Args)
139 ArgTys.push_back(Elt: Arg->getType());
140 FunctionType *FnType = FunctionType::get(Result: ResultTy, Params: ArgTys, isVarArg: false);
141 FunctionCallee LibcallFn =
142 M->getOrInsertFunction(Name: "__atomic_load", T: FnType, AttributeList: Attr);
143 CallInst *Call = Builder->CreateCall(Callee: LibcallFn, Args);
144 Call->setAttributes(Attr);
145 return std::make_pair(
146 x: Builder->CreateAlignedLoad(Ty, Ptr: AllocaResult, Align: AllocaAlignment),
147 y&: AllocaResult);
148}
149
150void AtomicInfo::EmitAtomicStoreLibcall(AtomicOrdering AO, Value *Source) {
151 LLVMContext &Ctx = getLLVMContext();
152 SmallVector<Value *, 6> Args;
153 AttributeList Attr;
154 Module *M = Builder->GetInsertBlock()->getModule();
155 const DataLayout &DL = M->getDataLayout();
156 Args.push_back(
157 Elt: ConstantInt::get(Ty: DL.getIntPtrType(C&: Ctx), V: this->getAtomicSizeInBits() / 8));
158
159 Value *PtrVal = getAtomicPointer();
160 PtrVal = Builder->CreateAddrSpaceCast(V: PtrVal, DestTy: PointerType::getUnqual(C&: Ctx));
161 Args.push_back(Elt: PtrVal);
162
163 auto CurrentIP = Builder->saveIP();
164 Builder->restoreIP(IP: AllocaIP);
165 Value *SourceAlloca = Builder->CreateAlloca(Ty: Source->getType());
166 Builder->restoreIP(IP: CurrentIP);
167 Builder->CreateStore(Val: Source, Ptr: SourceAlloca);
168 SourceAlloca = Builder->CreatePointerBitCastOrAddrSpaceCast(
169 V: SourceAlloca, DestTy: Builder->getPtrTy());
170 Args.push_back(Elt: SourceAlloca);
171
172 Constant *OrderingVal =
173 ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: (int)toCABI(AO));
174 Args.push_back(Elt: OrderingVal);
175
176 SmallVector<Type *, 6> ArgTys;
177 for (Value *Arg : Args)
178 ArgTys.push_back(Elt: Arg->getType());
179 FunctionType *FnType = FunctionType::get(Result: Type::getVoidTy(C&: Ctx), Params: ArgTys, isVarArg: false);
180 FunctionCallee LibcallFn =
181 M->getOrInsertFunction(Name: "__atomic_store", T: FnType, AttributeList: Attr);
182 CallInst *Call = Builder->CreateCall(Callee: LibcallFn, Args);
183 Call->setAttributes(Attr);
184}
185
186std::pair<Value *, Value *> AtomicInfo::EmitAtomicCompareExchange(
187 Value *ExpectedVal, Value *DesiredVal, AtomicOrdering Success,
188 AtomicOrdering Failure, bool IsVolatile, bool IsWeak) {
189 if (shouldUseLibcall())
190 return EmitAtomicCompareExchangeLibcall(ExpectedVal, DesiredVal, Success,
191 Failure);
192
193 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
194 Failure, IsVolatile, IsWeak);
195 return Res;
196}
197