1//===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass lowers atomic intrinsics to non-atomic form for use in a known
10// non-preemptible environment.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Utils/LowerAtomic.h"
15#include "llvm/IR/Function.h"
16#include "llvm/IR/IRBuilder.h"
17
18using namespace llvm;
19
20#define DEBUG_TYPE "loweratomic"
21
22bool llvm::lowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) {
23 IRBuilder<> Builder(CXI);
24 Value *Ptr = CXI->getPointerOperand();
25 Value *Cmp = CXI->getCompareOperand();
26 Value *Val = CXI->getNewValOperand();
27
28 auto [Orig, Equal] =
29 buildCmpXchgValue(Builder, Ptr, Cmp, Val, Alignment: CXI->getAlign());
30
31 Value *Res =
32 Builder.CreateInsertValue(Agg: PoisonValue::get(T: CXI->getType()), Val: Orig, Idxs: 0);
33 Res = Builder.CreateInsertValue(Agg: Res, Val: Equal, Idxs: 1);
34
35 CXI->replaceAllUsesWith(V: Res);
36 CXI->eraseFromParent();
37 return true;
38}
39
40std::pair<Value *, Value *> llvm::buildCmpXchgValue(IRBuilderBase &Builder,
41 Value *Ptr, Value *Cmp,
42 Value *Val,
43 Align Alignment) {
44 LoadInst *Orig = Builder.CreateAlignedLoad(Ty: Val->getType(), Ptr, Align: Alignment);
45 Value *Equal = Builder.CreateICmpEQ(LHS: Orig, RHS: Cmp);
46 Value *Res = Builder.CreateSelect(C: Equal, True: Val, False: Orig);
47 Builder.CreateAlignedStore(Val: Res, Ptr, Align: Alignment);
48
49 return {Orig, Equal};
50}
51
52Value *llvm::buildAtomicRMWValue(AtomicRMWInst::BinOp Op,
53 IRBuilderBase &Builder, Value *Loaded,
54 Value *Val) {
55 Value *NewVal;
56 switch (Op) {
57 case AtomicRMWInst::Xchg:
58 return Val;
59 case AtomicRMWInst::Add:
60 return Builder.CreateAdd(LHS: Loaded, RHS: Val, Name: "new");
61 case AtomicRMWInst::Sub:
62 return Builder.CreateSub(LHS: Loaded, RHS: Val, Name: "new");
63 case AtomicRMWInst::And:
64 return Builder.CreateAnd(LHS: Loaded, RHS: Val, Name: "new");
65 case AtomicRMWInst::Nand:
66 return Builder.CreateNot(V: Builder.CreateAnd(LHS: Loaded, RHS: Val), Name: "new");
67 case AtomicRMWInst::Or:
68 return Builder.CreateOr(LHS: Loaded, RHS: Val, Name: "new");
69 case AtomicRMWInst::Xor:
70 return Builder.CreateXor(LHS: Loaded, RHS: Val, Name: "new");
71 case AtomicRMWInst::Max:
72 NewVal = Builder.CreateICmpSGT(LHS: Loaded, RHS: Val);
73 return Builder.CreateSelect(C: NewVal, True: Loaded, False: Val, Name: "new");
74 case AtomicRMWInst::Min:
75 NewVal = Builder.CreateICmpSLE(LHS: Loaded, RHS: Val);
76 return Builder.CreateSelect(C: NewVal, True: Loaded, False: Val, Name: "new");
77 case AtomicRMWInst::UMax:
78 NewVal = Builder.CreateICmpUGT(LHS: Loaded, RHS: Val);
79 return Builder.CreateSelect(C: NewVal, True: Loaded, False: Val, Name: "new");
80 case AtomicRMWInst::UMin:
81 NewVal = Builder.CreateICmpULE(LHS: Loaded, RHS: Val);
82 return Builder.CreateSelect(C: NewVal, True: Loaded, False: Val, Name: "new");
83 case AtomicRMWInst::FAdd:
84 return Builder.CreateFAdd(L: Loaded, R: Val, Name: "new");
85 case AtomicRMWInst::FSub:
86 return Builder.CreateFSub(L: Loaded, R: Val, Name: "new");
87 case AtomicRMWInst::FMax:
88 return Builder.CreateMaxNum(LHS: Loaded, RHS: Val);
89 case AtomicRMWInst::FMin:
90 return Builder.CreateMinNum(LHS: Loaded, RHS: Val);
91 case AtomicRMWInst::FMaximum:
92 return Builder.CreateMaximum(LHS: Loaded, RHS: Val);
93 case AtomicRMWInst::FMinimum:
94 return Builder.CreateMinimum(LHS: Loaded, RHS: Val);
95 case AtomicRMWInst::UIncWrap: {
96 Constant *One = ConstantInt::get(Ty: Loaded->getType(), V: 1);
97 Value *Inc = Builder.CreateAdd(LHS: Loaded, RHS: One);
98 Value *Cmp = Builder.CreateICmpUGE(LHS: Loaded, RHS: Val);
99 Constant *Zero = ConstantInt::get(Ty: Loaded->getType(), V: 0);
100 return Builder.CreateSelect(C: Cmp, True: Zero, False: Inc, Name: "new");
101 }
102 case AtomicRMWInst::UDecWrap: {
103 Constant *Zero = ConstantInt::get(Ty: Loaded->getType(), V: 0);
104 Constant *One = ConstantInt::get(Ty: Loaded->getType(), V: 1);
105
106 Value *Dec = Builder.CreateSub(LHS: Loaded, RHS: One);
107 Value *CmpEq0 = Builder.CreateICmpEQ(LHS: Loaded, RHS: Zero);
108 Value *CmpOldGtVal = Builder.CreateICmpUGT(LHS: Loaded, RHS: Val);
109 Value *Or = Builder.CreateOr(LHS: CmpEq0, RHS: CmpOldGtVal);
110 return Builder.CreateSelect(C: Or, True: Val, False: Dec, Name: "new");
111 }
112 case AtomicRMWInst::USubCond: {
113 Value *Cmp = Builder.CreateICmpUGE(LHS: Loaded, RHS: Val);
114 Value *Sub = Builder.CreateSub(LHS: Loaded, RHS: Val);
115 return Builder.CreateSelect(C: Cmp, True: Sub, False: Loaded, Name: "new");
116 }
117 case AtomicRMWInst::USubSat:
118 return Builder.CreateIntrinsic(ID: Intrinsic::usub_sat, Types: Loaded->getType(),
119 Args: {Loaded, Val}, FMFSource: nullptr, Name: "new");
120 default:
121 llvm_unreachable("Unknown atomic op");
122 }
123}
124
125bool llvm::lowerAtomicRMWInst(AtomicRMWInst *RMWI) {
126 IRBuilder<> Builder(RMWI);
127 Builder.setIsFPConstrained(
128 RMWI->getFunction()->hasFnAttribute(Kind: Attribute::StrictFP));
129
130 Value *Ptr = RMWI->getPointerOperand();
131 Value *Val = RMWI->getValOperand();
132
133 LoadInst *Orig = Builder.CreateLoad(Ty: Val->getType(), Ptr);
134 Value *Res = buildAtomicRMWValue(Op: RMWI->getOperation(), Builder, Loaded: Orig, Val);
135 Builder.CreateStore(Val: Res, Ptr);
136 RMWI->replaceAllUsesWith(V: Orig);
137 RMWI->eraseFromParent();
138 return true;
139}
140